You're getting confused with your various temporary arrays. The undefined values are caused by out-of-bounds access on the line below your innermost loop.

I recommend that you stick to making a single array for the result of the multiplication. As you're probably aware, the hitch is that JavaScript doesn't allow you to initialize a multi-dimensional array. To make a two-dimensional array, you have to initialize a one-dimensional array, then iterate over its elements and initialize each one to a one-dimensional array.

function multiply(a, b) {
  var aNumRows = a.length, aNumCols = a[0].length,
      bNumRows = b.length, bNumCols = b[0].length,
      m = new Array(aNumRows);  // initialize array of rows
  for (var r = 0; r < aNumRows; ++r) {
    m[r] = new Array(bNumCols); // initialize the current row
    for (var c = 0; c < bNumCols; ++c) {
      m[r][c] = 0;             // initialize the current cell
      for (var i = 0; i < aNumCols; ++i) {
        m[r][c] += a[r][i] * b[i][c];
      }
    }
  }
  return m;
}

function display(m) {
  for (var r = 0; r < m.length; ++r) {
    document.write('&nbsp;&nbsp;'+m[r].join(' ')+'<br />');
  }
}

var a = [[8, 3], [2, 4], [3, 6]],
    b = [[1, 2, 3], [4, 6, 8]];
document.write('matrix a:<br />');
display(a);
document.write('matrix b:<br />');
display(b);
document.write('a * b =<br />');
display(multiply(a, b));

Array.fill

Consider using fill:

Array(9).fill().map(()=>Array(9).fill())

The idea here is that fill() will fill out the items with undefined, which is enough to get map to work on them.

You could also fill directly:

Array(9).fill(Array(9))

(important note: Array(9).fill(Array(9)) will fill each row of the array with the same array, so changing one row will change the other rows).

Alternatives to Array(9).fill() include

Array(...Array(9))
[].push(...Array(9))
[].concat(Array(9))
Array.from(Array(9))

We can rewrite the solution a bit more semantically as:

function array9() { return Array(9).fill(); }
array9().map(array9)

or

function array(n) { return Array(n).fill(); }
array(9).map(() => array(9))

Array.from provides us with an optional second mapping argument, so we have the alternative of writing

Array.from(Array(9), () => Array.from(Array(9));

or, if you prefer

function array9(map) { return Array.from(Array(9), map); }
array9(array9);

For verbose description and examples, see Mozilla's Docs on Array.prototype.fill() here.
and for Array.from(), here.

Note that neither Array.prototype.fill() nor Array.from() has support in Internet Explorer. A polyfill for IE is available at the above MDN links.

Partitioning

partition(Array(81), 9)

if you have a partition utility handy. Here's a quick recursive one:

function partition(a, n) {
  return a.length ? [a.splice(0, n)].concat(partition(a, n)) : [];
}  

Looping

We can loop a bit more efficiently with

var a = [], b;
while (a.push(b = []) < 9) while (b.push(null) < 9);

Taking advantage of the fact that push returns the new array length.

Practically? Yes. You can create an array of arrays which functions as an 2D array as every item is an array itself:

let items = [
  [1, 2],
  [3, 4],
  [5, 6]
];
console.log(items[0][0]); // 1
console.log(items[0][1]); // 2
console.log(items[1][0]); // 3
console.log(items[1][1]); // 4
console.log(items);

But technically this is just an array of arrays and not a “true” 2D array, as I. J. Kennedy pointed out.

It should be noted that you could keep nesting arrays into one another and so create “multidimensional” arrays.