If you want a result from a recursive function, all code paths through the function must return something. Your code isn't returning anything in the num!=1 case. It should be returning the result of calling itself, e.g. (see the *** line):

var fact=5;
function calfact(num)
{
 if(num!=1)
  {
   fact=fact*(num-1);
   num=num-1;
   return calfact(num); // ***
  }
 else
  {
   return fact;
  }
}

Your function is using a global variable, which isn't a great idea as it means the funtion isn't self-contained; and isn't a true factorial function, because you're effectively using two inputs (fact — the global  and num, the argument)

If you want a true factorial, you don't need a global variable, just work from the argument itself:

function factorial(num) {
    if (num < 0) {
        throw new Error("num must not be negative");
    }
    if (num <= 1) {
        // Both 1! and 0! are defined as 1
        return 1;
    }
    return num * factorial(num - 1);
}
console.log(factorial(5)); // 120

Or of course, more compactly:

function factorial(num) {
    if (num < 0) {
        throw new Error("num must not be negative");
    }
    return num <= 1 ? 1 : num * factorial(num - 1);
}

(More about 0!: https://en.wikipedia.org/wiki/Factorial)

You don't need recursion for that:

/**
 * Calculate factorial, optionally using a difference other than 1 with previous value.
 * Example: factorial(6, 2) // 6*4*2 = 48
 */
var factorial = function(n, d) {
  if (!d) {d = 1;}
  var product = 1;
  while (n > 1) {
    product *= n;
    n -= d;
  }
  return product;
};

console.log(factorial(6, 2)); // 48
console.log(factorial(6)); // 720

Note: Declare local variables inside the function with keyword 'var'. Otherwise they become globals and the second time you attempt to use a function may produce wrong results.