If you want a result from a recursive function, all code paths through the function must return something. Your code isn't returning anything in the num!=1 case. It should be returning the result of calling itself, e.g. (see the *** line):

var fact=5;
function calfact(num)
{
 if(num!=1)
  {
   fact=fact*(num-1);
   num=num-1;
   return calfact(num); // ***
  }
 else
  {
   return fact;
  }
}

Your function is using a global variable, which isn't a great idea as it means the funtion isn't self-contained; and isn't a true factorial function, because you're effectively using two inputs (fact — the global  and num, the argument)

If you want a true factorial, you don't need a global variable, just work from the argument itself:

function factorial(num) {
    if (num < 0) {
        throw new Error("num must not be negative");
    }
    if (num <= 1) {
        // Both 1! and 0! are defined as 1
        return 1;
    }
    return num * factorial(num - 1);
}
console.log(factorial(5)); // 120

Or of course, more compactly:

function factorial(num) {
    if (num < 0) {
        throw new Error("num must not be negative");
    }
    return num <= 1 ? 1 : num * factorial(num - 1);
}

(More about 0!: https://en.wikipedia.org/wiki/Factorial)