function randomIntFromInterval(min, max) { // min and max included 
  return Math.floor(Math.random() * (max - min + 1) + min);
}

const rndInt = randomIntFromInterval(1, 6);
console.log(rndInt);

What it does "extra" is it allows random intervals that do not start with 1. So you can get a random number from 10 to 15 for example. Flexibility.

What You really want to do is to get [1, 2, 3] array and shuffle it. I would do it like this:

const numbers = [];
const shuffled = [];

// prepare array of sorted numbers [1, 2, 3]
for(let i=0; i<scoreDisplay.length; i++)
  numbers.push(i + 1);

// shuffle the array by picking random position in sorted array and moving it to result array
while (numbers.length > 0)
  shuffled.push(numbers.splice(Math.floor(Math.random() * numbers.length), 1)[0]);

// print result
for(let i=0; i<scoreDisplay.length; i++)
    scoreDisplay[i].textContent = shuffled[i];