<!DOCTYPE HTML>
<html>
<head>
<title>A/V sync test for stream capturing</title>
<script src="/tests/SimpleTest/SimpleTest.js"></script>
<link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
<p>Following canvas will capture and show the video frame when the video becomes audible</p><br>
<canvas id="canvas" width="640" height="480"></canvas>
<script type="application/javascript">

/**
 * This test will capture stream before the video starts playing, and check if
 * A/V sync will keep sync during playing.
 */
add_task(async function testAVSyncForStreamCapturing() {
  createVideo();
  captureStreamFromVideo();
  await playMedia();
  await testAVSync();
  destroyVideo();
});

/**
 * This test will check if A/V is still on sync after we switch the media sink
 * from playback-based sink to mediatrack-based sink.
 */
add_task(async function testAVSyncWhenSwitchingMediaSink() {
  createVideo();
  await playMedia({resolveAfterReceivingTimeupdate : 5});
  captureStreamFromVideo();
  await testAVSync();
  destroyVideo();
});

/**
 * This test will check if A/V is still on sync after we change the playback
 * rate on the captured stream.
 */
add_task(async function testAVSyncWhenChangingPlaybackRate() {
  createVideo();
  captureStreamFromVideo();
  await playMedia();
  const playbackRates = [0.25, 0.5, 1.0, 1.5, 2.0];
  for (let rate of playbackRates) {
    setPlaybackRate(rate);
    // TODO : when playback rate set to 1.5+x, the A/V will become less stable
    // in testing so we raise the fuzzy frames for that, but also increase the
    // test times. As at that speed, precise A/V becomes trivial because we
    // can't really tell the difference. But it would be good for us to
    // investigate if we could make A/V sync work better at that extreme high
    // rate.
    if (rate >= 1.5) {
      await testAVSync({ expectedAVSyncTestTimes : 4, fuzzyFrames : 10});
    } else {
      await testAVSync({ expectedAVSyncTestTimes : 2 });
    }
  }
  destroyVideo();
});

/**
 * Following are helper functions
 */
const DEBUG = false;
function info_debug(msg) {
  if (DEBUG) {
    info(msg);
  }
}

function createVideo() {
  const video = document.createElement("video");
  // This video is special for testing A/V sync, it only produce audible sound
  // once per second, and when the sound comes out, you can check the position
  // of the square to know if the A/V keeps sync.
  video.src = "sync.webm";
  video.loop = true;
  video.controls = true;
  video.width = 640;
  video.height = 480;
  video.id = "video";
  document.body.appendChild(video);
}

function destroyVideo() {
  const video = document.getElementById("video");
  video.src = null;
  video.remove();
}

async function playMedia({ resolveAfterReceivingTimeupdate } = {}) {
  const video = document.getElementById("video");
  ok(await video.play().then(_=>true,_=>false), "video started playing");
  if (resolveAfterReceivingTimeupdate > 0) {
    // Play it for a while to ensure the clock growing on the normal audio sink.
    for (let idx = 0; idx < resolveAfterReceivingTimeupdate; idx++) {
      await new Promise(r => video.ontimeupdate = r);
    }
  }
}

async function captureStreamFromVideo() {
  const video = document.getElementById("video");
  let ac = new AudioContext();
  let analyser = ac.createAnalyser();
  analyser.frequencyBuf = new Float32Array(analyser.frequencyBinCount);
  analyser.smoothingTimeConstant = 0;
  analyser.fftSize = 2048; // 1024 bins
  let sourceNode = ac.createMediaElementSource(video);
  sourceNode.connect(analyser);
  analyser.connect(ac.destination);
  video.analyser = analyser;
}

// This method will capture the stream from the video element, and check if A/V
// keeps sync during capturing. `callback` parameter will be executed after
// finishing capturing.
// @param [optional] expectedAVSyncTestTimes
//        The amount of times that A/V sync test performs.
// @param [optional] fuzzyFrames
//        This will fuzz the result from +0 (perfect sync) to -X to +X frames.
async function testAVSync({ expectedAVSyncTestTimes = 5, fuzzyFrames = 5} = {}) {
  return new Promise(r => {
    const analyser = document.getElementById("video").analyser;
    let testIdx = 0;
    let hasDetectedAudibleFrame = false;
    // As we only want to detect the audible frame at the first moment when
    // sound becomes audible, so we always skip the first audible frame because
    // it might not be the start, but the tail part (where audio is being
    // decaying to silence) when we start detecting.
    let hasSkippedFirstFrame = false;
    analyser.notifyAnalysis = () => {
      let {frequencyBuf} = analyser;
      analyser.getFloatFrequencyData(frequencyBuf);
      if (checkIfBufferIsSilent(frequencyBuf)) {
        info_debug("no need to paint the silent frame");
        hasDetectedAudibleFrame = false;
        requestAnimationFrame(analyser.notifyAnalysis);
        return;
      }
      if (hasDetectedAudibleFrame) {
        info_debug("detected audible frame already");
        requestAnimationFrame(analyser.notifyAnalysis);
        return;
      }
      hasDetectedAudibleFrame = true;
      if (!hasSkippedFirstFrame) {
        info("skip the first audible frame");
        hasSkippedFirstFrame = true;
        requestAnimationFrame(analyser.notifyAnalysis);
        return;
      }
      const video = document.getElementById("video");
      info(`paint audible frame`);
      const cvs = document.getElementById("canvas");
      let context = cvs.getContext('2d');
      context.drawImage(video, 0, 0, 640, 480);
      if (checkIfAVIsOnSyncFuzzy(context, fuzzyFrames)) {
        ok(true, `test ${testIdx++} times, a/v is in sync!`);
      } else {
        ok(false, `test ${testIdx++} times, a/v is out of sync!`);
      }
      if (testIdx == expectedAVSyncTestTimes) {
        r();
        return;
      }
      requestAnimationFrame(analyser.notifyAnalysis);
    }
    analyser.notifyAnalysis();
  });
}

function checkIfBufferIsSilent(buffer) {
  for (let data of buffer) {
    // when sound is audible, its values are around -200 and the silence values
    // are around -800.
    if (data > -200) {
      return false;
    }
  }
  return true;
}

// This function will check the pixel data from the `context` to see if the
// square appears in the right place. As we can't control the exact timing
// of rendering video frames in the compositor, so the result would be fuzzy.
function checkIfAVIsOnSyncFuzzy(context, fuzzyFrames) {
  const squareLength = 48;
  // Canvas is 640*480, so perfect sync is the left-top corner when the square
  // shows up in the middle.
  const perfectSync =
      { x: 320 - squareLength/2.0 ,
        y: 240 - squareLength/2.0 };
  let isAVSyncFuzzy = false;
  // Get the whole partial section of image and detect where the square is.
  let imageData = context.getImageData(0, perfectSync.y, 640, squareLength);
  for (let i = 0; i < imageData.data.length; i += 4) {
    // If the pixel's color is red, then this position will be the left-top
    // corner of the square.
    if (isPixelColorRed(imageData.data[i], imageData.data[i+1],
                        imageData.data[i+2])) {
      const pos = ImageIdxToRelativeCoordinate(imageData, i);
      let diff = calculateFrameDiffenceInXAxis(pos.x, perfectSync.x);
      info(`find the square in diff=${diff}`);
      // Maybe we check A/V sync too early or too late, try to adjust the diff
      // to guess the previous correct position where the square should be.
      if (diff > fuzzyFrames) {
        diff = adjustFrameDiffBasedOnMediaTime(diff);
        const video = document.getElementById("video");
        info(`adjusted diff to ${diff} (time=${video.currentTime})`);
      }
      if (diff <= fuzzyFrames) {
        isAVSyncFuzzy = true;
      }
      context.putImageData(imageData, 0, 0);
      break;
    }
  }
  if (!isAVSyncFuzzy) {
    const ctx = document.getElementById('canvas');
    info(ctx.toDataURL());
  }
  return isAVSyncFuzzy;
}

// Input an imageData and its idx, then return a relative coordinate on the
// range of given imageData.
function ImageIdxToRelativeCoordinate(imageData, idx) {
  const offset = idx / 4; // RGBA
  return { x: offset % imageData.width, y: offset / imageData.width };
}

function calculateFrameDiffenceInXAxis(squareX, targetX) {
  const offsetX = Math.abs(targetX - squareX);
  const xSpeedPerFrame = 640 / 60; // video is 60fps
  return offsetX / xSpeedPerFrame;
}

function isPixelColorRed(r, g, b) {
  // As the rendering color would vary in the screen on different platforms, so
  // we won't strict the R should be 255, just check if it's larger than a
  // certain threshold.
  return r > 200 && g < 10 && b < 10;
}

function setPlaybackRate(rate) {
  const video = document.getElementById("video");
  info(`change playback rate from ${video.playbackRate} to ${rate}`);
  document.getElementById("video").playbackRate = rate;
}

function adjustFrameDiffBasedOnMediaTime(currentDiff) {
  // The audio wave can be simply regarded as being composed by "start", "peak"
  // and "tail". The "start" part is the sound gradually becoming louder and the
  // "tail" is gradually becoming silent. We want to check the "peak" part which
  // should happn on evert second regularly (1s, 2s, 3s ...) However, this check
  // is triggered by `requestAnimationFrame()` and we can't guarantee that
  // we're checking the peak part while the function is being called. Therefore,
  // we have to do an adjustment by the video time, to know if we're checking
  // the audio wave too early or too late in order to get a consistent result.
  const video = document.getElementById("video");
  const videoCurrentTimeFloatPortion = video.currentTime % 1;
  const timeOffset =
      videoCurrentTimeFloatPortion > 0.5 ?
          1 - videoCurrentTimeFloatPortion : // too early
          videoCurrentTimeFloatPortion;      // too late
  const frameOffset = timeOffset / 0.016; // 60fps, 1 frame=0.016s
  info(`timeOffset=${timeOffset}, frameOffset=${frameOffset}`);
  return Math.abs(currentDiff - frameOffset);
}

</script>
</head>
<body>
</body>
</html>
