Connected to E2E pipeline (#85)

* added RetrieveSentences.py * removed index * connet to E2E pipeline * updated code * Some modification * clean up * config * documentation * added documentation * update documentation * update doc * update doc * add js * updated documentation * changed doc * changed requirements.txt
2026-09-12 12:02:24 +08:00 · 2017-11-25 14:39:39 -05:00
parent 61c8c0e622
commit ad69d3abc2
15 changed files with 871 additions and 18 deletions
@@ -0,0 +1,182 @@
+/*
+Modifications Copyright 2017 Anserini
+
+The audio recording and analyzer visualization code was originally developed by Chris Wilson
+and modified for use in Anserini.
+
+Copyright 2013 Chris Wilson
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+window.AudioContext = window.AudioContext || window.webkitAudioContext;
+
+var audioContext = new AudioContext();
+var audioInput = null,
+  realAudioInput = null,
+  inputPoint = null,
+  audioRecorder = null;
+var rafID = null;
+var analyserContext = null;
+var canvasWidth, canvasHeight;
+var recIndex = 0;
+
+function saveAudio() {
+  //audioRecorder.exportWAV( doneEncoding );
+  // could get mono instead by saying
+   audioRecorder.exportMonoWAV( doneEncoding );
+}
+
+function gotBuffers(buffers) {
+  // the ONLY time gotBuffers is called is right after a new recording is completed.
+  audioRecorder.exportMonoWAV( doneEncoding );
+}
+
+function doneEncoding(blob) {
+  Recorder.speechToText(blob);
+  recIndex++;
+}
+
+function toggleRecording(e) {
+  if (e.classList.contains("recording")) {
+    // stop recording
+    audioRecorder.stop();
+    e.classList.remove("recording");
+    $('#question').text("Trying to understand your query...");
+    audioRecorder.getBuffers( gotBuffers );
+  } else {
+    // start recording
+    if (!audioRecorder)
+      return;
+    e.classList.add("recording");
+    $('#question').text("Listening...");
+    $('#answer').html("");
+    audioRecorder.clear();
+    audioRecorder.record();
+  }
+}
+
+function convertToMono(input) {
+  var splitter = audioContext.createChannelSplitter(2);
+  var merger = audioContext.createChannelMerger(2);
+
+  input.connect(splitter);
+  splitter.connect(merger, 0, 0);
+  splitter.connect(merger, 0, 1);
+  return merger;
+}
+
+function cancelAnalyserUpdates() {
+  window.cancelAnimationFrame(rafID);
+  rafID = null;
+}
+
+function updateAnalysers(time) {
+  if (!analyserContext) {
+    var canvas = document.getElementById("analyser");
+    canvasWidth = canvas.width;
+    canvasHeight = canvas.height;
+    analyserContext = canvas.getContext('2d');
+  }
+
+  // analyzer draw code here
+  {
+    var SPACING = 5;
+    var BAR_WIDTH = 3;
+    var numBars = Math.round(canvasWidth / SPACING);
+    var freqByteData = new Uint8Array(analyserNode.frequencyBinCount);
+
+    analyserNode.getByteFrequencyData(freqByteData);
+
+    analyserContext.clearRect(0, 0, canvasWidth, canvasHeight);
+    analyserContext.fillStyle = '#F6D565';
+    analyserContext.lineCap = 'round';
+    var multiplier = analyserNode.frequencyBinCount / numBars;
+
+    // Draw rectangle for each frequency bin.
+    for (var i = 0; i < numBars; ++i) {
+      var magnitude = 0;
+      var offset = Math.floor( i * multiplier );
+      // gotta sum/average the block, or we miss narrow-bandwidth spikes
+      for (var j = 0; j< multiplier; j++)
+        magnitude += freqByteData[offset + j];
+      magnitude = magnitude / multiplier;
+      var magnitude2 = freqByteData[i * multiplier];
+      analyserContext.fillStyle = "hsl( " + Math.round((i*360)/numBars) + ", 100%, 50%)";
+      analyserContext.fillRect(i * SPACING, canvasHeight, BAR_WIDTH, -magnitude);
+    }
+  }
+
+  rafID = window.requestAnimationFrame(updateAnalysers);
+}
+
+function toggleMono() {
+  if (audioInput != realAudioInput) {
+    audioInput.disconnect();
+    realAudioInput.disconnect();
+    audioInput = realAudioInput;
+  } else {
+    realAudioInput.disconnect();
+    audioInput = convertToMono( realAudioInput );
+  }
+
+  audioInput.connect(inputPoint);
+}
+
+function gotStream(stream) {
+  inputPoint = audioContext.createGain();
+
+  // Create an AudioNode from the stream.
+  realAudioInput = audioContext.createMediaStreamSource(stream);
+  audioInput = realAudioInput;
+  audioInput.connect(inputPoint);
+
+  analyserNode = audioContext.createAnalyser();
+  analyserNode.fftSize = 2048;
+  inputPoint.connect( analyserNode );
+
+  audioRecorder = new Recorder( inputPoint );
+
+  zeroGain = audioContext.createGain();
+  zeroGain.gain.value = 0.0;
+  inputPoint.connect( zeroGain );
+  zeroGain.connect( audioContext.destination );
+  updateAnalysers();
+}
+
+function initAudio() {
+  if (!navigator.getUserMedia)
+    navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia;
+  if (!navigator.cancelAnimationFrame)
+    navigator.cancelAnimationFrame = navigator.webkitCancelAnimationFrame || navigator.mozCancelAnimationFrame;
+  if (!navigator.requestAnimationFrame)
+    navigator.requestAnimationFrame = navigator.webkitRequestAnimationFrame || navigator.mozRequestAnimationFrame;
+
+  navigator.getUserMedia(
+    {
+      "audio": {
+        "mandatory": {
+          "googEchoCancellation": "false",
+          "googAutoGainControl": "false",
+          "googNoiseSuppression": "false",
+          "googHighpassFilter": "false"
+        },
+        "optional": []
+      }
+    }, gotStream, function(e) {
+      alert('Error getting audio');
+      console.log(e);
+    });
+}
+
+window.addEventListener('load', initAudio);