diff --git a/anserini_dependency/README.md b/anserini_dependency/README.md index 53ae9dc..0b51ecf 100644 --- a/anserini_dependency/README.md +++ b/anserini_dependency/README.md @@ -1,15 +1,20 @@ -## Retrieve Sentences +## Setup Retrieve Sentences and end2end QA pipeline -#### 1. Clone [Anserini](https://github.com/castorini/Anserini.git) and [Castor](https://github.com/castorini/Castor.git) +#### 1. Clone [Anserini](https://github.com/castorini/Anserini.git), [Castor](https://github.com/castorini/Castor.git), [data](https://github.com/castorini/data.git), and [models](https://github.com/castorini/models.git): ```bash git clone https://github.com/castorini/Anserini.git git clone https://github.com/castorini/Castor.git +git clone https://github.com/castorini/data.git +git clone https://github.com/castorini/models.git ``` Your directory structure should look like ``` +. ├── Anserini -└── Castor +├── Castor +├── data +└── models ``` #### 2. Compile Anserini @@ -17,15 +22,35 @@ Your directory structure should look like ```bash cd Anserini mvn package +cd .. ``` This creates `anserini-0.0.1-SNAPSHOT.jar` at `Anserini/target` +We highly recommend the use of [virtualenv](https://virtualenv.pypa.io/en/stable/) as the dependencies +are subjected to frequent changes. + +Install the dependency packages: + +``` +cd Castor +pip3 install -r requirements.txt +``` +Make sure that you have PyTorch installed. For more help, follow [these](https://github.com/castorini/Castor) steps. + #### 3. Download Dependencies - Download the TrecQA lucene index - Download the Google word2vec file from [here](https://drive.google.com/drive/folders/0B2u_nClt6NbzNWJkWExmaklYNTA?usp=sharing) -#### 4. Run the following command +#### 4. Additional files for pipeline: +As some of the files are too large to be uploaded onto GitHub, please download the following files from +[here](https://drive.google.com/drive/folders/0B2u_nClt6NbzNm1LdjlwUFdzQVE?usp=sharing) and place them +in the appropriate locations: + +- copy the contents of `word2vec` directory to `data/word2vec` +- copy `word2dfs.p` to `data/TrecQA/` + +### To run RetrieveSentences: ```bash python ./anserini_dependency/RetrieveSentences.py @@ -44,3 +69,60 @@ Possible parameters are: | `-k` | [1, inf) | 1 | top-k passages to be retrieved | Note: Either a query or a topic must be passed in as an argument; they can't be both empty. + + +__NB:__ The speech UI cannot be run in Ubuntu. To test the pipeline in Ubuntu, make the following changes: +- Comment out the JavaScript part and run the Bash script +- Make a REST API query to the endpoint using Postman, Curl etc. + +### To setup the demo + +#### 1. Installing libraries for demo + +```sh +cd anserini_dependency/js +npm install +cd ../.. +``` + +#### 2. Flask + +- Flask is used as the server for the API +- Copy `config.cfg.example` to `config.cfg` and make necessary changes, such as setting the index path and API keys. + + +#### 3. Run the Demo + +```sh +./run_ui.sh +``` + +### Additional Notes +- This is the documentation for the API call to send a question to the model and get back the predicted answer. +- The request body fields are: question(required )num_hits(optional) and k(optional). +``` + +# REQUEST: +HTTP Method: POST +Endpoint: [host]:[port]/answer +Content-Type: application/json +text of body in raw format: +{ + "question": "What is the birthdate of Einstein?", + "num_hits": 50, + "k": 30 +} +``` + +- The response body contains answers which is a list of objects with two fields - passage, score. +``` +# RESPONSE: +Content-Type: application/json +text of body in raw format: +{ + "answers": [ + {"passage": "Einstein was born in the 1800s", 'score': 0.976}, + {"passage": "Einstein was a physicist", 'score': 0.524} + ] +} +``` diff --git a/anserini_dependency/RetrieveSentences.py b/anserini_dependency/RetrieveSentences.py index 74dd1ef..87ff2d1 100644 --- a/anserini_dependency/RetrieveSentences.py +++ b/anserini_dependency/RetrieveSentences.py @@ -5,7 +5,7 @@ jnius_config.set_classpath("../Anserini/target/anserini-0.0.1-SNAPSHOT.jar") from jnius import autoclass -class CallRetrieveSentences: +class RetrieveSentences: """Python class built to call RetrieveSentences Attributes ---------- @@ -27,28 +27,51 @@ class CallRetrieveSentences: """ RetrieveSentences = autoclass("io.anserini.qa.RetrieveSentences") Args = autoclass("io.anserini.qa.RetrieveSentences$Args") - String = autoclass("java.lang.String") + self.String = autoclass("java.lang.String") self.args = Args() - index = String(args.index) + index = self.String(args.index) self.args.index = index - embeddings = String(args.embeddings) + embeddings = self.String(args.embeddings) self.args.embeddings = embeddings - topics = String(args.topics) + topics = self.String(args.topics) self.args.topics = topics - query = String(args.query) + query = self.String(args.query) self.args.query = query self.args.hits = int(args.hits) - scorer = String(args.scorer) + scorer = self.String(args.scorer) self.args.scorer = scorer self.args.k = int(args.k) self.rs = RetrieveSentences(self.args) - def getRankedPassages(self): + def getRankedPassages(self, query, index, hits, k): """ - Call RetrieveSentneces.getRankedPassages + Calls RetrieveSentences.getRankedPassages + + Parameters + ---------- + query : str + The query to be searched in the index + index: str + The index + hits: str + The number of document IDs to be returned + k: str + The number of passages to be returned """ - self.rs.getRankedPassages(self.args) + + scorer = self.rs.getRankedPassagesList(query, index, int(hits), int(k)) + candidate_passages_scores = [] + for i in range(0, scorer.size()): + candidate_passages_scores.append(scorer.get(i)) + + return candidate_passages_scores + + def getTermIdfJSON(self): + """ + Calls RetrieveSentences.getTermIdfJSON + """ + return self.rs.getTermIdfJSON() if __name__ == "__main__": parser = argparse.ArgumentParser(description='Retrieve Sentences') @@ -61,9 +84,7 @@ if __name__ == "__main__": parser.add_argument("-k", help="top-k passages to be retrieved", default=1) args_raw = parser.parse_args() - rs = CallRetrieveSentences(args_raw) - rs.getRankedPassages() - - + rs = RetrieveSentences(args_raw) + sc = rs.getRankedPassages(args_raw.query, args_raw.index, args_raw.hits, args_raw.k) diff --git a/anserini_dependency/api.py b/anserini_dependency/api.py new file mode 100755 index 0000000..f8ff62c --- /dev/null +++ b/anserini_dependency/api.py @@ -0,0 +1,122 @@ +import argparse +import configparser +import os +import sys + +from flask import Flask, jsonify, request +# FIXME: separate this out to a classifier class where we can switch out the models + +from RetrieveSentences import RetrieveSentences +from sm_cnn.bridge import SMModelBridge + +app = Flask(__name__) +rs = None + +@app.route("/", methods=['GET']) +def hello(): + return "Hello! The server is working properly... :)" + +@app.route('/answer', methods=['POST']) +def answer(): + try: + req = request.get_json(force=True) + question = req["question"] + num_hits = req.get('num_hits', 30) + k = req.get('k', 20) + print("Question: {}".format(question)) + # FIXME: get the answer from the PyTorch model here + answers = get_answers(question, num_hits, k) + answer_dict = {"answers": answers} + return jsonify(answer_dict) + except Exception as e: + print(e) + error_dict = {"error": "ERROR - could not parse the question or get answer. "} + return jsonify(error_dict) + +@app.route('/wit_ai_config', methods=['GET']) +def wit_ai_config(): + return jsonify({'WITAI_API_SECRET': app.config['Frontend']['witai_api_secret']}) + +# FIXME: separate this out to a classifier class where we can switch out the models +def get_answers(question, num_hits, k): + + parser = argparse.ArgumentParser(description='Retrieve Sentences') + parser.add_argument("-index", help="Lucene index", required=True) + parser.add_argument("-embeddings", help="Path of the word2vec index", default="") + parser.add_argument("-topics", help="topics file", default="") + parser.add_argument("-query", help="a single query", default="") + parser.add_argument("-hits", help="max number of hits to return", default=100) + parser.add_argument("-scorer", help="passage scores", default="Idf") + parser.add_argument("-k", help="top-k passages to be retrieved", default=1) + args_raw = parser.parse_args(["-query", question, "-hits", str(num_hits), "-scorer", + "Idf", "-k", str(k), "-index", app.config['Flask']['index']]) + + global rs + if rs == None: + rs = RetrieveSentences(args_raw) + candidate_passages_scores = rs.getRankedPassages(question, app.config['Flask']['index'], num_hits, k) + + candidate_sent_scores = [] + candidate_passages_sm = [] + + for ps in candidate_passages_scores: + ps_split = ps.split('\t') + candidate_passages_sm.append(ps_split[0]) + candidate_sent_scores.append((float(ps_split[1]), ps_split[0])) + + if app.config['Flask']['model'] == "sm": + path_to_castorini = os.getcwd() + "/.." + model = SMModelBridge(path_to_castorini + '/models/sm_model/sm_model.fixed_ext_feats_paper.puncts_stay', + path_to_castorini + '/data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache', + app.config['Flask']['index']) + + idf_json = rs.getTermIdfJSON() + flags = { + "punctuation": "", # ignoring for now you can {keep|remove} punctuation + "dash_words": "" # ignoring for now. you can {keep|split} words-with-hyphens + } + answers_list = model.rerank_candidate_answers(question, candidate_passages_sm, idf_json, flags) + sorted_answers = sorted(answers_list, key=lambda x: x[0], reverse=True) + else: + # the re-ranking model chosen is idf + sorted_answers = list(candidate_sent_scores) + + print("in idf:{}".format(sorted_answers)) + answers = [] + for score, sent in sorted_answers: + answers.append({'passage': sent, 'score': score}) + + return answers + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Start the Flask API at the specified host, port') + parser.add_argument('--config', help='config to use', required=False, type=str, default='config.cfg') + parser.add_argument("--debug", help="print debug info", action="store_true") + parser.add_argument("--model", help="[idf|sm]", default="idf") + args = parser.parse_args() + + if not os.path.isfile(args.config): + print("The configuration file ({}) does not exist!".format(args.config)) + sys.exit(1) + + config = configparser.ConfigParser() + config.read(args.config) + + for name, section in config.items(): + if name == 'DEFAULT': + continue + + app.config[name] = {} + for key, value in config.items(name): + app.config[name][key] = value + + app.config['Flask']['model'] = args.model + + print("Config: {}".format(args.config)) + print("Index: {}".format(app.config['Flask']['index'])) + print("Host: {}".format(app.config['Flask']['host'])) + print("Port: {}".format(app.config['Flask']['port'])) + print("Re-ranking Model: {}".format(app.config['Flask']['model'])) + print("Debug info: {}".format(args.debug)) + + app.run(debug=args.debug, host=app.config['Flask']['host'], port=int(app.config['Flask']['port'])) diff --git a/anserini_dependency/js/Icon.icns b/anserini_dependency/js/Icon.icns new file mode 100644 index 0000000..607318a Binary files /dev/null and b/anserini_dependency/js/Icon.icns differ diff --git a/anserini_dependency/js/IconTemplate.png b/anserini_dependency/js/IconTemplate.png new file mode 100644 index 0000000..bb85ee9 Binary files /dev/null and b/anserini_dependency/js/IconTemplate.png differ diff --git a/anserini_dependency/js/IconTemplate@2x.png b/anserini_dependency/js/IconTemplate@2x.png new file mode 100644 index 0000000..2e1e827 Binary files /dev/null and b/anserini_dependency/js/IconTemplate@2x.png differ diff --git a/anserini_dependency/js/index.html b/anserini_dependency/js/index.html new file mode 100644 index 0000000..96f16c8 --- /dev/null +++ b/anserini_dependency/js/index.html @@ -0,0 +1,71 @@ + + + Anserini Speech Demo + + + + + + + + + + + + +
+

What can I help you with?

+

+
+
+ + + + +
+ + diff --git a/anserini_dependency/js/index.js b/anserini_dependency/js/index.js new file mode 100644 index 0000000..b3c86b7 --- /dev/null +++ b/anserini_dependency/js/index.js @@ -0,0 +1,7 @@ +var menubar = require('menubar'); + +var mb = menubar(); + +mb.on('ready', function ready () { + console.log('Speech to text loaded..'); +}); diff --git a/anserini_dependency/js/package.json b/anserini_dependency/js/package.json new file mode 100644 index 0000000..7d3b5f6 --- /dev/null +++ b/anserini_dependency/js/package.json @@ -0,0 +1,17 @@ +{ + "name": "speech-qa-demo", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "build": "electron-packager . SpeechDemo --electron-version 1.6.2 --icon=Icon.icns", + "start": "electron ." + }, + "dependencies": { + "menubar": "^5.2.3" + }, + "devDependencies": { + "electron-packager": "^8.5.2", + "electron": "^1.6.2" + } +} diff --git a/anserini_dependency/js/recorder.js b/anserini_dependency/js/recorder.js new file mode 100644 index 0000000..8612da8 --- /dev/null +++ b/anserini_dependency/js/recorder.js @@ -0,0 +1,154 @@ +/*License (MIT) + + Copyright © 2013 Matt Diamond + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + to permit persons to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of + the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + */ + +function run() { + var WORKER_PATH = 'recorderWorker.js'; + + var Recorder = function(source, cfg) { + var config = cfg || {}; + var bufferLen = config.bufferLen || 4096; + this.context = source.context; + if(!this.context.createScriptProcessor){ + this.node = this.context.createJavaScriptNode(bufferLen, 2, 2); + } else { + this.node = this.context.createScriptProcessor(bufferLen, 2, 2); + } + + var worker = new Worker(config.workerPath || WORKER_PATH); + worker.postMessage({ + command: 'init', + config: { + sampleRate: this.context.sampleRate + } + }); + var recording = false, + currCallback; + + this.node.onaudioprocess = function(e) { + if (!recording) return; + worker.postMessage({ + command: 'record', + buffer: [ + e.inputBuffer.getChannelData(0), + e.inputBuffer.getChannelData(1) + ] + }); + }; + + this.configure = function(cfg) { + for (var prop in cfg){ + if (cfg.hasOwnProperty(prop)) { + config[prop] = cfg[prop]; + } + } + }; + + this.record = function() { + recording = true; + }; + + this.stop = function() { + recording = false; + }; + + this.clear = function() { + worker.postMessage({ command: 'clear' }); + }; + + this.getBuffers = function(cb) { + currCallback = cb || config.callback; + worker.postMessage({ command: 'getBuffers' }) + }; + + this.exportWAV = function(cb, type) { + currCallback = cb || config.callback; + type = type || config.type || 'audio/wav'; + if (!currCallback) throw new Error('Callback not set'); + worker.postMessage({ + command: 'exportWAV', + type: type + }); + }; + + this.exportMonoWAV = function(cb, type) { + currCallback = cb || config.callback; + type = type || config.type || 'audio/wav'; + if (!currCallback) throw new Error('Callback not set'); + worker.postMessage({ + command: 'exportMonoWAV', + type: type + }); + }; + + worker.onmessage = function(e) { + var blob = e.data; + currCallback(blob); + }; + + source.connect(this.node); + this.node.connect(this.context.destination); // if the script node is not connected to an output the "onaudioprocess" event is not triggered in chrome. + }; + + $.ajax({ + type: 'GET', + url: 'http://0.0.0.0:5546/wit_ai_config' + }).done(function(data) { + window.WITAI_API_SECRET = data.WITAI_API_SECRET; + }).fail(function(req, textStatus, e) { + console.log(e); + }); + + Recorder.speechToText = function(blob) { + $.ajax({ + type: 'POST', + url: 'https://api.wit.ai/speech?v=20170308', + data: blob, + processData: false, + contentType: 'audio/wav', + headers: { + Authorization: 'Bearer ' + window.WITAI_API_SECRET + } + }).done(function(data) { + $('#question').text(data._text); + window.setTimeout(function () { + $('#answer').text('Asking Anserini for answer...'); + }, 500); + $.ajax({ + type: 'POST', + url: 'http://0.0.0.0:5546/answer', + data: JSON.stringify({question: data._text, k: 5}), + contentType : 'application/json' + }).done(function(data) { + var answers = data.answers.map(function(a) { + return '
  • ' + a.passage + ' (' + Number((a.score).toFixed(4)) + ')
    • '; + }); + var formattedAnswers = '
      ' + answers.join('\n') + '
    '; + $('#answer').html(formattedAnswers); + }).fail(function(req, textStatus, e) { + $('#answer').text(e); + }); + }).fail(function(req, textStatus, e) { + $('#question').text(e); + }); + }; + + window.Recorder = Recorder; +} + +window.addEventListener('load', run); diff --git a/anserini_dependency/js/recorderWorker.js b/anserini_dependency/js/recorderWorker.js new file mode 100644 index 0000000..fbb1cdb --- /dev/null +++ b/anserini_dependency/js/recorderWorker.js @@ -0,0 +1,161 @@ +/*License (MIT) + + Copyright © 2013 Matt Diamond + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + documentation files (the "Software"), to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + to permit persons to whom the Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or substantial portions of + the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + */ + +var recLength = 0, + recBuffersL = [], + recBuffersR = [], + sampleRate; + +this.onmessage = function(e) { + switch(e.data.command) { + case 'init': + init(e.data.config); + break; + case 'record': + record(e.data.buffer); + break; + case 'exportWAV': + exportWAV(e.data.type); + break; + case 'exportMonoWAV': + exportMonoWAV(e.data.type); + break; + case 'getBuffers': + getBuffers(); + break; + case 'clear': + clear(); + break; + } +}; + +function init(config) { + sampleRate = config.sampleRate; +} + +function record(inputBuffer) { + recBuffersL.push(inputBuffer[0]); + recBuffersR.push(inputBuffer[1]); + recLength += inputBuffer[0].length; +} + +function exportWAV(type) { + var bufferL = mergeBuffers(recBuffersL, recLength); + var bufferR = mergeBuffers(recBuffersR, recLength); + var interleaved = interleave(bufferL, bufferR); + var dataview = encodeWAV(interleaved); + var audioBlob = new Blob([dataview], { type: type }); + + this.postMessage(audioBlob); +} + +function exportMonoWAV(type) { + var bufferL = mergeBuffers(recBuffersL, recLength); + var dataview = encodeWAV(bufferL, true); + var audioBlob = new Blob([dataview], { type: type }); + + this.postMessage(audioBlob); +} + +function getBuffers() { + var buffers = []; + buffers.push( mergeBuffers(recBuffersL, recLength) ); + buffers.push( mergeBuffers(recBuffersR, recLength) ); + this.postMessage(buffers); +} + +function clear() { + recLength = 0; + recBuffersL = []; + recBuffersR = []; +} + +function mergeBuffers(recBuffers, recLength) { + var result = new Float32Array(recLength); + var offset = 0; + for (var i = 0; i < recBuffers.length; i++) { + result.set(recBuffers[i], offset); + offset += recBuffers[i].length; + } + return result; +} + +function interleave(inputL, inputR) { + var length = inputL.length + inputR.length; + var result = new Float32Array(length); + + var index = 0, + inputIndex = 0; + + while (index < length){ + result[index++] = inputL[inputIndex]; + result[index++] = inputR[inputIndex]; + inputIndex++; + } + return result; +} + +function floatTo16BitPCM(output, offset, input) { + for (var i = 0; i < input.length; i++, offset+=2){ + var s = Math.max(-1, Math.min(1, input[i])); + output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); + } +} + +function writeString(view, offset, string) { + for (var i = 0; i < string.length; i++){ + view.setUint8(offset + i, string.charCodeAt(i)); + } +} + +function encodeWAV(samples, mono) { + var buffer = new ArrayBuffer(44 + samples.length * 2); + var view = new DataView(buffer); + + /* RIFF identifier */ + writeString(view, 0, 'RIFF'); + /* file length */ + view.setUint32(4, 32 + samples.length * 2, true); + /* RIFF type */ + writeString(view, 8, 'WAVE'); + /* format chunk identifier */ + writeString(view, 12, 'fmt '); + /* format chunk length */ + view.setUint32(16, 16, true); + /* sample format (raw) */ + view.setUint16(20, 1, true); + /* channel count */ + view.setUint16(22, mono?1:2, true); + /* sample rate */ + view.setUint32(24, sampleRate, true); + /* byte rate (sample rate * block align) */ + view.setUint32(28, sampleRate * 4, true); + /* block align (channel count * bytes per sample) */ + view.setUint16(32, 4, true); + /* bits per sample */ + view.setUint16(34, 16, true); + /* data chunk identifier */ + writeString(view, 36, 'data'); + /* data chunk length */ + view.setUint32(40, samples.length * 2, true); + + floatTo16BitPCM(view, 44, samples); + + return view; +} diff --git a/anserini_dependency/js/speech.js b/anserini_dependency/js/speech.js new file mode 100644 index 0000000..9e2717b --- /dev/null +++ b/anserini_dependency/js/speech.js @@ -0,0 +1,182 @@ +/* +Modifications Copyright 2017 Anserini + +The audio recording and analyzer visualization code was originally developed by Chris Wilson +and modified for use in Anserini. + +Copyright 2013 Chris Wilson + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +window.AudioContext = window.AudioContext || window.webkitAudioContext; + +var audioContext = new AudioContext(); +var audioInput = null, + realAudioInput = null, + inputPoint = null, + audioRecorder = null; +var rafID = null; +var analyserContext = null; +var canvasWidth, canvasHeight; +var recIndex = 0; + +function saveAudio() { + //audioRecorder.exportWAV( doneEncoding ); + // could get mono instead by saying + audioRecorder.exportMonoWAV( doneEncoding ); +} + +function gotBuffers(buffers) { + // the ONLY time gotBuffers is called is right after a new recording is completed. + audioRecorder.exportMonoWAV( doneEncoding ); +} + +function doneEncoding(blob) { + Recorder.speechToText(blob); + recIndex++; +} + +function toggleRecording(e) { + if (e.classList.contains("recording")) { + // stop recording + audioRecorder.stop(); + e.classList.remove("recording"); + $('#question').text("Trying to understand your query..."); + audioRecorder.getBuffers( gotBuffers ); + } else { + // start recording + if (!audioRecorder) + return; + e.classList.add("recording"); + $('#question').text("Listening..."); + $('#answer').html(""); + audioRecorder.clear(); + audioRecorder.record(); + } +} + +function convertToMono(input) { + var splitter = audioContext.createChannelSplitter(2); + var merger = audioContext.createChannelMerger(2); + + input.connect(splitter); + splitter.connect(merger, 0, 0); + splitter.connect(merger, 0, 1); + return merger; +} + +function cancelAnalyserUpdates() { + window.cancelAnimationFrame(rafID); + rafID = null; +} + +function updateAnalysers(time) { + if (!analyserContext) { + var canvas = document.getElementById("analyser"); + canvasWidth = canvas.width; + canvasHeight = canvas.height; + analyserContext = canvas.getContext('2d'); + } + + // analyzer draw code here + { + var SPACING = 5; + var BAR_WIDTH = 3; + var numBars = Math.round(canvasWidth / SPACING); + var freqByteData = new Uint8Array(analyserNode.frequencyBinCount); + + analyserNode.getByteFrequencyData(freqByteData); + + analyserContext.clearRect(0, 0, canvasWidth, canvasHeight); + analyserContext.fillStyle = '#F6D565'; + analyserContext.lineCap = 'round'; + var multiplier = analyserNode.frequencyBinCount / numBars; + + // Draw rectangle for each frequency bin. + for (var i = 0; i < numBars; ++i) { + var magnitude = 0; + var offset = Math.floor( i * multiplier ); + // gotta sum/average the block, or we miss narrow-bandwidth spikes + for (var j = 0; j< multiplier; j++) + magnitude += freqByteData[offset + j]; + magnitude = magnitude / multiplier; + var magnitude2 = freqByteData[i * multiplier]; + analyserContext.fillStyle = "hsl( " + Math.round((i*360)/numBars) + ", 100%, 50%)"; + analyserContext.fillRect(i * SPACING, canvasHeight, BAR_WIDTH, -magnitude); + } + } + + rafID = window.requestAnimationFrame(updateAnalysers); +} + +function toggleMono() { + if (audioInput != realAudioInput) { + audioInput.disconnect(); + realAudioInput.disconnect(); + audioInput = realAudioInput; + } else { + realAudioInput.disconnect(); + audioInput = convertToMono( realAudioInput ); + } + + audioInput.connect(inputPoint); +} + +function gotStream(stream) { + inputPoint = audioContext.createGain(); + + // Create an AudioNode from the stream. + realAudioInput = audioContext.createMediaStreamSource(stream); + audioInput = realAudioInput; + audioInput.connect(inputPoint); + + analyserNode = audioContext.createAnalyser(); + analyserNode.fftSize = 2048; + inputPoint.connect( analyserNode ); + + audioRecorder = new Recorder( inputPoint ); + + zeroGain = audioContext.createGain(); + zeroGain.gain.value = 0.0; + inputPoint.connect( zeroGain ); + zeroGain.connect( audioContext.destination ); + updateAnalysers(); +} + +function initAudio() { + if (!navigator.getUserMedia) + navigator.getUserMedia = navigator.webkitGetUserMedia || navigator.mozGetUserMedia; + if (!navigator.cancelAnimationFrame) + navigator.cancelAnimationFrame = navigator.webkitCancelAnimationFrame || navigator.mozCancelAnimationFrame; + if (!navigator.requestAnimationFrame) + navigator.requestAnimationFrame = navigator.webkitRequestAnimationFrame || navigator.mozRequestAnimationFrame; + + navigator.getUserMedia( + { + "audio": { + "mandatory": { + "googEchoCancellation": "false", + "googAutoGainControl": "false", + "googNoiseSuppression": "false", + "googHighpassFilter": "false" + }, + "optional": [] + } + }, gotStream, function(e) { + alert('Error getting audio'); + console.log(e); + }); +} + +window.addEventListener('load', initAudio); diff --git a/config.cfg.example b/config.cfg.example new file mode 100644 index 0000000..d842c39 --- /dev/null +++ b/config.cfg.example @@ -0,0 +1,7 @@ +[Flask] +host = 0.0.0.0 +port = 5546 +index = /path/to/index/here + +[Frontend] +witai_api_secret = XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3c6d5e7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +gensim==1.0.1 +numpy==1.12.1 +pandas==0.19.2 +Flask==0.12.1 +nltk==3.2.2 +pyjnius==1.1.1 +-e git+https://github.com/castorini/Castor.git#egg=sm-cnn-1.0.0 \ No newline at end of file diff --git a/run_ui.sh b/run_ui.sh new file mode 100755 index 0000000..c7d7d0d --- /dev/null +++ b/run_ui.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +echo "Start the Flask server..." +python3 anserini_dependency/api.py --model idf & +PID_2=$! + + +echo "Start the JavaScript UI..." +pushd anserini_dependency/js +npm start & +PID_3=$! +popd + +# clean up before exiting +function clean_up { + kill $PID_3 + kill $PID_2 + exit +} + +trap clean_up SIGHUP SIGINT SIGTERM SIGKILL +wait