From d1bd842cdba2c06bcacef5ace977f084da42575e Mon Sep 17 00:00:00 2001 From: wassname Date: Wed, 31 May 2017 21:58:21 +0800 Subject: [PATCH] init --- .gitignore | 119 ++ helpers/__init__.py | 1 + helpers/dataset_isledict.py | 104 ++ helpers/helpers.py | 257 +++++ helpers/layers.py | 306 ++++++ main.ipynb | 2040 +++++++++++++++++++++++++++++++++++ readme.md | 17 + requirements.txt | 9 + 8 files changed, 2853 insertions(+) create mode 100644 .gitignore create mode 100644 helpers/__init__.py create mode 100644 helpers/dataset_isledict.py create mode 100644 helpers/helpers.py create mode 100644 helpers/layers.py create mode 100644 main.ipynb create mode 100644 readme.md create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3a1e40d --- /dev/null +++ b/.gitignore @@ -0,0 +1,119 @@ + +# Created by https://www.gitignore.io/api/linux,python + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# End of https://www.gitignore.io/api/linux,python diff --git a/helpers/__init__.py b/helpers/__init__.py new file mode 100644 index 0000000..b9211a7 --- /dev/null +++ b/helpers/__init__.py @@ -0,0 +1 @@ +from .helpers import * diff --git a/helpers/dataset_isledict.py b/helpers/dataset_isledict.py new file mode 100644 index 0000000..a407d13 --- /dev/null +++ b/helpers/dataset_isledict.py @@ -0,0 +1,104 @@ + +import requests + +from pysle import isletool +import numpy as np +import itertools +from sklearn.model_selection import train_test_split +import re +import os + +from .helpers import CharacterTable + + + +def download_data_maybe(fname='ISLEdict.txt', url='http://isle.illinois.edu/sst/data/g2ps/English/ISLEdict.html', cache_subdir='datasets'): + + datadir_base = os.path.expanduser(os.path.join('~', '.keras')) + if not os.access(datadir_base, os.W_OK): + datadir_base = os.path.join('/tmp', '.keras') + datadir = os.path.join(datadir_base, cache_subdir) + if not os.path.exists(datadir): + os.makedirs(datadir) + fpath = os.path.join(datadir, fname) + + if not os.path.exists(fpath): + print('Downloading data from ', url, 'to', fpath) + r = requests.get(url) + assert r.status_code == 200 + from bs4 import BeautifulSoup + soup = BeautifulSoup(r.content, 'lxml') + with open(fpath, 'w') as fo: + fo.write(soup.text.strip()) + return fpath + + +def get_data(seed=42, test_size=0.20, verbose=0, maxlen_x=None, maxlen_y=None, blacklist='()0123456789%.?"-_', max_phonemes=np.inf, max_chars=np.inf, phon_sep='', unique_graphemes=False, unique_phonemes=True): + """Process ISLEDICT pronounciation dictionary to return unique phonemes two graphemes""" + + path = download_data_maybe() + + # load data + isleDict = isletool.LexicalTool(path) + X = [] + y = [] + for phrase in isleDict.data.keys(): + for pronounciation in zip(*isleDict.lookup(phrase)): + xx = [] + for syllableList, stressedSyllableList, stressedPhoneList in pronounciation: + xx += list(itertools.chain(*syllableList)) + y.append(phon_sep.join(xx)) + X.append(phrase) + if verbose: print('loaded entries {}'.format(len(X))) + + # filter out duplicate X's + if unique_phonemes: + y, X = zip(*dict(zip(y, X)).items()) + if verbose: print('removed duplicate phonemes leaving {}'.format(len(X))) + + # filter out duplicates Y's + if unique_graphemes: + X, y = zip(*dict(zip(X, y)).items()) + if verbose: print('removed duplicate graphemes leaving {}'.format(len(X))) + + # split data (we must set asside test data before cleanign so it's always the same) + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=seed) + + # filter out duplicate entries like 'HOUSE(2) or multi words CAT-DOG and CAT_DOG' + p = re.compile('[%s]' % (re.escape(blacklist))) + X_train, y_train = zip(*[(x, y) for x, y in zip(X_train, y_train) if not bool(p.findall(x))]) + X_test, y_test = zip(*[(x, y) for x, y in zip(X_test, y_test) if not bool(p.findall(x))]) + if verbose: + print('removed blacklisted entries leaving {}'.format(len(X_train) + len(X_test))) + + # filter out complex entries if needed + before_x = len(y_train) + X_train, y_train = zip(*[(x, y) for x, y in zip(X_train, y_train) if len(y) <= max_phonemes and len(x) <= max_chars]) + X_test, y_test = zip(*[(x, y) for x, y in zip(X_test, y_test) if len(y) <= max_phonemes and len(x) <= max_chars]) + if verbose: + print('restricted to less than {} phonemes leaving {} entries or {:2.2f}%'.format(max_phonemes, len(X_train) + len(X_test), len(X_train)/before_x*100)) + + # FIXME it's slow in the next few lines + # encode x and y and pad them + xtable = CharacterTable() + xtable.fit(X_test + X_train) + if maxlen_x: + xtable.maxlen = maxlen_x + X_train = xtable.encode(X_train) + X_test = xtable.encode(X_test) + + ytable = CharacterTable() + ytable.fit(y_test + y_train) + if maxlen_y: + ytable.maxlen = maxlen_y + y_train = ytable.encode(y_train) + y_test = ytable.encode(y_test) + + if verbose: + print('X_train shape:', X_train.shape) + print('X_test shape:', X_test.shape) + + print('y_train shape:', y_train.shape) + print('y_test shape:', y_test.shape) + + return (X_train, y_train), (X_test, y_test), (xtable, ytable) diff --git a/helpers/helpers.py b/helpers/helpers.py new file mode 100644 index 0000000..cfc9e5a --- /dev/null +++ b/helpers/helpers.py @@ -0,0 +1,257 @@ +import time +import re +import numpy as np +import nltk +import collections + + +# get short words +def cmudict_random_sample(n=100): + '''get words shorter or equal to word_length''' + cmudict = nltk.corpus.cmudict.dict() + pairs = cmudict.items() + # get random samples of cmudict of n=cutoff + samples = (np.random.sample(n) * len(cmudict)).astype(int) + pairs = [pairs[s] for s in samples] + cmudict = collections.OrderedDict(pairs) + return cmudict + +# get short words + + +def cmudict_short_words(word_length=3): + '''get words shorter or equal to word_length''' + cmudict_raw = nltk.corpus.cmudict.dict() + keys = np.array(cmudict_raw.keys()) + lens = np.array([len(k) for k in keys]) + vals = np.array(cmudict_raw.values()) + inds = np.argwhere(lens <= word_length) + vals2 = vals[inds] + keys2 = keys[inds] + cmudict = dict(np.dstack((keys2.flatten(), vals2.flatten()))[0]) # must be a better way! + return cmudict + + +def cmudict_short_phones(phone_length=3): + '''get words shorter or equal to word_length''' + cmudict_raw = nltk.corpus.cmudict.dict() + vals = np.array(cmudict_raw.values()) + lens = np.array([len(p[0]) for p in vals]) + keys = np.array(cmudict_raw.keys()) + inds = np.argwhere(lens <= phone_length) + vals2 = vals[inds] + keys2 = keys[inds] + cmudict = dict(np.dstack((keys2.flatten(), vals2.flatten()))[0]) # must be a better way! + return cmudict + + +def unique(seq, idfun=None): + '''get only unique items in list''' + # order preserving + if idfun is None: + def idfun(x): return x + seen = {} + result = [] + for item in seq: + marker = idfun(item) + # in old Python versions: + # if seen.has_key(marker) + # but in new ones: + if marker in seen: + continue + seen[marker] = 1 + result.append(item) + return result + + +def split_string(word, inds): + '''split string at indices''' + splitword = [] + inds = np.sort(inds) + for i in range(len(inds) - 1): + j = inds[i] + k = inds[i + 1] + if k == 0 and j == len(word): + continue + part = word[j:k] + splitword.append(part) + return splitword + + +def remove_stress_arp(s): + return re.sub('\d+', '', s) + + +def findall(pattern, target): + inds = [] + while pattern in target: + inds.append(target.index(pattern)) + target = target.replace(pattern, '_' * len(pattern), 1) + return inds + + +def dist_in_letters(word, phones, phon_ind, match, lind): + '''find distance between pheonom position and it's match + provide: + word e.g. 'baby' + phones e.g. [u'B', u'EY1', u'B', u'IY0'] + phon_ind e.g. 2 + match e.g. B + lind: positve of match in word e.g. 2 + ''' + dists = [] + letters_p_phon = 1.0 * len(word) / len(phones) + phon_lind = phon_ind * letters_p_phon + dist1 = abs(lind - phon_lind) + dist2 = abs(lind + len(match) - phon_lind - len(match)) + dists.append(dist1) + dists.append(dist2) + + return min(dists) + + +def write_cmudict_sample(n=200): + import numpy as np + cmudict = nltk.corpus.cmudict.dict() + outf = 'cmudict_samples' + str(time.time()) + '.txt' + fo = open(outf, 'w') + samples = (np.random.random_sample(100) * len(cmudict)).astype(int) + cmuk = cmudict.keys() + for s in samples: + k = cmuk[s] + vs = cmudict[k] + for v in vs: + + vstr = ' '.join(v) + fo.write('{} {} {}\n'.format(k, k, vstr)) + fo.close() + print("Please go to {} and place space between the letters that match the peonomes, this will be the positive test".format(outf)) + + +def empty_tree(input_list): + """Recursively iterate through values in nested lists.""" + if input_list: + for item in input_list: + if not isinstance(item, list) or not empty_tree(item): + return False + return True + +import itertools +# based on https://github.com/fchollet/keras/blob/master/examples/addition_rnn.py +# note: can't make sparse 3d matrices + + +class CharacterTable(object): + ''' + Given a set of characters: + + Encode them to a one hot integer representation + + Decode the one hot integer representation to their character output + + Decode a vector of probabilities to their character output + ''' + + def __init__(self, chars='', maxlen=None, null_char=' ', left_pad=False): + self.chars = sorted(set([null_char] + list(chars))) + self.char_indices = dict((c, i) for i, c in enumerate(self.chars)) + self.indices_char = dict((i, c) for i, c in enumerate(self.chars)) + self.maxlen = maxlen + self.left_pad = left_pad + self.null_char = null_char + + def fit(self, Cs, null_char=' '): + """Determine chars and maxlen by fitting to data""" + self.chars = sorted(set(itertools.chain([null_char], *Cs))) + self.char_indices = dict((c, i) for i, c in enumerate(self.chars)) + self.indices_char = dict((i, c) for i, c in enumerate(self.chars)) + self.maxlen = max(len(c) for c in Cs) + self.null_char = null_char + + def encode(self, Cs, maxlen=None): + """Pass in an array of arrays to convert to integers""" + maxlen = maxlen if maxlen else self.maxlen + n = len(Cs) + X = np.zeros((n, maxlen, len(self.chars)), dtype=np.bool) + for j, C in enumerate(Cs): + if self.left_pad: + C = [self.null_char] * (maxlen - len(C)) + list(C) + else: + C = list(C) + [self.null_char] * (maxlen - len(C)) + for i, c in enumerate(C): + X[j, i, self.char_indices[c]] = True + return X + + def decode(self, Xs, calc_argmax=True): + if calc_argmax: + Xs = Xs.argmax(axis=-1) + return np.array(list([self.indices_char[x] for x in X] for X in Xs)) + + +# show_results +from IPython.core.display import display, HTML +m=1.2 +lighten=lambda x:1-(1/m-x/m) + +def show_results(ytable, y_pred,y_test=None,X_test=None,xtable=None): + """Show results which are darker when more confident""" + html = '' + html += '' + html += '' + p_pred = ytable.decode(y_pred) + conf = y_pred.max(-1) + for i in range(p_pred.shape[0]): + html += '' + + if X_test is not None: + p_test = xtable.decode(X_test) + html+='' + + html+='' + + if y_test is not None: + html+='' + html += '' + html += '
pronunciationguessspelling
' + for j in range(p_test.shape[1]): + c=p_test[i][p_test.shape[1]-j-1] + html+='{c:}'.format(c=c,a=1) + html+='' + for j in range(p_pred.shape[1]): + c=p_pred[i][j] + a=lighten(conf[i][j]) + html+='{c:}'.format(c=c,a=a) + html+='' + p_test = ytable.decode(y_test) + for j in range(p_test.shape[1]): + c=p_test[i][j] + html+='{c:}'.format(c=c,a=1) + html+='
' + return HTML(html) + +# test +# r=np.random.random((10,8,30))**20 +# show_results(ytable, r) + + +class weighted_categorical_crossentropy(object): + """ + A weighted version of keras.objectives.categorical_crossentropy + + Variables: + weights: numpy array of shape (C,) where C is the number of classes + + Usage: + loss = weighted_categorical_crossentropy(weights).loss + model.compile(loss=loss,optimizer='adam') + """ + + def __init__(self,weights): + self.weights = K.variable(weights) + + def loss(self,y_true, y_pred): + # scale preds so that the class probas of each sample sum to 1 + y_pred /= y_pred.sum(axis=-1, keepdims=True) + # clip + y_pred = K.clip(y_pred, K.epsilon(), 1) + # calc + loss = y_true*K.log(y_pred)*self.weights + loss =-K.sum(loss,-1) + return loss diff --git a/helpers/layers.py b/helpers/layers.py new file mode 100644 index 0000000..1f4ed1c --- /dev/null +++ b/helpers/layers.py @@ -0,0 +1,306 @@ +""" +A keras attention layer that wraps RNN layers. + +Based on tensorflows [attention_decoder](https://github.com/tensorflow/tensorflow/blob/c8a45a8e236776bed1d14fd71f3b6755bd63cc58/tensorflow/python/ops/seq2seq.py#L506) +and [Grammar as a Foreign Language](https://arxiv.org/abs/1412.7449). + +date: 20161101 +author: wassname +url: +""" +from __future__ import absolute_import + +from keras import backend as K +from keras.engine import InputSpec +from keras.layers import LSTM, activations, Wrapper, Recurrent + +class Attention(Wrapper): + """ + This wrapper will provide an attention layer to a recurrent layer. + + # Arguments: + layer: `Recurrent` instance with consume_less='gpu' or 'mem' + + # Examples: + + ```python + model = Sequential() + model.add(LSTM(10, return_sequences=True), batch_input_shape=(4, 5, 10)) + model.add(TFAttentionRNNWrapper(LSTM(10, return_sequences=True, consume_less='gpu'))) + model.add(Dense(5)) + model.add(Activation('softmax')) + model.compile(loss='categorical_crossentropy', optimizer='rmsprop') + ``` + + # References + - [Grammar as a Foreign Language](https://arxiv.org/abs/1412.7449) + + + """ + def __init__(self, layer, **kwargs): + assert isinstance(layer, Recurrent) + if layer.get_config()['consume_less']=='cpu': + raise Exception("AttentionLSTMWrapper doesn't support RNN's with consume_less='cpu'") + self.supports_masking = True + super(Attention, self).__init__(layer, **kwargs) + + def build(self, input_shape): + assert len(input_shape) >= 3 + self.input_spec = [InputSpec(shape=input_shape)] + nb_samples, nb_time, input_dim = input_shape + + if not self.layer.built: + self.layer.build(input_shape) + self.layer.built = True + + super(Attention, self).build() + + self.W1 = self.layer.init((input_dim, input_dim, 1, 1), name='{}_W1'.format(self.name)) + self.W2 = self.layer.init((self.layer.output_dim, input_dim), name='{}_W2'.format(self.name)) + self.b2 = K.zeros((input_dim,), name='{}_b2'.format(self.name)) + self.W3 = self.layer.init((input_dim*2, input_dim), name='{}_W3'.format(self.name)) + self.b3 = K.zeros((input_dim,), name='{}_b3'.format(self.name)) + self.V = self.layer.init((input_dim,), name='{}_V'.format(self.name)) + + self.trainable_weights = [self.W1, self.W2, self.W3, self.V, self.b2, self.b3] + + def get_output_shape_for(self, input_shape): + return self.layer.get_output_shape_for(input_shape) + + def step(self, x, states): + # This is based on [tensorflows implementation](https://github.com/tensorflow/tensorflow/blob/c8a45a8e236776bed1d14fd71f3b6755bd63cc58/tensorflow/python/ops/seq2seq.py#L506). + # First, we calculate new attention masks: + # attn = softmax(V^T * tanh(W2 * X +b2 + W1 * h)) + # and we make the input as a concatenation of the input and weighted inputs which is then + # transformed back to the shape x of using W3 + # x = W3*(x+X*attn)+b3 + # Then, we run the cell on a combination of the input and previous attention masks: + # h, state = cell(x, h). + + nb_samples, nb_time, input_dim = self.input_spec[0].shape + h = states[0] + X = states[-1] + xW1 = states[-2] + + Xr = K.reshape(X,(-1,nb_time,1,input_dim)) + hW2 = K.dot(h,self.W2)+self.b2 + hW2 = K.reshape(hW2,(-1,1,1,input_dim)) + u = K.tanh(xW1+hW2) + a = K.sum(self.V*u,[2,3]) + a = K.softmax(a) + a = K.reshape(a,(-1, nb_time, 1, 1)) + + # Weight attention vector by attention + Xa = K.sum(a*Xr,[1,2]) + Xa = K.reshape(Xa,(-1,input_dim)) + + # Merge input and attention weighted inputs into one vector of the right size. + x = K.dot(K.concatenate([x,Xa],1),self.W3)+self.b3 + + h, new_states = self.layer.step(x, states) + return h, new_states + + def get_constants(self, x): + constants = self.layer.get_constants(x) + + # Calculate K.dot(x, W2) only once per sequence by making it a constant + nb_samples, nb_time, input_dim = self.input_spec[0].shape + Xr = K.reshape(x,(-1,nb_time,input_dim,1)) + Xrt = K.permute_dimensions(Xr, (0, 2, 1, 3)) + xW1t = K.conv2d(Xrt,self.W1,border_mode='same') + xW1 = K.permute_dimensions(xW1t, (0, 2, 3, 1)) + constants.append(xW1) + + # we need to supply the full sequence of inputs to step (as the attention_vector) + constants.append(x) + + return constants + + def call(self, x, mask=None): + # input shape: (nb_samples, time (padded with zeros), input_dim) + input_shape = self.input_spec[0].shape + if K._BACKEND == 'tensorflow': + if not input_shape[1]: + raise Exception('When using TensorFlow, you should define ' + 'explicitly the number of timesteps of ' + 'your sequences.\n' + 'If your first layer is an Embedding, ' + 'make sure to pass it an "input_length" ' + 'argument. Otherwise, make sure ' + 'the first layer has ' + 'an "input_shape" or "batch_input_shape" ' + 'argument, including the time axis. ' + 'Found input shape at layer ' + self.name + + ': ' + str(input_shape)) + + if self.layer.stateful: + initial_states = self.layer.states + else: + initial_states = self.layer.get_initial_states(x) + constants = self.get_constants(x) + preprocessed_input = self.layer.preprocess_input(x) + + + last_output, outputs, states = K.rnn(self.step, preprocessed_input, + initial_states, + go_backwards=self.layer.go_backwards, + mask=mask, + constants=constants, + unroll=self.layer.unroll, + input_length=input_shape[1]) + if self.layer.stateful: + self.updates = [] + for i in range(len(states)): + self.updates.append((self.layer.states[i], states[i])) + + if self.layer.return_sequences: + return outputs + else: + return last_output + + + + +# this is a copy of tensorflow with simplified matrix algebra, need to check I didn't make a mistkae +class SimplifiedAttention(Wrapper): + def __init__(self, layer, attn_activation='tanh', **kwargs): + assert isinstance(layer, Recurrent) + if not layer.return_sequences: + raise Exception("AttentionLSTMWrapper doesn't support RNN's with return_sequences=False") + + self.supports_masking = True + super(SimplifiedAttention, self).__init__(layer, **kwargs) + + def build(self, input_shape): + assert len(input_shape) >= 3 + self.input_spec = [InputSpec(shape=input_shape)] + nb_samples, nb_time, input_dim = input_shape + + if not self.layer.built: + self.layer.build(input_shape) + self.layer.built = True + + super(SimplifiedAttention, self).build() + +# self.W1 = self.layer.init((input_dim, input_dim, 1, 1), name='{}_W1'.format(self.name)) + self.W1 = self.layer.init((input_dim, input_dim), name='{}_W1'.format(self.name)) +# self.W2 = self.layer.init((input_dim,nb_time), name='{}_W2'.format(self.name)) + self.W2 = self.layer.init((nb_time,input_dim,input_dim), name='{}_W2'.format(self.name)) + self.b2 = K.zeros((input_dim,), name='{}_b2'.format(self.name)) + self.W3 = self.layer.init((input_dim*2, input_dim), name='{}_W3'.format(self.name)) + self.b3 = K.zeros((input_dim,), name='{}_b3'.format(self.name)) + self.V = self.layer.init((input_dim,), name='{}_V'.format(self.name)) + + self.trainable_weights = [self.W1, self.W2, self.W3, self.V, self.b2, self.b3] + + def get_output_shape_for(self, input_shape): + return self.layer.get_output_shape_for(input_shape) + + def step(self, x, states): + + # First, we calculate new attention masks: + # attn = softmax(V^T * tanh(W2 * inputs + W1 * prev_h)) + # and then weight the previous state by the attention + # prev_h = prev_h * attn + # and we make the input as a concatenation of the input and weighted inputs which is then + # transformed back to the shape x of using W3 + # x = W3*(x+X*attn)+b3 + # Then, we run the cell on a combination of the input and previous attention masks: + # h, state = cell(x, h). + + nb_samples, nb_time, input_dim = self.input_spec[0].shape + h,c,B_U,B_W,xW2,X = states + +# # as in tensorflow +# Xr = K.reshape(X,(-1,nb_time,input_dim,1)) +# Xrt = K.permute_dimensions(Xr, (0, 2, 1, 3)) +# xW1t = K.conv2d(Xrt,self.W1,border_mode='same') # could be cached +# xW1 = K.permute_dimensions(xW1t, (0, 2, 1, 3)) + + # or (input_dim,input_dim)x(nb_samples, nb_time, input_dim)=>(nb_samples, nb_time, input_dim) + # same value once reshaped, need to take away the extra dims + xW1 = K.dot(X,self.W1) + + # assert hW1.shape == Xr.shape + hW2 = K.dot(h,self.W2)+self.b2 +# xW2 = K.reshape(xW2,(-1,1,input_dim,1)) + u = K.tanh(xW1+hW2) + a = K.sum(self.V*u,-1) + # assert a.shape==(nb_samples,nb_time) + a = K.softmax(a) + a = K.reshape(a,(-1, nb_time, 1)) + Xa = K.sum(a*X,1) + Xa = K.reshape(Xa,(-1,input_dim)) + + # Merge input and previous attentions into one vector of the right size. + # TODO, deal with the consume_less='cpu' flag which reshapes x + x = K.dot(K.concatenate([x,Xa],1),self.W3)+self.b3 + # assert x.shape == (nb_samples,input_dim) + + + + + h, new_states = self.layer.step(x, [h,c,B_U,B_W]) +# new_states.append(a) +# Tracer()() + return h, new_states + + def get_constants(self, x): + constants = self.layer.get_constants(x) + # Calculate K.dot(x, W2) only once per sequence by making it a constant +# # as in tensorflow +# self.W1 = self.layer.init((input_dim, input_dim, 1, 1), name='{}_W1'.format(self.name)) +# Xr = K.reshape(x,(-1,nb_time,input_dim,1)) +# Xrt = K.permute_dimensions(Xr, (0, 2, 1, 3)) +# xW1t = K.conv2d(Xrt,self.W1,border_mode='same') # could be cached +# xW1 = K.permute_dimensions(xW1t, (0, 2, 1, 3)) + + # or just +# self.W1 = self.layer.init((input_dim, input_dim), name='{}_W1'.format(self.name)) + xW1 = K.dot(x,self.W1) + + constants.append(xW1) + # the need to provide X to the step function too so it can be weighted to produce the inputs + constants.append(x) + return constants + + def call(self, x, mask=None): + # input shape: (nb_samples, time (padded with zeros), input_dim) + input_shape = self.input_spec[0].shape + if K._BACKEND == 'tensorflow': + if not input_shape[1]: + raise Exception('When using TensorFlow, you should define ' + 'explicitly the number of timesteps of ' + 'your sequences.\n' + 'If your first layer is an Embedding, ' + 'make sure to pass it an "input_length" ' + 'argument. Otherwise, make sure ' + 'the first layer has ' + 'an "input_shape" or "batch_input_shape" ' + 'argument, including the time axis. ' + 'Found input shape at layer ' + self.name + + ': ' + str(input_shape)) + + if self.layer.stateful: + initial_states = self.layer.states + else: + initial_states = self.layer.get_initial_states(x)#+[K.ones((input_shape[0],input_shape[1]))] + constants = self.get_constants(x) + preprocessed_input = self.layer.preprocess_input(x) + + last_output, outputs, states = K.rnn(self.step, preprocessed_input, + initial_states, + go_backwards=self.layer.go_backwards, + mask=mask, + constants=constants, + unroll=self.layer.unroll, + input_length=input_shape[1]) + if self.layer.stateful: + self.updates = [] + for i in range(len(states)): + self.updates.append((self.layer.states[i], states[i])) + + if self.layer.return_sequences: + return outputs + else: + return last_output diff --git a/main.ipynb b/main.ipynb new file mode 100644 index 0000000..80115c4 --- /dev/null +++ b/main.ipynb @@ -0,0 +1,2040 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:18.096716Z", + "start_time": "2017-05-31T21:12:18.093219+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "import os\n", + "# os.environ['THEANO_FLAGS']='mode=FAST_RUN,device=gpu,floatX=float32'\n", + "os.environ['KERAS_BACKEND']='tensorflow'\n", + "os.sys.path.append('..')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:19.641072Z", + "start_time": "2017-05-31T21:12:18.501412+08:00" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using TensorFlow backend.\n" + ] + } + ], + "source": [ + "# keras\n", + "import keras\n", + "# from keras.engine.training import slice_X\n", + "from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, recurrent\n", + "from keras.layers import InputLayer, Embedding, Lambda, Reshape, InputLayer, Dropout\n", + "from keras.models import Sequential\n", + "from keras.preprocessing import sequence\n", + "\n", + "from keras.wrappers.scikit_learn import KerasClassifier\n", + "from keras.layers.wrappers import Bidirectional, TimeDistributed\n", + "\n", + "# from keras.engine.training import slice_X\n", + "from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, recurrent\n", + "from keras.layers import InputLayer, Embedding\n", + "from keras.models import Sequential\n", + "\n", + "# helper\n", + "from keras_tqdm import TQDMNotebookCallback\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:19.879772Z", + "start_time": "2017-05-31T21:12:19.642763+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# sklearn\n", + "import sklearn\n", + "# from sklearn.model_selection import GridSearchCV\n", + "from sklearn.model_selection import train_test_split" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:21.353170Z", + "start_time": "2017-05-31T21:12:19.881395+08:00" + } + }, + "outputs": [], + "source": [ + "# pylab imports\n", + "from matplotlib import pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import scipy as sp\n", + "import seaborn as sns\n", + "%matplotlib inline\n", + "\n", + "# io & utils\n", + "from path import Path\n", + "from pprint import pprint\n", + "import json\n", + "import arrow\n", + "from tqdm import tqdm_notebook as tqdm\n", + "\n", + "import re\n", + "import string\n", + "import itertools\n", + "import collections" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:21.357163Z", + "start_time": "2017-05-31T21:12:21.354813+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "seed=1337 # for reproducibility\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:23.178886Z", + "start_time": "2017-05-31T21:12:23.173158+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "import sklearn\n", + "\n", + "import numpy as np\n", + "import scipy as sp\n", + "import pandas as pd\n", + "from matplotlib import pyplot as plt\n", + "%matplotlib inline\n", + "\n", + "from tqdm import tqdm_notebook as tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:24.615825Z", + "start_time": "2017-05-31T21:12:24.613324+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "import os\n", + "os.sys.path.append('.')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:26.654677Z", + "start_time": "2017-05-31T21:12:26.413393+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "from helpers import dataset_isledict, dataset_cmudict, show_results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# About\n", + "\n", + "# What?\n", + "\n", + "This project tried to build a spelling guesser, or technically a phoneme2grapheme translator.\n", + "\n", + "# What, what?\n", + "\n", + "Image you hear someone say a word for the first time:\n", + "\n", + "- DYE-uh-REE-a\n", + "\n", + "How is it spelt? We know the answer\n", + "\n", + "- DYE-uh-REE-a => \"Diarrhoea\" \n", + "\n", + "\n", + "But what if we didn't, here's a word you probobly don't know:\n", + "\n", + "- PHA-NG-AH-RAY => \n", + "\n", + "[answer](https://en.wikipedia.org/wiki/Whangarei)\n", + "\n", + "So we are building something that can take the pronounciation and guess the spelling.\n", + "\n", + "\n", + "# Data?\n", + "\n", + "We want a dictionary of data like\n", + "\n", + "pron | spelling \n", + "--|--\n", + "DYE-uh-REE-a | Diarrhoea\n", + "PHA-NG-AH-RAY | -\n", + " AR-kən-saw | Arkansas\n", + " \n", + "The best data comes from the CMU Pronouncing Dictionary with 140k entries\n", + "\n", + "Pronounciation|Spelling\n", + " -------| ------- |-----------\n", + "AA D|odd\n", + "AE T|at\n", + "HH AH T|hut\n", + "AO T|ought\n", + "K AW|cow\n", + "HH AY D|hide\n", + "B IY|be\n", + "CH IY Z|cheese\n", + "D IY|dee\n", + "DH IY|thee\n", + "EH D|Ed\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " \n", + "# Why?\n", + "\n", + "It can be used to \n", + "\n", + "- generate spelling for words that are not in the dictionary\n", + "- identify hard to spell words in english\n", + "- make a consistent spelling scheme for english?\n", + "- other things?\n", + "- no one's done it\n", + "\n", + "\n", + "# How\n", + "\n", + "I'm not happy with my results, so what do you suggest?\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Metrics\n", + "\n", + "The first steps is to decide on a metric and generate dummy benchmarks. That way we know what we have to beat.\n", + "\n", + "Our metric is \n", + "\n", + " Charector error rate: CER\n", + " bad: 50% DYE-uh-REE-a =>\tDirrrhoaa\n", + " good: 0% DYE-uh-REE-a =>\tDiarrhoea" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:27.665462Z", + "start_time": "2017-05-31T21:12:27.644129+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "def CER(y_pred,y_test):\n", + " \"\"\"\n", + " A keras metric for Charecter error rate\n", + " \n", + " inputs:\n", + " y_pred - an array with shape samples*index*classes\n", + " y_test - as above \n", + " \"\"\"\n", + " return 1-(y_pred.argmax(-1)==y_test.argmax(-1)).sum()/(y_test.shape[0]*y_test.shape[1])\n", + "# print ('Char error rate {:%}'.format(CER(y_pred,y_test)))\n", + "\n", + "from keras.utils.np_utils import to_categorical\n", + "# good\n", + "assert CER(\n", + " np.array([to_categorical([0,2,1,4,0],5)]),\n", + " np.array([to_categorical([0,2,1,4,0],5)])\n", + ") ==0\n", + "\n", + "# bad\n", + "assert CER(\n", + " np.array([to_categorical([0,2,1,4],5)]),\n", + " np.array([to_categorical([0,2,0,2],5)])\n", + ")==0.5\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# load data\n", + "\n", + "\n", + "My input data is the [CMUDict](http://www.speech.cs.cmu.edu/cgi-bin/cmudict) which maps english to the 39 ARPAbet phonemes. I then convert this to IPA." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "I wrote helper functions to load the data\n", + "this is boring so I will leave it out - jeremy style\n", + "\n", + "- download\n", + "- parse a text file\n", + "- remove duplicates\n", + "- turn charectors into numbers and back\n", + " - a=>0, 0=>a, b=>2, ... etc\n", + "- turn phonemes into numbers and back\n", + " - AH => 0, 0=>AH, ... etc\n", + "- split into 2 sets :( it should be three since we shouldn't use the test set for trying model archetectures\n", + "\n", + "\n", + "We will make it faster by restricting it to sequences of 10 or under\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:34.164036Z", + "start_time": "2017-05-31T21:12:28.540247+08:00" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "loaded entries 282842\n", + "removed duplicate phonemes leaving 260707\n", + "removed duplicate graphemes leaving 236077\n", + "removed blacklisted entries leaving 158184\n", + "restricted to less than 10 phonemes leaving 113368 entries or 71.81%\n", + "X_train shape: (90879, 10, 29)\n", + "X_test shape: (22489, 10, 29)\n", + "y_train shape: (90879, 10, 43)\n", + "y_test shape: (22489, 10, 43)\n" + ] + } + ], + "source": [ + "# grab a small subset of the data\n", + "(y_train, X_train),(y_test,X_test),(ytable,xtable) = dataset_isledict.get_data(\n", + " verbose=1,\n", + " max_phonemes=10,\n", + " max_chars=10,\n", + "# invert=False,\n", + "# x_hot=True,\n", + " unique_graphemes=True,\n", + " unique_phonemes=True\n", + " \n", + ")\n", + "X_train = X_train[:,::-1,:]\n", + "X_test = X_test[:,::-1,:]\n", + "\n", + "nb_chars = len(ytable.chars)\n", + "nb_phons = len(xtable.chars)\n", + "maxlen_x = xtable.maxlen\n", + "maxlen_y = ytable.maxlen\n", + "batch_size = 300" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:34.351422Z", + "start_time": "2017-05-31T21:12:34.165937+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# lets just do a sanity test\n", + "assert (X_train.sum(-1)==1).all(),'all classes must sum to one'\n", + "assert (y_train.sum(-1)==1).all(),'all classes must sum to one'\n", + "assert (X_train.max(-1)==1).all(),'should have max value of 1'\n", + "assert (X_train.min(-1)==0).all(), 'should have min of 0'" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:34.373609Z", + "start_time": "2017-05-31T21:12:34.353066+08:00" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[(' ̩nmɚ˺diˈb', 'biederman '),\n", + " (' ̩nɹæˈm', 'marron '),\n", + " (' ŋæˈɹps', 'sprang '),\n", + " (' ieˌwlɔˈh', 'hallway '),\n", + " (' ɹɛˌʃtmɹɑˈ', 'armchair ')]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# view data\n", + "a=[''.join(x) for x in xtable.decode(X_train[:5])]\n", + "b=[''.join(y) for y in ytable.decode(y_train[:5])]\n", + "list(zip(a,b))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Dummy model\n", + "\n", + "Our best dummy models get 60% CER or above\n", + "\n", + "We have to a little massaging because scikit-learn works with 2d arrays while keras uses >2d. I use argmax, to_categorical, and reshape for this." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:34.400892Z", + "start_time": "2017-05-31T21:12:34.375495+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "def to_categorical_3d(data2,n=None):\n", + " data=data2.astype(np.int)\n", + " return np.array([to_categorical(data[i],n) for i in range(len(data))])" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:35.095824Z", + "start_time": "2017-05-31T21:12:34.402710+08:00" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stratified : CER=87.83%\n", + "most_frequent : CER=69.55%\n", + "prior : CER=69.55%\n", + "uniform : CER=96.58%\n" + ] + } + ], + "source": [ + "from sklearn.dummy import DummyClassifier\n", + "for strategy in ['stratified', 'most_frequent', 'prior', 'uniform']:\n", + " clf = DummyClassifier(strategy=strategy,random_state=0)\n", + " clf.fit(X_train.reshape((-1,nb_phons)), y_train.reshape((-1,nb_chars)).argmax(-1))\n", + " \n", + " # on a simple classifier CER=1-score\n", + " score = clf.score(X_test.reshape((-1,nb_phons)), y_test.reshape((-1,nb_chars)).argmax(-1)) \n", + " print('{strategy:20.20s}: CER={CER:2.2%}'.format(strategy=strategy, CER=1-score))\n", + " \n", + "# # we get the same if we reshape from (329628,) to (27469, 12) \n", + "# # then make it categorical (27469, 12, 29)\n", + "# y_pred = clf.predict(X_test.reshape((-1,nb_phons))) \n", + "# y_pred_2d = y_pred.reshape((-1,maxlen_y))\n", + "# y_pred_3d = to_categorical_3d(y_pred_2d)\n", + "# cer = CER(y_pred_3d, y_test)\n", + "# print('CER={CER:2.2%}'.format(CER=cer))" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:36.399606Z", + "start_time": "2017-05-31T21:12:35.097567+08:00" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mean : CER=88.23%\n", + "median : CER=71.10%\n", + "quantile : CER=71.80%\n", + "constant : CER=69.55%\n" + ] + } + ], + "source": [ + "from sklearn.dummy import DummyRegressor\n", + "for strategy in ['mean', 'median', 'quantile', 'constant']:\n", + " clf = DummyRegressor(strategy=strategy, constant=np.zeros((10,1)), quantile=0.555)\n", + " clf.fit(X_train.argmax(-1), y_train.argmax(-1))\n", + " score = clf.score(X_test.argmax(-1), y_test.argmax(-1)) \n", + " y_pred_2d = clf.predict(X_test.argmax(-1)) \n", + " y_pred_3d=to_categorical_3d(y_pred_2d)\n", + " cer = CER(y_pred_3d, y_test)\n", + " print('{strategy:20.20s}: CER={CER:2.2%}'.format(strategy=strategy, CER=cer))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Helpers\n", + "\n", + "- loss\n", + " - dice_coef loss I recommend trying this for unbalanced classes (e.g. more e's), when kld and categorical cross-entropy don't work\n", + "- display\n", + " - something to show our results as as table\n", + " - progress logger to show predictons as we go\n", + " - tqdm_keras to have fast progress bars" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T10:56:24.624473Z", + "start_time": "2017-05-31T18:56:24.601558+08:00" + }, + "code_folding": [] + }, + "source": [ + "\n", + "### Dice loss\n", + "\n", + "Use it when you have unbalanced classes and the inbuild `kullback_leibler_divergence` wont work for you\n", + "\n", + "$$ L = \\frac{2*\\sum{|A*B|}}{(\\sum{A^2}+\\sum{B^2})} $$" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:37.733647Z", + "start_time": "2017-05-31T21:12:37.715496+08:00" + }, + "code_folding": [ + 0, + 6 + ], + "collapsed": true + }, + "outputs": [], + "source": [ + "# define custom loss and metric functions \n", + "from keras import backend as K\n", + "smooth = 1\n", + "\n", + "def dice_coef(y_true, y_pred, smooth=1):\n", + " \"\"\"\n", + " Dice = (2*|X & Y|)/ (|X|+ |Y|)\n", + " = 2*sum(|A*B|)/(sum(A^2)+sum(B^2))\n", + " ref: https://arxiv.org/pdf/1606.04797v1.pdf\n", + " \"\"\"\n", + " intersection = K.sum(K.abs(y_true * y_pred), axis=-1)\n", + " return (2. * intersection + smooth) / (K.sum(K.square(y_true),-1) + K.sum(K.square(y_pred),-1) + smooth)\n", + "\n", + "def dice_coef_loss(y_true, y_pred):\n", + " return 1-dice_coef(y_true, y_pred)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T11:09:11.210024Z", + "start_time": "2017-05-31T19:09:11.207001+08:00" + } + }, + "source": [ + "### PredictionLogger\n", + "With keras you can make custom callback so that you can see the prediction after every epoch. You will see it in action soon." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:38.139069Z", + "start_time": "2017-05-31T21:12:38.116574+08:00" + }, + "code_folding": [], + "collapsed": true + }, + "outputs": [], + "source": [ + "# PredictionLogger\n", + "import keras\n", + "from keras.callbacks import Callback\n", + "class PredictionLogger(keras.callbacks.Callback):\n", + " def __init__(self, n=1):\n", + " self.n = n\n", + "\n", + " def on_epoch_end(self, batch, logs={}):\n", + " y_pred = self.model.predict(self.model.validation_data[0][:self.n])\n", + " p_pred = ytable.decode(y_pred)\n", + " p_test = ytable.decode(self.model.validation_data[1][:self.n])\n", + " text_pred=[''.join(ls) for ls in p_pred]\n", + " text_true=[''.join(ls) for ls in p_test]\n", + " print('')\n", + " pprint(list(zip(text_true,text_pred)))\n", + "\n", + "prediction_logger = PredictionLogger(n=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# quick cnn" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2017-05-31T13:56:28.661Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:91: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n", + " \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "979614b4dc3d4173939f9170f29e31aa" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f82184647f2a47ef941e34773fc3ce34" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'cocksaa '),\n", + " ('brassfield', 'brasseell '),\n", + " ('flashbulb ', 'flashelll ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "43397189fa0348ffbc1bdb12126b48d4" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'cocksooe '),\n", + " ('brassfield', 'brasfeel '),\n", + " ('flashbulb ', 'flasblll ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6979fdb8f58f49938030ad9fff3dcf22" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'cocksooe '),\n", + " ('brassfield', 'brassfill '),\n", + " ('flashbulb ', 'flachbll ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0ce28a1779df4fa89417a222e0120de5" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'cocksooe '),\n", + " ('brassfield', 'brassfeld '),\n", + " ('flashbulb ', 'flashbll ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a0fd1109b8d9473cb462923aa886f04f" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'cocksooe '),\n", + " ('brassfield', 'brassfelld'),\n", + " ('flashbulb ', 'flashbel ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f4e6e2861f454e6f9a4609738885697f" + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from keras.layers import Embedding, Convolution1D, GlobalMaxPooling1D, Reshape, Flatten, Dropout\n", + "import keras\n", + "model = Sequential()\n", + "\n", + "# we start off with an efficient embedding layer which maps\n", + "# our vocab indices into embedding_dims dimensions\n", + "model.add(Embedding(nb_phons,\n", + " 90,\n", + " input_length=maxlen_x,\n", + " dropout=0.2))\n", + "\n", + "# we add a Convolution1D, which will learn nb_filter\n", + "# word group filters of size filter_length:\n", + "model.add(Convolution1D(nb_filter=256,\n", + " filter_length=3,\n", + " border_mode='valid',\n", + " activation='relu',\n", + " subsample_length=1))\n", + "model.add(Dropout(0.2))\n", + "# we add a Convolution1D, which will learn nb_filter\n", + "# word group filters of size filter_length:\n", + "model.add(Convolution1D(nb_filter=256,\n", + " filter_length=3,\n", + " border_mode='valid',\n", + " activation='relu',\n", + " subsample_length=1))\n", + "\n", + "# model.add(Dropout(0.2))\n", + "# # # we add a Convolution1D, which will learn nb_filter\n", + "# # # word group filters of size filter_length:\n", + "# model.add(Convolution1D(nb_filter=256,\n", + "# filter_length=3,\n", + "# border_mode='valid',\n", + "# activation='relu',\n", + "# subsample_length=1))\n", + "\n", + "# we use max pooling:\n", + "# we add a Convolution1D, which will learn nb_filter\n", + "# word group filters of size filter_length:\n", + "# model.add(Convolution1D(nb_filter=1,\n", + "# filter_length=1,\n", + "# border_mode='valid',\n", + "# activation='relu',\n", + "# subsample_length=1))\n", + "\n", + "# classifier\n", + "model.add(Flatten())\n", + "model.add(Dense(maxlen_y*nb_chars))\n", + "model.add(Reshape((maxlen_y,nb_chars)))\n", + "\n", + "model.add(Activation('softmax'))\n", + "model.compile(loss=dice_coef_loss,\n", + " optimizer='nadam', #keras.optimizers.Nadam(lr=0.2),\n", + " metrics=['accuracy'])\n", + "# model.summary()\n", + "\n", + "history = model.fit(\n", + " X_train.argmax(-1), \n", + " y_train, \n", + " batch_size=batch_size, \n", + " nb_epoch=20, \n", + " verbose=0, \n", + " validation_data=[X_test.argmax(-1),y_test],\n", + " callbacks=[\n", + " keras.callbacks.EarlyStopping(patience=2),\n", + " PredictionLogger(n=3),\n", + " TQDMNotebookCallback()\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "pd.DataFrame(history.history).plot()\n", + "\n", + "scores = model.evaluate(X_test.argmax(-1),y_test, verbose=0)\n", + "score = dict(zip(model.metrics_names,scores))\n", + "\n", + "y_pred = model.predict(X_test.argmax(-1), verbose=0, batch_size=batch_size)\n", + "\n", + "print ('Accuracy {:%}'.format(score['acc']))\n", + "print ('Word error rate {:%}'.format(WER(y_pred,y_test)))\n", + "print ('Char error rate {:%}'.format(CER(y_pred,y_test)))\n", + "n=10\n", + "show_results(ytable=ytable, y_pred=y_pred[:n],y_test=y_test[:n],xtable=xtable,X_test=X_test[:n])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "## not dice loss" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:21:06.877899Z", + "start_time": "2017-05-31T21:18:22.102370+08:00" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:91: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n", + " \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fa6eee8ea3c34d0f9c866b318eeef023" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4cd533cca244486f8cc3e13badbb6ab9" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'cocksoo '),\n", + " ('brassfield', 'brassfll '),\n", + " ('flashbulb ', 'flacbbrl ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6414418895034ade84cbc49ccc2420f2" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsore '),\n", + " ('brassfield', 'brassflld '),\n", + " ('flashbulb ', 'flasbbll ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "390e14aade624148aaa23c054dc36340" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsaae '),\n", + " ('brassfield', 'brasfflld '),\n", + " ('flashbulb ', 'flachbal ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0ad525f54dc644d2acaea18f12c4c829" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsoie '),\n", + " ('brassfield', 'brassflldd'),\n", + " ('flashbulb ', 'flashblll ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d97a7349400e4705a83c97c1bb0fe86c" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccooie '),\n", + " ('brassfield', 'brasffeldd'),\n", + " ('flashbulb ', 'flashble ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b0c5eaca7e5948269b85ce8422965d98" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsoie '),\n", + " ('brassfield', 'brasffildd'),\n", + " ('flashbulb ', 'flashblel ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "173a4491acf34bfd81f022478db27444" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsoid '),\n", + " ('brassfield', 'brassfeld '),\n", + " ('flashbulb ', 'flashbull ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8c082c1d40104c638d9598d3d78fa87f" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsoiee '),\n", + " ('brassfield', 'brassfildd'),\n", + " ('flashbulb ', 'flashbubl ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "40ae35cec16e4b09a8fedf58d8569747" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsoid '),\n", + " ('brassfield', 'brassfeldd'),\n", + " ('flashbulb ', 'flashblbl ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cf371b13c8e94e73ba5f4de50bcf1ee8" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccooid '),\n", + " ('brassfield', 'brassfeldd'),\n", + " ('flashbulb ', 'flashbubl ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ba8710565fde43ee8cc82dcf8f04dfc5" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsoid '),\n", + " ('brassfield', 'brassfeldd'),\n", + " ('flashbulb ', 'flashbubl ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "20e5206de5a24d5a93a02aaf934cb90c" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsoid '),\n", + " ('brassfield', 'brasffildd'),\n", + " ('flashbulb ', 'flashbeb ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9db1001bea404fff864cddfe6173fb2f" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccooid '),\n", + " ('brassfield', 'brassfildd'),\n", + " ('flashbulb ', 'flashbubl ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "23247602272943edb4ca7d0fe4c6074c" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'cocksoid '),\n", + " ('brassfield', 'brassfeldd'),\n", + " ('flashbulb ', 'flashbubl ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9f3c7f4b5c254858a24f3e168deda9d2" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'cocksoid '),\n", + " ('brassfield', 'brassieldd'),\n", + " ('flashbulb ', 'flashbll ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "855d83bec35e4bea91f2c774bceed2f9" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'cocksoid '),\n", + " ('brassfield', 'brassfeld '),\n", + " ('flashbulb ', 'flashber ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e2c028655aae44fba30ca4c5b6026178" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsoid '),\n", + " ('brassfield', 'brassiildd'),\n", + " ('flashbulb ', 'flashblll ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0a7c000529cb414182eef4d3a03df9a6" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccooid '),\n", + " ('brassfield', 'brassfilld'),\n", + " ('flashbulb ', 'flashblb ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a004b016a1b84c5e8f2816adeb3869fd" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'coccsoid '),\n", + " ('brassfield', 'brassfildd'),\n", + " ('flashbulb ', 'flashblb ')]\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "cc92f1c303b04838ab4b25bbb6fdcca0" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "[('caucasoid ', 'cocksoid '),\n", + " ('brassfield', 'brassfildd'),\n", + " ('flashbulb ', 'flashblb ')]\n" + ] + } + ], + "source": [ + "from keras.layers import Embedding, Convolution1D, GlobalMaxPooling1D, Reshape, Flatten, Dropout\n", + "import keras\n", + "model = Sequential()\n", + "\n", + "# we start off with an efficient embedding layer which maps\n", + "# our vocab indices into embedding_dims dimensions\n", + "model.add(Embedding(nb_phons,\n", + " 90,\n", + " input_length=maxlen_x,\n", + " dropout=0.2))\n", + "\n", + "# we add a Convolution1D, which will learn nb_filter\n", + "# word group filters of size filter_length:\n", + "model.add(Convolution1D(nb_filter=256,\n", + " filter_length=3,\n", + " border_mode='valid',\n", + " activation='relu',\n", + " subsample_length=1))\n", + "model.add(Dropout(0.2))\n", + "# we add a Convolution1D, which will learn nb_filter\n", + "# word group filters of size filter_length:\n", + "model.add(Convolution1D(nb_filter=256,\n", + " filter_length=3,\n", + " border_mode='valid',\n", + " activation='relu',\n", + " subsample_length=1))\n", + "\n", + "# model.add(Dropout(0.2))\n", + "# # # we add a Convolution1D, which will learn nb_filter\n", + "# # # word group filters of size filter_length:\n", + "# model.add(Convolution1D(nb_filter=256,\n", + "# filter_length=3,\n", + "# border_mode='valid',\n", + "# activation='relu',\n", + "# subsample_length=1))\n", + "\n", + "# we use max pooling:\n", + "# we add a Convolution1D, which will learn nb_filter\n", + "# word group filters of size filter_length:\n", + "# model.add(Convolution1D(nb_filter=1,\n", + "# filter_length=1,\n", + "# border_mode='valid',\n", + "# activation='relu',\n", + "# subsample_length=1))\n", + "\n", + "# classifier\n", + "model.add(Flatten())\n", + "model.add(Dense(maxlen_y*nb_chars))\n", + "model.add(Reshape((maxlen_y,nb_chars)))\n", + "\n", + "model.add(Activation('softmax'))\n", + "model.compile(loss='categorical_crossentropy',\n", + " optimizer='nadam', #keras.optimizers.Nadam(lr=0.2),\n", + " metrics=['accuracy'])\n", + "# model.summary()\n", + "\n", + "history = model.fit(\n", + " X_train.argmax(-1), \n", + " y_train, \n", + " batch_size=batch_size, \n", + " nb_epoch=20, \n", + " verbose=0, \n", + " validation_data=[X_test.argmax(-1),y_test],\n", + " callbacks=[\n", + " keras.callbacks.EarlyStopping(patience=2),\n", + " PredictionLogger(n=3),\n", + " TQDMNotebookCallback()\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:41:31.428117Z", + "start_time": "2017-05-31T21:41:29.002297+08:00" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy 77.108367%\n", + "Word error rate 80.812842%\n", + "Char error rate 22.891636%\n" + ] + }, + { + "data": { + "text/html": [ + "
pronunciationguessspelling
kˈɑkəsˌɔidcocksoid caucasoid
bɹˈæsfˌildbrassfilddbrassfield
flˈæʃbl̩b flashblb flashbulb
hˈɛstiə hestia hestia
kɹˈæbi crabyy crabby
tɹədˈus troduse traduce
təlˈulə talulaa tallulah
bˈitn̩ beatn beaton
hˈɑlɪtʃɛk halicck holecek
beidˈɔiə bedooy bedoya
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD3CAYAAAAALt/WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8XGed7/HPKVM1oy5XWZbr4xTbKU4lDeMQEhJKsoFs\nbmgLG1qAC9yFyxbKkoXAwgIJJQnlBpaShQWWECAJJCGJE4KT2I4dl8e2ZMtFLupd0865f5zRaCRL\nlmxrNDP27/16zevUmfnNaPSdZ57TDNd1EUIIUbzMfBcghBDi5EiQCyFEkZMgF0KIIidBLoQQRU6C\nXAghipw93U+YTKbcjo7+6X7aE1ZREUbqzR2pN7eKrV4ovpqnq96amqgx3rJpb5HbtjXdT3lSpN7c\nknpzq9jqheKruRDqla4VIYQochLkQghR5CTIhRCiyEmQCyFEkZMgF0KIIidBLoQQRU6CXAghity0\nB/lD2x9DTp0rhBBTZ9qD/Mcv/5oj/S3T/bRCCHHKykvXyu7uvfl4WiGEOCVN+7lWwAvyi2evysdT\nCyEKyM+f2MUL24+MmGdZBqnUiXe/XrBsBm9ZvfiY6/T19XLXXXfS29tDa2sLN974FpYuXcbdd38V\nx3GoqZnBZz7zeXbt2nXUvEAgeMK15cqkglwpdRHwJa31VaPm/y/g40AK+IHW+jsTPZbP8rGnS1rk\nQoj82b9/P2vWvJYrr1xNa2sLd9xxO8FgiM9+9t+or1/Aww//D3v27OHf//0LR81Talm+yz/KhEGu\nlPoE8Dagb4zFXwHOAnqBrUqpB7XWHcd6vIUVdexobSSWihOw/CdSsxDiFPGW1YuPaj3X1ERpaenJ\n6fNWVlby85//lKeeepJwuIRkMkl7exv19QsAuP76NwGMOa8QTaaPvAG4cZxlm4AyIAgYwIS/h5ZW\nLcDFpal736SLFEKIqfTggz/m7LNX8OlPf57Vq9fgui7V1dXs2+f1Fvz4xw/w1FNPjjmvEE3YItda\n/1IpVT/O4leAl/Ba67/SWndO9HhLqrxvt5bUYV5Vc87kK82jmppovks4LlJvbkm9uZfrmq+77hru\nvPNOnn76caLRKH6/jzvv/Dxf+tK/YZomNTU1fPCD72Xp0nruumvkPL//6J6EfL/HxmT26U4H+YNa\n64uz5q0Afg5chNe18mO8MP/FsR6rvb/Tfd9vP8WK6rN474p3nEzt02I6fuZNJak3t6Te3Cu2mqer\n3lxdWKILGAAGtNYp4AhQMdGdKsPllAfK2N3dJAcGCSHEFDjuIFdK3aqUul1r3QTcB6xVSq0FyoEH\nJvMYC0rr6In30j54zO2iQgghJmFSux9qrfcAF6fHf5o1/17g3uN90gVl89nQspndXU1UhSqP9+5C\nCCGy5OXIzgVldYAc4SmEEFMhL0FeG5mLZVgS5EIIMQXyEuR+y0dtZA77e5pJpBL5KEEIIU4ZeTsf\neX1ZHSk3xb7e5nyVIIQ4Tf3+97/lO9+5J99lTJm8BfnC0nQ/eVdTvkoQQohTQl7OfghQXzYfkA2e\nQpzOfrXrYTYc2TxinmUapJwTP8bk3BnLuXHx9ZNa92c/+zGPP/4YlmWxcuW5fOADH2bTpo1885tf\nx7ZtgsEgd975JVpbW/niFz+HZdk4jsNnPnMnM2fOOuEap1regrwqWEHUF5EzIQoh8mL//r2sX/8i\n9977AyzL4p/+6RM8++wzbNy4ntWr1/CWt9zK2rVP093dwwsv/JUzzjiLD3zgI7z88gb6+nrzXf4I\neQtywzCoL6tjc+tWOmNdlAfK8lWKECJPblx8/VGt5+k65H3nzh1ceunl2LYXgytXnsPu3Q287W3v\n4kc/+gEf+cj7qamZwZlnns3117+Rn/zkh3z84x+ipCTCe9/7wZzXdzzyevHlBel+cmmVCyGm25Il\nS9m69RWSySSu67Jx4wbmzZvPY4/9nuuuu5577rmPBQsW8tBDv2Lt2qdYufJcvvGN7/DqV7+Gn/zk\nh/kuf4S8tchh+MCgxu4mzpmxPJ+lCCFOM7W1dSxfvpL3v//duK7LihUrueKKq9i6dQt33XUnoVAI\nwzD4xCf+Cdd1ufPOz/DDH34fx3H40Ic+lu/yR8hrkNdF52FgSItcCDGtrrvuhsz4LbfcNmLZWWed\nzf33P3DUfb7zne/nuqwTlteulaAdYE5kFnt79pNyUvksRQghilZegxy8fvKEk+RA78F8lyKEEEUp\n70Eu+5MLIcTJyXuQL5AjPIUQ4qTkPchnhKsJ2yFpkQshxAnKe5Cbhkl9aR2tA230xAvraCkhhCgG\neQ9y8M6ECLBHWuVCCHHcCiLI5QhPIUQhuuOO22lq2pPvMiaU1wOChtSXzgOgUVrkQpxWWn7xID0v\nvjBiXpNlkko5J/yY0VUXUHPzLSdbWlEpiCAP+8LMCs+gqXsvjutgGgXxQ0EIcYr6x3/8B26++RbO\nPfd8tm/fyre+9Q3Kyyvo7e2htbWFG298C29+899M+DhPPvknfvvbXzEwEMMwDL7wha9QVlbG1772\nZbZt20IikeTd776dyy678qh5l19+1ZS9nkkFuVLqIuBLWuurRs2/APgPwAAOAbdprQdPpJD6sjoO\nHTzCwb7DzI3MPpGHEEIUmZqbbzmq9TwdZz+84YY38Yc/PMy5557P7373W847bxULFy7iyitX09ra\nwh133D6pIN+3by/3338/vb1Jvvzlf2Pdur8QCATp6urku9/9Ed3d3fzXf/0Ex3GPmjeVQT5h01cp\n9Qnge0Bw1HwD+C7wLq31ZcAjwPwTLUT6yYUQ0+Wiiy5h27YtdHd3sWnTBq6//o08/fSf+dd//Rce\neOD7JJPJST1ORUUln/zkJ/nCFz5HQ8Mukskke/c2cdZZKwAoLS3l7//+/WPOm0qT6cNoAG4cY/5S\noA34qFLqKaBSa61PtJAFcoSnEGKamKbJq1+9hq985S4uv/wqHnzwx5x99go+/enPs3r1Glx34isU\n9fb28v3v38fXvvY1PvnJfyYQCOC6LvX19WzfvjWzzsc+dseY86bShF0rWutfKqXqx1hUDVwK3AHs\nAh5WSr2otX5iosesqYkeNa+qqoTg+gB7+/aNuTyfCq2eiUi9uSX15t501Pz2t9/KmjVrePTRR9m/\nfz933nknTz/9ONFoFL/fR1lZAL/fpqIiPGY91dURVq06n7e+9a3Ytk1paSkDA9284x23smXLRj78\n4dtJpVJ88IMf5Iorrjhq3lS+RmMy3zzpIH9Qa31x1rxlwC+01svT0x8FfFrrL0/wcO54/V/f2HA/\nOzp28e+Xf46wLzTJl5Bb03W1kqki9eaW1Jt7xVbzdNVbUxM1xlt2MnutNAIRpdRirfUu4HLgpE7Y\nu6C0jh0du2jq3scZVUtP5qGEEGJKbN36Ct/+9t1HzX/Na147qQ2i0+G4g1wpdSsQ0Vrfr5R6N/DT\n9IbP57TWvzuZYoauGLS7u0mCXAhREM4882y++c37813GMU0qyLXWe4CL0+M/zZr/BHDhVBVTP3Qm\nRNngKYQQk1ZQR95E/RGqg5Xs6fIODBJCCDGxggpy8HZD7E8O0NLfmu9ShBCiKBRckNeXSfeKEEIc\nj4IL8gXSTy6EKBATnf3wb/7mBmKx2PQVNI6COGlWtrmR2fhMWw7VF+I08NwTDTRuPzJinmmZOCdx\n9sOFy2Zw6epFJ1taUSm4ILdNm3nRWnZ3NTGYjBG0A/kuSQhxipmqsx8OOXiwmS9+8V9JpVIYhsFH\nPvJ/WLJkKV/4wufYv38fsViMm2++hde97vXcd9+32LDhJVKpJFdeuZrbbnvnSb+eggty8LpXGrv2\nsLdnP0srTq9vViFOJ5euXnRU67mYzn445Fvf+jo333wLl19+FTt3au666/Pcc8+9bNy4nvvuewDD\nMFi37nkA/vjHR7jnnvuoqqrm97//7ZS8nsIM8rL5sM87E6IEuRBiql100SV8+9vfyJz98CtfuZt7\n7/0mTz31JOFwyaTPfjhkz549rFx5HgBLliiOHDlMOFzChz/8cb785X+jv7+P1772WgA+/enPc++9\n99DW1sbFF186Ja+nQINcNngKIXJnvLMfvvnNf8P69S/yl7+sPa7Hq6+vZ9OmDVx22ZXs3KmprKyi\ntbUVrbfxxS9+hVgsxk03vZ6rr34dTz75OJ/97BcAuO22m1mz5hpmzTq5azAUZJCXB8ooD5Sxu7sJ\n13UxjHHPFSOEECfk9a9/A295yxt58MFfc/BgM1/72pd5/PHHiEQiWJZFPB6f9GN98IP/my996U5+\n9rMfk0wm+dSn/oWqqira29t43/v+DtM0ueWW2/D7/ZSWlnL77e8kEAhwwQUXM3PmrJN+LZM6++EU\nG/fsh9m+t/k/2dCymc9d8n+pDlVOQ1ljkzOx5ZbUm1vFVi8UX83FfvbDnKovq2NDy2b2dDXlNciF\nEKe3U/Lsh9NlYdYVg1bNOjfP1QghTlfFcPbDgjuyc0htZC6WYckGTyGEmEDBBrnf8lEbmcP+nmYS\nqUS+yxFCiIJVsEEOXj95yk2xr7c536UIIUTBKuggz5xAq6spz5UIIUThKuwgz9rgKYQQYmwFHeRV\nwQqivoicCVEIIY6hoIPcMAzqy+roiHXSGevKdzlCCFGQCjrIYbifXFrlQggxtkkFuVLqIqXUn4+x\n/H6l1F1TVlWWoRNoNXbLBk8hhBjLhEGulPoE8D0gOM7y9wLLp7iujLroPAwMaZELIcQ4JtMibwBu\nHGuBUupS4CLgvqksKlvQDjAnMou9PftJOalcPY0QQhStCc+1orX+pVKqfvR8pdRs4DPAm4G3HM+T\n1tREj2d1zpi5mAMNB+n3dbGwcv5x3XcqHG+9+Sb15pbUm3vFVnO+6z2Zk2bdDFQDvwdmAWGl1Hat\n9QMT3fF4T/k4y++ddH1903aiqek9E6KcUjO3pN7cKrZ6ofhqnsbT2I677ISDXGt9N3A3gFLqncCy\nyYT4icg+wvPK2qm5NJIQQpwqjnv3Q6XUrUqp23NRzHhmhKsJ2SE5wlMIIcYwqRa51noPcHF6/Kdj\nLH9gSqsaxTRMFpTWsbVd0xPvJeqP5PLphBCiqBT8AUFD6tP7k++RVrkQQoxQNEEuR3gKIcTYiibI\n60vnAdAoLXIhhBihaII87AszMzyDpu69OK6T73KEEKJgFE2Qg3felVgqzsG+w/kuRQghCkZxBbn0\nkwshxFGKK8jlikFCCHGUogry2SUz8Vt+uYanEEJkKaogNw2T+ug8DvUfoT8xkO9yhBCiIBRVkMNw\n90pT9748VyKEEIWhCIM8fQItuWKQEEIARRjk9aV1GBg827yO9sGOfJcjhBB5V3RBHvVHeOOia+mM\ndXHPxu/SE+/Nd0lCCJFXRRfkAFfPv4qr667iSH8r33r5+wwkB/NdkhBC5E1RBjnAGxddy6WzL2Rf\nzwHu2/QA8VQi3yUJIUReFG2QG4bB3y67kXNqlrOzs5EfbPmJXJxZCHFaKtogB2+/8nee9bcsq1jC\n5tat/GT7f8sJtYQQp52iDnIAn2nz98vfTn1pHX899BK/2vkwruvmuywhhJg2RR/kAEE7wPtXvotZ\nJTN5cv9aHtnzeL5LEkKIaTPtQX73v/2JwYGp3zAZ8ZXwoXPeQ1Wwgod3P8bT+5+b8ucQQohCNO1B\n3tk+wMZ1uTm8vjxQxh3n/D1Rf4Sf7/gNLx7akJPnEUKIQjKpIFdKXaSU+vMY8/9WKfVXpdSzSql7\nlVITPl6kNMDmF/fT3xc/gXInNiNczR0r30PQDvDDbf/FK63bcvI8QghRKCYMXqXUJ4DvAcFR80PA\nncCrtdavAsqA6yd6vMvXLCWZcNjwfO7OKV4bncP7VrwLy7D43iv/ya7O3Tl7LiGEyLfJtMgbgBvH\nmB8DLtVa96enbWDCQyzPvWgekdIAWzY009sTm3ylx2lx+QLec/ZtpFyHezf9P/b3NOfsuYQQIp+M\nyeyqp5SqBx7UWl88zvIPAdcB12mtJ3pAd8Nf9/Lbn7/Mqkvrue6m5cdb83FZ27SOe55/gNJglM+v\n/jizojNy+nxCCJEjxngL7JN51HSf+JeBpcBNkwhxAGbPL6OsIsT655tYtnIW0bLgxHc60RrDZ3Dz\n0jfy8x3/w+ee+DofO/8DlAfKJn3/mpooLS09Oatvqkm9uSX15l6x1Txd9dbURMdddrJ7rdyH13f+\npqwulglZlsmqV83HcVxefHbPSZYwsStrL+X6Ba+lbbCDezZ+j95EX86fUwghpstxB7lS6lal1O1K\nqfOAdwPLgSeUUn9WSr15so+z+MyZVFSF0ZsP0dk+6e+AE/a6+tfw6trLONR3mO+8/P8YTOauf14I\nIabTpLpWtNZ7gIvT4z/NWnTCLXrTNLjg8noe+5+tvPjsHtbccOaJPtSkGIbBjUuupy/Zz7pD6/nu\n5h/xvpXvwmeeVO+SEELkXV4P0V+oaqieEWHnliO0t+S+u8M0TG5bdjPLq89ke8dOHtjyMxJOMufP\nK4QQuZTXIDcMr1UO8MLaPdPynJZp8e6z/hdLyheysWUzn/vLl3mu+QU5Ba4Qomjl/aRZ8xdXMWN2\nlEbdQsuh6dlS7bN8vG/FO1k973J6Er38ZPsvuHPdV3np8EY5Da4QoujkPcgNw+DCKxYA8MIze6bt\neYN2kJuW3MBnL/4El825iNaBdn6w5afc9cI32Ny6VU6FK4QoGnkPcoDa+gpm15bR1NDG4ebuaX3u\nimA5f7vsJj590T9wwczzaO49xL2bHuCrL32bHR0N01qLEEKciIII8uxW+bqn83NelJpwFe886xb+\n8cKPsrLmbHZ3N/GNDffx+T9/gz3duTsvjBBCnKyC2fduTl05tfUV7N/TQfPeTubUleenjsgsbl/+\ndpq69/HbxkfZfHg7mw9vZ0X1WVy/8LXMjczOS11CCDGegmiRD8lulee7j3p+6TzuOOc9fPbVH2Vh\nWT2bWrfwxXVf54EtP+NIf2teaxNCiGwF0yIHmDmnlPmLqmhqaGP/ng7mLajMd0mcOWMpHzvv/Wxp\n285vGx/lhcMbeOnIy1wy+wKurX8NFcH8/HIQQoghBRXkABdcXk9TQxvrnt5NbX0FhjHuCb+mjWEY\nnF19BmdWKTa2vMLDjY/ybPNf+euhl7hi7iVcPf8qSv3jn9BGCCFyqeCCvGZWlIWqhkbdQtOuNuqX\nVOe7pAzTMDlvxgpWVp/FusMb+P3uP/LEvmd4ct9a6kprOaNyKWdULmVBaR2WaeW7XCHEaaLgghzg\ngsvqadQtvPDMHuYvriqIVnk2y7S4ZPYqVs08h780r+PFwy+zu7uJpu59PLLncYJWgCUVizLBXhMq\nvNcghDh1FGSQV9aUsOSsGezccoRG3cKiZYV5MQifaXNF7aVcUXspA8lBdnY0sK19J9vbd7C5dSub\nW7cCUBWs5IzKJZxRuZSlFYsJ+0J5rlwIcSopyCAHWPWqenZtPcILz+xhwdIaTLOwW7QhO8iKmrNY\nUXMWAK0D7Wxv38G29h3ojl2sbf4ra5v/ioFBfWmdF+xVivnRWumGEUKclIIN8vLKMGr5LLZvOsSu\nrYdZevasfJd0XKpDlVw292Ium3sxKSdFU89+trXvYHv7DnZ37WV3dxO/3/MnQnYQVbGYZZVLmBGq\nIeIvIeLzbhLwQojJKNggB69VvuOVw7z4bBOLzpiBZRXUbu+TZpkWC8vms7BsPq9fcDX9iQF2dDaw\nrU2zrX0HG1teYWPLK0fdL2SHiPpKKPGVEPGXEPWVEPFHKPGFifoiI0I/4o8QsPx5eHVCiHwr6CCP\nlgU545zZbFnfzI5XDnPGylPjqMqwL8Q5NWdzTs3ZuK5Ly0AbOzsa6Ix10ZvooyfRR1+8Lz3eS+tg\n+6TOyugzfVSGypgZmsncyCzmRuYwNzKL6lAVplGcX4JCiIkVdJADnH/JfLZvOsSLz+5h6VkzsexT\nK5AMw2BGuJoZ4fF3s3Rch8HkID2JPnrTAd+b6M0aH5rfS1e8m02tW9jUuiVzf7/pY05kNnNH3GYR\nsmWjqxCngoIP8pJogLPOncOmF/azbdNBzj5vbr5LmnamYRL2hQn7wswM1xxz3ZqaKA0HmjnQe5AD\nvQfZ33OQ5r6D7Os5cNTJv6qCFcyJzKY2MjszlNa7EMWn4IMc4NyL69i6sZmXnmti2fJZ2D7ZCHgs\npf4opZVRzqhcmpmXdJIc7m9hf08zB/oOcqDnIAf6Do7YTRKGW++zSmZQE/J+KdSEqqgJVRG0g/l4\nOUKICRRFkIdL/CxfVcuGv+xly4ZmVl44L98lFR3btDPdKtm64z2Tar0DRP0RL9xD1dSEvXCvCVdT\nE6omJCEvRN5MKsiVUhcBX9JaXzVq/g3Ap4Ek8AOt9XenvMK0cy6cx5b1B1j/l72cec5sfP6i+A4q\neOO13tsG2mkZaOPIQCst/W20DLTS0t/K7q4mGrv2HPU4EV9JuvVenRXwVcwIV0tfvBA5NmEaKqU+\nAbwN6Bs13wd8DbggvexZpdRDWuvDuSg0GPKx4oJ5vLh2D5tfOsB5l8zPxdMIvNb7zJIZzCw5+oja\npJOkbbCDlv5WWgaGAt4L/D3d+2jsajrqPlF/hBmhGmaGq5kRrmFGuJqZ4RqqQlXT8XKEOOVNplnb\nANwI/Oeo+WcAu7TWHQBKqbXAFcAvprTCLCtW1bL5xf1seH4fZ507h0DQl6unEuOwTZuZ4ZoxN7qm\nnJQX8lnhfqS/hSP9rTR27aGha+TVnwwMZpRUURWoSu+544X8jFANFcGy497o6rouSTdFPBUfvjkJ\nYqk4CSfBjFA1VaH8nxpZiKk2YZBrrX+plKofY1Ep0JU13QOUTeZJa2pO/JSvl71mCY//bhsbn9/H\ndTcux5yGg4ROpt58yGe9sygHFhw1P55KcKS3leaewxzsOcLBnsMc7D1Cc88RtrZrtrbrEev7LB+z\nIjXMjs6gPFBKPJVgMBUjnowTS8WJZYYxYqlEehif8IIkM0uqOXvmMpbPVJw9Q1EaPP73Sj4PuVds\nNee73pPpaO4GsquPAp2TuWNLS88JP+kCVU3ZcyHWP7+X5v2drLnhTKJludvQVlMTPal6p1sh1xsg\nwoJAhAWBRZDebb6mJsreg0c40t/K4XTr/Uh/i9ea721hX1fzmI9lmzZ+04ff8hOw/JQGI/gsP37T\nR8Dy47O8od/047f8WKbFgZ5mdnQ28HjjWh5vXAvA3MhsVMViVMViFpcvJGgHjvkaCvn9HUux1QvF\nV/N01XusL4uTCfJtwBKlVCXQi9et8pWTeLxJ8fktbnrHeTz1yA4atrfw8x+8yFXXLi3YMySKiYXs\nEPNL5zG/dOTeSK7r0h3voTfR54Wz6Sdg+fCZvhM+D43jOuzrOcD29p3ojl00du3hQO9Bntj3DKZh\nUl9alwn2BWV12ObUbFR3XIeeeB/d8R664924rkuJr4Ro+jQLASsgpzoWJ+y4P6VKqVuBiNb6fqXU\nx4BH8a79+QOt9YGpLnAsgaCPq994JvMWHGLtn3by2P9s5YyVHbxqzWJ8so/5KcMwDMoCpZQFSqfs\nMU3DzHxpXFO/mkQqQWNXE7pjF7pjV2avnD/s+RN+08fi8oWoSi/YR++66bousVSM7ngPXbGedEin\nb1nTXfFueuN9uIzf7WObNhHfMc6n44+MWB6yg3Lglsgw8nCRY3cqf4Z0tPXxp99so/VILxVVYda8\n4UyqZ0am7PHlZ15uFVq9/YkBdnY2ZoL9UN/wTlglvjBLqurpHuinO9ZNd7yHuJM45uMFLL+3i6e/\nlNJAND0exTSMrFMrDJ9ioSfRRzwVn7BO0zAp8YWpi9aypHwhSysWMS8696hwL7T3dzKKreZp7FoZ\n9ydb0Qc5QCrp8Jc/N7D5xQNYlsElqxdx9nlzp+SnqnyocqvQ6+2KdXuh3u4Fe0esE9MwifoilAai\nlPmHwzmaDuoyf6k37Y9M2Oc+lngqkTmXTk+ij75EH73x3qPOtdMV66ZtsCNzv6AVZHH5ApZUeMFe\nG5nDzBllBf3+jqXQPxOjSZBPsaZdbTzxu+0MDiSYv7iKV1+nCIVP7tSu8qHKrWKq13VdSsptejsT\nBdOt0RnrYmdHIzs6GtjZ2UDLQFtmWcgOcdaMJcwP17GkYhFzI7OnpG7HdeiO99A20EHbYDvtgx20\np79QSnwllPjCRMYYBifZHVRMnwmQIM+Jvt4Yj/92GweaOimJ+Fl9/RnU1lec8OPJhyq3pN6p1THY\nyc7ORnZ2NLCjs5HWrGAP2yEWp7thlpQvZE5k1pjB6rhOprXfNtBO+2An7YPt3vRgBx2DnaTc1HHX\nZhomYTt0VMgPnW+/xA5T4gszp6aKeK931a2wL4zf9BX0hmAJ8lw9geuy8a/7WPf0bhzH5bxL6lh1\nWf0JXZii0P9xR5N6c6vY6jXCCZ5v2MSOzgZ2djTSNtieWVZih1lcsZDZJTPpjHXRPuC1rNtjneOe\n/z7qj1AVrKQyWJ4eVlAVqqAyWIGB1+/fl+inLz0cOs3y6Hn9iYFjbvzNZhlWOtRDhOwQ4fQt5Msa\nT4d+yA4StkPYpu0dZ5CKEU+ljzkYNZ09Pz7GunEnjomJbdr4TBvb8uEzbXzm0NDGNn1EwiGcOPgs\nO73u6PV8BOwAActPwBoaDo/7Lf9kf6mcXkE+5HBzN396aCvdnYPMnFPKmjecQWn58Z33o9j+caXe\n3Cr2etsGOtiZDvUdnQ2ZLpEhZf4olcHKTDhXBiuoSt8qghX4rak5mtpxHfqTA5mA740Ph7zhd2jr\n7qI/OUB/coCBxAD9yUH6k/0MJAZInsCvgckwMDLBOjR0XZeEkyDhJEk6SeJOgkQqMekvocnyjn9I\nh/xRoe8N77js7adnkAPEY0mefnQHO7cewR+wuOKapSw5c+ak71/s/7iFTurNrYnqbRtop3WgnYpg\nGRWBcnxTFNQnY6Ka46kEA8kBBtJB359IB35ykP6ENz/pJvGbQyHohWR2QGe3jIemfaY96S6clJMi\n4SRJOAlKKwIcbunMTCdSXuh7XwAJ4k4y0+IfOgI5lhpvODw++lfRz9/6nXGLO+VPIegP2LzmhjOY\nt6CSpx89K6JhAAAcm0lEQVTbwZ8e2sb+3R1cdvUSfH7Z51yc3qpClUV3/hm/5cNv+ab0+ILjZZkW\nlmkRJEB1OIobntovQNd1STrJTLAPpmLHXP+UD3LwDixRy2cxc24pf/zNVrZvPsTBA11c/YYzqZlV\nXOd0EEKc+gzDwGf58Fk+IpRMuH5h7EM1Tcorw9z49vNYeWEtXe0D/OpH6/ndLzax7eWDDPRPfBCG\nEEIUotOiRZ7NskwuXb2Y2vpKnv9zA3sb2tnb0I7xCMypK2fh0hoWLK2mJHr8B3IIIUQ+nHZBPqRu\nYSV1Cyvp6uincUcrjbqFA02dHGjq5Jk/7mTW3FIWLK1h1SX1+S5VCCGO6bQN8iFlFWHOvaiOcy+q\no7d7kN07Wmnc0crBfZ0cOtDNX55soHpmhIVLq1moaqionri/SgghptNpH+TZIqVBlq+qZfmqWvr7\n4uzZ1cr+3R3s3tFK6+Fe1j2zh/KqMAtVNQuX1lA9M1LQR5wJIU4PEuTjCJf4OXPlHK5co9i/r52m\nXW007mhlX2M765/by/rn9hItC2Za6jPnlkqoCyHyQoJ8EgJBH0vPnsXSs2eRiKfY29hO444Wmna1\n8fIL+3n5hf2URP0sXFrDomU1zKotk1AXQkwbCfLj5PNbLFrmBXYq6bB/TwcNuoXdO1rZ/NIBNr90\ngHBkZKibpoS6ECJ3JMhPgmWbzF9cxfzFVaRe53CgqYOG7V6ov7L+AK+sP0CoxJcJ9dnzyjDN02rX\nfSHENJAgnyKWZVK3sIq6hVVccY1D895OGnULjbqVLRua2bKhmWDYx8Kl1SxaVsOcunIJdSHElJAg\nzwHLMpm3oJJ5Cyq5/LVLaN7b5XW/6Ba2bjzI1o0HCYZsFqRb6nPqyk/oFLtCCAES5Dlnmia19RXU\n1ldw+dVLOLivMx3qrWx7+SDbXj5IIGizYEk18xZWMqeunHDJyV3VSAhxepEgn0amaTB3fgVz51dw\n2ZolHDrQReP2Fhp1C9s3H2L75kMAlFeFmTOvjDl15cyZVy6nCxBCHJMEeZ6YpsGceV5Qv2rNYg43\nd9O8t5Pmvd4RpUNdMABlFSFmZwV7tCyY5+qFEIVkwiBXSpnAt4GVQAx4j9Z6V9byjwO3Ag7wBa31\nr3NU6ynLMAxmzS1j1twyzrtkPo7j0HKol+Z9nRzc28nB/V1s33SI7Zu8Fnu0LDjcYq/zgl32Wxfi\n9DWZFvmbgKDW+hKl1MXAV4E3AiilyoGPAIuBEmAjIEF+kkzTZOacUmbOKeXci+pwHJe2I71ei31f\nJwf3daFfOYx+5TAAkdIAc+aVM7uujGVnzqandxDLMjAtM2toyv7sQpyiJrzUm1LqP4B1WusH09MH\ntNZz0+M+4AngDXhB/ozWesEEzznt15Y71biOy5FDPTQ1tNHU2EZTQxv9fROfT90wvD1qLDt9s4Zu\nBrZtYdle6Nu2SSjsp6wiRHlF2BtWesNgKP+XAhPiNHVSl3orBbqyplNKKVtrnUxP7wO2AhbwxclU\ncypd8zBfTJ/BgmXVLFhWjeu6dLT107y3k4HeOL29MVIpByflkko5pFIuzojh8LJEIkVscOSyY323\n+wMW0dIgkbIg0dIg0bIg0bIA0bIgkdIgobDvuLp5CvX9HY/Um3vFVvN01VtTM/7VzCYT5N1A9iOY\nWSF+LTAbGGqFP6qUelZrve5EChUnxjAMKqtLqKwumZIPleO4DPTH6ekapLc7Rk/XoHfr9obdXYO0\ntfSNeV/bNtMh74V7aUWI8ooQZZVhSsuD2LZcJ1WIqTaZIH8WuAH4ebqPfHPWsg5gAIhprV2lVCdQ\nPvVliulkmgYlkQAlkQDMPXq567rEBpMjAr63K5YJ+p6uQTrb+sd87GhpgLJ0N01ZRYi6BVUYlrcB\nVw6Kmnqu6+I6TvaMkcP0OjgODK2bnnZHDXEdXMcF1xk1P30/xxlz6KZSo+Z50zgurpPy5qcc3PTj\nxsN++noHcbNrdQFc77lc9+jXMTR/1Gs7psn8cnQd3JQDTmp46LiZaddJ0e4zGeyPp5c5kEoNv9ZU\nyqvHNDFMCywTwzTBsjDM9LhpYVhHDw3T8u5necOa97x9/JcyiT7yob1WVuD10bwLuA7YpbV+SCn1\nOeB1eHutrAU+obU+1oO68rMpdwql3kQ8SXfXIF3tA3R1pG/t/XR1DNDXe3R/vmF4YV5WGfZa8BUh\nyiq9vvlIaeC4T2fguq73D5VMDt8ch6EwGAqHTCi4LuB6/6Skg8N10uukg9BxKS8P0X6kEzeR8G7J\nBG48gTM0nUjgJuKjpseYl0plAi4TkNlh544MRC/0htbNWua92JHhNdkgE9PDMKbkb/Kq3/xy3G+e\nCYM8ByTIc2gq6s2EoJtuKTkpL0iyWxpOKr3MGW6JZFpfbqbV5aaSuMl0oKa8QI3HUvT0Jenpd4gl\nLdp6UvQMGvQmTGKpsbpeXPyk8JPATxy/G8fvxPE5MfypAXzJQXzJfnyJfnzxAex4L2YqOcbjnBzv\nP8XAmIrt9ekWGUOtMmOodWYMzzNNDMMcOW0aXovNNL2AGGpVGsbIbRPpcZ/fJpFIjWh9Dq9nDG8+\ny9SRfv5RQwwjUyemkalreL4x8vWMfn0jhkMtzfTrtrLmGwZl5WG6uwdHvDYMvOc0huoe43WPWH+C\n1vZkcm+oJT30WizLq8HKbl1bVNVEae8YyLScvaEx/DoNY9QvleHWvPc/MrrFn/UrJms4//ILT2pj\npygCTiKO09dH/0AnAwfbcWODOIPp29B4LIYzODhqWWzEOm56ejpadaH0LbsvLmn66PeVZm4DvlIG\nfSXErSAxK0ivWZre/QbvNs5OND4S+EkSMFPpm4NhQAoDBxMXg5Rr4GJ681wDZ/TQNXAgazwdjqZD\nwAa/DUEfBPwmgaBFMGARDNoEwz6CYT+hEj/BcIBQNIgV8GP4fBg+P4ZtT9t+/8XWEAGoqoniFFHN\ngaootnPsejNfYjmqQYK8gLiOg9PfT6qvj1RfH07/qGF6fqo/Pd7fT6qvF6e/Hzc+8e6H4zH8fsxA\nEDMYxIxGMQJBTJ9vRP/ciNaUZY5smaRbV5nxoXUty7vZNtg2hmVj2N60N25TXhWluzfurTPGcsO2\nvHGfD8OycAyT2GCSwf4EA/1xBvoTI8a96TgDAwkG+hK0DyS8Tr+J3gPDOy2xaZlYtoFlmfgsE9My\nMrtpmpaB32/T2z3I4ECCzoEETv/QF14qfRv77+AP2ITCPoIh7xYI2tg+E9tn4fNZ2D4zPRw57vNb\n2LbpDX0WvvR9pvsAMNd1SSUdkkmHZCI1PEwMzzMtg0DQe23+gE0gaMt2j2kiQX4SnEScZGcnqZ5e\n3HgMJxbzWrRHjce98fT00LibXpaZNzAw+Sc3DMxQCKukBHv2HKySEsxwCSVVZcSxMsFsDAV0IOAN\ng0fPN/J4Ot2KmijJ42h9WXiX4fNOLDbxhbAdxyU26AU8gJUO5qGDpYbGJ3uwVHYL13VdkomU9+Ux\nkL71JxgcSDI4kGAgMz186+kaxHFO/teOZZv4fF7thmFgmt7NSA9Nwxv3B2xSqVRmevR6Q18IyaTj\nBXU6nBPJ1IjpZHIS34ZjsH1mJtwD6XAPBG38QXvM+YGQj2gkiOu6crTycZAgH4Prujh9fSQ7O+jY\nF6Nr9wGSnR3eraODRIc37vT2nviTGIYXooEApj+AEYlihcOYJSVYJSVY4RJvPOxNe+NhzJKINwyH\nxwzgYvwpnUumaRAK+wmFp/6MkoZh4PPb+Pw2peWhSd3HdV3isRTxWJJkMh2a8RSJRGo4RNPjiYST\nHqbDdGg8nl6WTOGkXBzHTX+pODjO8LTjuOkdTI59bMDI10TmV4FtW4RK/Ni2lfn1YNtmZpk3PrSu\nSSrl7c0UG0wQjyXT496ttztGe2zsXVbH4vNblEQDRNK3kmiASOnwMBIN4A9MXxdVPjiO11CIx1Mk\n4smT3o/8lOOmUiSOHCbW3EyyYyig29PjnSQ7O47ZVWEEgtgV5QTn1WGXV2BFo5lANgN+bzwQwPB7\nw6PGA34M+/gOnBGnBsMwMq3P6TD0xe662eHuDge+4+JCJqxz2RXifYmNDPjYYJJYLEFsMEl80Psl\nk4g7dLT20dsz/m6s4LX2I6XBMcM+GPKld7100zv1DA8dx9sryUnv7ePt/DP0BciI+0F6u6lhZG1n\nNTLzDcPgyIEeenoG0vOMo+7jupBIpLwv7HgyHczD04l4asS8eHpeMjHyV9Cnv3rD+O/FCf1Fikiq\np4fY/n3p235i+/cRbz6Am0gcvbJhYEWj+GfNxq6owK6opGzuTGL+MHa5N21XVGCFJtf6EqJQGOm9\nO/J5USrvS8xHIHjs0zxk/6pMJlL09sTo64nR2xOjtzs93j0871hhX0xsn4nfb+P325REAvj9lveL\nL2Dh9x/7QLpTJsjdZJL4wYNZoe0Fd6qrc8R6hm3jnzOXQG0t/rm1+Kqq0iFdgV1W7m10yyJdFULk\nj+2zKK8MU14ZHnedZCJFX68X7kOhHxtMpr+8GB6mtwlkxjEwTG+5md7F0TTTw+xdGN3h1rnL0PEH\n6SkXIiUBenoHhw5FyByY5B2q4GLgdRV53XBWOqC96eHxk9uAXZRBnuzpJta0h9i+/ZnQjh86CKnU\niPXsyipKVqwkUDuPQO08/LXz8M+c6e2JIcQpLuU4dPXGONTeT99ggv7BZNYwSd+ANx5PpvDZJn7b\n8oY+E59t4bdN/HZ63GcevY7l7V0ztB4YJB3vvD3JlJO+eeOZeY43L5W9zHEz66ZSDoGQj56eGKl0\n90/KcTLj3rSbtWz0PG9dbxdwb4Ould74620Ezpqf3gCcWZ7ZaExmg/GQTJcLpI8XG+qGgdBAgv7+\neCa4hw9C9cbddDeO67g4Q9077nBXTmYcr1/8qGXp7p+7/8+rx/1bF1WQx/btpf3RP9DzwroRoW0E\nAgTn16cDuxZ/7TwCc2uxSibeq0GIbCnHoW8w6YXdQCI97g37BhOYlkUslsC2TGzLwJc+W6Sd3l3R\nG0/PT0/70uvaWfMMgxGBl0q5Y4ZcMuUFWXbQJYfWTToMxFIjQjq73sF4auIXLAqaASP2LhpPwQe5\n67r0b91Cx2OP0L/lFQD8s+cQWXUBgXl1BGrn4auuzusudOLYkimHRNK7xZOprHGHg12DtLT2ZqZH\nrJdwSKRDzjugMN3Pawy3mLwPOSNaVJnpUcsGE6nhUB44NcMv6LcoCdrUlIcoCdpUlIWwDIgEfYSD\nNiVBm3DQNzwM2fhti0T6TJjxUe9/POtvFk/vlhjP+vtl3wfIfGFZQ19cpoltp+eZw19mw8tHrm+Z\nBpWVJfT2DHot5awWs2UZw/PSf18rfR/T8JZZVvrvznBL1zuVy3BLd3i+Sypr3DtLwvDyVHo30Uz3\nDKQPhh3qsvHGKyrCdHb2j1wnPT6Uv2ZWq984xuc3e9nQ805GwQa5m0zS8+I6Oh59hNi+vQCElioq\nXnctJWevkOA+Sa7rfVBjiRTxhEMskSIWTxFPpjLz4slU+p/XIZ7w5g/9Q8cy/9wp75aeP3Sf7Ps6\nBX7uDy/8fMwoD3lhF/KCriQTfj5KQt747BlR2tr7SCYdElkt6kRyuNU8PO59ESWTwy1ob563B4lv\nVOBlxtP7t9uW4QVh1rQ1ajoUGK4zHLCxR+11UozbeKaqZsswmI7jkWpqorQE8ttdW3BBnhoYoPuZ\np+j402Mk29vBMIisupDKa15HcMHCfJc3ZVKO97N4IJbM3Lyf0d5P6bH6AlOO99N6xLQz8j4+v01n\n92A6YJ10KKeIJYbDOJ5MEYtPbcAagN/n9aX6bZNIyIcvOtyn6svqb82M+0zKS0Mk4sl0f+sY69gm\nlml6fYxuVsspM53d/ziyzzEznR4P+KxMIJdkWqU21nE0CmpqorSE5eIaorAUTJAnOjrofPyPdD31\nJM7AAIbfT/nqNVRcfQ2+mpp8lzemWDxFW/cg3X1xBmJJ+rNCeSCWGjWdvdwL1OlimQaBdMgG/Ral\nJX4CPouAz8TvszLLhsctAraZFczW8AYwn0nAttKhO7zctibuxxtLMbYYhSg0eQ/y2P59dDz2CN1/\nfR5SKazSUqquuZbyq1ZjRSJ5q8t1XXoHEnTt62RXUxtt3THaugZp6x7MDHsHxtgXfRyW6f0MDgW8\nIA0HbEIBOzMMBux069MY0Tc4NG6b5lHzLMvrKxy6HqdlGtRUR+jrHcRvDwf06J/bQohTS16C3HVd\nBrZvo/3RP9D/inedCt+sWVS+9lqil1yC6Zv6w6nHqqGjJ0brqHDODLsHiSfGPr+EzzapLA0yf2aE\nytIgZZEA4YD3M30orLNDOhSw8dvmtBzJ6bVwJbiFOJ1Me5C3PP0Me3/xa2J7m4D0BszXvo6SFStz\nugGzfzDJ7kPdNB7ooqG5m8bm7nFb1CVBm1kVYarKgtTOKiXsM6kqDVJVFqSqNEj0OK9LKYQQuTTt\nQb7jq1/3NmCev4qKa64ltHDRlD+H47g0t/bR0Dwc2gdb+0ZcDqC6LMiy+RXMKA9lArqqNEBlaZBQ\nYPhtkT5cIUShm/Ygr735JuxzL8I/Y8aUPWZXX5zG5i4am7tpONDF7kM9xLL2Bw74LFRdOQvnlLFo\nTikL55RSFglM2fMLIUQ+TXuQz7/t1pNu4R5s6+OVxnYa0uHd2jU4YvnsqjCL5pSxcG4pi+aUMbe6\nZNLnmxZCiGKT971Wjpfe28FXHtyYOeqqJGizYlEVC+d4ob1gdpTwBGdXE0KIU0lRBfmRzgG+9Wvv\nMP23XaM4s97r45YNj0KI09mEQa6UMoFvAyuBGPAerfWurOXXAp/BO7jvJeCDWuspPyZ7IJbknv/e\nRO9Agne8TnHlOXOn+imEEKIoTWZ/vzcBQa31JcD/Bb46tEApFQX+Hbhea30RsAeonuoiHcfl/oe2\ncKC1jzXn10qICyFElsl0rVwGPAKgtX5eKbUqa9mlwGbgq0qphcD3tNYtEz3gsa49N5YHHt7Cyw1t\nnLO0hjveeu60X5n7eOvNN6k3t6Te3Cu2mvNd72SCvBToyppOKaVsrXUSr/X9auAcoBd4Rin1F631\njmM94PHstfLcKwf55ZO7mFkZ5t3XLaO9ffIXcJ0KxbYfudSbW1Jv7hVbzdNV77G+LCbTtO0Gsh/B\nTIc4QBvwgtb6kNa6F3gaL9SnRMOBLh74w3ZCAZsP37ScEtkbRQghjjKZIH8WuA5AKXUxXlfKkPXA\n2UqpaqWUDVwMbJ2Kwtq7B7nnV5tJOS7vf9NZzK6Sq/0IIcRYJtO18mvgaqXUc3h7prxLKfUxYJfW\n+iGl1KeAR9Pr/lxr/crJFhWLp7j7l5vo7ovzt2uWcPaCqpN9SCGEOGVNGORaawd436jZ27OWPwg8\nOFUFOa7L93+3lb2He7li5WzWnF87VQ8thBCnpII73+lvn93Di7qFpbVl3PZaJQf7CCHEBAoqyF/c\nfoTfrN1NdVmQD9y4XC6IIIQQk1AwSdl0qIfvPbyVgN/iwzetoDSc+4tLCCHEqaAggryrN8bdv9xE\nIulw+w1nUjsjf5d4E0KIYpP3IE8kU3zzV5vp6Ilx45ULOXdJYV5oWQghClVeg9x1XX74iKahuZuL\nz5rJdRfPz2c5QghRlPIa5I+s28tzrxxiwexS3nXtMtlDRQghTkDegnzjrlb++8kGKqIBPnTTcny2\nla9ShBCiqOUlyA+09HLfQ1vw2SYfumk55XL9TCGEOGHTHuRDe6jE4in+7vVnUD+rdLpLEEKIU8q0\nB/mXfvQiLZ2DvOFV9Vx4xszpfnohhDjlTHuQb25o5XxVwxsuWzDdTy2EEKekab/48lXn1/LWKxdh\nyh4qQggxJaa9Rf7xW88n4Jc9VIQQYqrk/chOIYQQJ0eCXAghipwEuRBCFDkJciGEKHIS5EIIUeQk\nyIUQoshJkAshRJGTIBdCiCJnuK6b7xqEEEKcBGmRCyFEkZMgF0KIIidBLoQQRU6CXAghipwEuRBC\nFDkJciGEKHIS5EIIUeRydoUgpZQJfBtYCcSA92itd2Ut/3vgvUASuFNr/XCuapkMpZQP+AFQDwTS\nNT2UtfyjwHuAlvSs92qt9XTXmU0ptR7oTk/u1lq/K2tZob2/7wTemZ4MAucAs7TWnenl3wAuA3rS\n67xRa901zWWSruUi4Eta66uUUouBBwAXeAX4oNbayVo3BPwYmIFX+zu01i1HP+q01XsOcA+Qwvu/\ne7vW+vCo9cf93OSh3nOBh4Gd6cXf0Vr/V9a6hfb+PgjMSi+qB57XWt+Sta4B7Gf49fxFa/2pXNeY\ny0u9vQkIaq0vUUpdDHwVeCOAUmoW8GFgFd4/9Vql1B+11rEc1jOR24A2rfXblFKVwEbgoazl5+P9\nU7yUl+pGUUoFAUNrfdUYywru/dVaP4AXiCilvgX8YCjE084HrtFat05/dcOUUp8A3gb0pWf9B/DP\nWus/K6XuxfsM/zrrLu8HNmutP6uUugX4Z+Ajeaz3G8CHtNYblVLvBT4JfCxr/XE/N9NhjHrPB/5D\na/3Vce5SUO/vUGgrpSqAJ4GPjrrLImC91vqG6aoRctu1chnwCIDW+nm8UBlyIfCs1jqWbnXtAlbk\nsJbJ+AXwL+lxA68lm+184FNKqbVKqZx/w07CSiCslHpMKfVE+stySCG+vwAopVYBZ2mt78+aZwJL\ngPuVUs8qpf4ubwVCA3Bj1vT5wFPp8T8Aa0atn/mcj7M810bXe4vWemN63AYGR61/rM/NdBjr/X29\nUupppdT3lVLRUesX2vs75HPAPVrrg6Pmnw/MVUo9qZT6vVJK5bxCchvkpUD2T+OUUsoeZ1kPUJbD\nWiakte7VWvekP0j/jffNn+1B4H3AauAypdT1013jKP3AV4Br8Or6SSG/v1n+Ee+fIFsJXnfAbcDr\ngA8opfLyxaO1/iWQyJplaK2HzmMx1vuY/V5P+/s8ut6hYFFKXQrcAXxt1F2O9bnJuTHe33XAP2it\nrwAagc+MuktBvb8ASqkZwGtI/8Ic5SDwRa31q4Ev4HUL5Vwug7wbyP52NbXWyXGWRYHsn9l5oZSa\nh/dz6T+11j/Nmm8AX9dat2qt48DvgHPzVOaQHcCPtdau1noH0AbMTi8r1Pe3HFBa6ydHLeoHvqG1\n7tda9wBP4LUcC4GTNT7W+5j9XhfK+/xW4F7g9WP0Jx/rc5MPv87qrvw1R/9fFdz7C/wN8FOtdWqM\nZS8CvwHQWq8F5qTzI6dyGeTPAtcBpH++bc5atg64XCkVVEqVAWfgbUjKG6XUTOAx4JNa6x+MWlwK\nvKKUiqT/KKuBfPeV/x3edgeUUnPwahz6mVdw72/aFcDjY8xfCjyrlLLSG50vA9ZPa2Xj26CUuio9\nfi3wzKjlmc/5OMunlVLqNryW+FVa68YxVjnW5yYfHlVKXZgefw1H/18V1Pubtgavm2csnwH+N4BS\naiWwL+sXXc7k8ifVr4GrlVLP4fU5v0sp9TFgl9b6IaXU3Xh/FBP4J6316L686faPQAXwL0qpob7y\n7wIlWuv7lVL/iNdajwGPa61/n6c6h3wfeEAptRZvj4q/Az6slCrU9xdA4f189iZGfh7+E3ge72fs\nj7TWW/JU42gfB76rlPID2/C63VBKPQZcD3wH+GH67xAHbs1XoUopC7gb2Av8Kt09+5TW+jNKqR/h\ndRce9bnJ+qWcD+8H7lFKJYBDwO1QmO9vlhGfYxhR713Aj5VSr8fbzvbO6ShITmMrhBBFTg4IEkKI\nIidBLoQQRU6CXAghipwEuRBCFDkJciGEKHIS5EIIUeQkyIUQosj9f59B/LrZX3aKAAAAAElFTkSu\nQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "pd.DataFrame(history.history).plot()\n", + "\n", + "scores = model.evaluate(X_test.argmax(-1),y_test, verbose=0)\n", + "score = dict(zip(model.metrics_names,scores))\n", + "\n", + "y_pred = model.predict(X_test.argmax(-1), verbose=0, batch_size=batch_size)\n", + "\n", + "print ('Accuracy {:%}'.format(score['acc']))\n", + "print ('Word error rate {:%}'.format(WER(y_pred,y_test)))\n", + "print ('Char error rate {:%}'.format(CER(y_pred,y_test)))\n", + "n=10\n", + "show_results(ytable=ytable, y_pred=y_pred[:n],y_test=y_test[:n],xtable=xtable,X_test=X_test[:n])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using a sequence2sequence rnn with attention\n", + "\n", + "This notebook uses the seq2seq translator with \"attention\" for alignment.\n", + "\n", + "We choose this because this model has had great results (a 24% word error rate) for grapheme two phoneme translation. We are doing it the other way round: phoneme to grapheme. This is a bit harder but the same model should work well.\n", + "\n", + "I tried lots of anttenion layer for keras but in the end I wrote by own by porting from tensorflow. This is based on [tensorflows implementation](https://github.com/tensorflow/tensorflow/blob/c8a45a8e236776bed1d14fd71f3b6755bd63cc58/tensorflow/python/ops/seq2seq.py#L506).\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:12:38.787465Z", + "start_time": "2017-05-31T21:12:38.771473+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "from helpers.layers import Attention, SimplifiedAttention" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:43:01.328277Z", + "start_time": "2017-05-31T21:43:01.299637+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# set model parameters\n", + "nb_chars = len(ytable.chars)\n", + "nb_phons = len(xtable.chars)\n", + "maxlen_x = xtable.maxlen\n", + "maxlen_y = ytable.maxlen\n", + "batch_size = 300\n", + "hidden_nodes = 128\n", + "RNN=keras.layers.recurrent.GRU" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:43:23.180329Z", + "start_time": "2017-05-31T21:43:23.162669+08:00" + } + }, + "outputs": [], + "source": [ + "import tensorflow\n", + "import keras\n", + "\n", + "# you probobly need these exact versions\n", + "assert keras.__version__=='1.2.2'\n", + "assert tensorflow.__version__=='1.0.0'" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:51:44.163211Z", + "start_time": "2017-05-31T21:44:57.287710+08:00" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e2227220291146e498a030837abb0ee4" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a509f424c4954a58af4b6d39a598abf1" + } + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8700/|/[loss: 2.597, acc: 0.308] 12%|| 8700/72703 [06:29<46:19, 23.03it/s]" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0mPredictionLogger\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0mTQDMNotebookCallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEarlyStopping\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'loss'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 31\u001b[0m ]\n\u001b[1;32m 32\u001b[0m )\n", + "\u001b[0;32m/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m 670\u001b[0m \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 671\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 672\u001b[0;31m initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m 673\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 674\u001b[0m def evaluate(self, x, y, batch_size=32, verbose=1,\n", + "\u001b[0;32m/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch)\u001b[0m\n\u001b[1;32m 1194\u001b[0m \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1195\u001b[0m \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m 1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1198\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, nb_epoch, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m 889\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 890\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 891\u001b[0;31m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 892\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m 1941\u001b[0m \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1942\u001b[0m updated = session.run(self.outputs + [self.updates_op],\n\u001b[0;32m-> 1943\u001b[0;31m feed_dict=feed_dict)\n\u001b[0m\u001b[1;32m 1944\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1945\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 765\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 766\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 768\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 769\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 963\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 964\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m 966\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 967\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1013\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1014\u001b[0m return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m 1016\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1017\u001b[0m return self._do_call(_prun_fn, self._session, handle, feed_dict,\n", + "\u001b[0;32m/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1021\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1023\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1024\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m 1002\u001b[0m return tf_session.TF_Run(session, options,\n\u001b[1;32m 1003\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m status, run_metadata)\n\u001b[0m\u001b[1;32m 1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1006\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + " \r", + "9000/|/[loss: 2.586, acc: 0.310] 12%|| 9000/72703 [06:49<46:05, 23.03it/s]" + ] + } + ], + "source": [ + "model = Sequential()\n", + "# hidden_nodes = 512\n", + "\n", + "# Encode, bidirectional keeps the accuracy up at the end of the word\n", + "model.add(Bidirectional(RNN(hidden_nodes, return_sequences=True), batch_input_shape=(batch_size, maxlen_x, nb_phons)))\n", + "# model.add(RNN(hidden_nodes, return_sequences=True, batch_input_shape=(batch_size, maxlen_x, nb_phons)))\n", + "model.add(Dropout(0.2))\n", + "\n", + "# Decode with attention\n", + "model.add(Attention(RNN(hidden_nodes, return_sequences=True, consume_less='mem')))\n", + "model.add(Dropout(0.2))\n", + "\n", + "# # I could add some more RNN models\n", + "# model.add(RNN(hidden_nodes, return_sequences=True, consume_less='mem'))\n", + "\n", + "\n", + "\n", + "# classifier\n", + "model.add(TimeDistributed(Dense(nb_chars)))\n", + "\n", + "model.add(Activation('softmax'))\n", + "model.compile(loss='categorical_crossentropy',\n", + " optimizer='adam',\n", + " metrics=['accuracy'])\n", + "# model.summary()\n", + "\n", + "history = model.fit(X_train, y_train, \n", + " batch_size=batch_size, \n", + " nb_epoch=60, \n", + " verbose=0, \n", + " validation_split=0.2,\n", + " callbacks=[\n", + " keras.callbacks.EarlyStopping(patience=2),\n", + " PredictionLogger(n=3),\n", + " TQDMNotebookCallback(),\n", + " keras.callbacks.EarlyStopping('loss',2)\n", + " ]\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.761591Z", + "start_time": "2017-05-31T12:47:48.562Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "ts = arrow.utcnow().format('YYYYMM-DDHHmmss')\n", + "model_file = './models/{}_{}.hdf'.format('seq2seq_attention', ts)\n", + "model.save(model_file)\n", + "model_file" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:52:51.247279Z", + "start_time": "2017-05-31T21:52:51.039077+08:00" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 81, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD3CAYAAAAALt/WAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8XGed7/HPKVM1oy5XWZbr4xTbKU4lDeMQEhJKsoFs\nbmgLG1qAC9yFyxbKkoXAwgIJJQnlBpaShQWWECAJJCGJE4KT2I4dl8e2ZMtFLupd0865f5zRaCRL\nlmxrNDP27/16zevUmfnNaPSdZ57TDNd1EUIIUbzMfBcghBDi5EiQCyFEkZMgF0KIIidBLoQQRU6C\nXAghipw93U+YTKbcjo7+6X7aE1ZREUbqzR2pN7eKrV4ovpqnq96amqgx3rJpb5HbtjXdT3lSpN7c\nknpzq9jqheKruRDqla4VIYQochLkQghR5CTIhRCiyEmQCyFEkZMgF0KIIidBLoQQRU6CXAghity0\nB/lD2x9DTp0rhBBTZ9qD/Mcv/5oj/S3T/bRCCHHKykvXyu7uvfl4WiGEOCVN+7lWwAvyi2evysdT\nCyEKyM+f2MUL24+MmGdZBqnUiXe/XrBsBm9ZvfiY6/T19XLXXXfS29tDa2sLN974FpYuXcbdd38V\nx3GoqZnBZz7zeXbt2nXUvEAgeMK15cqkglwpdRHwJa31VaPm/y/g40AK+IHW+jsTPZbP8rGnS1rk\nQoj82b9/P2vWvJYrr1xNa2sLd9xxO8FgiM9+9t+or1/Aww//D3v27OHf//0LR81Talm+yz/KhEGu\nlPoE8Dagb4zFXwHOAnqBrUqpB7XWHcd6vIUVdexobSSWihOw/CdSsxDiFPGW1YuPaj3X1ERpaenJ\n6fNWVlby85//lKeeepJwuIRkMkl7exv19QsAuP76NwGMOa8QTaaPvAG4cZxlm4AyIAgYwIS/h5ZW\nLcDFpal736SLFEKIqfTggz/m7LNX8OlPf57Vq9fgui7V1dXs2+f1Fvz4xw/w1FNPjjmvEE3YItda\n/1IpVT/O4leAl/Ba67/SWndO9HhLqrxvt5bUYV5Vc87kK82jmppovks4LlJvbkm9uZfrmq+77hru\nvPNOnn76caLRKH6/jzvv/Dxf+tK/YZomNTU1fPCD72Xp0nruumvkPL//6J6EfL/HxmT26U4H+YNa\n64uz5q0Afg5chNe18mO8MP/FsR6rvb/Tfd9vP8WK6rN474p3nEzt02I6fuZNJak3t6Te3Cu2mqer\n3lxdWKILGAAGtNYp4AhQMdGdKsPllAfK2N3dJAcGCSHEFDjuIFdK3aqUul1r3QTcB6xVSq0FyoEH\nJvMYC0rr6In30j54zO2iQgghJmFSux9qrfcAF6fHf5o1/17g3uN90gVl89nQspndXU1UhSqP9+5C\nCCGy5OXIzgVldYAc4SmEEFMhL0FeG5mLZVgS5EIIMQXyEuR+y0dtZA77e5pJpBL5KEEIIU4ZeTsf\neX1ZHSk3xb7e5nyVIIQ4Tf3+97/lO9+5J99lTJm8BfnC0nQ/eVdTvkoQQohTQl7OfghQXzYfkA2e\nQpzOfrXrYTYc2TxinmUapJwTP8bk3BnLuXHx9ZNa92c/+zGPP/4YlmWxcuW5fOADH2bTpo1885tf\nx7ZtgsEgd975JVpbW/niFz+HZdk4jsNnPnMnM2fOOuEap1regrwqWEHUF5EzIQoh8mL//r2sX/8i\n9977AyzL4p/+6RM8++wzbNy4ntWr1/CWt9zK2rVP093dwwsv/JUzzjiLD3zgI7z88gb6+nrzXf4I\neQtywzCoL6tjc+tWOmNdlAfK8lWKECJPblx8/VGt5+k65H3nzh1ceunl2LYXgytXnsPu3Q287W3v\n4kc/+gEf+cj7qamZwZlnns3117+Rn/zkh3z84x+ipCTCe9/7wZzXdzzyevHlBel+cmmVCyGm25Il\nS9m69RWSySSu67Jx4wbmzZvPY4/9nuuuu5577rmPBQsW8tBDv2Lt2qdYufJcvvGN7/DqV7+Gn/zk\nh/kuf4S8tchh+MCgxu4mzpmxPJ+lCCFOM7W1dSxfvpL3v//duK7LihUrueKKq9i6dQt33XUnoVAI\nwzD4xCf+Cdd1ufPOz/DDH34fx3H40Ic+lu/yR8hrkNdF52FgSItcCDGtrrvuhsz4LbfcNmLZWWed\nzf33P3DUfb7zne/nuqwTlteulaAdYE5kFnt79pNyUvksRQghilZegxy8fvKEk+RA78F8lyKEEEUp\n70Eu+5MLIcTJyXuQL5AjPIUQ4qTkPchnhKsJ2yFpkQshxAnKe5Cbhkl9aR2tA230xAvraCkhhCgG\neQ9y8M6ECLBHWuVCCHHcCiLI5QhPIUQhuuOO22lq2pPvMiaU1wOChtSXzgOgUVrkQpxWWn7xID0v\nvjBiXpNlkko5J/yY0VUXUHPzLSdbWlEpiCAP+8LMCs+gqXsvjutgGgXxQ0EIcYr6x3/8B26++RbO\nPfd8tm/fyre+9Q3Kyyvo7e2htbWFG298C29+899M+DhPPvknfvvbXzEwEMMwDL7wha9QVlbG1772\nZbZt20IikeTd776dyy678qh5l19+1ZS9nkkFuVLqIuBLWuurRs2/APgPwAAOAbdprQdPpJD6sjoO\nHTzCwb7DzI3MPpGHEEIUmZqbbzmq9TwdZz+84YY38Yc/PMy5557P7373W847bxULFy7iyitX09ra\nwh133D6pIN+3by/3338/vb1Jvvzlf2Pdur8QCATp6urku9/9Ed3d3fzXf/0Ex3GPmjeVQT5h01cp\n9Qnge0Bw1HwD+C7wLq31ZcAjwPwTLUT6yYUQ0+Wiiy5h27YtdHd3sWnTBq6//o08/fSf+dd//Rce\neOD7JJPJST1ORUUln/zkJ/nCFz5HQ8Mukskke/c2cdZZKwAoLS3l7//+/WPOm0qT6cNoAG4cY/5S\noA34qFLqKaBSa61PtJAFcoSnEGKamKbJq1+9hq985S4uv/wqHnzwx5x99go+/enPs3r1Glx34isU\n9fb28v3v38fXvvY1PvnJfyYQCOC6LvX19WzfvjWzzsc+dseY86bShF0rWutfKqXqx1hUDVwK3AHs\nAh5WSr2otX5iosesqYkeNa+qqoTg+gB7+/aNuTyfCq2eiUi9uSX15t501Pz2t9/KmjVrePTRR9m/\nfz933nknTz/9ONFoFL/fR1lZAL/fpqIiPGY91dURVq06n7e+9a3Ytk1paSkDA9284x23smXLRj78\n4dtJpVJ88IMf5Iorrjhq3lS+RmMy3zzpIH9Qa31x1rxlwC+01svT0x8FfFrrL0/wcO54/V/f2HA/\nOzp28e+Xf46wLzTJl5Bb03W1kqki9eaW1Jt7xVbzdNVbUxM1xlt2MnutNAIRpdRirfUu4HLgpE7Y\nu6C0jh0du2jq3scZVUtP5qGEEGJKbN36Ct/+9t1HzX/Na147qQ2i0+G4g1wpdSsQ0Vrfr5R6N/DT\n9IbP57TWvzuZYoauGLS7u0mCXAhREM4882y++c37813GMU0qyLXWe4CL0+M/zZr/BHDhVBVTP3Qm\nRNngKYQQk1ZQR95E/RGqg5Xs6fIODBJCCDGxggpy8HZD7E8O0NLfmu9ShBCiKBRckNeXSfeKEEIc\nj4IL8gXSTy6EKBATnf3wb/7mBmKx2PQVNI6COGlWtrmR2fhMWw7VF+I08NwTDTRuPzJinmmZOCdx\n9sOFy2Zw6epFJ1taUSm4ILdNm3nRWnZ3NTGYjBG0A/kuSQhxipmqsx8OOXiwmS9+8V9JpVIYhsFH\nPvJ/WLJkKV/4wufYv38fsViMm2++hde97vXcd9+32LDhJVKpJFdeuZrbbnvnSb+eggty8LpXGrv2\nsLdnP0srTq9vViFOJ5euXnRU67mYzn445Fvf+jo333wLl19+FTt3au666/Pcc8+9bNy4nvvuewDD\nMFi37nkA/vjHR7jnnvuoqqrm97//7ZS8nsIM8rL5sM87E6IEuRBiql100SV8+9vfyJz98CtfuZt7\n7/0mTz31JOFwyaTPfjhkz549rFx5HgBLliiOHDlMOFzChz/8cb785X+jv7+P1772WgA+/enPc++9\n99DW1sbFF186Ja+nQINcNngKIXJnvLMfvvnNf8P69S/yl7+sPa7Hq6+vZ9OmDVx22ZXs3KmprKyi\ntbUVrbfxxS9+hVgsxk03vZ6rr34dTz75OJ/97BcAuO22m1mz5hpmzTq5azAUZJCXB8ooD5Sxu7sJ\n13UxjHHPFSOEECfk9a9/A295yxt58MFfc/BgM1/72pd5/PHHiEQiWJZFPB6f9GN98IP/my996U5+\n9rMfk0wm+dSn/oWqqira29t43/v+DtM0ueWW2/D7/ZSWlnL77e8kEAhwwQUXM3PmrJN+LZM6++EU\nG/fsh9m+t/k/2dCymc9d8n+pDlVOQ1ljkzOx5ZbUm1vFVi8UX83FfvbDnKovq2NDy2b2dDXlNciF\nEKe3U/Lsh9NlYdYVg1bNOjfP1QghTlfFcPbDgjuyc0htZC6WYckGTyGEmEDBBrnf8lEbmcP+nmYS\nqUS+yxFCiIJVsEEOXj95yk2xr7c536UIIUTBKuggz5xAq6spz5UIIUThKuwgz9rgKYQQYmwFHeRV\nwQqivoicCVEIIY6hoIPcMAzqy+roiHXSGevKdzlCCFGQCjrIYbifXFrlQggxtkkFuVLqIqXUn4+x\n/H6l1F1TVlWWoRNoNXbLBk8hhBjLhEGulPoE8D0gOM7y9wLLp7iujLroPAwMaZELIcQ4JtMibwBu\nHGuBUupS4CLgvqksKlvQDjAnMou9PftJOalcPY0QQhStCc+1orX+pVKqfvR8pdRs4DPAm4G3HM+T\n1tREj2d1zpi5mAMNB+n3dbGwcv5x3XcqHG+9+Sb15pbUm3vFVnO+6z2Zk2bdDFQDvwdmAWGl1Hat\n9QMT3fF4T/k4y++ddH1903aiqek9E6KcUjO3pN7cKrZ6ofhqnsbT2I677ISDXGt9N3A3gFLqncCy\nyYT4icg+wvPK2qm5NJIQQpwqjnv3Q6XUrUqp23NRzHhmhKsJ2SE5wlMIIcYwqRa51noPcHF6/Kdj\nLH9gSqsaxTRMFpTWsbVd0xPvJeqP5PLphBCiqBT8AUFD6tP7k++RVrkQQoxQNEEuR3gKIcTYiibI\n60vnAdAoLXIhhBihaII87AszMzyDpu69OK6T73KEEKJgFE2Qg3felVgqzsG+w/kuRQghCkZxBbn0\nkwshxFGKK8jlikFCCHGUogry2SUz8Vt+uYanEEJkKaogNw2T+ug8DvUfoT8xkO9yhBCiIBRVkMNw\n90pT9748VyKEEIWhCIM8fQItuWKQEEIARRjk9aV1GBg827yO9sGOfJcjhBB5V3RBHvVHeOOia+mM\ndXHPxu/SE+/Nd0lCCJFXRRfkAFfPv4qr667iSH8r33r5+wwkB/NdkhBC5E1RBjnAGxddy6WzL2Rf\nzwHu2/QA8VQi3yUJIUReFG2QG4bB3y67kXNqlrOzs5EfbPmJXJxZCHFaKtogB2+/8nee9bcsq1jC\n5tat/GT7f8sJtYQQp52iDnIAn2nz98vfTn1pHX899BK/2vkwruvmuywhhJg2RR/kAEE7wPtXvotZ\nJTN5cv9aHtnzeL5LEkKIaTPtQX73v/2JwYGp3zAZ8ZXwoXPeQ1Wwgod3P8bT+5+b8ucQQohCNO1B\n3tk+wMZ1uTm8vjxQxh3n/D1Rf4Sf7/gNLx7akJPnEUKIQjKpIFdKXaSU+vMY8/9WKfVXpdSzSql7\nlVITPl6kNMDmF/fT3xc/gXInNiNczR0r30PQDvDDbf/FK63bcvI8QghRKCYMXqXUJ4DvAcFR80PA\nncCrtdavAsqA6yd6vMvXLCWZcNjwfO7OKV4bncP7VrwLy7D43iv/ya7O3Tl7LiGEyLfJtMgbgBvH\nmB8DLtVa96enbWDCQyzPvWgekdIAWzY009sTm3ylx2lx+QLec/ZtpFyHezf9P/b3NOfsuYQQIp+M\nyeyqp5SqBx7UWl88zvIPAdcB12mtJ3pAd8Nf9/Lbn7/Mqkvrue6m5cdb83FZ27SOe55/gNJglM+v\n/jizojNy+nxCCJEjxngL7JN51HSf+JeBpcBNkwhxAGbPL6OsIsT655tYtnIW0bLgxHc60RrDZ3Dz\n0jfy8x3/w+ee+DofO/8DlAfKJn3/mpooLS09Oatvqkm9uSX15l6x1Txd9dbURMdddrJ7rdyH13f+\npqwulglZlsmqV83HcVxefHbPSZYwsStrL+X6Ba+lbbCDezZ+j95EX86fUwghpstxB7lS6lal1O1K\nqfOAdwPLgSeUUn9WSr15so+z+MyZVFSF0ZsP0dk+6e+AE/a6+tfw6trLONR3mO+8/P8YTOauf14I\nIabTpLpWtNZ7gIvT4z/NWnTCLXrTNLjg8noe+5+tvPjsHtbccOaJPtSkGIbBjUuupy/Zz7pD6/nu\n5h/xvpXvwmeeVO+SEELkXV4P0V+oaqieEWHnliO0t+S+u8M0TG5bdjPLq89ke8dOHtjyMxJOMufP\nK4QQuZTXIDcMr1UO8MLaPdPynJZp8e6z/hdLyheysWUzn/vLl3mu+QU5Ba4Qomjl/aRZ8xdXMWN2\nlEbdQsuh6dlS7bN8vG/FO1k973J6Er38ZPsvuHPdV3np8EY5Da4QoujkPcgNw+DCKxYA8MIze6bt\neYN2kJuW3MBnL/4El825iNaBdn6w5afc9cI32Ny6VU6FK4QoGnkPcoDa+gpm15bR1NDG4ebuaX3u\nimA5f7vsJj590T9wwczzaO49xL2bHuCrL32bHR0N01qLEEKciIII8uxW+bqn83NelJpwFe886xb+\n8cKPsrLmbHZ3N/GNDffx+T9/gz3duTsvjBBCnKyC2fduTl05tfUV7N/TQfPeTubUleenjsgsbl/+\ndpq69/HbxkfZfHg7mw9vZ0X1WVy/8LXMjczOS11CCDGegmiRD8lulee7j3p+6TzuOOc9fPbVH2Vh\nWT2bWrfwxXVf54EtP+NIf2teaxNCiGwF0yIHmDmnlPmLqmhqaGP/ng7mLajMd0mcOWMpHzvv/Wxp\n285vGx/lhcMbeOnIy1wy+wKurX8NFcH8/HIQQoghBRXkABdcXk9TQxvrnt5NbX0FhjHuCb+mjWEY\nnF19BmdWKTa2vMLDjY/ybPNf+euhl7hi7iVcPf8qSv3jn9BGCCFyqeCCvGZWlIWqhkbdQtOuNuqX\nVOe7pAzTMDlvxgpWVp/FusMb+P3uP/LEvmd4ct9a6kprOaNyKWdULmVBaR2WaeW7XCHEaaLgghzg\ngsvqadQtvPDMHuYvriqIVnk2y7S4ZPYqVs08h780r+PFwy+zu7uJpu59PLLncYJWgCUVizLBXhMq\nvNcghDh1FGSQV9aUsOSsGezccoRG3cKiZYV5MQifaXNF7aVcUXspA8lBdnY0sK19J9vbd7C5dSub\nW7cCUBWs5IzKJZxRuZSlFYsJ+0J5rlwIcSopyCAHWPWqenZtPcILz+xhwdIaTLOwW7QhO8iKmrNY\nUXMWAK0D7Wxv38G29h3ojl2sbf4ra5v/ioFBfWmdF+xVivnRWumGEUKclIIN8vLKMGr5LLZvOsSu\nrYdZevasfJd0XKpDlVw292Ium3sxKSdFU89+trXvYHv7DnZ37WV3dxO/3/MnQnYQVbGYZZVLmBGq\nIeIvIeLzbhLwQojJKNggB69VvuOVw7z4bBOLzpiBZRXUbu+TZpkWC8vms7BsPq9fcDX9iQF2dDaw\nrU2zrX0HG1teYWPLK0fdL2SHiPpKKPGVEPGXEPWVEPFHKPGFifoiI0I/4o8QsPx5eHVCiHwr6CCP\nlgU545zZbFnfzI5XDnPGylPjqMqwL8Q5NWdzTs3ZuK5Ly0AbOzsa6Ix10ZvooyfRR1+8Lz3eS+tg\n+6TOyugzfVSGypgZmsncyCzmRuYwNzKL6lAVplGcX4JCiIkVdJADnH/JfLZvOsSLz+5h6VkzsexT\nK5AMw2BGuJoZ4fF3s3Rch8HkID2JPnrTAd+b6M0aH5rfS1e8m02tW9jUuiVzf7/pY05kNnNH3GYR\nsmWjqxCngoIP8pJogLPOncOmF/azbdNBzj5vbr5LmnamYRL2hQn7wswM1xxz3ZqaKA0HmjnQe5AD\nvQfZ33OQ5r6D7Os5cNTJv6qCFcyJzKY2MjszlNa7EMWn4IMc4NyL69i6sZmXnmti2fJZ2D7ZCHgs\npf4opZVRzqhcmpmXdJIc7m9hf08zB/oOcqDnIAf6Do7YTRKGW++zSmZQE/J+KdSEqqgJVRG0g/l4\nOUKICRRFkIdL/CxfVcuGv+xly4ZmVl44L98lFR3btDPdKtm64z2Tar0DRP0RL9xD1dSEvXCvCVdT\nE6omJCEvRN5MKsiVUhcBX9JaXzVq/g3Ap4Ek8AOt9XenvMK0cy6cx5b1B1j/l72cec5sfP6i+A4q\neOO13tsG2mkZaOPIQCst/W20DLTS0t/K7q4mGrv2HPU4EV9JuvVenRXwVcwIV0tfvBA5NmEaKqU+\nAbwN6Bs13wd8DbggvexZpdRDWuvDuSg0GPKx4oJ5vLh2D5tfOsB5l8zPxdMIvNb7zJIZzCw5+oja\npJOkbbCDlv5WWgaGAt4L/D3d+2jsajrqPlF/hBmhGmaGq5kRrmFGuJqZ4RqqQlXT8XKEOOVNplnb\nANwI/Oeo+WcAu7TWHQBKqbXAFcAvprTCLCtW1bL5xf1seH4fZ507h0DQl6unEuOwTZuZ4ZoxN7qm\nnJQX8lnhfqS/hSP9rTR27aGha+TVnwwMZpRUURWoSu+544X8jFANFcGy497o6rouSTdFPBUfvjkJ\nYqk4CSfBjFA1VaH8nxpZiKk2YZBrrX+plKofY1Ep0JU13QOUTeZJa2pO/JSvl71mCY//bhsbn9/H\ndTcux5yGg4ROpt58yGe9sygHFhw1P55KcKS3leaewxzsOcLBnsMc7D1Cc88RtrZrtrbrEev7LB+z\nIjXMjs6gPFBKPJVgMBUjnowTS8WJZYYxYqlEehif8IIkM0uqOXvmMpbPVJw9Q1EaPP73Sj4PuVds\nNee73pPpaO4GsquPAp2TuWNLS88JP+kCVU3ZcyHWP7+X5v2drLnhTKJludvQVlMTPal6p1sh1xsg\nwoJAhAWBRZDebb6mJsreg0c40t/K4XTr/Uh/i9ea721hX1fzmI9lmzZ+04ff8hOw/JQGI/gsP37T\nR8Dy47O8od/047f8WKbFgZ5mdnQ28HjjWh5vXAvA3MhsVMViVMViFpcvJGgHjvkaCvn9HUux1QvF\nV/N01XusL4uTCfJtwBKlVCXQi9et8pWTeLxJ8fktbnrHeTz1yA4atrfw8x+8yFXXLi3YMySKiYXs\nEPNL5zG/dOTeSK7r0h3voTfR54Wz6Sdg+fCZvhM+D43jOuzrOcD29p3ojl00du3hQO9Bntj3DKZh\nUl9alwn2BWV12ObUbFR3XIeeeB/d8R664924rkuJr4Ro+jQLASsgpzoWJ+y4P6VKqVuBiNb6fqXU\nx4BH8a79+QOt9YGpLnAsgaCPq994JvMWHGLtn3by2P9s5YyVHbxqzWJ8so/5KcMwDMoCpZQFSqfs\nMU3DzHxpXFO/mkQqQWNXE7pjF7pjV2avnD/s+RN+08fi8oWoSi/YR++66bousVSM7ngPXbGedEin\nb1nTXfFueuN9uIzf7WObNhHfMc6n44+MWB6yg3Lglsgw8nCRY3cqf4Z0tPXxp99so/VILxVVYda8\n4UyqZ0am7PHlZ15uFVq9/YkBdnY2ZoL9UN/wTlglvjBLqurpHuinO9ZNd7yHuJM45uMFLL+3i6e/\nlNJAND0exTSMrFMrDJ9ioSfRRzwVn7BO0zAp8YWpi9aypHwhSysWMS8696hwL7T3dzKKreZp7FoZ\n9ydb0Qc5QCrp8Jc/N7D5xQNYlsElqxdx9nlzp+SnqnyocqvQ6+2KdXuh3u4Fe0esE9MwifoilAai\nlPmHwzmaDuoyf6k37Y9M2Oc+lngqkTmXTk+ij75EH73x3qPOtdMV66ZtsCNzv6AVZHH5ApZUeMFe\nG5nDzBllBf3+jqXQPxOjSZBPsaZdbTzxu+0MDiSYv7iKV1+nCIVP7tSu8qHKrWKq13VdSsptejsT\nBdOt0RnrYmdHIzs6GtjZ2UDLQFtmWcgOcdaMJcwP17GkYhFzI7OnpG7HdeiO99A20EHbYDvtgx20\np79QSnwllPjCRMYYBifZHVRMnwmQIM+Jvt4Yj/92GweaOimJ+Fl9/RnU1lec8OPJhyq3pN6p1THY\nyc7ORnZ2NLCjs5HWrGAP2yEWp7thlpQvZE5k1pjB6rhOprXfNtBO+2An7YPt3vRgBx2DnaTc1HHX\nZhomYTt0VMgPnW+/xA5T4gszp6aKeK931a2wL4zf9BX0hmAJ8lw9geuy8a/7WPf0bhzH5bxL6lh1\nWf0JXZii0P9xR5N6c6vY6jXCCZ5v2MSOzgZ2djTSNtieWVZih1lcsZDZJTPpjHXRPuC1rNtjneOe\n/z7qj1AVrKQyWJ4eVlAVqqAyWIGB1+/fl+inLz0cOs3y6Hn9iYFjbvzNZhlWOtRDhOwQ4fQt5Msa\nT4d+yA4StkPYpu0dZ5CKEU+ljzkYNZ09Pz7GunEnjomJbdr4TBvb8uEzbXzm0NDGNn1EwiGcOPgs\nO73u6PV8BOwAActPwBoaDo/7Lf9kf6mcXkE+5HBzN396aCvdnYPMnFPKmjecQWn58Z33o9j+caXe\n3Cr2etsGOtiZDvUdnQ2ZLpEhZf4olcHKTDhXBiuoSt8qghX4rak5mtpxHfqTA5mA740Ph7zhd2jr\n7qI/OUB/coCBxAD9yUH6k/0MJAZInsCvgckwMDLBOjR0XZeEkyDhJEk6SeJOgkQqMekvocnyjn9I\nh/xRoe8N77js7adnkAPEY0mefnQHO7cewR+wuOKapSw5c+ak71/s/7iFTurNrYnqbRtop3WgnYpg\nGRWBcnxTFNQnY6Ka46kEA8kBBtJB359IB35ykP6ENz/pJvGbQyHohWR2QGe3jIemfaY96S6clJMi\n4SRJOAlKKwIcbunMTCdSXuh7XwAJ4k4y0+IfOgI5lhpvODw++lfRz9/6nXGLO+VPIegP2LzmhjOY\nt6CSpx89K6JhAAAcm0lEQVTbwZ8e2sb+3R1cdvUSfH7Z51yc3qpClUV3/hm/5cNv+ab0+ILjZZkW\nlmkRJEB1OIobntovQNd1STrJTLAPpmLHXP+UD3LwDixRy2cxc24pf/zNVrZvPsTBA11c/YYzqZlV\nXOd0EEKc+gzDwGf58Fk+IpRMuH5h7EM1Tcorw9z49vNYeWEtXe0D/OpH6/ndLzax7eWDDPRPfBCG\nEEIUotOiRZ7NskwuXb2Y2vpKnv9zA3sb2tnb0I7xCMypK2fh0hoWLK2mJHr8B3IIIUQ+nHZBPqRu\nYSV1Cyvp6uincUcrjbqFA02dHGjq5Jk/7mTW3FIWLK1h1SX1+S5VCCGO6bQN8iFlFWHOvaiOcy+q\no7d7kN07Wmnc0crBfZ0cOtDNX55soHpmhIVLq1moaqionri/SgghptNpH+TZIqVBlq+qZfmqWvr7\n4uzZ1cr+3R3s3tFK6+Fe1j2zh/KqMAtVNQuX1lA9M1LQR5wJIU4PEuTjCJf4OXPlHK5co9i/r52m\nXW007mhlX2M765/by/rn9hItC2Za6jPnlkqoCyHyQoJ8EgJBH0vPnsXSs2eRiKfY29hO444Wmna1\n8fIL+3n5hf2URP0sXFrDomU1zKotk1AXQkwbCfLj5PNbLFrmBXYq6bB/TwcNuoXdO1rZ/NIBNr90\ngHBkZKibpoS6ECJ3JMhPgmWbzF9cxfzFVaRe53CgqYOG7V6ov7L+AK+sP0CoxJcJ9dnzyjDN02rX\nfSHENJAgnyKWZVK3sIq6hVVccY1D895OGnULjbqVLRua2bKhmWDYx8Kl1SxaVsOcunIJdSHElJAg\nzwHLMpm3oJJ5Cyq5/LVLaN7b5XW/6Ba2bjzI1o0HCYZsFqRb6nPqyk/oFLtCCAES5Dlnmia19RXU\n1ldw+dVLOLivMx3qrWx7+SDbXj5IIGizYEk18xZWMqeunHDJyV3VSAhxepEgn0amaTB3fgVz51dw\n2ZolHDrQReP2Fhp1C9s3H2L75kMAlFeFmTOvjDl15cyZVy6nCxBCHJMEeZ6YpsGceV5Qv2rNYg43\nd9O8t5Pmvd4RpUNdMABlFSFmZwV7tCyY5+qFEIVkwiBXSpnAt4GVQAx4j9Z6V9byjwO3Ag7wBa31\nr3NU6ynLMAxmzS1j1twyzrtkPo7j0HKol+Z9nRzc28nB/V1s33SI7Zu8Fnu0LDjcYq/zgl32Wxfi\n9DWZFvmbgKDW+hKl1MXAV4E3AiilyoGPAIuBEmAjIEF+kkzTZOacUmbOKeXci+pwHJe2I71ei31f\nJwf3daFfOYx+5TAAkdIAc+aVM7uujGVnzqandxDLMjAtM2toyv7sQpyiJrzUm1LqP4B1WusH09MH\ntNZz0+M+4AngDXhB/ozWesEEzznt15Y71biOy5FDPTQ1tNHU2EZTQxv9fROfT90wvD1qLDt9s4Zu\nBrZtYdle6Nu2SSjsp6wiRHlF2BtWesNgKP+XAhPiNHVSl3orBbqyplNKKVtrnUxP7wO2AhbwxclU\ncypd8zBfTJ/BgmXVLFhWjeu6dLT107y3k4HeOL29MVIpByflkko5pFIuzojh8LJEIkVscOSyY323\n+wMW0dIgkbIg0dIg0bIg0bIA0bIgkdIgobDvuLp5CvX9HY/Um3vFVvN01VtTM/7VzCYT5N1A9iOY\nWSF+LTAbGGqFP6qUelZrve5EChUnxjAMKqtLqKwumZIPleO4DPTH6ekapLc7Rk/XoHfr9obdXYO0\ntfSNeV/bNtMh74V7aUWI8ooQZZVhSsuD2LZcJ1WIqTaZIH8WuAH4ebqPfHPWsg5gAIhprV2lVCdQ\nPvVliulkmgYlkQAlkQDMPXq567rEBpMjAr63K5YJ+p6uQTrb+sd87GhpgLJ0N01ZRYi6BVUYlrcB\nVw6Kmnqu6+I6TvaMkcP0OjgODK2bnnZHDXEdXMcF1xk1P30/xxlz6KZSo+Z50zgurpPy5qcc3PTj\nxsN++noHcbNrdQFc77lc9+jXMTR/1Gs7psn8cnQd3JQDTmp46LiZaddJ0e4zGeyPp5c5kEoNv9ZU\nyqvHNDFMCywTwzTBsjDM9LhpYVhHDw3T8u5necOa97x9/JcyiT7yob1WVuD10bwLuA7YpbV+SCn1\nOeB1eHutrAU+obU+1oO68rMpdwql3kQ8SXfXIF3tA3R1pG/t/XR1DNDXe3R/vmF4YV5WGfZa8BUh\nyiq9vvlIaeC4T2fguq73D5VMDt8ch6EwGAqHTCi4LuB6/6Skg8N10uukg9BxKS8P0X6kEzeR8G7J\nBG48gTM0nUjgJuKjpseYl0plAi4TkNlh544MRC/0htbNWua92JHhNdkgE9PDMKbkb/Kq3/xy3G+e\nCYM8ByTIc2gq6s2EoJtuKTkpL0iyWxpOKr3MGW6JZFpfbqbV5aaSuMl0oKa8QI3HUvT0Jenpd4gl\nLdp6UvQMGvQmTGKpsbpeXPyk8JPATxy/G8fvxPE5MfypAXzJQXzJfnyJfnzxAex4L2YqOcbjnBzv\nP8XAmIrt9ekWGUOtMmOodWYMzzNNDMMcOW0aXovNNL2AGGpVGsbIbRPpcZ/fJpFIjWh9Dq9nDG8+\ny9SRfv5RQwwjUyemkalreL4x8vWMfn0jhkMtzfTrtrLmGwZl5WG6uwdHvDYMvOc0huoe43WPWH+C\n1vZkcm+oJT30WizLq8HKbl1bVNVEae8YyLScvaEx/DoNY9QvleHWvPc/MrrFn/UrJms4//ILT2pj\npygCTiKO09dH/0AnAwfbcWODOIPp29B4LIYzODhqWWzEOm56ejpadaH0LbsvLmn66PeVZm4DvlIG\nfSXErSAxK0ivWZre/QbvNs5OND4S+EkSMFPpm4NhQAoDBxMXg5Rr4GJ681wDZ/TQNXAgazwdjqZD\nwAa/DUEfBPwmgaBFMGARDNoEwz6CYT+hEj/BcIBQNIgV8GP4fBg+P4ZtT9t+/8XWEAGoqoniFFHN\ngaootnPsejNfYjmqQYK8gLiOg9PfT6qvj1RfH07/qGF6fqo/Pd7fT6qvF6e/Hzc+8e6H4zH8fsxA\nEDMYxIxGMQJBTJ9vRP/ciNaUZY5smaRbV5nxoXUty7vZNtg2hmVj2N60N25TXhWluzfurTPGcsO2\nvHGfD8OycAyT2GCSwf4EA/1xBvoTI8a96TgDAwkG+hK0DyS8Tr+J3gPDOy2xaZlYtoFlmfgsE9My\nMrtpmpaB32/T2z3I4ECCzoEETv/QF14qfRv77+AP2ITCPoIh7xYI2tg+E9tn4fNZ2D4zPRw57vNb\n2LbpDX0WvvR9pvsAMNd1SSUdkkmHZCI1PEwMzzMtg0DQe23+gE0gaMt2j2kiQX4SnEScZGcnqZ5e\n3HgMJxbzWrRHjce98fT00LibXpaZNzAw+Sc3DMxQCKukBHv2HKySEsxwCSVVZcSxMsFsDAV0IOAN\ng0fPN/J4Ot2KmijJ42h9WXiX4fNOLDbxhbAdxyU26AU8gJUO5qGDpYbGJ3uwVHYL13VdkomU9+Ux\nkL71JxgcSDI4kGAgMz186+kaxHFO/teOZZv4fF7thmFgmt7NSA9Nwxv3B2xSqVRmevR6Q18IyaTj\nBXU6nBPJ1IjpZHIS34ZjsH1mJtwD6XAPBG38QXvM+YGQj2gkiOu6crTycZAgH4Prujh9fSQ7O+jY\nF6Nr9wGSnR3eraODRIc37vT2nviTGIYXooEApj+AEYlihcOYJSVYJSVY4RJvPOxNe+NhzJKINwyH\nxwzgYvwpnUumaRAK+wmFp/6MkoZh4PPb+Pw2peWhSd3HdV3isRTxWJJkMh2a8RSJRGo4RNPjiYST\nHqbDdGg8nl6WTOGkXBzHTX+pODjO8LTjuOkdTI59bMDI10TmV4FtW4RK/Ni2lfn1YNtmZpk3PrSu\nSSrl7c0UG0wQjyXT496ttztGe2zsXVbH4vNblEQDRNK3kmiASOnwMBIN4A9MXxdVPjiO11CIx1Mk\n4smT3o/8lOOmUiSOHCbW3EyyYyig29PjnSQ7O47ZVWEEgtgV5QTn1WGXV2BFo5lANgN+bzwQwPB7\nw6PGA34M+/gOnBGnBsMwMq3P6TD0xe662eHuDge+4+JCJqxz2RXifYmNDPjYYJJYLEFsMEl80Psl\nk4g7dLT20dsz/m6s4LX2I6XBMcM+GPKld7100zv1DA8dx9sryUnv7ePt/DP0BciI+0F6u6lhZG1n\nNTLzDcPgyIEeenoG0vOMo+7jupBIpLwv7HgyHczD04l4asS8eHpeMjHyV9Cnv3rD+O/FCf1Fikiq\np4fY/n3p235i+/cRbz6Am0gcvbJhYEWj+GfNxq6owK6opGzuTGL+MHa5N21XVGCFJtf6EqJQGOm9\nO/J5USrvS8xHIHjs0zxk/6pMJlL09sTo64nR2xOjtzs93j0871hhX0xsn4nfb+P325REAvj9lveL\nL2Dh9x/7QLpTJsjdZJL4wYNZoe0Fd6qrc8R6hm3jnzOXQG0t/rm1+Kqq0iFdgV1W7m10yyJdFULk\nj+2zKK8MU14ZHnedZCJFX68X7kOhHxtMpr+8GB6mtwlkxjEwTG+5md7F0TTTw+xdGN3h1rnL0PEH\n6SkXIiUBenoHhw5FyByY5B2q4GLgdRV53XBWOqC96eHxk9uAXZRBnuzpJta0h9i+/ZnQjh86CKnU\niPXsyipKVqwkUDuPQO08/LXz8M+c6e2JIcQpLuU4dPXGONTeT99ggv7BZNYwSd+ANx5PpvDZJn7b\n8oY+E59t4bdN/HZ63GcevY7l7V0ztB4YJB3vvD3JlJO+eeOZeY43L5W9zHEz66ZSDoGQj56eGKl0\n90/KcTLj3rSbtWz0PG9dbxdwb4Ould74620Ezpqf3gCcWZ7ZaExmg/GQTJcLpI8XG+qGgdBAgv7+\neCa4hw9C9cbddDeO67g4Q9077nBXTmYcr1/8qGXp7p+7/8+rx/1bF1WQx/btpf3RP9DzwroRoW0E\nAgTn16cDuxZ/7TwCc2uxSibeq0GIbCnHoW8w6YXdQCI97g37BhOYlkUslsC2TGzLwJc+W6Sd3l3R\nG0/PT0/70uvaWfMMgxGBl0q5Y4ZcMuUFWXbQJYfWTToMxFIjQjq73sF4auIXLAqaASP2LhpPwQe5\n67r0b91Cx2OP0L/lFQD8s+cQWXUBgXl1BGrn4auuzusudOLYkimHRNK7xZOprHGHg12DtLT2ZqZH\nrJdwSKRDzjugMN3Pawy3mLwPOSNaVJnpUcsGE6nhUB44NcMv6LcoCdrUlIcoCdpUlIWwDIgEfYSD\nNiVBm3DQNzwM2fhti0T6TJjxUe9/POtvFk/vlhjP+vtl3wfIfGFZQ19cpoltp+eZw19mw8tHrm+Z\nBpWVJfT2DHot5awWs2UZw/PSf18rfR/T8JZZVvrvznBL1zuVy3BLd3i+Sypr3DtLwvDyVHo30Uz3\nDKQPhh3qsvHGKyrCdHb2j1wnPT6Uv2ZWq984xuc3e9nQ805GwQa5m0zS8+I6Oh59hNi+vQCElioq\nXnctJWevkOA+Sa7rfVBjiRTxhEMskSIWTxFPpjLz4slU+p/XIZ7w5g/9Q8cy/9wp75aeP3Sf7Ps6\nBX7uDy/8fMwoD3lhF/KCriQTfj5KQt747BlR2tr7SCYdElkt6kRyuNU8PO59ESWTwy1ob563B4lv\nVOBlxtP7t9uW4QVh1rQ1ajoUGK4zHLCxR+11UozbeKaqZsswmI7jkWpqorQE8ttdW3BBnhoYoPuZ\np+j402Mk29vBMIisupDKa15HcMHCfJc3ZVKO97N4IJbM3Lyf0d5P6bH6AlOO99N6xLQz8j4+v01n\n92A6YJ10KKeIJYbDOJ5MEYtPbcAagN/n9aX6bZNIyIcvOtyn6svqb82M+0zKS0Mk4sl0f+sY69gm\nlml6fYxuVsspM53d/ziyzzEznR4P+KxMIJdkWqU21nE0CmpqorSE5eIaorAUTJAnOjrofPyPdD31\nJM7AAIbfT/nqNVRcfQ2+mpp8lzemWDxFW/cg3X1xBmJJ+rNCeSCWGjWdvdwL1OlimQaBdMgG/Ral\nJX4CPouAz8TvszLLhsctAraZFczW8AYwn0nAttKhO7zctibuxxtLMbYYhSg0eQ/y2P59dDz2CN1/\nfR5SKazSUqquuZbyq1ZjRSJ5q8t1XXoHEnTt62RXUxtt3THaugZp6x7MDHsHxtgXfRyW6f0MDgW8\nIA0HbEIBOzMMBux069MY0Tc4NG6b5lHzLMvrKxy6HqdlGtRUR+jrHcRvDwf06J/bQohTS16C3HVd\nBrZvo/3RP9D/inedCt+sWVS+9lqil1yC6Zv6w6nHqqGjJ0brqHDODLsHiSfGPr+EzzapLA0yf2aE\nytIgZZEA4YD3M30orLNDOhSw8dvmtBzJ6bVwJbiFOJ1Me5C3PP0Me3/xa2J7m4D0BszXvo6SFStz\nugGzfzDJ7kPdNB7ooqG5m8bm7nFb1CVBm1kVYarKgtTOKiXsM6kqDVJVFqSqNEj0OK9LKYQQuTTt\nQb7jq1/3NmCev4qKa64ltHDRlD+H47g0t/bR0Dwc2gdb+0ZcDqC6LMiy+RXMKA9lArqqNEBlaZBQ\nYPhtkT5cIUShm/Ygr735JuxzL8I/Y8aUPWZXX5zG5i4am7tpONDF7kM9xLL2Bw74LFRdOQvnlLFo\nTikL55RSFglM2fMLIUQ+TXuQz7/t1pNu4R5s6+OVxnYa0uHd2jU4YvnsqjCL5pSxcG4pi+aUMbe6\nZNLnmxZCiGKT971Wjpfe28FXHtyYOeqqJGizYlEVC+d4ob1gdpTwBGdXE0KIU0lRBfmRzgG+9Wvv\nMP23XaM4s97r45YNj0KI09mEQa6UMoFvAyuBGPAerfWurOXXAp/BO7jvJeCDWuspPyZ7IJbknv/e\nRO9Agne8TnHlOXOn+imEEKIoTWZ/vzcBQa31JcD/Bb46tEApFQX+Hbhea30RsAeonuoiHcfl/oe2\ncKC1jzXn10qICyFElsl0rVwGPAKgtX5eKbUqa9mlwGbgq0qphcD3tNYtEz3gsa49N5YHHt7Cyw1t\nnLO0hjveeu60X5n7eOvNN6k3t6Te3Cu2mvNd72SCvBToyppOKaVsrXUSr/X9auAcoBd4Rin1F631\njmM94PHstfLcKwf55ZO7mFkZ5t3XLaO9ffIXcJ0KxbYfudSbW1Jv7hVbzdNV77G+LCbTtO0Gsh/B\nTIc4QBvwgtb6kNa6F3gaL9SnRMOBLh74w3ZCAZsP37ScEtkbRQghjjKZIH8WuA5AKXUxXlfKkPXA\n2UqpaqWUDVwMbJ2Kwtq7B7nnV5tJOS7vf9NZzK6Sq/0IIcRYJtO18mvgaqXUc3h7prxLKfUxYJfW\n+iGl1KeAR9Pr/lxr/crJFhWLp7j7l5vo7ovzt2uWcPaCqpN9SCGEOGVNGORaawd436jZ27OWPwg8\nOFUFOa7L93+3lb2He7li5WzWnF87VQ8thBCnpII73+lvn93Di7qFpbVl3PZaJQf7CCHEBAoqyF/c\nfoTfrN1NdVmQD9y4XC6IIIQQk1AwSdl0qIfvPbyVgN/iwzetoDSc+4tLCCHEqaAggryrN8bdv9xE\nIulw+w1nUjsjf5d4E0KIYpP3IE8kU3zzV5vp6Ilx45ULOXdJYV5oWQghClVeg9x1XX74iKahuZuL\nz5rJdRfPz2c5QghRlPIa5I+s28tzrxxiwexS3nXtMtlDRQghTkDegnzjrlb++8kGKqIBPnTTcny2\nla9ShBCiqOUlyA+09HLfQ1vw2SYfumk55XL9TCGEOGHTHuRDe6jE4in+7vVnUD+rdLpLEEKIU8q0\nB/mXfvQiLZ2DvOFV9Vx4xszpfnohhDjlTHuQb25o5XxVwxsuWzDdTy2EEKekab/48lXn1/LWKxdh\nyh4qQggxJaa9Rf7xW88n4Jc9VIQQYqrk/chOIYQQJ0eCXAghipwEuRBCFDkJciGEKHIS5EIIUeQk\nyIUQoshJkAshRJGTIBdCiCJnuK6b7xqEEEKcBGmRCyFEkZMgF0KIIidBLoQQRU6CXAghipwEuRBC\nFDkJciGEKHIS5EIIUeRydoUgpZQJfBtYCcSA92itd2Ut/3vgvUASuFNr/XCuapkMpZQP+AFQDwTS\nNT2UtfyjwHuAlvSs92qt9XTXmU0ptR7oTk/u1lq/K2tZob2/7wTemZ4MAucAs7TWnenl3wAuA3rS\n67xRa901zWWSruUi4Eta66uUUouBBwAXeAX4oNbayVo3BPwYmIFX+zu01i1HP+q01XsOcA+Qwvu/\ne7vW+vCo9cf93OSh3nOBh4Gd6cXf0Vr/V9a6hfb+PgjMSi+qB57XWt+Sta4B7Gf49fxFa/2pXNeY\ny0u9vQkIaq0vUUpdDHwVeCOAUmoW8GFgFd4/9Vql1B+11rEc1jOR24A2rfXblFKVwEbgoazl5+P9\nU7yUl+pGUUoFAUNrfdUYywru/dVaP4AXiCilvgX8YCjE084HrtFat05/dcOUUp8A3gb0pWf9B/DP\nWus/K6XuxfsM/zrrLu8HNmutP6uUugX4Z+Ajeaz3G8CHtNYblVLvBT4JfCxr/XE/N9NhjHrPB/5D\na/3Vce5SUO/vUGgrpSqAJ4GPjrrLImC91vqG6aoRctu1chnwCIDW+nm8UBlyIfCs1jqWbnXtAlbk\nsJbJ+AXwL+lxA68lm+184FNKqbVKqZx/w07CSiCslHpMKfVE+stySCG+vwAopVYBZ2mt78+aZwJL\ngPuVUs8qpf4ubwVCA3Bj1vT5wFPp8T8Aa0atn/mcj7M810bXe4vWemN63AYGR61/rM/NdBjr/X29\nUupppdT3lVLRUesX2vs75HPAPVrrg6Pmnw/MVUo9qZT6vVJK5bxCchvkpUD2T+OUUsoeZ1kPUJbD\nWiakte7VWvekP0j/jffNn+1B4H3AauAypdT1013jKP3AV4Br8Or6SSG/v1n+Ee+fIFsJXnfAbcDr\ngA8opfLyxaO1/iWQyJplaK2HzmMx1vuY/V5P+/s8ut6hYFFKXQrcAXxt1F2O9bnJuTHe33XAP2it\nrwAagc+MuktBvb8ASqkZwGtI/8Ic5SDwRa31q4Ev4HUL5Vwug7wbyP52NbXWyXGWRYHsn9l5oZSa\nh/dz6T+11j/Nmm8AX9dat2qt48DvgHPzVOaQHcCPtdau1noH0AbMTi8r1Pe3HFBa6ydHLeoHvqG1\n7tda9wBP4LUcC4GTNT7W+5j9XhfK+/xW4F7g9WP0Jx/rc5MPv87qrvw1R/9fFdz7C/wN8FOtdWqM\nZS8CvwHQWq8F5qTzI6dyGeTPAtcBpH++bc5atg64XCkVVEqVAWfgbUjKG6XUTOAx4JNa6x+MWlwK\nvKKUiqT/KKuBfPeV/x3edgeUUnPwahz6mVdw72/aFcDjY8xfCjyrlLLSG50vA9ZPa2Xj26CUuio9\nfi3wzKjlmc/5OMunlVLqNryW+FVa68YxVjnW5yYfHlVKXZgefw1H/18V1Pubtgavm2csnwH+N4BS\naiWwL+sXXc7k8ifVr4GrlVLP4fU5v0sp9TFgl9b6IaXU3Xh/FBP4J6316L686faPQAXwL0qpob7y\n7wIlWuv7lVL/iNdajwGPa61/n6c6h3wfeEAptRZvj4q/Az6slCrU9xdA4f189iZGfh7+E3ge72fs\nj7TWW/JU42gfB76rlPID2/C63VBKPQZcD3wH+GH67xAHbs1XoUopC7gb2Av8Kt09+5TW+jNKqR/h\ndRce9bnJ+qWcD+8H7lFKJYBDwO1QmO9vlhGfYxhR713Aj5VSr8fbzvbO6ShITmMrhBBFTg4IEkKI\nIidBLoQQRU6CXAghipwEuRBCFDkJciGEKHIS5EIIUeQkyIUQosj9f59B/LrZX3aKAAAAAElFTkSu\nQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "pd.DataFrame(history.history).plot()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "scores = model.evaluate(X_test,y_test, verbose=1, batch_size=batch_size)\n", + "score = dict(zip(model.metrics_names,scores))\n", + "print()\n", + "print ('Accuracy {:%}'.format(score['acc']))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:55:27.793104Z", + "start_time": "2017-05-31T21:55:27.776484+08:00" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T13:55:55.223865Z", + "start_time": "2017-05-31T21:55:49.009738+08:00" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "300/300 [==============================] - 5s\n", + "Word error rate 100.000000%\n", + "Char error rate 64.100000%\n" + ] + }, + { + "data": { + "text/html": [ + "
pronunciationguessspelling
kˈɑkəsˌɔidoaaaeee caucasoid
bɹˈæsfˌildaaeeee brassfield
flˈæʃbl̩b aeeee flashbulb
hˈɛstiə eeeeee hestia
kɹˈæbi aeeee crabby
tɹədˈus oaeee traduce
təlˈulə aeeee tallulah
bˈitn̩ aaeee beaton
hˈɑlɪtʃɛk oeeeee holecek
beidˈɔiə aeeee bedoya
sɪvˈiɹ aeeee severe
nɑɪsˈɪpi oeeee nicippe
lˈoʊɹntʃəɹnnnnnnnn launcher
bɹɪtˈɔil oeeee britoil
ˈɔɹtnɚ reeee ortner
bˈɪljn̩ɵs onnnnnn billionths
sˈɛmijˌɑn oeeeee semillon
lˈɔntʃɪŋ oeennnn launching
zˌubɪlˈɑgəeeee zubillaga
ɹɪfɹˈɛʃɪz eeeee refreshes
ˈɪgnəɹn̩s onnnnnn ignorance
tɪsˈɪpəs oeeee ctesippus
vɪtkˈɔfskieeeen witkowski
dˈɑdɹɪdʒ aeeee doddridge
tˈɛnn̩t oaennn tennent
dˈɪŋk eeeee dink
ˈeiljənəɹ aennne alienor
dˈuln̩ aaeee doolan
hˈʌlki aeeee hulky
mˈɛmwˌɑɹz aaaeee memoirs
kɑɹdɑɹˈɛliaaaae cardarelli
fˈɝmɚ eeeee firmer
kwˈɑpjɑ aaaee kuopio
kɚˈɛktɪd eeeeee corrected
ɹˈɔɵmn̩ aaann rothman
stɹʊk aeeee strook
ɹˈɛnkwɪst oennnnn renquist
ɛspɪnˈoʊzəoonnnnn espinosa
bˈɪgɪnz eeeee biggins
pəɹfˈɛkʃn̩nnnnn perfection
ˈændəɹsn̩ annnnnn andersen
ˈænɪnæt aaann aninat
ˈɔlɹɪd aeeee alred
ɑɹkˈoʊlə ooooe arcola
pɹˈimiəm aaeee premium
bˈɑlmʊŋ ooeee balmung
ɹˌɑbˈʌstəseaeee robustas
dɪsˈɔɹdɚz rreeee disorders
ˈæsɪtˌæl oaeee acetal
bjˈuz eeeee buse
wˈeif aeeee waif
ˈæɹənˌoʊs ooonnn arenose
kwˈeiswˈeieeeeee kweisui
ɪmbˈɑdiɪŋ eeeeee embodying
mˈʌlkɚn raeee mulkern
kˈɪmɪtʃ oeeeee kimmich
tˈupəlˌoʊ oooee tupelo
tɹɑɪˈʌmf oeeee triumph
pɹˈaʊd˺ɚ aeeee prouder
wˈɛdʒ eeeee wedge
ɹˈɛznɪk oeeee reznik
flˈeid aaeee flayed
ˈæsɪlɪn oeeee asselin
dɪkɹˈitl̩ oeeee decretal
sɪndˈɛt˺ɪkeeeeeee syndetic
spˈænɪʃ oeeee spanish
ɹˈæli aeeee rallye
aʊtʃˈoʊn ooonn outshone
sˈɑsəɹˌɑɪtoaaaaae saussurite
ɪspˈaʊzd aaeee espoused
bɹˈɪklɪn oeeee bricklin
ɛskˈɑləp aaeee escallop
hˈɔɹdz aaeee hordes
kˈæɹəkɚ aaaee caraker
wˈɑɪthˌɝstoeeeeee whitehurst
mˈɛɹɪmæk aaeee merrimac
tʃɑɪnˈi eeeee chinee
hˈænlɪn oeenn hanlin
dɑɪˈækənl̩annnnn diaconal
kwˈɪnin oeeee quinine
bɹʊjˈɛɹ oeeee bruyeres
pitsˈutoʊ oooee pizzuto
liˈændəɹ aaaaen leander
əblˈɑɪdʒɪŋiiieee obliging
mˈɔɹfju aaeee morphew
glˈækn̩z aaannn glackens
bukˈeiz aeeee bouquets
hˈɝɹmɪt oeeee hermit
mˈʌsl̩ aeeee mussell
mˈʌt˺əɹ aaaee mutter
pəhˈoʊki ooee pahokee
gɑlˈɑsoʊ oooee galasso
ˈoʊvɚlˌɑk oeeee overlock
tˈɪŋktʃəɹ onnnnnn tincture
pjˈudʒɪn oeeee pugin
spˌɛʃəlˌi eeeeee especially
hˈʊfpɹˌɪntonnnee hoofprint
pˈɑntɪk oeeeee pontic
pɹˈɑɪsɪz oeeee prices
nˈeivi eeeee navy
gɹˈɪmʃˌɔ ooeeee grimshaw
pɹəvˈɑɪzoʊooeee proviso
sˈoʊlɪtɹɑnonnnnn solitron
ˈænəɵˌoʊl oooon anethole
pɑmpˈɛɹoʊ ooeee pampero
ˈɑɪməs eeeee imus
hˈoʊfəɹ aaaee hofer
smˈɑɹts ooeee smarts
mɔɹtˈimɔɹ oeeee mortimore
blˈɛkmn̩ aannn blechman
ʃˈæfɹn̩ aaann shafran
tn̩dɹˈoʊ ooee tondreau
ˈɝdʒn̩tli oeennnn urgently
ˌɑksˈænə aaann oksana
flˈɪntʃiɹ oeennn flintshire
ʃɹˈɛŋk reeee schrenk
plˈæzə aaae plaza
dˈunədɪn eennnn dunedin
dɪsmˈɪʃn̩ onnnn dismission
skwˈɪfi eeeee squiffy
skˈɑɪwˌei oeeeee skyway
fˈæʃɪst oeeee fascist
pˈɑləpˌɛɹiaaaaeee polypary
ɑɪˈælmənəsaanaaa ialmenus
ˈɑzmn̩dsn̩nnnnnnnn osmundson
ˈɑɪlˌæʃɪz aaeeee eyelashes
gˈæsp aaeee gasp
ɪnwˈɑɪnd onnnn inwind
kˈoʊldli oeeee coldly
bˈækʃi aeeee bakshi
æbˈɛsɪv eeeee abessive
tʃɪkˈɔin ooeee chicoine
skˌɪmz aeeee skims
kɹɪsp eeeee crisp
hˈɑɪdɹˌɑɪdrnaaeee hydride
bɹˈæmlɪt oeeee bramlett
jˈɛsəf aeeee iosif
gˈɑɹdhˌaʊsoooaaae guardhouse
sˈɪliəs eeeee syleus
zˈɪŋə eeeee zinga
tˈɪŋkɚd aeeee tinkered
tˈægəɹ aaaee tagger
ˈʌltəmˌoʊ ooaeee ultimo
ʃˈɛɹɪl eeeee sherrill
poʊstwˈoʊɹroeeee postwar
lˈɔɹiəts oeeeee laureates
dʒˈɑskɪn oonne joskin
kˈinɪŋ eeeee keening
slˈoʊli oeeee slowly
spoʊsˈɑtoʊooee sposato
wˌɑt˺əɹˈi aeeeee wateree
æmfˈɪbiə eeeee amphibia
dʒˌæpənˈizaaaaa japanese
əntɹˈu aaeee untrue
pˈɛɹəbl̩ oeeee parable
kʊɹzˈɑwə aaaa kurzawa
sˈʌmwˌɛɹ aaeee somewhere
zˈɪlviə aeeee zilvia
oʊsˈɔɹioʊ oooee osorio
swˈɪtʃɚ reeeee switcher
ɚˈɛndsˌi oeeeee arendsee
kɑɹvɑjˈæl eeee carvajal
blˈæknɪs oeenn blackness
pɚˈɪsiˌɛn oeeeee parisienne
kˈænjn̩z aannnn canyons
pɔɹtˈɛndz onnnn portends
lˈuɪn eeeee lewin
slˈɪmɪŋ eeeee slimming
mʊɹˈɑt aaeee murat
tˈɑɪmn̩ aaennn timon
hˈimɪn eeeee hemin
tˈɝɹnkˌi oaeeee turnkey
ɹɪdˈiməbl̩oeeeee redeemable
lˈɑkəs aaeee lochus
skˈɛɹiɚ eeeee scarier
ˌænəkjˈuʒəaaaa anacusia
stɹˈut oeeee stroot
tɑɪmˈiəs aeeee timaeus
səbmˈɝs eeeee submerse
stˈɑlwəɹɵ raaaaee stalworth
ˈiniəs aeeee oeneus
ləzˈɪɹ eeee lazear
bˈɑoʊ aeeee booe
ɪkspˈændɪdnnnnn expanded
zəɹkˈɑnɪk onnnn zirconic
ˈɑbleit oeeee oblate
gˈɪbi eeeee gibby
sˈʌmnɚz aeeeee sumners
tˈeiln̩ aaeee taillon
tɹˈɪbjut ooeee tribute
pˈʌzl̩mn̩tnnnnnnn puzzlement
lˈændfˌɪlzaaeaeen landfills
mˈɑnɪtɹɪs onnnnnn monitress
pˈupoʊ oaee pupo
ɪʃikˈɑwə aaaee ishikawa
kɹɪmˈinz oeeee cremeens
sɚɹv eeeee serve
hˈɛɵklɪf oeeee heathcliff
mˈɪdsˌɛʃn̩onnnnee midsession
gælwˈidʒn̩onnne galwegian
pitɹˈʌʃkə oaeee pietruszka
ˈidi eeeee edyie
stɹˈits oeeee streets
wˈɪd˺ɚ eeeee widder
ɹɑpˈoʊzoʊ ooooo rapozo
sɛt eeeeee sett
plˈaʊn̩ oaenn plauen
lˈɛpəɹdɪs eeeeee leopardess
pˈɑkɚni aaeee pokorney
flˈɪntʃɪŋ onnnnnn flinching
wˈɪləɹd aaeee willard
klˈɑɹkɪn oonnn clarkin
fˈɝsts eeeee firsts
dʊk eeeee doek
klˈoʊʒəɹ ooeee closure
sˈɪkl̩ aeeee sickle
kˈɑɹəkɚ aaaee karraker
kˈeivəɹ aaeee caver
kˈʌpl̩z aaeee cupples
ɹˈɪd˺ɚ eeeee ridder
lˈoʊn eeeee lone
hˈɑs eeeee hoss
ˈædɑ aaeee adah
pɹˈɑfɪsˌɑɪiaaeee prophesy
ɑɹˈɑnjɑ aaaee aranha
mɑɹsˈili oeeee marsili
nˈɛtwˌɝks reeeeee networks
gˈɑnkɑɹz aaaaee gancarz
bɑɹkˈɑɪ oaeee barkai
stˈɪŋks oeeee stinks
bɹˈʌʃi eeeee brushy
ˈɑstɚgɚd reeeee ostergard
jɔɹˈi eeeee youree
sˈizɪk raeee cizik
səhˈidʒin̩onneee sahagian
əlˈuvin̩ aaeee alluvion
mɪnˈoʊn̩ oonnnn minoan
nˈiəlɪɵ oeeee neolith
dʒɪnˈændɹinnnnnnn gynandry
lˈaʊd eeeee loud
snˈʊks aeeee snooks
ɪkstɹˈusɪvoonnee extrusive
bˈɛd˺əgˌɑɹeeeeeee bedeguar
ˈeikn̩ aaeee aken
kɹˈɑpi aaeee croppie
kɹˈudʒɚ reeee cruger
hˈʊlsmn̩ oonnn huelsmann
flˈʌŋk raeee flunk
pˌɑpɑjˈɑn aeee popayan
ˈɑknəs aaeee ocnus
kˈɪŋsɑlvɚ ooeeee kingsolver
əfˈɔɹdəd aaaee afforded
hˈɑtbˌɛdz aeeeee hotbeds
zˈʌs eeeee xus
əndˈu eeeee undue
zˈɑd˺ə aaaee zada
kn̩sˈoʊlz oooee consoles
ˈɛkɚsn̩ aaeene eckerson
flˈæt˺əɹi aaeeee flattery
hˈeidʒˈoʊ oeeee heijo
pɚˈɛnil̩z eeeeee perennials
kˈɑkshˌɛd aaaee cockshead
wˈein̩z aaeeee wayans
ˈækjɚətli aeeeee accurately
plumˈɑsɪtieeeee plumosity
bˈaʊmɚt oeeee baumert
kɹˈɛhn̩ oaenn crehan
aʊtlˈæstɪdoaeee outlasted
kɹˈɛst eeeee crest
leik eeeee laigh
vˈækjuəsliaaaaa vacuously
ˈægnˌɛli eeeee agnelli
ənˈæptnɛs onnnnn unaptness
sˈitoʊ ooeee sito
tˈɔk aeeee talk
hˈɛdgˌɪɹ oeeee headgear
flˈɑɪəɹ aaeee flyer
nˈuwˌeiv oaeee newwave
ˈɪt˺əɹn̩s onnnnnn iterance
dʒˈɑnzi aaeee johnsey
ʃwˈɪɹ eeeee schweer
wˈikfˌɪʃ oeeee weakfish
tɑɪfˈin̩ oeene typhoean
hɪdʒˈɑɪɹə oeeee hejira
dˈɑltn̩ aaennn dolton
ˈægəts aeeee agates
hˈit˺ɪdli eeeeee heatedly
wɑɹˈɑs aaaee huaras
kn̩fˈɛʃn̩ onnnn confession
pˈʊɹim aaeee purim
ɹimˈɪt˺ɪd oeeeee remitted
kjˌuziˈænəaaaee cusiana
pˌɝɹmɪtˈi eeeee permittee
pˈoʊzoʊs oooo pozos
ɹˌifˌɛnsd annnnn refenced
neisˈut ooeee nasute
ˌoʊnəm aaeee onum
tɹiˈoʊmf ooeee triomphe
tˈɛɹələ aaeee pteryla
moʊlˈɑnoʊ oonnn molano
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 93, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "n=batch_size\n", + "y_pred = model.predict(X_test[:n], verbose=1, batch_size=batch_size)\n", + "\n", + "print()\n", + "print ('Word error rate {:%}'.format(WER(y_pred[:n],y_test[:n])))\n", + "print ('Char error rate {:%}'.format(CER(y_pred[:n],y_test[:n])))\n", + "\n", + "show_results(ytable=ytable, y_pred=y_pred[:n],y_test=y_test[:n],xtable=xtable,X_test=X_test[:n])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.764655Z", + "start_time": "2017-05-31T12:47:48.567Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# accuracy by class\n", + "r=(y_test-y_pred).reshape((-1,y_test.shape[-1]))\n", + "plt.plot(np.abs(r).mean(-1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.769006Z", + "start_time": "2017-05-31T12:47:48.569Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.769709Z", + "start_time": "2017-05-31T12:47:48.572Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# load best\n", + "# model.load_weights('./models/phone_to_respelling_201610-16142229.hdf')\n", + "# model = keras.models.load_model(model_checkpoint.filepath,custom_objects=dict(dice_coef_loss=dice_coef_loss))\n", + "model = keras.models.load_model('./models/phone_to_respelling_201610-19071704.hdf',custom_objects=dict(dice_coef_loss=dice_coef_loss))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.770678Z", + "start_time": "2017-05-31T12:47:48.575Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "scores = model.evaluate(X_test,y_test, batch_size=batch_size, verbose=1)\n", + "score = dict(zip(model.metrics_names,scores))\n", + "score" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.771783Z", + "start_time": "2017-05-31T12:47:48.578Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# model.save('./models/phone_to_respelling_201610-16142229_acc-{:2.2f}.hdf'.format(score['acc']))\n", + "# model.save_weights('./models/phone_to_respelling_201610-16142229_acc-{:2.2f}_weights.hdf'.format(score['acc']))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### visualise results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.772461Z", + "start_time": "2017-05-31T12:47:48.580Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "n=10\n", + "y_pred = model.predict(X_test) \n", + "show_results(ytable, y_pred[:n],y_test[:n])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.773273Z", + "start_time": "2017-05-31T12:47:48.583Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "def word_error_rate(y_true,y_pred):\n", + " # calc word error rate (compare 23.3-33.89 for grap2phon https://github.com/cmusphinx/g2p-seq2seq)\n", + " # like https://github.com/cmusphinx/g2p-seq2seq/blob/master/g2p_seq2seq/g2p.py#L317\n", + " c=(y_pred.argmax(-1)==y_true.argmax(-1)).all(-1).sum()\n", + " return (1-c/len(y_true))\n", + "\n", + "from leven import levenshtein \n", + "def element_error_rate(y_true, y_pred, ytable=ytable):\n", + " \"\"\"Returns the word error rate of the supplied hypothesis with respect to\n", + " the reference string.\"\"\"\n", + " c_test = [''.join(x).strip() for x in ytable.decode(y_true)]\n", + " c_pred = [''.join(x).strip() for x in ytable.decode(y_pred)]\n", + " distance = np.array([levenshtein(pred,test) for pred,test in zip(c_pred,c_test)])\n", + " elems = np.array([len(x) for x in c_test])\n", + " error_rate = distance / len(elems)\n", + " return error_rate.sum()\n", + "\n", + "wer = word_error_rate(y_test, y_pred)\n", + "print('word error rate {:2.2%}'.format(wer))\n", + "\n", + "eer = element_error_rate(y_test, y_pred)\n", + "print('element (char or phon) error rate {:2.2%}'.format(eer))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.774032Z", + "start_time": "2017-05-31T12:47:48.585Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "a=[[1,2,3],[1,2,3]]\n", + "b=[[1,2,3],[2,0,3]]\n", + "a=np.array([keras.utils.np_utils.to_categorical(x,4) for x in a])\n", + "b=np.array([keras.utils.np_utils.to_categorical(x,4) for x in b])\n", + "assert word_error_rate(a,b)==0.5\n", + "assert element_error_rate(a,b)==1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.774687Z", + "start_time": "2017-05-31T12:47:48.586Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# accuracy for each char\n", + "report = sklearn.metrics.classification_report(y_test.argmax(-1).flatten(), y_pred.argmax(-1).flatten(), target_names=ytable.chars)\n", + "print(report)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.775418Z", + "start_time": "2017-05-31T12:47:48.588Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# show confidence\n", + "conf = y_pred.max(-1)\n", + "plt.imshow(conf[:n],interpolation='none',vmin=0,vmax=1)\n", + "plt.colorbar()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-05-31T12:47:59.776286Z", + "start_time": "2017-05-31T12:47:48.589Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# TODO make this better, maybe just repeat the phoneme?\n", + "def show_possibilities(y_vis):\n", + " \"\"\"\n", + " Given a y_prediction, dispay the most commonly predicted chars for each input\n", + " \n", + " e.g. 'AY2 ' gives 'AOU' for p>0.1\n", + " \"\"\"\n", + " for k in range(len(xtable.chars)):\n", + "\n", + " # grab top choices\n", + " choices = np.array([np.argsort(y_vis[k,j])[::-1] for j in range(y_vis.shape[1])])\n", + " # and thier probs\n", + " probs = np.array([[y_vis[k][i][choices[i][j]] for i in range(choices.shape[0])] for j in range(choices.shape[1])]).T\n", + "\n", + " # print('most common chars (in order) p<0.1 for:')\n", + " html='

\"{}\" most likely chars (p>0.1)

'.format(xtable.chars[k])\n", + " res = ytable.decode(choices,calc_argmax=False)\n", + " for i in range(res.shape[0]):\n", + " html+='
'\n", + " for j in range(res.shape[1]):\n", + " c=res[i,j]\n", + " a=np.sqrt(probs[i,j])\n", + " if a>np.sqrt(0.1):\n", + " html+='{c:}'.format(c=c,a=a)\n", + " # pprint([''.join(y) for y in ytable.decode(choices,calc_argmax=False)])\n", + " display(HTML(html))\n", + "\n", + "\n", + "# lets see what each phonemes predicts\n", + "# x_vis = xtable.encode(np.array([xtable.chars]).T)\n", + "x_vis = xtable.encode(np.array([[x]*xtable.maxlen for x in xtable.chars]))\n", + "y_vis = model.predict(x_vis)\n", + "show_possibilities(y_vis) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "hide_input": false, + "kernelspec": { + "display_name": "jupyter3", + "language": "python", + "name": "jupyter3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.0" + }, + "toc": { + "toc_cell": false, + "toc_number_sections": true, + "toc_threshold": 6, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..e91488b --- /dev/null +++ b/readme.md @@ -0,0 +1,17 @@ +# phon2graph + +Using a sequence 2 sequence model with attention to convert from pronunciation to spelling. + + +## results + +It reaches a character error rate of <20%, and here are the results (lighter letters show where the model was uncertain). The machine has made some reasonable mistakes. + + +
pronunciationguessspelling
kˈɑkəsˌɔidcocksoid caucasoid
bɹˈæsfˌildbrassfilddbrassfield
flˈæʃbl̩b flashblb flashbulb
hˈɛstiə hestia hestia
kɹˈæbi crabyy crabby
tɹədˈus troduse traduce
təlˈulə talulaa tallulah
bˈitn̩ beatn beaton
hˈɑlɪtʃɛk halicck holecek
beidˈɔiə bedooy bedoya
+ +## installation + +Install the requirements `pip install --upgrade -r requirements.txt` + +Then start a jupyter notebook and open main.ipynb diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c9bbaee --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +keras==1.2.2 +tensorflow-gpu==1.0.0 + +# data processing +leven==1.0.4 +https://github.com/timmahrt/pysle/archive/e47ab5630679451b719787f5579968756bd7a644.zip +https://github.com/datalogai/recurrentshop/archive/6f709f1aabd5156b184a06852b8edb9ceee5bb21.zip +https://github.com/farizrahman4u/seq2seq/archive/1b1ae455fcebd55b16eb6e1c77a58aabfe85892a.zip +keras-tqdm==2.0.1