init

.gitignore

@@ -0,0 +1,119 @@
+
+# Created by https://www.gitignore.io/api/linux,python
+
+### Linux ###
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# End of https://www.gitignore.io/api/linux,python

helpers/__init__.py

@@ -0,0 +1 @@
+from .helpers import *

helpers/dataset_isledict.py

@@ -0,0 +1,104 @@
+
+import requests
+
+from pysle import isletool
+import numpy as np
+import itertools
+from sklearn.model_selection import train_test_split
+import re
+import os
+
+from .helpers import CharacterTable
+
+
+
+def download_data_maybe(fname='ISLEdict.txt', url='http://isle.illinois.edu/sst/data/g2ps/English/ISLEdict.html', cache_subdir='datasets'):
+
+    datadir_base = os.path.expanduser(os.path.join('~', '.keras'))
+    if not os.access(datadir_base, os.W_OK):
+        datadir_base = os.path.join('/tmp', '.keras')
+    datadir = os.path.join(datadir_base, cache_subdir)
+    if not os.path.exists(datadir):
+        os.makedirs(datadir)
+    fpath = os.path.join(datadir, fname)
+
+    if not os.path.exists(fpath):
+        print('Downloading data from ', url, 'to', fpath)
+        r = requests.get(url)
+        assert r.status_code == 200
+        from bs4 import BeautifulSoup
+        soup = BeautifulSoup(r.content, 'lxml')
+        with open(fpath, 'w') as fo:
+            fo.write(soup.text.strip())
+    return fpath
+
+
+def get_data(seed=42, test_size=0.20, verbose=0, maxlen_x=None, maxlen_y=None, blacklist='()0123456789%.?"-_', max_phonemes=np.inf, max_chars=np.inf, phon_sep='', unique_graphemes=False, unique_phonemes=True):
+    """Process ISLEDICT pronounciation dictionary to return unique phonemes two graphemes"""
+
+    path = download_data_maybe()
+
+    # load data
+    isleDict = isletool.LexicalTool(path)
+    X = []
+    y = []
+    for phrase in isleDict.data.keys():
+        for pronounciation in zip(*isleDict.lookup(phrase)):
+            xx = []
+            for syllableList, stressedSyllableList, stressedPhoneList in pronounciation:
+                xx += list(itertools.chain(*syllableList))
+            y.append(phon_sep.join(xx))
+            X.append(phrase)
+    if verbose: print('loaded entries {}'.format(len(X)))
+
+    # filter out duplicate X's
+    if unique_phonemes:
+        y, X = zip(*dict(zip(y, X)).items())
+        if verbose: print('removed duplicate phonemes leaving {}'.format(len(X)))
+
+    # filter out duplicates Y's
+    if unique_graphemes:
+        X, y = zip(*dict(zip(X, y)).items())
+        if verbose: print('removed duplicate graphemes leaving {}'.format(len(X)))
+
+    # split data (we must set asside test data before cleanign so it's always the same)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=seed)
+
+    # filter out duplicate entries like 'HOUSE(2) or multi words CAT-DOG and CAT_DOG'
+    p = re.compile('[%s]' % (re.escape(blacklist)))
+    X_train, y_train = zip(*[(x, y) for x, y in zip(X_train, y_train) if not bool(p.findall(x))])
+    X_test, y_test = zip(*[(x, y) for x, y in zip(X_test, y_test) if not bool(p.findall(x))])
+    if verbose:
+        print('removed blacklisted entries leaving {}'.format(len(X_train) + len(X_test)))
+
+    # filter out complex entries if needed
+    before_x = len(y_train)
+    X_train, y_train = zip(*[(x, y) for x, y in zip(X_train, y_train) if len(y) <= max_phonemes and len(x) <= max_chars])
+    X_test, y_test = zip(*[(x, y) for x, y in zip(X_test, y_test) if len(y) <= max_phonemes and len(x) <= max_chars])
+    if verbose:
+        print('restricted to less than {} phonemes leaving {} entries or {:2.2f}%'.format(max_phonemes, len(X_train) + len(X_test), len(X_train)/before_x*100))
+
+    # FIXME it's slow in the next few lines
+    # encode x and y and pad them
+    xtable = CharacterTable()
+    xtable.fit(X_test + X_train)
+    if maxlen_x:
+        xtable.maxlen = maxlen_x
+    X_train = xtable.encode(X_train)
+    X_test = xtable.encode(X_test)
+
+    ytable = CharacterTable()
+    ytable.fit(y_test + y_train)
+    if maxlen_y:
+        ytable.maxlen = maxlen_y
+    y_train = ytable.encode(y_train)
+    y_test = ytable.encode(y_test)
+
+    if verbose:
+        print('X_train shape:', X_train.shape)
+        print('X_test shape:', X_test.shape)
+
+        print('y_train shape:', y_train.shape)
+        print('y_test shape:', y_test.shape)
+
+    return (X_train, y_train), (X_test, y_test), (xtable, ytable)

helpers/helpers.py

@@ -0,0 +1,257 @@
+import time
+import re
+import numpy as np
+import nltk
+import collections
+
+
+# get short words
+def cmudict_random_sample(n=100):
+    '''get words shorter or equal to word_length'''
+    cmudict = nltk.corpus.cmudict.dict()
+    pairs = cmudict.items()
+    # get random samples of cmudict of n=cutoff
+    samples = (np.random.sample(n) * len(cmudict)).astype(int)
+    pairs = [pairs[s] for s in samples]
+    cmudict = collections.OrderedDict(pairs)
+    return cmudict
+
+# get short words
+
+
+def cmudict_short_words(word_length=3):
+    '''get words shorter or equal to word_length'''
+    cmudict_raw = nltk.corpus.cmudict.dict()
+    keys = np.array(cmudict_raw.keys())
+    lens = np.array([len(k) for k in keys])
+    vals = np.array(cmudict_raw.values())
+    inds = np.argwhere(lens <= word_length)
+    vals2 = vals[inds]
+    keys2 = keys[inds]
+    cmudict = dict(np.dstack((keys2.flatten(), vals2.flatten()))[0])  # must be a better way!
+    return cmudict
+
+
+def cmudict_short_phones(phone_length=3):
+    '''get words shorter or equal to word_length'''
+    cmudict_raw = nltk.corpus.cmudict.dict()
+    vals = np.array(cmudict_raw.values())
+    lens = np.array([len(p[0]) for p in vals])
+    keys = np.array(cmudict_raw.keys())
+    inds = np.argwhere(lens <= phone_length)
+    vals2 = vals[inds]
+    keys2 = keys[inds]
+    cmudict = dict(np.dstack((keys2.flatten(), vals2.flatten()))[0])  # must be a better way!
+    return cmudict
+
+
+def unique(seq, idfun=None):
+    '''get only unique items in list'''
+    # order preserving
+    if idfun is None:
+        def idfun(x): return x
+    seen = {}
+    result = []
+    for item in seq:
+        marker = idfun(item)
+        # in old Python versions:
+        # if seen.has_key(marker)
+        # but in new ones:
+        if marker in seen:
+            continue
+        seen[marker] = 1
+        result.append(item)
+    return result
+
+
+def split_string(word, inds):
+    '''split string at indices'''
+    splitword = []
+    inds = np.sort(inds)
+    for i in range(len(inds) - 1):
+        j = inds[i]
+        k = inds[i + 1]
+        if k == 0 and j == len(word):
+            continue
+        part = word[j:k]
+        splitword.append(part)
+    return splitword
+
+
+def remove_stress_arp(s):
+    return re.sub('\d+', '', s)
+
+
+def findall(pattern, target):
+    inds = []
+    while pattern in target:
+        inds.append(target.index(pattern))
+        target = target.replace(pattern, '_' * len(pattern), 1)
+    return inds
+
+
+def dist_in_letters(word, phones, phon_ind, match, lind):
+    '''find distance between pheonom position and it's match
+    provide:
+        word e.g. 'baby'
+        phones e.g. [u'B', u'EY1', u'B', u'IY0']
+        phon_ind e.g. 2
+        match e.g. B
+        lind: positve of match in word e.g. 2
+    '''
+    dists = []
+    letters_p_phon = 1.0 * len(word) / len(phones)
+    phon_lind = phon_ind * letters_p_phon
+    dist1 = abs(lind - phon_lind)
+    dist2 = abs(lind + len(match) - phon_lind - len(match))
+    dists.append(dist1)
+    dists.append(dist2)
+
+    return min(dists)
+
+
+def write_cmudict_sample(n=200):
+    import numpy as np
+    cmudict = nltk.corpus.cmudict.dict()
+    outf = 'cmudict_samples' + str(time.time()) + '.txt'
+    fo = open(outf, 'w')
+    samples = (np.random.random_sample(100) * len(cmudict)).astype(int)
+    cmuk = cmudict.keys()
+    for s in samples:
+        k = cmuk[s]
+        vs = cmudict[k]
+        for v in vs:
+
+            vstr = ' '.join(v)
+            fo.write('{}  {}  {}\n'.format(k, k, vstr))
+    fo.close()
+    print("Please go to {} and place space between the letters that match the peonomes, this will be the positive test".format(outf))
+
+
+def empty_tree(input_list):
+    """Recursively iterate through values in nested lists."""
+    if input_list:
+        for item in input_list:
+            if not isinstance(item, list) or not empty_tree(item):
+                return False
+    return True
+
+import itertools
+# based on https://github.com/fchollet/keras/blob/master/examples/addition_rnn.py
+# note: can't make sparse 3d matrices
+
+
+class CharacterTable(object):
+    '''
+    Given a set of characters:
+    + Encode them to a one hot integer representation
+    + Decode the one hot integer representation to their character output
+    + Decode a vector of probabilities to their character output
+    '''
+
+    def __init__(self, chars='', maxlen=None, null_char=' ', left_pad=False):
+        self.chars = sorted(set([null_char] + list(chars)))
+        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
+        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))
+        self.maxlen = maxlen
+        self.left_pad = left_pad
+        self.null_char = null_char
+
+    def fit(self, Cs, null_char=' '):
+        """Determine chars and maxlen by fitting to data"""
+        self.chars = sorted(set(itertools.chain([null_char], *Cs)))
+        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
+        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))
+        self.maxlen = max(len(c) for c in Cs)
+        self.null_char = null_char
+
+    def encode(self, Cs, maxlen=None):
+        """Pass in an array of arrays to convert to integers"""
+        maxlen = maxlen if maxlen else self.maxlen
+        n = len(Cs)
+        X = np.zeros((n, maxlen, len(self.chars)), dtype=np.bool)
+        for j, C in enumerate(Cs):
+            if self.left_pad:
+                C = [self.null_char] * (maxlen - len(C)) + list(C)
+            else:
+                C = list(C) + [self.null_char] * (maxlen - len(C))
+                for i, c in enumerate(C):
+                    X[j, i, self.char_indices[c]] = True
+        return X
+
+    def decode(self, Xs, calc_argmax=True):
+        if calc_argmax:
+            Xs = Xs.argmax(axis=-1)
+        return np.array(list([self.indices_char[x] for x in X] for X in Xs))
+
+
+# show_results
+from IPython.core.display import display, HTML
+m=1.2
+lighten=lambda x:1-(1/m-x/m)
+
+def show_results(ytable, y_pred,y_test=None,X_test=None,xtable=None):
+    """Show results which are darker when more confident"""
+    html = '<table><tbody><thead>'
+    html += '<tr><th>pronunciation</th><th>guess</th><th>spelling</th></tr>'
+    html += '</thead>'
+    p_pred = ytable.decode(y_pred)
+    conf = y_pred.max(-1)
+    for i in range(p_pred.shape[0]):
+        html += '<tr>'
+
+        if X_test is not None:
+            p_test = xtable.decode(X_test)
+            html+='<td>'
+            for j in range(p_test.shape[1]):
+                c=p_test[i][p_test.shape[1]-j-1]
+                html+='<span style="color:rgba(0,0,0,{a:1.1f})">{c:}</span>'.format(c=c,a=1)
+            html+='</td>'
+
+        html+='<td>'
+        for j in range(p_pred.shape[1]):
+            c=p_pred[i][j]
+            a=lighten(conf[i][j])
+            html+='<span style="color:rgba(0,0,0,{a:1.1f})">{c:}</span>'.format(c=c,a=a)
+        html+='</td>'
+
+        if y_test is not None:
+            html+='<td>'
+            p_test = ytable.decode(y_test)
+            for j in range(p_test.shape[1]):
+                c=p_test[i][j]
+                html+='<span style="color:rgba(0,0,0,{a:1.1f})">{c:}</span>'.format(c=c,a=1)
+            html+='</td>'
+        html += '</tr>'
+    html += '</tbody></table>'
+    return HTML(html)
+
+# test
+# r=np.random.random((10,8,30))**20
+# show_results(ytable, r)
+
+
+class weighted_categorical_crossentropy(object):
+    """
+    A weighted version of keras.objectives.categorical_crossentropy
+
+    Variables:
+        weights: numpy array of shape (C,) where C is the number of classes
+
+    Usage:
+        loss = weighted_categorical_crossentropy(weights).loss
+        model.compile(loss=loss,optimizer='adam')
+    """
+
+    def __init__(self,weights):
+        self.weights = K.variable(weights)
+
+    def loss(self,y_true, y_pred):
+        # scale preds so that the class probas of each sample sum to 1
+        y_pred /= y_pred.sum(axis=-1, keepdims=True)
+        # clip
+        y_pred = K.clip(y_pred, K.epsilon(), 1)
+        # calc
+        loss = y_true*K.log(y_pred)*self.weights
+        loss =-K.sum(loss,-1)
+        return loss

helpers/layers.py

@@ -0,0 +1,306 @@
+"""
+A keras attention layer that wraps RNN layers.
+
+Based on tensorflows [attention_decoder](https://github.com/tensorflow/tensorflow/blob/c8a45a8e236776bed1d14fd71f3b6755bd63cc58/tensorflow/python/ops/seq2seq.py#L506)
+and [Grammar as a Foreign Language](https://arxiv.org/abs/1412.7449).
+
+date: 20161101
+author: wassname
+url:
+"""
+from __future__ import absolute_import
+
+from keras import backend as K
+from keras.engine import InputSpec
+from keras.layers import LSTM, activations, Wrapper, Recurrent
+
+class Attention(Wrapper):
+    """
+    This wrapper will provide an attention layer to a recurrent layer.
+
+    # Arguments:
+        layer: `Recurrent` instance with consume_less='gpu' or 'mem'
+
+    # Examples:
+
+    ```python
+    model = Sequential()
+    model.add(LSTM(10, return_sequences=True), batch_input_shape=(4, 5, 10))
+    model.add(TFAttentionRNNWrapper(LSTM(10, return_sequences=True, consume_less='gpu')))
+    model.add(Dense(5))
+    model.add(Activation('softmax'))
+    model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
+    ```
+
+    # References
+    - [Grammar as a Foreign Language](https://arxiv.org/abs/1412.7449)
+
+
+    """
+    def __init__(self, layer, **kwargs):
+        assert isinstance(layer, Recurrent)
+        if layer.get_config()['consume_less']=='cpu':
+            raise Exception("AttentionLSTMWrapper doesn't support RNN's with consume_less='cpu'")
+        self.supports_masking = True
+        super(Attention, self).__init__(layer, **kwargs)
+
+    def build(self, input_shape):
+        assert len(input_shape) >= 3
+        self.input_spec = [InputSpec(shape=input_shape)]
+        nb_samples, nb_time, input_dim = input_shape
+
+        if not self.layer.built:
+            self.layer.build(input_shape)
+            self.layer.built = True
+
+        super(Attention, self).build()
+
+        self.W1 = self.layer.init((input_dim, input_dim, 1, 1), name='{}_W1'.format(self.name))
+        self.W2 = self.layer.init((self.layer.output_dim, input_dim), name='{}_W2'.format(self.name))
+        self.b2 = K.zeros((input_dim,), name='{}_b2'.format(self.name))
+        self.W3 = self.layer.init((input_dim*2, input_dim), name='{}_W3'.format(self.name))
+        self.b3 = K.zeros((input_dim,), name='{}_b3'.format(self.name))
+        self.V = self.layer.init((input_dim,), name='{}_V'.format(self.name))
+
+        self.trainable_weights = [self.W1, self.W2, self.W3, self.V, self.b2, self.b3]
+
+    def get_output_shape_for(self, input_shape):
+        return self.layer.get_output_shape_for(input_shape)
+
+    def step(self, x, states):
+        # This is based on [tensorflows implementation](https://github.com/tensorflow/tensorflow/blob/c8a45a8e236776bed1d14fd71f3b6755bd63cc58/tensorflow/python/ops/seq2seq.py#L506).
+        # First, we calculate new attention masks:
+        #   attn = softmax(V^T * tanh(W2 * X +b2 + W1 * h))
+        # and we make the input as a concatenation of the input and weighted inputs which is then
+        # transformed back to the shape x of using W3
+        #   x = W3*(x+X*attn)+b3
+        # Then, we run the cell on a combination of the input and previous attention masks:
+        #   h, state = cell(x, h).
+
+        nb_samples, nb_time, input_dim = self.input_spec[0].shape
+        h = states[0]
+        X = states[-1]
+        xW1 = states[-2]
+
+        Xr = K.reshape(X,(-1,nb_time,1,input_dim))
+        hW2 = K.dot(h,self.W2)+self.b2
+        hW2 = K.reshape(hW2,(-1,1,1,input_dim))
+        u = K.tanh(xW1+hW2)
+        a = K.sum(self.V*u,[2,3])
+        a = K.softmax(a)
+        a = K.reshape(a,(-1, nb_time, 1, 1))
+
+        # Weight attention vector by attention
+        Xa = K.sum(a*Xr,[1,2])
+        Xa = K.reshape(Xa,(-1,input_dim))
+
+        # Merge input and attention weighted inputs into one vector of the right size.
+        x = K.dot(K.concatenate([x,Xa],1),self.W3)+self.b3
+
+        h, new_states = self.layer.step(x, states)
+        return h, new_states
+
+    def get_constants(self, x):
+        constants = self.layer.get_constants(x)
+
+        # Calculate K.dot(x, W2) only once per sequence by making it a constant
+        nb_samples, nb_time, input_dim = self.input_spec[0].shape
+        Xr = K.reshape(x,(-1,nb_time,input_dim,1))
+        Xrt = K.permute_dimensions(Xr, (0, 2, 1, 3))
+        xW1t = K.conv2d(Xrt,self.W1,border_mode='same')
+        xW1 = K.permute_dimensions(xW1t, (0, 2, 3, 1))
+        constants.append(xW1)
+
+        # we need to supply the full sequence of inputs to step (as the attention_vector)
+        constants.append(x)
+
+        return constants
+
+    def call(self, x, mask=None):
+        # input shape: (nb_samples, time (padded with zeros), input_dim)
+        input_shape = self.input_spec[0].shape
+        if K._BACKEND == 'tensorflow':
+            if not input_shape[1]:
+                raise Exception('When using TensorFlow, you should define '
+                                'explicitly the number of timesteps of '
+                                'your sequences.\n'
+                                'If your first layer is an Embedding, '
+                                'make sure to pass it an "input_length" '
+                                'argument. Otherwise, make sure '
+                                'the first layer has '
+                                'an "input_shape" or "batch_input_shape" '
+                                'argument, including the time axis. '
+                                'Found input shape at layer ' + self.name +
+                                ': ' + str(input_shape))
+
+        if self.layer.stateful:
+            initial_states = self.layer.states
+        else:
+            initial_states = self.layer.get_initial_states(x)
+        constants = self.get_constants(x)
+        preprocessed_input = self.layer.preprocess_input(x)
+
+
+        last_output, outputs, states = K.rnn(self.step, preprocessed_input,
+                                             initial_states,
+                                             go_backwards=self.layer.go_backwards,
+                                             mask=mask,
+                                             constants=constants,
+                                             unroll=self.layer.unroll,
+                                             input_length=input_shape[1])
+        if self.layer.stateful:
+            self.updates = []
+            for i in range(len(states)):
+                self.updates.append((self.layer.states[i], states[i]))
+
+        if self.layer.return_sequences:
+            return outputs
+        else:
+            return last_output
+
+
+
+
+# this is a copy of tensorflow with simplified matrix algebra, need to check I didn't make a mistkae
+class SimplifiedAttention(Wrapper):
+    def __init__(self, layer, attn_activation='tanh', **kwargs):
+        assert isinstance(layer, Recurrent)
+        if not layer.return_sequences:
+            raise Exception("AttentionLSTMWrapper doesn't support RNN's with return_sequences=False")
+
+        self.supports_masking = True
+        super(SimplifiedAttention, self).__init__(layer, **kwargs)
+
+    def build(self, input_shape):
+        assert len(input_shape) >= 3
+        self.input_spec = [InputSpec(shape=input_shape)]
+        nb_samples, nb_time, input_dim = input_shape
+
+        if not self.layer.built:
+            self.layer.build(input_shape)
+            self.layer.built = True
+
+        super(SimplifiedAttention, self).build()
+
+#         self.W1 = self.layer.init((input_dim, input_dim, 1, 1), name='{}_W1'.format(self.name))
+        self.W1 = self.layer.init((input_dim, input_dim), name='{}_W1'.format(self.name))
+#         self.W2 = self.layer.init((input_dim,nb_time), name='{}_W2'.format(self.name))
+        self.W2 = self.layer.init((nb_time,input_dim,input_dim), name='{}_W2'.format(self.name))
+        self.b2 = K.zeros((input_dim,), name='{}_b2'.format(self.name))
+        self.W3 = self.layer.init((input_dim*2, input_dim), name='{}_W3'.format(self.name))
+        self.b3 = K.zeros((input_dim,), name='{}_b3'.format(self.name))
+        self.V = self.layer.init((input_dim,), name='{}_V'.format(self.name))
+
+        self.trainable_weights = [self.W1, self.W2, self.W3, self.V, self.b2, self.b3]
+
+    def get_output_shape_for(self, input_shape):
+        return self.layer.get_output_shape_for(input_shape)
+
+    def step(self, x, states):
+
+        # First, we calculate new attention masks:
+        #   attn = softmax(V^T * tanh(W2 * inputs + W1 * prev_h))
+        # and then weight the previous state by the attention
+        #   prev_h = prev_h * attn
+        # and we make the input as a concatenation of the input and weighted inputs which is then
+        # transformed back to the shape x of using W3
+        #   x = W3*(x+X*attn)+b3
+        # Then, we run the cell on a combination of the input and previous attention masks:
+        #   h, state = cell(x, h).
+
+        nb_samples, nb_time, input_dim = self.input_spec[0].shape
+        h,c,B_U,B_W,xW2,X = states
+
+#         # as in tensorflow
+#         Xr = K.reshape(X,(-1,nb_time,input_dim,1))
+#         Xrt = K.permute_dimensions(Xr, (0, 2, 1, 3))
+#         xW1t = K.conv2d(Xrt,self.W1,border_mode='same') # could be cached
+#         xW1 = K.permute_dimensions(xW1t, (0, 2, 1, 3))
+
+        # or (input_dim,input_dim)x(nb_samples, nb_time, input_dim)=>(nb_samples, nb_time, input_dim)
+        # same value once reshaped, need to take away the extra dims
+        xW1 = K.dot(X,self.W1)
+
+        # assert hW1.shape == Xr.shape
+        hW2 = K.dot(h,self.W2)+self.b2
+#         xW2 = K.reshape(xW2,(-1,1,input_dim,1))
+        u = K.tanh(xW1+hW2)
+        a = K.sum(self.V*u,-1)
+        # assert a.shape==(nb_samples,nb_time)
+        a = K.softmax(a)
+        a = K.reshape(a,(-1, nb_time, 1))
+        Xa = K.sum(a*X,1)
+        Xa = K.reshape(Xa,(-1,input_dim))
+
+        # Merge input and previous attentions into one vector of the right size.
+        # TODO, deal with the consume_less='cpu' flag which reshapes x
+        x = K.dot(K.concatenate([x,Xa],1),self.W3)+self.b3
+        # assert x.shape == (nb_samples,input_dim)
+
+
+
+
+        h, new_states = self.layer.step(x, [h,c,B_U,B_W])
+#         new_states.append(a)
+#         Tracer()()
+        return h, new_states
+
+    def get_constants(self, x):
+        constants = self.layer.get_constants(x)
+        # Calculate K.dot(x, W2) only once per sequence by making it a constant
+#         # as in tensorflow
+#         self.W1 = self.layer.init((input_dim, input_dim, 1, 1), name='{}_W1'.format(self.name))
+#         Xr = K.reshape(x,(-1,nb_time,input_dim,1))
+#         Xrt = K.permute_dimensions(Xr, (0, 2, 1, 3))
+#         xW1t = K.conv2d(Xrt,self.W1,border_mode='same') # could be cached
+#         xW1 = K.permute_dimensions(xW1t, (0, 2, 1, 3))
+
+        # or just
+#         self.W1 = self.layer.init((input_dim, input_dim), name='{}_W1'.format(self.name))
+        xW1 = K.dot(x,self.W1)
+
+        constants.append(xW1)
+        # the need to provide X to the step function too so it can be weighted to produce the inputs
+        constants.append(x)
+        return constants
+
+    def call(self, x, mask=None):
+        # input shape: (nb_samples, time (padded with zeros), input_dim)
+        input_shape = self.input_spec[0].shape
+        if K._BACKEND == 'tensorflow':
+            if not input_shape[1]:
+                raise Exception('When using TensorFlow, you should define '
+                                'explicitly the number of timesteps of '
+                                'your sequences.\n'
+                                'If your first layer is an Embedding, '
+                                'make sure to pass it an "input_length" '
+                                'argument. Otherwise, make sure '
+                                'the first layer has '
+                                'an "input_shape" or "batch_input_shape" '
+                                'argument, including the time axis. '
+                                'Found input shape at layer ' + self.name +
+                                ': ' + str(input_shape))
+
+        if self.layer.stateful:
+            initial_states = self.layer.states
+        else:
+            initial_states = self.layer.get_initial_states(x)#+[K.ones((input_shape[0],input_shape[1]))]
+        constants = self.get_constants(x)
+        preprocessed_input = self.layer.preprocess_input(x)
+
+        last_output, outputs, states = K.rnn(self.step, preprocessed_input,
+                                             initial_states,
+                                             go_backwards=self.layer.go_backwards,
+                                             mask=mask,
+                                             constants=constants,
+                                             unroll=self.layer.unroll,
+                                             input_length=input_shape[1])
+        if self.layer.stateful:
+            self.updates = []
+            for i in range(len(states)):
+                self.updates.append((self.layer.states[i], states[i]))
+
+        if self.layer.return_sequences:
+            return outputs
+        else:
+            return last_output

requirements.txt

@@ -0,0 +1,9 @@
+keras==1.2.2
+tensorflow-gpu==1.0.0
+
+# data processing
+leven==1.0.4
+https://github.com/timmahrt/pysle/archive/e47ab5630679451b719787f5579968756bd7a644.zip
+https://github.com/datalogai/recurrentshop/archive/6f709f1aabd5156b184a06852b8edb9ceee5bb21.zip
+https://github.com/farizrahman4u/seq2seq/archive/1b1ae455fcebd55b16eb6e1c77a58aabfe85892a.zip
+keras-tqdm==2.0.1