init

prepare.py

@@ -0,0 +1,99 @@
+import json
+import numpy as np
+import os
+from PIL import Image
+from keras.layers import Embedding
+from keras.preprocessing.text import Tokenizer
+from keras.preprocessing.sequence import pad_sequences
+
+EMBEDDING_DIM = 50
+tokenizer = Tokenizer()
+
+
+def load_data(path):
+    f = open(path, 'r')
+    data = []
+    for l in f:
+        jn = json.loads(l)
+        s = jn['sentence']
+        idn = jn['identifier']
+        la = int(jn['label'] == 'true')
+        data.append([idn, s, la])
+    return data
+
+
+def tokenize_data(sdata, mxlen):
+    texts = [t[1] for t in sdata]
+    tokenizer.fit_on_texts(texts)
+    seqs = tokenizer.texts_to_sequences(texts)
+    seqs = pad_sequences(seqs, mxlen)
+    data = {}
+    for k in range(len(sdata)):
+        data[sdata[k][0]] = [seqs[k], sdata[k][2]]
+    return data
+
+
+def load_images(path, sdata, debug=False):
+    data = {}
+    cnt = 0
+    N = 1000
+    for lists in os.listdir(path):
+        p = os.path.join(path, lists)
+        for f in os.listdir(p):
+            cnt += 1
+            if debug and cnt > N:
+                break
+            im_path = os.path.join(p, f)
+            im = Image.open(im_path)
+            im = im.convert('RGB')
+            im = im.resize((200, 50))
+            im = np.array(im)
+            idf = f[f.find('-') + 1:f.rfind('-')]
+            data[f] = [im] + sdata[idf]
+    ims, ws, labels = [], [], []
+    for key in data:
+        ims.append(data[key][0])
+        ws.append(data[key][1])
+        labels.append(data[key][2])
+    data.clear()
+    idx = np.arange(0, len(ims), 1)
+    np.random.shuffle(idx)
+    ims = [ims[t] for t in idx]
+    ws = [ws[t] for t in idx]
+    labels = [labels[t] for t in idx]
+    ims = np.array(ims, dtype=np.float32)
+    ws = np.array(ws, dtype=np.float32)
+    labels = np.array(labels, dtype=np.float32)
+    return ims, ws, labels
+
+
+def get_embeddings_index():
+    embeddings_index = {}
+    path = r'C:\local\word2vec\glove.6B.50d.txt'
+    f = open(path, 'r', errors='ignore')
+    for line in f:
+        values = line.split()
+        word = values[0]
+        coefs = np.asarray(values[1:], dtype='float32')
+        embeddings_index[word] = coefs
+    f.close()
+    return embeddings_index
+
+
+def get_embedding_matrix(word_index, embeddings_index):
+    embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))
+    for word, i in word_index.items():
+        embedding_vector = embeddings_index.get(word)
+        if embedding_vector is not None:
+            # words not found in embedding index will be all-zeros.
+            embedding_matrix[i] = embedding_vector
+    return embedding_matrix
+
+
+def embedding_layer(word_index, embedding_index, sequence_len):
+    embedding_matrix = get_embedding_matrix(word_index, embedding_index)
+    return Embedding(len(word_index) + 1,
+                     EMBEDDING_DIM,
+                     weights=[embedding_matrix],
+                     input_length=sequence_len,
+                     trainable=False)

train.py

@@ -0,0 +1,118 @@
+import numpy as np
+import keras
+from keras.models import Sequential, Model
+from keras.layers import Dense, Dropout, Activation, Flatten, Input, Embedding,\
+    LSTM, Bidirectional, Lambda, Concatenate, Add
+from keras.layers.convolutional import Conv2D, MaxPooling2D, AveragePooling2D
+from keras.layers.normalization import BatchNormalization
+from keras.optimizers import Adam
+import gc
+import prepare
+import subprocess
+
+mxlen = 32
+embedding_dim = 50
+lstm_unit = 128
+MLP_unit = 128
+epochs = 100
+
+train_json = 'nlvr\\train\\train.json'
+train_img_folder = 'nlvr\\train\\images'
+data = prepare.load_data(train_json)
+data = prepare.tokenize_data(data, mxlen)
+imgs, ws, labels = prepare.load_images(train_img_folder, data, debug=True)
+data.clear()
+imgs_mean = np.mean(imgs)
+imgs_std = np.std(imgs - imgs_mean)
+imgs = (imgs - imgs_mean) / imgs_std
+
+epochs = 100
+batch_size = 64
+
+
+def bn_layer(x, conv_unit):
+    def f(inputs):
+        md = Conv2D(x, (conv_unit, conv_unit), padding='same')(inputs)
+        md = BatchNormalization()(md)
+        return Activation('relu')(md)
+    return f
+
+
+def conv_net(inputs):
+    model = bn_layer(32, 3)(inputs)
+    model = MaxPooling2D((2, 2), 2)(model)
+    model = bn_layer(32, 3)(model)
+    model = MaxPooling2D((2, 2), 2)(model)
+    model = bn_layer(32, 3)(model)
+    model = MaxPooling2D((2, 2), 2)(model)
+    model = bn_layer(32, 3)(model)
+    model = MaxPooling2D((2, 2), 2)(model)
+    model = bn_layer(64, 3)(model)
+    return model
+
+
+input1 = Input((50, 200, 3))
+input2 = Input((mxlen,))
+cnn_features = conv_net(input1)
+embedding_layer = prepare.embedding_layer(prepare.tokenizer.word_index, prepare.get_embeddings_index(), mxlen)
+embedding = embedding_layer(input2)
+bi_lstm = Bidirectional(LSTM(lstm_unit, implementation=2, return_sequences=False))
+lstm_encode = bi_lstm(embedding)
+shapes = cnn_features.shape
+w, h = shapes[1], shapes[2]
+features = []
+for k1 in range(w):
+    for k2 in range(h):
+        def get_feature(t):
+            return t[:, k1, k2, :]
+        get_feature_layer = Lambda(get_feature)
+        features.append(get_feature_layer(cnn_features))
+
+relations = []
+concat = Concatenate()
+for feature1 in features:
+    for feature2 in features:
+        relations.append(concat([feature1, feature2, lstm_encode]))
+
+
+def get_dense(n):
+    r = []
+    for k in range(n):
+        r.append(Dense(MLP_unit, activation='relu'))
+    return r
+
+
+def get_MLP(n, denses):
+    def g(x):
+        d = x
+        for k in range(n):
+            d = denses[k](d)
+        return d
+    return g
+
+
+def dropout_dense(x):
+    y = Dense(MLP_unit)(x)
+    y = Dropout(0.5)(y)
+    y = Activation('relu')(y)
+    return y
+
+g_MLP = get_MLP(4, get_dense(4))
+f_MLP = get_MLP(2, get_dense(2))
+
+mid_relations = []
+for r in relations:
+    mid_relations.append(g_MLP(r))
+combined_relation = Add()(mid_relations)
+
+rn = dropout_dense(combined_relation)
+rn = dropout_dense(rn)
+pred = Dense(1, activation='sigmoid')(rn)
+
+model = Model(inputs=[input1, input2], outputs=pred)
+optimizer = Adam(lr=3e-5)
+model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
+model.fit([imgs, ws], labels, validation_split=0.1, epochs=epochs)
+model.save('model')
+gc.collect()
+subprocess.Popen("rundll32.exe powrprof.dll,SetSuspendState 0,1,0")