mirror of
https://github.com/wassname/relation-network.git
synced 2026-08-21 11:21:13 +08:00
138 lines
3.5 KiB
Python
138 lines
3.5 KiB
Python
import numpy as np
|
|
import keras
|
|
from keras.models import Sequential, Model
|
|
from keras.layers import Dense, Dropout, Activation, Flatten, Input, Embedding,\
|
|
LSTM, Bidirectional, Lambda, Concatenate, Add
|
|
from keras.layers.convolutional import Conv2D, MaxPooling2D, AveragePooling2D
|
|
from keras.layers.normalization import BatchNormalization
|
|
from keras.optimizers import Adam
|
|
import gc
|
|
import prepare
|
|
import subprocess
|
|
|
|
mxlen = 32
|
|
embedding_dim = 50
|
|
lstm_unit = 128
|
|
MLP_unit = 128
|
|
epochs = 100
|
|
|
|
train_json = 'nlvr\\train\\train.json'
|
|
train_img_folder = 'nlvr\\train\\images'
|
|
data = prepare.load_data(train_json)
|
|
data = prepare.tokenize_data(data, mxlen)
|
|
imgs, ws, labels = prepare.load_images(train_img_folder, data, debug=True)
|
|
data.clear()
|
|
imgs_mean = np.mean(imgs)
|
|
imgs_std = np.std(imgs - imgs_mean)
|
|
imgs = (imgs - imgs_mean) / imgs_std
|
|
|
|
epochs = 100
|
|
batch_size = 64
|
|
|
|
|
|
def bn_layer(x, conv_unit):
|
|
def f(inputs):
|
|
md = Conv2D(x, (conv_unit, conv_unit), padding='same')(inputs)
|
|
md = BatchNormalization()(md)
|
|
return Activation('relu')(md)
|
|
return f
|
|
|
|
|
|
def conv_net(inputs):
|
|
model = bn_layer(32, 3)(inputs)
|
|
model = MaxPooling2D((2, 2), 2)(model)
|
|
model = bn_layer(32, 3)(model)
|
|
model = MaxPooling2D((2, 2), 2)(model)
|
|
model = bn_layer(32, 3)(model)
|
|
model = MaxPooling2D((2, 2), 2)(model)
|
|
model = bn_layer(32, 3)(model)
|
|
model = MaxPooling2D((2, 2), 2)(model)
|
|
model = bn_layer(64, 3)(model)
|
|
return model
|
|
|
|
|
|
input1 = Input((50, 200, 3))
|
|
input2 = Input((mxlen,))
|
|
cnn_features = conv_net(input1)
|
|
embedding_layer = prepare.embedding_layer(prepare.tokenizer.word_index, prepare.get_embeddings_index(), mxlen)
|
|
embedding = embedding_layer(input2)
|
|
bi_lstm = Bidirectional(LSTM(lstm_unit, implementation=2, return_sequences=False))
|
|
lstm_encode = bi_lstm(embedding)
|
|
shapes = cnn_features.shape
|
|
w, h = shapes[1], shapes[2]
|
|
|
|
def slice_1(t):
|
|
return t[:, 0, :, :]
|
|
|
|
def slice_2(t):
|
|
return t[:, 1:, :, :]
|
|
|
|
def slice_3(t):
|
|
return t[:, 0, :]
|
|
|
|
def slice_4(t):
|
|
return t[:, 1:, :]
|
|
|
|
slice_layer1 = Lambda(slice_1)
|
|
slice_layer2 = Lambda(slice_2)
|
|
slice_layer3 = Lambda(slice_3)
|
|
slice_layer4 = Lambda(slice_4)
|
|
|
|
features = []
|
|
for k1 in range(w):
|
|
features1 = slice_layer1(cnn_features)
|
|
cnn_features = slice_layer2(cnn_features)
|
|
for k2 in range(h):
|
|
features2 = slice_layer3(features1)
|
|
features1 = slice_layer4(features1)
|
|
features.append(features2)
|
|
|
|
relations = []
|
|
concat = Concatenate()
|
|
for feature1 in features:
|
|
for feature2 in features:
|
|
relations.append(concat([feature1, feature2, lstm_encode]))
|
|
|
|
|
|
def get_dense(n):
|
|
r = []
|
|
for k in range(n):
|
|
r.append(Dense(MLP_unit, activation='relu'))
|
|
return r
|
|
|
|
|
|
def get_MLP(n, denses):
|
|
def g(x):
|
|
d = x
|
|
for k in range(n):
|
|
d = denses[k](d)
|
|
return d
|
|
return g
|
|
|
|
|
|
def dropout_dense(x):
|
|
y = Dense(MLP_unit)(x)
|
|
y = Dropout(0.5)(y)
|
|
y = Activation('relu')(y)
|
|
return y
|
|
|
|
g_MLP = get_MLP(4, get_dense(4))
|
|
f_MLP = get_MLP(2, get_dense(2))
|
|
|
|
mid_relations = []
|
|
for r in relations:
|
|
mid_relations.append(g_MLP(r))
|
|
combined_relation = Add()(mid_relations)
|
|
|
|
rn = dropout_dense(combined_relation)
|
|
rn = dropout_dense(rn)
|
|
pred = Dense(1, activation='sigmoid')(rn)
|
|
|
|
model = Model(inputs=[input1, input2], outputs=pred)
|
|
optimizer = Adam(lr=3e-5)
|
|
model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
|
|
model.fit([imgs, ws], labels, validation_split=0.1, epochs=epochs)
|
|
model.save('model')
|
|
gc.collect()
|
|
subprocess.Popen("rundll32.exe powrprof.dll,SetSuspendState 0,1,0")
|