From 7d0a78d1a79ff982291e8b81b1be285961216459 Mon Sep 17 00:00:00 2001 From: Peng Shi Date: Tue, 25 Apr 2017 03:53:36 +0800 Subject: [PATCH] Kim cnn (#21) Reimplement Kim's sentence classification model #12 --- kim_cnn/README.md | 130 ++++++++++++++++++ kim_cnn/bucket.py | 87 ++++++++++++ kim_cnn/config/sst-1.cfg | 34 +++++ kim_cnn/config/sst-2.cfg | 34 +++++ kim_cnn/config/trec.cfg | 34 +++++ kim_cnn/configurable.py | 166 +++++++++++++++++++++++ kim_cnn/dataset.py | 145 ++++++++++++++++++++ kim_cnn/etc/__init__.py | 2 + kim_cnn/etc/kmeans.py | 180 +++++++++++++++++++++++++ kim_cnn/etc/utils.py | 30 +++++ kim_cnn/example.py | 42 ++++++ kim_cnn/getData.sh | 8 ++ kim_cnn/main.py | 81 ++++++++++++ kim_cnn/model/__init__.py | 1 + kim_cnn/model/cnnText/__init__.py | 1 + kim_cnn/model/cnnText/cnntext.py | 87 ++++++++++++ kim_cnn/network/__init__.py | 1 + kim_cnn/network/cnnTextNetwork.py | 211 ++++++++++++++++++++++++++++++ kim_cnn/vocab.py | 165 +++++++++++++++++++++++ 19 files changed, 1439 insertions(+) create mode 100644 kim_cnn/README.md create mode 100644 kim_cnn/bucket.py create mode 100644 kim_cnn/config/sst-1.cfg create mode 100644 kim_cnn/config/sst-2.cfg create mode 100644 kim_cnn/config/trec.cfg create mode 100644 kim_cnn/configurable.py create mode 100644 kim_cnn/dataset.py create mode 100644 kim_cnn/etc/__init__.py create mode 100644 kim_cnn/etc/kmeans.py create mode 100644 kim_cnn/etc/utils.py create mode 100644 kim_cnn/example.py create mode 100644 kim_cnn/getData.sh create mode 100644 kim_cnn/main.py create mode 100644 kim_cnn/model/__init__.py create mode 100644 kim_cnn/model/cnnText/__init__.py create mode 100644 kim_cnn/model/cnnText/cnntext.py create mode 100644 kim_cnn/network/__init__.py create mode 100644 kim_cnn/network/cnnTextNetwork.py create mode 100644 kim_cnn/vocab.py diff --git a/kim_cnn/README.md b/kim_cnn/README.md new file mode 100644 index 0000000..1db8d06 --- /dev/null +++ b/kim_cnn/README.md @@ -0,0 +1,130 @@ +# text-classification-cnn +Implementation for Convolutional Neural Networks for Sentence Classification of [Kim (2014)](https://arxiv.org/abs/1408.5882) with PyTorch. + +## Project Structure + +``` +text-classification-cnn + ├── config + │ └── classification.cfg + ├── etc + │ ├── kmeans.py + │ └── utils.py + ├── model + │ └── cnnText + │ └── cnntext.py + ├── network + │ └── cnnTextNetwork.py + ├── data + ├── saves + ├── README.md + ├── getData.sh + ├── bucket.py + ├── configurable.py + ├── dataset.py + ├── example.py + ├── vocab.py + └── main.py + +``` + +## Model Type + +- rand: All words are randomly initialized and then modified during training. +- static: A model with pre-trained vectors from [word2vec](https://code.google.com/archive/p/word2vec/). All words -- including the unknown ones that are randomly initialized -- are kept static and only the other parameters of the model are learned. +- non-static: Same as above but the pretrained vectors are fine-tuned for each task. +- multichannel: A model with two sets of word vectors. Each set of vectors is treated as a 'channel' and each filter is applied to both channels, but gradients are back-propagated only through one of the channels. Hence the model is able to fine-tune one set of vectors while keeping the other static. Both channels are initialized with word2vec. + + + +## Quick Start + +Run + +``` +bash getData.sh +``` + +to get dataset. + +To run the model on [TREC](http://cogcomp.cs.illinois.edu/Data/QA/QC/) dataset on [rand](Model Type), just run the following code. + +``` +python main.py --config_file config/trec.cfg --model_type CNNText --train +``` + +The file will be saved in + +``` +saves/model_file +``` +You can modify these parameters under these [instructions](Configurable File) + +To test the model, you can use the following command. + +``` +python main.py --config_file config/trec.cfg --model_type CNNText --test --restore_from saves/trec/model_file +``` + +You need to specify the config file and the path to model file here. Note: The path need to be the same as the declaration in the config file. + +## Configurable File + +- model_type: **CNNText** in this case. +- mode: **rand**, **static**, **non-static** and **multichannel** which are specified [here](Model Type) +- save_dir: the path you want to save the model parameters +- word_file: all words appearing in the dataset +- target_file: all labels appearing in the dataset +- data_dir: the path of dataset +- train_file: the name of the training file +- valid_file: the name of the validation file +- test_file: the name of the test file +- save_model_file: the name you want to use for the model parameters +- restore_from: this option will be used we you want to restore file for validation and testing +- embed_file: embedding file +- use_gpu: use GPU or not +- words_dim: the dimension of the words. This is same with pre-trained word embedding. +- n_bkts: the bucket number for training dataset. Group the sentence according to the lengths +- n_valid_bkts: the bucket number for testing dataset. +- dataset_type: the dataset you used. **TREC**, **SST-1** and **SST-2** in this case. +- min_occur_count: set the word as *UNK* according to its frequency. +- learning_rate: leanring rate +- epoch_decay: decay the learning rate 0.75 for every *epoch_dacay* epoch +- valid_interval: validate for every *valid_interval* iterations +- train_batch_size: the token number for each batch in training +- test_batch_size: the token number for each batch in testing + + +## Dataset and Embeddings + +We experiment the model on the following three datasets. + +- TREC: We use the 1-5000 for training, 5001-5452 for validation, and original test dataset for testing. +- SST-1: Keep the original splits and train with phrase level dataset and test on sentence level dataset. +- SST-2: Same as above. + +Furthermore, we filter the word embeddings to fit specific dataset. These file can be found in dir *data*. + +For self-defined data, please keep the format as + +``` +label1 sentence +label2 sentence +``` + +And you can use [this](http://ocp59jkku.bkt.clouddn.com/filterVec.py) script to filter the pre-trained word embeddings. + +Or you can modify the *reading_dataset* in *dataset.py*, *add_train_file* in *vocab.py* and *example.py* to fit your own dataset. + + +## Results + + +|dataset|rand|static|non-static|multichannel| +|---|---|---|---|---| +|TREC|91.98|90.32|92.62|93.36| +|SST-1|42.59|46.33|44.32|47.32| +|SST-2|82.20|86.42|85.43|84.39| + +We do not tune the parameters for each dataset. And the implementation is simplified from the original version on regularization. + diff --git a/kim_cnn/bucket.py b/kim_cnn/bucket.py new file mode 100644 index 0000000..9587a86 --- /dev/null +++ b/kim_cnn/bucket.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +from configurable import Configurable +import numpy as np + + +class Bucket(Configurable): + """ + """ + def __init__(self, *args, **kwargs): + """ + + :param args: + :param kwargs: + """ + super(Bucket, self).__init__(*args, **kwargs) + self._size = None + self._data = None + self._sents = None + self._target = None + + def set_size(self, size): + self._size = size + self._data = [] + self._sents = [] + self._target = [] + + def add(self, example): + # TODO: After finalize, we can not add data anymore + if example.length > self._size: #and self._size != -1: + # TODO: we may support size = -1 in the future + raise ValueError("Bucket of size %d received sequence of len %d" % (self._size, example.length)) + self._data.append(example.data['words']) + self._sents.append(example.sent['words']) + self._target.append(example.data['targets']) + return len(self._data)-1 + + def finalize(self): + if self._data is None: + raise ValueError("You need to set size before finalize it") + if len(self._data) > 0: + shape = (len(self._data), self._size, len(self._data[-1][-1])) + data = np.zeros(shape, dtype=np.int64) + for i, datum in enumerate(self._data): + try: + datum = np.array(datum) + data[i,0:len(datum)] = datum + except: + print("sentence %d has Error with data :"%(i+1)) + print(datum) + exit() + self._data = data + self._sents = np.array(self._sents) + self._target = np.array(self._target) + + + else: + print("Finalize Error in bucket") + exit() + print("Bucket %s is %d x %d" % ((self._name,) + self._data.shape[0:2])) + + + def __len__(self): + return len(self._data) + + @property + def size(self): + return self._size + @property + def data(self): + return self._data + @property + def sents(self): + return self._sents + @property + def target(self): + return self._target + + + + + + + + diff --git a/kim_cnn/config/sst-1.cfg b/kim_cnn/config/sst-1.cfg new file mode 100644 index 0000000..95fbdfd --- /dev/null +++ b/kim_cnn/config/sst-1.cfg @@ -0,0 +1,34 @@ +[OS] +model_type = CNNText +mode = multichannel +save_dir = saves/sst-1 +word_file = %(save_dir)s/words.txt +target_file = %(save_dir)s/targets.txt +data_dir = data +train_file = %(data_dir)s/stsa.fine.phrases.train +valid_file = %(data_dir)s/stsa.fine.dev +test_file = %(data_dir)s/stsa.fine.test +save_model_file = %(save_dir)s/model_file +restore_from = %(save_dir)s/model_file +embed_file = %(data_dir)s/word2vec.sst-1 +use_gpu = False + +[Sizes] +words_dim = 300 + +[Dataset] +n_bkts = 10 +n_valid_bkts = 3 +dataset_type = SST-1 +min_occur_count = 2 + +[Learning rate] +learning_rate = 1e-3 +epoch_decay = 30 +dropout = 0.5 + +[Training] +log_interval = 10 +valid_interval = 100 +train_batch_size = 2000 +test_batch_size = 2000 \ No newline at end of file diff --git a/kim_cnn/config/sst-2.cfg b/kim_cnn/config/sst-2.cfg new file mode 100644 index 0000000..78663a1 --- /dev/null +++ b/kim_cnn/config/sst-2.cfg @@ -0,0 +1,34 @@ +[OS] +model_type = CNNText +mode = multichannel +save_dir = saves/sst-2 +word_file = %(save_dir)s/words.txt +target_file = %(save_dir)s/targets.txt +data_dir = data +train_file = %(data_dir)s/stsa.binary.phrases.train +valid_file = %(data_dir)s/stsa.binary.dev +test_file = %(data_dir)s/stsa.binary.test +save_model_file = %(save_dir)s/model_file +restore_from = %(save_dir)s/model_file +embed_file = %(data_dir)s/word2vec.sst-2 +use_gpu = False + +[Sizes] +words_dim = 300 + +[Dataset] +n_bkts = 10 +n_valid_bkts = 3 +dataset_type = SST-2 +min_occur_count = 2 + +[Learning rate] +learning_rate = 2e-3 +epoch_decay = 30 +dropout = 0.5 + +[Training] +log_interval = 10 +valid_interval = 100 +train_batch_size = 2000 +test_batch_size = 2000 \ No newline at end of file diff --git a/kim_cnn/config/trec.cfg b/kim_cnn/config/trec.cfg new file mode 100644 index 0000000..c0758a2 --- /dev/null +++ b/kim_cnn/config/trec.cfg @@ -0,0 +1,34 @@ +[OS] +model_type = CNNText +mode = rand +save_dir = saves/trec +word_file = %(save_dir)s/words.txt +target_file = %(save_dir)s/targets.txt +data_dir = data +train_file = %(data_dir)s/train.trec +valid_file = %(data_dir)s/validate.trec +test_file = %(data_dir)s/test.trec +save_model_file = %(save_dir)s/model_file +restore_from = %(save_dir)s/model_file +embed_file = %(data_dir)s/word2vec.trec +use_gpu = False + +[Sizes] +words_dim = 300 + +[Dataset] +n_bkts = 10 +n_valid_bkts = 3 +dataset_type = TREC +min_occur_count = 2 + +[Learning rate] +learning_rate = 2e-3 +epoch_decay = 30 +dropout = 0.5 + +[Training] +log_interval = 10 +valid_interval = 100 +train_batch_size = 2000 +test_batch_size = 2000 \ No newline at end of file diff --git a/kim_cnn/configurable.py b/kim_cnn/configurable.py new file mode 100644 index 0000000..aa1fe07 --- /dev/null +++ b/kim_cnn/configurable.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import argparse + +from ConfigParser import SafeConfigParser + + +class Configurable(object): + """ + Configuration processing for the network + """ + def __init__(self, *args, **kwargs): + self._name = kwargs.pop("name", "Unknown") + if args and kwargs: + raise TypeError('Configurable must take either a config parser or keyword args') + if len(args) > 1: + raise TypeError('Configurable takes at most one argument') + if args: + self._config = args[0] + else: + self._config = self._configure(**kwargs) + return + + @property + def name(self): + return self._name + + + def _configure(self, **kwargs): + config = SafeConfigParser() + config_file = kwargs.pop("config_file", "") + config.read(config_file) + # Override the config setting if the (k,v) specified in command line + for option, value in kwargs.iteritems(): + assigned = False + for section in config.sections(): + if option in config.options(section): + config.set(section, option, str(value)) + assigned = True + break + if not assigned: + raise ValueError("%s is not a valid option" % option) + return config + + argparser = argparse.ArgumentParser() + argparser.add_argument('--config_file') + + # ====== + # [OS] + @property + def model_type(self): + return self._config.get('OS', 'model_type') + argparser.add_argument('--model_type') + @property + def mode(self): + return self._config.get('OS', 'mode') + argparser.add_argument('--mode') + @property + def save_dir(self): + return self._config.get('OS', 'save_dir') + argparser.add_argument('--save_dir') + @property + def word_file(self): + return self._config.get('OS', 'word_file') + argparser.add_argument('--word_file') + @property + def target_file(self): + return self._config.get('OS', 'target_file') + argparser.add_argument('--target_file') + @property + def train_file(self): + return self._config.get('OS', 'train_file') + argparser.add_argument('--train_file') + @property + def valid_file(self): + return self._config.get('OS', 'valid_file') + argparser.add_argument('--valid_file') + @property + def test_file(self): + return self._config.get('OS', 'test_file') + argparser.add_argument('--test_file') + @property + def save_model_file(self): + return self._config.get('OS', 'save_model_file') + argparser.add_argument('--save_model_file') + @property + def restore_from(self): + return self._config.get('OS', 'restore_from') + argparser.add_argument('--restore_from') + @property + def embed_file(self): + return self._config.get('OS', 'embed_file') + argparser.add_argument('--embed_file') + @property + def use_gpu(self): + return self._config.getboolean('OS', 'use_gpu') + argparser.add_argument('--use_gpu') + + + # [Dataset] + @property + def n_bkts(self): + return self._config.getint('Dataset', 'n_bkts') + argparser.add_argument('--n_bkts') + @property + def n_valid_bkts(self): + return self._config.getint('Dataset', 'n_valid_bkts') + argparser.add_argument('--n_valid_bkts') + @property + def dataset_type(self): + return self._config.get('Dataset', 'dataset_type') + argparser.add_argument('--dataset_type') + @property + def min_occur_count(self): + return self._config.getint('Dataset', 'min_occur_count') + argparser.add_argument('--min_occur_count') + + + + # [Learning rate] + @property + def learning_rate(self): + return self._config.getfloat('Learning rate', 'learning_rate') + argparser.add_argument('--learning_rate') + @property + def epoch_decay(self): + return self._config.getint('Learning rate', 'epoch_decay') + argparser.add_argument('--epoch_decay') + @property + def dropout(self): + return self._config.getfloat('Learning rate', 'dropout') + argparser.add_argument('--dropout') + + # [Sizes] + @property + def words_dim(self): + return self._config.getint('Sizes', 'words_dim') + argparser.add_argument('--words_dim') + + + # [Training] + @property + def log_interval(self): + return self._config.getint('Training', 'log_interval') + argparser.add_argument('--log_interval') + @property + def valid_interval(self): + return self._config.getint('Training', 'valid_interval') + argparser.add_argument('--valid_interval') + @property + def train_batch_size(self): + return self._config.getint('Training', 'train_batch_size') + argparser.add_argument('--train_batch_size') + @property + def test_batch_size(self): + return self._config.getint('Training', 'test_batch_size') + argparser.add_argument('--test_batch_size') + + + diff --git a/kim_cnn/dataset.py b/kim_cnn/dataset.py new file mode 100644 index 0000000..ea7c561 --- /dev/null +++ b/kim_cnn/dataset.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from configurable import Configurable +from bucket import Bucket +from example import Example +from collections import Counter +from etc.kmeans import KMeans +import numpy as np +from etc.utils import clean_str, clean_str_sst + +class Dataset(Configurable): + """ + Dataset Class: + - Store Data + - Generate Minibatch + - Padding + """ + def __init__(self, filename, vocabs, *args, **kwargs): + + super(Dataset, self).__init__(*args, **kwargs) + self._train = (filename == self.train_file) + self.vocabs = vocabs + self.buckets = [Bucket(self._config, name='Sents-%d' % i) for i in xrange(self.n_bkts)] + self.id2position = [] + self.len2bkts = {} + self.vocabs = vocabs + self.reading_dataset(filename) + self._finalize() + + def _finalize(self): + for bucket in self.buckets: + bucket.finalize() + + + + + @property + def n_bkts(self): + if self._train: + return super(Dataset, self).n_bkts + else: + return super(Dataset, self).n_valid_bkts + + + def reading_dataset(self, filename): + """ + :param filename: + :return: + """ + if self.dataset_type == 'SST-1' or self.dataset_type == 'SST-2': + with open(filename) as f: + buff = [] + for line_num, line in enumerate(f): + line = clean_str_sst(line).split() + if len(line) > 1: + buff.append(line) + self._process_buff(buff) + else: + with open(filename) as f: + buff = [] + for line_num, line in enumerate(f): + line = clean_str(line).split() + if line: + buff.append(line) + self._process_buff(buff) + return + + def _process_buff(self, buff): + """ + :param buff: + :return: + """ + len_cntr = Counter() + for sent in buff: + len_cntr[len(sent)] += 1 + bkts_splits = KMeans(self.n_bkts, len_cntr).splits + # Count the sents length + # Use k-means to splits the sents into n_bkts parts + + # reset bucket size + # map the lenth to bkts id + prev_size = -1 + for bkt_idx, size in enumerate(bkts_splits): + self.buckets[bkt_idx].set_size(size) + self.len2bkts.update(zip(range(prev_size+1, size+1), [bkt_idx] * (size-prev_size))) + prev_size = size + # map all length from min to max to bkts id + # some of lengths do not appear in the data set + for sent in buff: + # Add the sent to the specific bucket according to their length + # Construct the sent into example first + # And then push them into buckets + bkt_idx = self.len2bkts[len(sent)] + example = Example(sent, self._config) + example.convert(self.vocabs) + # save to bucket + idx = self.buckets[bkt_idx].add(example) + self.id2position.append((bkt_idx, idx)) + + + + + + + + def minibatch(self, batch_size, input_idx, target_idx, shuffle=True): + minibatches = [] + for bkt_idx, bucket in enumerate(self.buckets): + if batch_size == 0: + print("Please Specify the batch size") + exit() + else: + n_tokens = len(bucket) * bucket.size + n_splits = max(n_tokens // batch_size, 1) + + if shuffle: + range_func = np.random.permutation + else: + range_func = np.arange + arr_sp = np.array_split(range_func(len(bucket)), n_splits) + for bkt_mb in arr_sp: + minibatches.append((bkt_idx, bkt_mb)) + if shuffle: + np.random.shuffle(minibatches) + + for bkt_idx, bkt_mb in minibatches: + data = self.buckets[bkt_idx].data[bkt_mb] + sents = self.buckets[bkt_idx].sents[bkt_mb] + target = self.buckets[bkt_idx].target[bkt_mb] + maxlen = np.max(np.sum(np.greater(data[:,:,0], 0), axis=1)) + # Do not use dynamic index like conll_index + # For word, set 0 data = [(fea1, fea2, fea3), (fea1, fea2, fea3), ...] + # For target, target = [(target1,), (target2,), ...] + feed_dict = { + 'text' : data[:,:maxlen, input_idx], + 'label' : target[:, target_idx], + 'batch_size' : len(target) + } + yield feed_dict + + @property + def sentsNum(self): + return len(self.id2position) + diff --git a/kim_cnn/etc/__init__.py b/kim_cnn/etc/__init__.py new file mode 100644 index 0000000..3f0eef9 --- /dev/null +++ b/kim_cnn/etc/__init__.py @@ -0,0 +1,2 @@ +from kmeans import KMeans +from utils import clean_str, clean_str_sst \ No newline at end of file diff --git a/kim_cnn/etc/kmeans.py b/kim_cnn/etc/kmeans.py new file mode 100644 index 0000000..4624859 --- /dev/null +++ b/kim_cnn/etc/kmeans.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +""" +Adapted from Bi-Affine Parser code +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from collections import Counter + +import numpy as np + + +# *************************************************************** +class KMeans(object): + """""" + + # ============================================================= + def __init__(self, k, len_cntr): + """""" + + # Error checking + if len(len_cntr) < k: + raise ValueError('Trying to sort %d data points into %d buckets' % (len(len_cntr), k)) + + # Initialize variables + self._k = k + self._len_cntr = len_cntr + self._lengths = sorted(self._len_cntr.keys()) + self._splits = [] + self._split2len_idx = {} + self._len2split_idx = {} + self._split_cntr = Counter() + + # Initialize the splits evenly + lengths = [] + for length, count in self._len_cntr.items(): + lengths.extend([length] * count) + lengths.sort() + self._splits = [np.max(split) for split in np.array_split(lengths, self._k)] + + i = len(self._splits) - 1 + while i > 0: + while self._splits[i - 1] >= self._splits[i] or self._splits[i - 1] not in self._len_cntr: + self._splits[i - 1] -= 1 + i -= 1 + + i = 1 + while i < len(self._splits) - 1: + while self._splits[i] <= self._splits[i - 1] or self._splits[i] not in self._len_cntr: + self._splits[i] += 1 + i += 1 + + # Reindex everything + split_idx = 0 + split = self._splits[split_idx] + for len_idx, length in enumerate(self._lengths): + count = self._len_cntr[length] + self._split_cntr[split] += count + if length == split: + self._split2len_idx[split] = len_idx + split_idx += 1 + if split_idx < len(self._splits): + split = self._splits[split_idx] + self._split_cntr[split] = 0 + elif length > split: + raise IndexError() + + # Iterate + old_splits = None + # print('0) Initial splits: %s; Initial mass: %d' % (self._splits, self.get_mass())) + i = 0 + while self._splits != old_splits: + old_splits = list(self._splits) + self.recenter() + i += 1 + # print('%d) Final splits: %s; Final mass: %d' % (i, self._splits, self.get_mass())) + + self.reindex() + return + + # ============================================================= + def recenter(self): + """""" + + for split_idx in xrange(len(self._splits)): + split = self._splits[split_idx] + len_idx = self._split2len_idx[split] + if split == self._splits[-1]: + continue + right_split = self._splits[split_idx + 1] + + # Try shifting the centroid to the left + if len_idx > 0 and self._lengths[len_idx - 1] not in self._split_cntr: + new_split = self._lengths[len_idx - 1] + left_delta = self._len_cntr[split] * (right_split - new_split) - self._split_cntr[split] * (split - new_split) + if left_delta < 0: + self._splits[split_idx] = new_split + self._split2len_idx[new_split] = len_idx - 1 + del self._split2len_idx[split] + self._split_cntr[split] -= self._len_cntr[split] + self._split_cntr[right_split] += self._len_cntr[split] + self._split_cntr[new_split] = self._split_cntr[split] + del self._split_cntr[split] + + # Try shifting the centroid to the right + elif len_idx < len(self._lengths) - 2 and self._lengths[len_idx + 1] not in self._split_cntr: + new_split = self._lengths[len_idx + 1] + right_delta = self._split_cntr[split] * (new_split - split) - self._len_cntr[split] * (new_split - split) + if right_delta <= 0: + self._splits[split_idx] = new_split + self._split2len_idx[new_split] = len_idx + 1 + del self._split2len_idx[split] + self._split_cntr[split] += self._len_cntr[split] + self._split_cntr[right_split] -= self._len_cntr[split] + self._split_cntr[new_split] = self._split_cntr[split] + del self._split_cntr[split] + return + + # ============================================================= + + def get_mass(self): + """""" + + mass = 0 + split_idx = 0 + split = self._splits[split_idx] + for len_idx, length in enumerate(self._lengths): + count = self._len_cntr[length] + mass += split * count + if length == split: + split_idx += 1 + if split_idx < len(self._splits): + split = self._splits[split_idx] + return mass + + # ============================================================= + def reindex(self): + """""" + + self._len2split_idx = {} + last_split = -1 + for split_idx, split in enumerate(self._splits): + self._len2split_idx.update(dict(zip(range(last_split + 1, split), [split_idx] * (split - (last_split + 1))))) + return + + # ============================================================= + + def __len__(self): + return self._k + + def __iter__(self): + return (split for split in self.splits) + + def __getitem__(self, key): + return self._splits[key] + + # ============================================================= + @property + def splits(self): + return self._splits + + @property + def len2split_idx(self): + return self._len2split_idx + + +# *************************************************************** +if __name__ == '__main__': + """""" + + len_cntr = Counter() + for i in xrange(10000): + len_cntr[1 + int(10 ** (1 + np.random.randn()))] += 1 + print(len_cntr) + kmeans = KMeans(10, len_cntr) + print(kmeans.splits) \ No newline at end of file diff --git a/kim_cnn/etc/utils.py b/kim_cnn/etc/utils.py new file mode 100644 index 0000000..bbcf4b9 --- /dev/null +++ b/kim_cnn/etc/utils.py @@ -0,0 +1,30 @@ +import re + + +def clean_str(string): + """ + Tokenization/string cleaning for all datasets except for SST. + """ + string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string) + string = re.sub(r"\'s", " \'s", string) + string = re.sub(r"\'ve", " \'ve", string) + string = re.sub(r"n\'t", " n\'t", string) + string = re.sub(r"\'re", " \'re", string) + string = re.sub(r"\'d", " \'d", string) + string = re.sub(r"\'ll", " \'ll", string) + string = re.sub(r",", " , ", string) + string = re.sub(r"!", " ! ", string) + string = re.sub(r"\(", " ( ", string) + string = re.sub(r"\)", " ) ", string) + string = re.sub(r"\?", " ? ", string) + string = re.sub(r"\s{2,}", " ", string) + return string.strip() + + +def clean_str_sst(string): + """ + Tokenization/string cleaning for the SST dataset + """ + string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string) + string = re.sub(r"\s{2,}", " ", string) + return string.strip() \ No newline at end of file diff --git a/kim_cnn/example.py b/kim_cnn/example.py new file mode 100644 index 0000000..3e640f1 --- /dev/null +++ b/kim_cnn/example.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + + +from configurable import Configurable +class Example(Configurable): + """ + + """ + def __init__(self, sent, *args, **kwargs): + super(Example, self).__init__(*args, **kwargs) + self.length = len(sent) + self.sent = None + self.data = None + # original word in this setting "TREC" + # TODO: for different dataset, the original data will have different format + # TODO: this data format is related to output. Leave for future work + # self.feature = None + # # Convert each of the features to one-hot representation: (n_word, n_feature) + # self.target = None + # # Convert each of the targets to one-hot representation: (n_word, n_target) + if self.dataset_type == "TREC": + self.data = {} + self.sent = {} + self.sent["words"] = sent[2:] + self.sent["targets"] = sent[0] + else: + self.data = {} + self.sent = {} + self.sent["words"] = sent[1:] + self.sent["targets"] = sent[0] + + def convert(self, vocabs): + words, target = vocabs + self.data["words"] = [] + self.data["targets"] = target[self.sent["targets"]] + for word in self.sent["words"]: + self.data["words"].append(words[word]) + + + diff --git a/kim_cnn/getData.sh b/kim_cnn/getData.sh new file mode 100644 index 0000000..73e9a89 --- /dev/null +++ b/kim_cnn/getData.sh @@ -0,0 +1,8 @@ +mkdir data +mkdir saves +wget http://ocp59jkku.bkt.clouddn.com/sst-1.zip -P data/ +wget http://ocp59jkku.bkt.clouddn.com/sst-2.zip -P data/ +wget http://ocp59jkku.bkt.clouddn.com/trec.zip -P data/ +unzip data/sst-1.zip -d data/ +unzip data/sst-2.zip -d data/ +unzip data/trec.zip -d data/ diff --git a/kim_cnn/main.py b/kim_cnn/main.py new file mode 100644 index 0000000..357596b --- /dev/null +++ b/kim_cnn/main.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import model +from network import cnnTextNetwork +from configurable import Configurable +import torch +import numpy as np +import os + +if __name__=='__main__': + + import argparse + torch.manual_seed(3435) + np.random.seed(3435) + if torch.cuda.is_available(): + torch.cuda.manual_seed(3435) + + + argparser = argparse.ArgumentParser() + argparser.add_argument('--train', action='store_true') + argparser.add_argument('--validate', action='store_true') + argparser.add_argument('--test', action='store_true') + argparser.add_argument('--load', action='store_true') + + args, extra_args = argparser.parse_known_args() + # args.train = True/False ... + # extra_args['--some': "xxxx"] + cargs = {k: v for (k, v) in vars(Configurable.argparser.parse_args(extra_args)).iteritems() if v is not None} + + if 'model_type' not in cargs: + print("You need to specify the model_type") + exit() + print('*** '+cargs['model_type']+" ***") + + if args.load and 'restore_from' in cargs: + print("Loading model from [%s]..." % (cargs['restore_from'])) + try: + m = torch.load(cargs['restore_from']) + cargs.pop(cargs['restore_from'], "") + except: + print("The model doesn't exist") + exit() + elif args.validate or args.test: + print("Loading model from [%s]..." % (cargs['restore_from'])) + try: + m = torch.load(cargs['restore_from']) + cargs.pop(cargs['restore_from'], "") + except: + print("The model doesn't exist") + exit() + else: + m = getattr(model, cargs['model_type']) + + + network = None + + if cargs['model_type'] == "CNNText": + cargs.pop("model_type", "") + network = cnnTextNetwork(args, m, **cargs) + else: + print("The model type is not supported") + exit() + + if not os.path.exists(network.save_dir): + os.mkdir(network.save_dir) + + + # if torch.cuda.is_available(): + # torch.cuda.manual_seed_all(1) + + + + if args.train: + network.train() + elif args.validate: + print("### The accuracy for validate set: ") + print(network.test(validate=True)) + elif args.test: + print("### The accuracy for test set: ") + print(network.test(validate=False)) \ No newline at end of file diff --git a/kim_cnn/model/__init__.py b/kim_cnn/model/__init__.py new file mode 100644 index 0000000..29b8b5e --- /dev/null +++ b/kim_cnn/model/__init__.py @@ -0,0 +1 @@ +from cnnText import * \ No newline at end of file diff --git a/kim_cnn/model/cnnText/__init__.py b/kim_cnn/model/cnnText/__init__.py new file mode 100644 index 0000000..accb687 --- /dev/null +++ b/kim_cnn/model/cnnText/__init__.py @@ -0,0 +1 @@ +from cnntext import CNNText \ No newline at end of file diff --git a/kim_cnn/model/cnnText/cnntext.py b/kim_cnn/model/cnnText/cnntext.py new file mode 100644 index 0000000..5156d5b --- /dev/null +++ b/kim_cnn/model/cnnText/cnntext.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import torch +import torch.nn as nn + +import torch.nn.functional as F + +from configurable import Configurable + +class CNNText(nn.Module): + """ + Model class for the computational graph + """ + def __init__(self, args): + super(CNNText, self).__init__() + + + #input_channel = args['input_channels'] + output_channel = args['output_channels'] + target_class = args['target_class'] + words_num = args['words_num'] + words_dim = args['words_dim'] + embeds_num = args['embeds_num'] + embeds_dim = args['embeds_dim'] + Ks = args['kernel_sizes'] + self.mode = args['mode'] + if self.mode == 'multichannel': + input_channel = 2 + else: + input_channel = 1 + self.use_gpu = args['use_gpu'] + self.embed = nn.Embedding(words_num, words_dim) + self.static_embed = nn.Embedding(embeds_num, embeds_dim) + self.static_embed.weight.data.copy_(torch.from_numpy(args['embeds'])) + self.non_static_embed = nn.Embedding(embeds_num, embeds_dim) + self.non_static_embed.weight.data.copy_(torch.from_numpy(args['embeds'])) + self.static_embed.weight.requires_grad = False + + self.conv1 = nn.Conv2d(input_channel, output_channel, (3, words_dim), padding=(2, 0)) + self.conv2 = nn.Conv2d(input_channel, output_channel, (4, words_dim), padding=(3, 0)) + self.conv3 = nn.Conv2d(input_channel, output_channel, (5, words_dim), padding=(4, 0)) + #self.convs1 = [nn.Conv2d(input_channel, output_channel, (K, words_dim), padding=(K-1, 0)) for K in Ks] + + self.dropout = nn.Dropout(args['dropout']) + self.fc1 = nn.Linear(len(Ks) * output_channel, target_class) + + def conv_and_pool(self, x, conv): + x = F.relu(conv(x)).squeeze(3) # (batch_size, output_channel, feature_map_dim) + x = F.max_pool1d(x, x.size(2)).squeeze(2) + return x + + def forward(self, x): + #if self.use_gpu: + # self.conv1s = [model.cuda() for model in self.convs1] + if self.mode == 'rand': + words = x[:,:,0] + word_input = self.embed(words) # (batch, sent_len, embed_dim) + x = word_input.unsqueeze(1) # (batch, channel_input, sent_len, embed_dim) + elif self.mode == 'static': + static_words = x[:,:,1] + static_input = self.static_embed(static_words) + x = static_input.unsqueeze(1) # (batch, channel_input, sent_len, embed_dim) + elif self.mode == 'non-static': + non_static_words = x[:, :, 1] + non_static_input = self.non_static_embed(non_static_words) + x = non_static_input.unsqueeze(1) # (batch, channel_input, sent_len, embed_dim) + elif self.mode == 'multichannel': + words = x[:, :, 1] + word_input = self.non_static_embed(words) # (batch, sent_len, embed_dim) + static_words = x[:, :, 1] + static_input = self.static_embed(static_words) + x = torch.stack([word_input, static_input], dim=1)# (batch, channel_input, sent_len, embed_dim) + else: + print("Unsupported Mode") + exit() + #x = word_input.unsqueeze(1) # (batch, channel_input, sent_len, embed_dim) + #x = [F.relu(conv(x)).squeeze(3) for conv in self.convs1] + x = [F.relu(self.conv1(x)).squeeze(3), F.relu(self.conv2(x)).squeeze(3), F.relu(self.conv3(x)).squeeze(3)] + # (batch, channel_output, ~=(sent_len)) * len(Ks) + x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling + # (batch, channel_output) * len(Ks) + x = torch.cat(x, 1) # (batch, channel_output * len(Ks)) + x = self.dropout(x) + logit = self.fc1(x) # (batch, target_size) + return logit + diff --git a/kim_cnn/network/__init__.py b/kim_cnn/network/__init__.py new file mode 100644 index 0000000..7269ea9 --- /dev/null +++ b/kim_cnn/network/__init__.py @@ -0,0 +1 @@ +from cnnTextNetwork import cnnTextNetwork \ No newline at end of file diff --git a/kim_cnn/network/cnnTextNetwork.py b/kim_cnn/network/cnnTextNetwork.py new file mode 100644 index 0000000..0e82542 --- /dev/null +++ b/kim_cnn/network/cnnTextNetwork.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from configurable import Configurable +from vocab import Vocab +from dataset import Dataset +import os +import sys +import torch +import torch.nn.functional as F +from torch.autograd import Variable + + +class cnnTextNetwork(Configurable): + """ + Network class + - build the vocabulary + - build the dataset + - control the training + - control the validation and testing + - save the model and store the best result + """ + + def __init__(self, option, model, *args, **cargs): + + '''check args?''' + super(cnnTextNetwork, self).__init__(*args, **cargs) + if not os.path.isdir(self.save_dir): + os.mkdir(self.save_dir) + + with open(os.path.join(self.save_dir, 'config_file'), 'w') as f: + self._config.write(f) + + + self._vocabs = [] + vocab_file = [(self.word_file, 'Words'), + (self.target_file, "Targets")] + + for i, (vocab_file, name) in enumerate(vocab_file): + vocab = Vocab(vocab_file, self._config, + name = name, + load_embed_file = (not i), + lower_case = (not i) + ) + self._vocabs.append(vocab) + + print("################## Data ##################") + print("There are %d words in training set" % (len(self.words) - 2)) + print("There are %d targets in training set" % (len(self.targets) - 2)) + print("Loading training set ...") + self._trainset = Dataset(self.train_file, self._vocabs, self._config, name="Trainset") + print("There are %d sentences in training set" % (self._trainset.sentsNum)) + print("Loading validation set ...") + self._validset = Dataset(self.valid_file, self._vocabs, self._config, name="Validset") + print("There are %d sentences in validation set" % (self._validset.sentsNum)) + print("Loading testing set ...") + self._testset = Dataset(self.test_file, self._vocabs, self._config, name="Testset") + print("There are %d sentences in testing set" % (self._testset.sentsNum)) + + self.args = {#'input_channels':2, + 'kernel_sizes':[3,4,5], + 'words_num': len(self.words), + 'words_dim': self.words_dim, + 'target_class': len(self.targets), + 'output_channels': 100, + 'dropout': self.dropout, + 'embeds_num' : self.words.embeds_size, + 'embeds_dim' : self.words_dim, # Embedding size must be the same with words size + 'embeds':self.words.pretrained_embeddings, + 'use_gpu': self.use_gpu, + 'mode': self.mode} + + self.model = model + return + + + def train_minibatch(self): + return self._trainset.minibatch(self.train_batch_size, self.input_idx, self.target_idx, shuffle=True) + + def valid_minibatch(self): + return self._validset.minibatch(self.test_batch_size, self.input_idx, self.target_idx, shuffle=False) + + def test_minibatch(self): + return self._testset.minibatch(self.test_batch_size, self.input_idx, self.target_idx, shuffle=False) + + def train(self): + # if torch.cuda.is_available(): # and use_cuda + # self.model.cuda() + if self.use_gpu: + self.model = self.model(self.args).cuda() + else: + self.model = self.model(self.args) + parameter = filter(lambda p: p.requires_grad, self.model.parameters()) + optimizer = torch.optim.Adam(parameter, lr=self.learning_rate) + # The optimizer doesn't have adaptive learning rate + + step = 0 + best_score = 0 + valid_accuracy = 0 + test_accuracy = 0 + + acc_corrects = 0 # count the corrects for one log_interval + acc_sents = 0 # count sents number for one log_interval + + epoch = 0 + while True: + for batch in self.train_minibatch(): + self.model.train() + feature, target = batch['text'], batch['label'] + # Sanity check + # for sent in feature: + # for word in sent: + # word_str = self.words._idx2str[word[0]] + # embed_str = self.words._embed2str[word[1]] + # if word_str != embed_str: + # print(word_str, embed_str) + ## + if self.use_gpu: + feature = Variable(torch.from_numpy(feature).cuda()) + target = Variable(torch.from_numpy(target).cuda())[:, 0] + else: + feature = Variable(torch.from_numpy(feature)) + target = Variable(torch.from_numpy(target))[:, 0] + + # if torch.cuda.is_available(): + # feature, target = feature.cuda(), target.cuda() + optimizer.zero_grad() # Clears the gradients of all optimized Variable + logit = self.model(feature) + loss = F.cross_entropy(logit, target) + loss.backward() + optimizer.step() + step += 1 + preds = torch.max(logit, 1)[1].view(target.size()) # get the index + acc_corrects += (preds.cpu().data == target.cpu().data).sum() + acc_sents += batch['batch_size'] + # if step % self.log_interval == 0: + # accuracy = float(acc_corrects) / float(acc_sents) * 100.0 + # print("## [Batch %d] Accuracy : %5.2f" % (step, accuracy)) + # acc_corrects = 0 + # acc_sents = 0 + + if step == 1 or step % self.valid_interval == 0: + accuracy = self.test(validate=True) + print("## Validation: %5.2f" % (accuracy)) + if accuracy > best_score: + best_score = accuracy + valid_accuracy = accuracy + print("## Update Model ##") + torch.save(self.model, self.save_model_file) + + print("## Currently the best validation: Accucacy %5.2f" % (valid_accuracy)) + + epoch += 1 + accuracy = float(acc_corrects) / float(acc_sents) * 100 + print("[EPOCH] %d Accuracy: %5.2f" % (epoch, accuracy)) + acc_corrects = 0 + acc_sents = 0 + if (epoch % self.epoch_decay == 0): + lr = self.learning_rate * (0.75 ** (epoch // self.epoch_decay)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + + def test(self, validate=False): + self.model.eval() + if validate: + dataset = self._validset + minibatch = self.valid_minibatch + else: + dataset = self._testset + minibatch = self.test_minibatch + + test_corrects = 0 + test_sents = 0 + for batch in minibatch(): + # TODO: Prediton to Text + feature, target = batch['text'], batch['label'] + if self.use_gpu: + feature = Variable(torch.from_numpy(feature).cuda()) + else: + feature = Variable(torch.from_numpy(feature)) + target = Variable(torch.from_numpy(target))[:,0] + # if torch.cuda.is_available(): + # feature, target = feature.cuda(), target.cuda() + + logit = self.model(feature) + preds = torch.max(logit, 1)[1].view(target.size()) # get the index + test_corrects += (preds.cpu().data == target.data).sum() + test_sents += batch['batch_size'] + return float(test_corrects) / float(test_sents) * 100.0 + + @property + def words(self): + return self._vocabs[0] + + @property + def targets(self): + return self._vocabs[1] + + + @property + def input_idx(self): + return (0, 1) + + + @property + def target_idx(self): + return (0,) + + + diff --git a/kim_cnn/vocab.py b/kim_cnn/vocab.py new file mode 100644 index 0000000..6676a3b --- /dev/null +++ b/kim_cnn/vocab.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from configurable import Configurable +from collections import Counter +import numpy as np + +from etc.utils import clean_str, clean_str_sst + +class Vocab(Configurable): + """ + Vocab for + - id-str converting + - word embedding / lookup + """ + def __init__(self, vocab_file, *args, **kwargs): + """ + vocab_file: the name of the vocab_file + args: it should be self._config + kwargs: some options for Vocab, before super, they should be pop out + because these are not options for the global settings + """ + self.vocab_file = vocab_file + load_embed_file = kwargs.pop('load_embed_file', False) + self.lower_case = kwargs.pop('lower_case', False) + super(Vocab, self).__init__(*args, **kwargs) + + self.SPECIAL_TOKENS = ('', '') + self.START_IDX = len(self.SPECIAL_TOKENS) + self.PAD, self.UNK = range(self.START_IDX) + self.pretrained_embeddings = None + self._count = Counter() # Count the number of vocab + + self._str2idx = dict(zip(self.SPECIAL_TOKENS, range(self.START_IDX))) + self._idx2str = dict(zip(range(self.START_IDX), self.SPECIAL_TOKENS)) + + self._str2embed = {} + self._embed2str = {} + + self.add_train_file() + self.save_vocab_file() + if load_embed_file: + self.load_embed_file() + + + + def add_train_file(self): + if self.dataset_type == 'TREC': + with open(self.train_file) as f: + for line_num, line in enumerate(f): + line = clean_str(line).split() + if line: + if self.name == 'Targets': + self.add(line[0]) + if self.name == 'Words': + for word in line[2:]: + self.add(word) + else: + with open(self.train_file) as f: + for line_num, line in enumerate(f): + line = clean_str(line).split() + if line: + if self.name == 'Targets': + self.add(line[0]) + if self.name == 'Words': + for word in line[1:]: + self.add(word) + + self.index_vocab() + + def add(self, item): + if self.lower_case: + item = item.lower() + + self._count[item] += 1 + return + + def index_vocab(self): + """ + Sorted the vocabs by frequency and assign id to them + Process: + - Get all the words with same frequency from the Counter + - Sort those words + - Assign ID to them + - Go back to first step + """ + cur_idx = self.START_IDX + buff = [] + for word_and_count in self._count.most_common(): + if (not buff) or (buff[-1][1]==word_and_count[1]): + buff.append(word_and_count) + else: + buff.sort() + for word, count in buff: + if count >= self.min_occur_count and (word not in self._str2idx): + self._str2idx[word] = cur_idx + self._idx2str[cur_idx] = word + cur_idx += 1 + buff = [word_and_count] + buff.sort() + for word, count in buff: + if count >= self.min_occur_count and word not in self._str2idx: + self._str2idx[word] = cur_idx + self._idx2str[cur_idx] = word + cur_idx += 1 + return + + + def save_vocab_file(self): + """ + save the words on the file + """ + with open(self.vocab_file, "w") as f: + for word_and_count in self._count.most_common(): + f.write('%s\t%d\n' %(word_and_count)) + return + + def load_embed_file(self): + self._str2embed = dict(zip(self.SPECIAL_TOKENS, range(self.START_IDX))) + self._embed2str = dict(zip(range(self.START_IDX), self.SPECIAL_TOKENS)) + embeds = [[0] * self.words_dim, [0] * self.words_dim] + with open(self.embed_file) as f: + cur_idx = self.START_IDX + for line_num, line in enumerate(f): + line = line.strip().split() + if line: + try: + if self.dataset_type != 'SST-1' or self.dataset_type != 'SST-2': + self._str2embed[clean_str(line[0])] = cur_idx + self._embed2str[cur_idx] = clean_str(line[0]) + else: + self._str2embed[clean_str_sst(line[0])] = cur_idx + self._embed2str[cur_idx] = clean_str_sst(line[0]) + embeds.append(line[1:]) + cur_idx += 1 + except: + raise ValueError('The embedding file is misformatted at line %d' % (line_num+1)) + # Randomly initialize the pre-trained vector for those words not in pre-train-file + for word in self._str2idx.keys(): + if word not in self._str2embed.keys(): + self._str2embed[word] = cur_idx + self._embed2str[cur_idx] = word + embeds.append(list(np.random.uniform(-1, 1, self.words_dim))) + cur_idx += 1 + self.pretrained_embeddings = np.array(embeds, dtype=np.float64) + del embeds + return + + + def __getitem__(self, key): + if isinstance(key, basestring): + # Convert the lower case + if self.lower_case: + key = key.lower() + if self.pretrained_embeddings is not None: + return (self._str2idx.get(key, self.UNK), self._str2embed.get(key, self.UNK)) + else: + return (self._str2idx.get(key, self.UNK),) + + def __len__(self): + return len(self._str2idx) + + @property + def embeds_size(self): + return len(self._embed2str)