From 84225d208c15b41e07616f5c681e258fe309846b Mon Sep 17 00:00:00 2001 From: Ashutosh-Adhikari Date: Tue, 29 Jan 2019 09:52:20 -0500 Subject: [PATCH] Add char_quantize for AAPD (#173) --- datasets/aapd.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datasets/aapd.py b/datasets/aapd.py index fb41923..88ab18d 100644 --- a/datasets/aapd.py +++ b/datasets/aapd.py @@ -9,6 +9,13 @@ from torchtext.vocab import Vectors from datasets.reuters import clean_string, clean_string_fl, split_sents +def char_quantize(string, max_length=1000): + identity = np.identity(len(AAPDCharQuantized.ALPHABET)) + quantized_string = np.array([identity[AAPDCharQuantized.ALPHABET[char]] for char in list(string.lower()) if char in AAPDCharQuantized.ALPHABET], dtype=np.float32) + if len(quantized_string) > max_length: + return quantized_string[:max_length] + else: + return np.concatenate((quantized_string, np.zeros((max_length - len(quantized_string), len(AAPDCharQuantized.ALPHABET)), dtype=np.float32))) def process_labels(string): """