turkish cleaner and data preprocessor

This commit is contained in:
erogol
2020-03-17 14:47:59 +01:00
parent fd4e6d0245
commit fa795347a9
2 changed files with 26 additions and 0 deletions
+8
View File
@@ -91,6 +91,14 @@ def transliteration_cleaners(text):
return text
# TODO: elaborate it
def basic_turkish_cleaners(text):
'''Pipeline for Turkish text'''
text = text.replace("I", "ı")
text = lowercase(text)
text = collapse_whitespace(text)
def english_cleaners(text):
'''Pipeline for English text, including number and abbreviation expansion.'''
text = convert_to_ascii(text)