From 50203f92eb879ed1945a4feed757300c1b25d642 Mon Sep 17 00:00:00 2001 From: Thomas Lemoine Date: Fri, 24 Mar 2023 21:22:22 -0400 Subject: [PATCH] smaller token blocks --- src/text_splitter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/text_splitter.py b/src/text_splitter.py index 5285607..bcf0c7a 100644 --- a/src/text_splitter.py +++ b/src/text_splitter.py @@ -26,7 +26,7 @@ def split_into_sentences(text: str) -> List[str]: class TokenSplitter: """Splits text into blocks of tokens according to chatgpt's tokenizer.""" - def __init__(self, min_tokens: int = 500, max_tokens: int = 750): + def __init__(self, min_tokens: int = 200, max_tokens: int = 300): self.encoding = tiktoken.get_encoding("cl100k_base") self.min_tokens = min_tokens self.max_tokens = max_tokens