From 11dc2dd6c16ae9962c863ac0d321d584f2afe29d Mon Sep 17 00:00:00 2001 From: henri123lemoine Date: Tue, 27 Jun 2023 07:26:41 -0400 Subject: [PATCH] Fixed entry typing and renamed var for clarity --- src/dataset/update_dataset.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dataset/update_dataset.py b/src/dataset/update_dataset.py index d720815..580c509 100644 --- a/src/dataset/update_dataset.py +++ b/src/dataset/update_dataset.py @@ -1,6 +1,6 @@ # dataset/update_dataset.py -from typing import Dict, List +from typing import Dict, List, Union import numpy as np from tqdm.auto import tqdm import openai @@ -71,7 +71,7 @@ class ARDUpdater: logger.error(f"Entry validation failed: {str(e)}", exc_info=True) return None - def validate_entry(self, entry: Dict[str, str | list], len_lower_limit: int = 0): + def validate_entry(self, entry: Dict[str, Union[str, list]], char_len_lower_limit: int = 0): metadata_types = { 'id': str, 'source': str, @@ -86,8 +86,8 @@ class ARDUpdater: if not isinstance(entry.get(metadata_type), metadata_type_type): raise ValueError(f"Entry metadata '{metadata_type}' is not of type '{metadata_type_type}' or is missing.") - if len(entry['text']) < len_lower_limit: - raise ValueError(f"Entry text is too short (< {len_lower_limit} tokens).") + if len(entry['text']) < char_len_lower_limit: + raise ValueError(f"Entry text is too short (< {char_len_lower_limit} characters).") def get_embeddings(self, chunks): embeddings = np.zeros((len(chunks), EMBEDDINGS_DIMS))