wassname/stampy-chat
1
0
Fork 0
mirror of https://github.com/wassname/stampy-chat.git synced 2026-09-17 12:50:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Updated dataset/semantic search/main for ease of use.

Browse Source
This commit is contained in:
henri123lemoine
2023-03-14 08:45:33 -04:00
parent 344ed281ee
commit e7979073e7
4 changed files with 9 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Embeddings Search/dataset.py
+1 -13
View File
@@ -1,9 +1,7 @@
import jsonlines
import numpy as np
from typing import List, Dict, Tuple
import re
import time
import random
import pickle
import openai
from tenacity import (
@@ -13,22 +11,12 @@ from tenacity import (
) # for exponential backoff
import config
from text_splitter import TextSplitter, split_into_sentences
from settings import PATH_TO_DATA, PATH_TO_EMBEDDINGS, PATH_TO_DATASET, EMBEDDING_MODEL
LEN_EMBEDDINGS = 1536
PATH_TO_DATA = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\alignment_texts.jsonl"
PATH_TO_EMBEDDINGS = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\embeddings.npy"
PATH_TO_DATASET = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\dataset.pkl"
COMPLETIONS_MODEL = "text-davinci-003"
EMBEDDING_MODEL = "text-embedding-ada-002"
openai.api_key = config.OPENAI_API_KEY
MAX_LEN_PROMPT = 5000
error_count_dict = {
"Entry has no source.": 0,
"Entry has no title.": 0,
src/Embeddings Search/main.py
+1 -25
View File
@@ -1,37 +1,13 @@
import jsonlines
import numpy as np
from typing import List, Dict, Tuple
import re
import time
import random
import pickle
import openai
from tenacity import (
retry,
stop_after_attempt,
wait_random_exponential,
) # for exponential backoff
import config
from dataset import Dataset
from semantic_search import AlignmentSearch
from settings import DATA_PATH
LEN_EMBEDDINGS = 1536
PATH_TO_DATA = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\alignment_texts.jsonl"
PATH_TO_EMBEDDINGS = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\embeddings.npy"
PATH_TO_DATASET = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\dataset.pkl"
COMPLETIONS_MODEL = "text-davinci-003"
EMBEDDING_MODEL = "text-embedding-ada-002"
from settings import PATH_TO_DATASET
openai.api_key = config.OPENAI_API_KEY
MAX_LEN_PROMPT = 5000
def main():
with open(PATH_TO_DATASET, 'rb') as f:
src/Embeddings Search/semantic_search.py
+3 -17
View File
@@ -1,9 +1,5 @@
import jsonlines
import numpy as np
from typing import List, Dict, Tuple
import re
import time
import random
from typing import List, Tuple
import pickle
import openai
from tenacity import (
@@ -11,23 +7,13 @@ from tenacity import (
stop_after_attempt,
wait_random_exponential,
) # for exponential backoff
from dataset import Dataset
import config
LEN_EMBEDDINGS = 1536
PATH_TO_DATA = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\alignment_texts.jsonl"
PATH_TO_EMBEDDINGS = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\embeddings.npy"
PATH_TO_DATASET = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\dataset.pkl"
COMPLETIONS_MODEL = "text-davinci-003"
EMBEDDING_MODEL = "text-embedding-ada-002"
from dataset import Dataset
from settings import PATH_TO_DATASET, EMBEDDING_MODEL, COMPLETIONS_MODEL, MAX_LEN_PROMPT
openai.api_key = config.OPENAI_API_KEY
MAX_LEN_PROMPT = 5000
class AlignmentSearch:
def __init__(self,
src/Embeddings Search/settings.py
+4 -2
View File
@@ -2,6 +2,8 @@ EMBEDDING_MODEL = "text-embedding-ada-002"
COMPLETIONS_MODEL = "text-davinci-003"
LEN_EMBEDDINGS = 1536
MAX_LEN_PROMPT = 5000
MAX_LEN_PROMPT = 8191
DATA_PATH = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\data\alignment_texts.jsonl" # Path to the dataset .jsonl file.
PATH_TO_DATA = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\data\alignment_texts.jsonl" # Path to the dataset .jsonl file.
PATH_TO_EMBEDDINGS = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\embeddings.npy" # Path to the saved embeddings (.npy) file.
PATH_TO_DATASET = r"C:\Users\Henri\Documents\GitHub\AlignmentSearch\src\Embeddings Search\data\dataset.pkl" # Path to the saved dataset (.pkl) file.