diff --git a/api/chat.py b/api/chat.py index fb5378e..a49b207 100644 --- a/api/chat.py +++ b/api/chat.py @@ -11,114 +11,146 @@ import tiktoken EMBEDDING_MODEL = "text-embedding-ada-002" COMPLETIONS_MODEL = "gpt-3.5-turbo" # COMPLETIONS_MODEL = "gpt-4" -MODERATION_ENDPOINT = "https://api.openai.com/v1/moderations" -# OpenAI parameters -LEN_EMBEDDINGS = 1536 -MAX_TOKEN_LEN_PROMPT = 8191 if COMPLETIONS_MODEL == 'gpt-4' else 4095 -TRUNCATE_CONTEXT_LEN = 2300 if COMPLETIONS_MODEL == 'gpt-4' else 1500 -TRUNCATE_HISTORY_LEN = 500 -MAX_RESPONSE_LEN = 900 +# parameters + +# NOTE: All this is approximate, there's bits I'm intentionally not counting. Leave a buffer beyond what you might expect. +NUM_TOKENS = 8191 if COMPLETIONS_MODEL == 'gpt-4' else 4095 +PROMPT_FRACTION = 0.25 # the (approximate) fraction of num_tokens to use for non-context prompt text before truncating +CONTEXT_FRACTION = 0.45 # the (approximate) fraction of num_tokens to use for context text before truncating + +ENCODER = tiktoken.get_encoding("cl100k_base") # --------------------------------- prompt code -------------------------------- -def limit_tokens(text: str, max_tokens: int, encoding_name: str = "cl100k_base") -> str: - encoding = tiktoken.get_encoding(encoding_name) - tokens = encoding.encode(text)[:max_tokens] - return encoding.decode(tokens) -def construct_prompt(query: str, history: List[Dict[str, str]], context: List[Block], encoding_name: str = "cl100k_base"): + +# limit a string to a certain number of tokens +def cap(text: str, max_tokens: int) -> str: + + if max_tokens <= 0: return "..." + + encoded_text = ENCODER.encode(text) + + if len(encoded_text) <= max_tokens: return text + else: return ENCODER.decode(encoded_text[:max_tokens]) + " ..." + + + + +def construct_prompt(query: str, history: List[Dict[str, str]], context: List[Block]) -> List[Dict[str, str]]: # History takes the format: history=[ - # {"role": "system", "content": "You are a helpful assistant."}, # {"role": "user", "content": "Who won the world series in 2020?"}, # {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."}, # {"role": "user", "content": "Where was it played?"} # {"role": "assistant", "content": "Los Angeles, California."} # ] - # Encoder to count tokens - enc = tiktoken.get_encoding(encoding_name) - total_tokens = 0 - + token_count = 0 prompt = [] system_prompt = "You are a helpful assistant knowledgeable about AI Alignment and Saftey." - total_tokens += len(enc.encode(system_prompt)) + token_count += len(ENCODER.encode(system_prompt)) + prompt.append({"role": "system", "content": system_prompt}) # Get past user queries - past_user_queries = "\nQ: ".join([message["content"] for message in history if message["role"] == "user"][-5:]) - past_user_queries = f"My previous queries in our conversation have been:\n" + past_user_queries + past_user_queries = [message["content"] for message in history if message["role"] == "user"][-5 * 2:] # get the last 5 user queries + if len(past_user_queries) > 0: + for i, q in enumerate(past_user_queries): + prompt.append({"role": "user", "content": "Q: " + q}) + token_count += len(ENCODER.encode("Q: " + q)) + + # for all but the last query, just add the system message mentioning that there has been a response. + if i < len(past_user_queries) - 1: + response = "the assistant's response has been left out for brevity." + prompt.append({"role": "system", "content": response}) + token_count += len(ENCODER.encode(response)) + + # Add the response to the latest query, if there was one. Possibly truncate it. + if len(history) > 0 and history[-1]["role"] == "assistant": + last_response = cap(history[-1]["content"], int(NUM_TOKENS * PROMPT_FRACTION) - token_count) + prompt.append({"role": "assistant", "content": last_response}) + token_count += len(ENCODER.encode(last_response)) + + + + + + + # Instruction prompt - instruction_context_query_prompt = \ + main_prompt = \ "Please give a clear and coherent answer to my question (written after \"Q:\") " \ "using the following sources. Each source is labeled with a letter. Feel free to " \ - "use the sources in any order, and try to use multiple sources in your answer." + "use the sources in any order, and try to use multiple sources in your answer.\n\n" + + token_count = len(ENCODER.encode(main_prompt)) # Context from top-k blocks - context_prompt = "" - for i, block in enumerate(context): - context_prompt += f"[{chr(ord('a') + i)}] {block.title} - {block.author} - {block.date}\n\n{block.text}\n\n\n" - context_prompt = context_prompt[:-2] # trim last two newlines - context_prompt = limit_tokens(context_prompt, TRUNCATE_CONTEXT_LEN) # truncate the context_prompt to max TRUNCATE_CONTEXT tokens - context_prompt += "\n" if (context_prompt[-1] != "\n") else "" + block_str = f"[{chr(ord('a') + i)}] {block.title} - {block.author} - {block.date}\n{block.text}\n\n" + block_tc = len(ENCODER.encode(block_str)) - # Question prompt - question_prompt = f"In your answer, please cite any claims you make back to each source " \ + if token_count + block_tc > int(NUM_TOKENS * CONTEXT_FRACTION): + main_prompt += cap(block_str, int(NUM_TOKENS * CONTEXT_FRACTION) - token_count) + break + else: + main_prompt += block_str + token_count += block_tc + + main_prompt = main_prompt.strip() + "\n\n\n" + + + + + + + + main_prompt += f"In your answer, please cite any claims you make back to each source " \ f"using the format: [a], [b], etc. If you use multiple sources to make a claim " \ f"cite all of them. For example: \"AGI is concerning [c, d, e].\"\n\nQ: " + query - instruction_context_query_prompt = f"{instruction_context_query_prompt}\n\n{context_prompt}\n\n{question_prompt}" + prompt.append({"role": "user", "content": main_prompt}) - total_tokens += len(enc.encode(past_user_queries)) - total_tokens += len(enc.encode(history[-2]["content"])) if (len(history) >= 2) else 0 - total_tokens += len(enc.encode(history[-1]["content"])) if (len(history) >= 1) else 0 - total_tokens += len(enc.encode(instruction_context_query_prompt)) - - # If the prompt is too long, truncate the last answer - if total_tokens > MAX_TOKEN_LEN_PROMPT - TRUNCATE_HISTORY_LEN: - tokens_left = MAX_TOKEN_LEN_PROMPT - total_tokens - print(f"WARNING: Prompt is too long! Prompt length: {total_tokens} tokens") - last_assistant_reply_trunctated = limit_tokens(prompt[-1]["content"], tokens_left) - prompt[-1]["content"] = f"{last_assistant_reply_trunctated}" - - prompt.append({"role": "system", "content": system_prompt}) - prompt.append({"role": "user", "content": past_user_queries}) - prompt.extend(history[-2:]) - prompt.append({"role": "user", "content": instruction_context_query_prompt}) - - return prompt, MAX_TOKEN_LEN_PROMPT - (total_tokens + 50) # add 50 tokens for safety + return prompt # ------------------------------------------------------------------------------ - -def normal_completion(prompt: List[Dict[str, str]], max_tokens_completion: int) -> str: - try: - return openai.ChatCompletion.create( - model=COMPLETIONS_MODEL, - messages=prompt, - max_tokens=max_tokens_completion - )["choices"][0]["message"]["content"] - except Exception as e: - print(e) - return "I'm sorry, I failed to process your query. Please try again. If the problem persists, please contact the administrator." # returns either (True, reply string, embeddings) or (False, error message string, None) -def talk_to_robot(dataset_dict, query: str, history: List[Dict[str, str]] = [], k: int = 10): +def talk_to_robot(dataset_dict, query: str, history: List[Dict[str, str]], k: int = 10): + # 1. Find the most relevant blocks from the Alignment Research Dataset top_k_blocks: List[Block] = get_top_k_blocks(dataset_dict, query, k) - # 2. Generate a prompt for the ChatCompletions API - prompt, max_tokens_completion = construct_prompt(query, history, top_k_blocks) - # print(" ------------------------------ prompt: -----------------------------") - # for message in prompt: - # print(f"{message['role']}: {message['content']}\n\n") - # if we were to error out, return something like this - # return (False, "Example error message", None) - - # 3. Answer the user query - return (True, normal_completion(prompt, max_tokens_completion), top_k_blocks) + # 2. Generate a prompt + prompt = construct_prompt(query, history, top_k_blocks) + print('\n' * 10) + print(" ------------------------------ prompt: -----------------------------") + for message in prompt: + print(f"----------- {message['role']}: ------------------") + print(message['content']) + + print('\n' * 10) + + + + # 3. Count number of tokens left for completion (-50 for a buffer) + max_tokens_completion = NUM_TOKENS - sum([len(ENCODER.encode(message["content"]) + ENCODER.encode(message["role"])) for message in prompt]) - 50 + + + # 4. Answer the user query + try: + return (True, openai.ChatCompletion.create( + model=COMPLETIONS_MODEL, + messages=prompt, + max_tokens=max_tokens_completion + )["choices"][0]["message"]["content"], top_k_blocks) + except Exception as e: + print(e) + return (False, "Error: " + str(e), None) + diff --git a/api/get_blocks.py b/api/get_blocks.py index d4af74e..5f6b3dd 100644 --- a/api/get_blocks.py +++ b/api/get_blocks.py @@ -75,7 +75,7 @@ def get_top_k_blocks(data, user_query: str, k: int = 10) -> List[Block]: blocks = [Block(*block) for block in top_k_metadata_and_text] # for all blocks that are "the same" (same title, author, date, url, tags), - # combine their text with "\n\n.....\n\n" in between. Return them in order such + # combine their text with "....." in between. Return them in order such # that the combined block has the minimum index of the blocks combined. key = lambda bi: (bi[0].title or "", bi[0].author or "", bi[0].date or "", bi[0].url or "", bi[0].tags or "") @@ -91,7 +91,7 @@ def get_top_k_blocks(data, user_query: str, k: int = 10) -> List[Block]: group = group[:3] # limit to a max of 3 blocks from any one source - text = "\n\n\n.....\n\n\n".join([block[0].text for block in group]) + text = "\n.....\n".join([block[0].text for block in group]) min_index = min([block[1] for block in group]) diff --git a/api/main.py b/api/main.py index bdac5fe..662b507 100644 --- a/api/main.py +++ b/api/main.py @@ -48,9 +48,11 @@ def semantic(): @app.route('/chat', methods=['POST']) @cross_origin() def chat(): + query = request.json['query'] + history = request.json['history'] - is_valid, response, context = talk_to_robot(dataset_dict, query) + is_valid, response, context = talk_to_robot(dataset_dict, query, history) if is_valid: return jsonify({'response': response, 'citations': [{'title': block.title, 'author': block.author, 'date': block.date, 'url': block.url} for block in context]})