mirror of
https://github.com/wassname/stampy-chat.git
synced 2026-09-10 12:40:44 +08:00
fix bug in prompt
This commit is contained in:
Fraser
+103
-71
@@ -11,114 +11,146 @@ import tiktoken
|
||||
EMBEDDING_MODEL = "text-embedding-ada-002"
|
||||
COMPLETIONS_MODEL = "gpt-3.5-turbo"
|
||||
# COMPLETIONS_MODEL = "gpt-4"
|
||||
MODERATION_ENDPOINT = "https://api.openai.com/v1/moderations"
|
||||
|
||||
# OpenAI parameters
|
||||
LEN_EMBEDDINGS = 1536
|
||||
MAX_TOKEN_LEN_PROMPT = 8191 if COMPLETIONS_MODEL == 'gpt-4' else 4095
|
||||
TRUNCATE_CONTEXT_LEN = 2300 if COMPLETIONS_MODEL == 'gpt-4' else 1500
|
||||
TRUNCATE_HISTORY_LEN = 500
|
||||
MAX_RESPONSE_LEN = 900
|
||||
# parameters
|
||||
|
||||
# NOTE: All this is approximate, there's bits I'm intentionally not counting. Leave a buffer beyond what you might expect.
|
||||
NUM_TOKENS = 8191 if COMPLETIONS_MODEL == 'gpt-4' else 4095
|
||||
PROMPT_FRACTION = 0.25 # the (approximate) fraction of num_tokens to use for non-context prompt text before truncating
|
||||
CONTEXT_FRACTION = 0.45 # the (approximate) fraction of num_tokens to use for context text before truncating
|
||||
|
||||
ENCODER = tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
# --------------------------------- prompt code --------------------------------
|
||||
|
||||
def limit_tokens(text: str, max_tokens: int, encoding_name: str = "cl100k_base") -> str:
|
||||
encoding = tiktoken.get_encoding(encoding_name)
|
||||
tokens = encoding.encode(text)[:max_tokens]
|
||||
return encoding.decode(tokens)
|
||||
|
||||
def construct_prompt(query: str, history: List[Dict[str, str]], context: List[Block], encoding_name: str = "cl100k_base"):
|
||||
|
||||
# limit a string to a certain number of tokens
|
||||
def cap(text: str, max_tokens: int) -> str:
|
||||
|
||||
if max_tokens <= 0: return "..."
|
||||
|
||||
encoded_text = ENCODER.encode(text)
|
||||
|
||||
if len(encoded_text) <= max_tokens: return text
|
||||
else: return ENCODER.decode(encoded_text[:max_tokens]) + " ..."
|
||||
|
||||
|
||||
|
||||
|
||||
def construct_prompt(query: str, history: List[Dict[str, str]], context: List[Block]) -> List[Dict[str, str]]:
|
||||
|
||||
# History takes the format: history=[
|
||||
# {"role": "system", "content": "You are a helpful assistant."},
|
||||
# {"role": "user", "content": "Who won the world series in 2020?"},
|
||||
# {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."},
|
||||
# {"role": "user", "content": "Where was it played?"}
|
||||
# {"role": "assistant", "content": "Los Angeles, California."}
|
||||
# ]
|
||||
|
||||
# Encoder to count tokens
|
||||
enc = tiktoken.get_encoding(encoding_name)
|
||||
total_tokens = 0
|
||||
|
||||
token_count = 0
|
||||
prompt = []
|
||||
|
||||
system_prompt = "You are a helpful assistant knowledgeable about AI Alignment and Saftey."
|
||||
total_tokens += len(enc.encode(system_prompt))
|
||||
token_count += len(ENCODER.encode(system_prompt))
|
||||
prompt.append({"role": "system", "content": system_prompt})
|
||||
|
||||
# Get past user queries
|
||||
past_user_queries = "\nQ: ".join([message["content"] for message in history if message["role"] == "user"][-5:])
|
||||
past_user_queries = f"My previous queries in our conversation have been:\n" + past_user_queries
|
||||
past_user_queries = [message["content"] for message in history if message["role"] == "user"][-5 * 2:] # get the last 5 user queries
|
||||
if len(past_user_queries) > 0:
|
||||
for i, q in enumerate(past_user_queries):
|
||||
prompt.append({"role": "user", "content": "Q: " + q})
|
||||
token_count += len(ENCODER.encode("Q: " + q))
|
||||
|
||||
# for all but the last query, just add the system message mentioning that there has been a response.
|
||||
if i < len(past_user_queries) - 1:
|
||||
response = "the assistant's response has been left out for brevity."
|
||||
prompt.append({"role": "system", "content": response})
|
||||
token_count += len(ENCODER.encode(response))
|
||||
|
||||
# Add the response to the latest query, if there was one. Possibly truncate it.
|
||||
if len(history) > 0 and history[-1]["role"] == "assistant":
|
||||
last_response = cap(history[-1]["content"], int(NUM_TOKENS * PROMPT_FRACTION) - token_count)
|
||||
prompt.append({"role": "assistant", "content": last_response})
|
||||
token_count += len(ENCODER.encode(last_response))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Instruction prompt
|
||||
instruction_context_query_prompt = \
|
||||
main_prompt = \
|
||||
"Please give a clear and coherent answer to my question (written after \"Q:\") " \
|
||||
"using the following sources. Each source is labeled with a letter. Feel free to " \
|
||||
"use the sources in any order, and try to use multiple sources in your answer."
|
||||
"use the sources in any order, and try to use multiple sources in your answer.\n\n"
|
||||
|
||||
token_count = len(ENCODER.encode(main_prompt))
|
||||
|
||||
# Context from top-k blocks
|
||||
context_prompt = ""
|
||||
|
||||
for i, block in enumerate(context):
|
||||
context_prompt += f"[{chr(ord('a') + i)}] {block.title} - {block.author} - {block.date}\n\n{block.text}\n\n\n"
|
||||
context_prompt = context_prompt[:-2] # trim last two newlines
|
||||
context_prompt = limit_tokens(context_prompt, TRUNCATE_CONTEXT_LEN) # truncate the context_prompt to max TRUNCATE_CONTEXT tokens
|
||||
context_prompt += "\n" if (context_prompt[-1] != "\n") else ""
|
||||
block_str = f"[{chr(ord('a') + i)}] {block.title} - {block.author} - {block.date}\n{block.text}\n\n"
|
||||
block_tc = len(ENCODER.encode(block_str))
|
||||
|
||||
# Question prompt
|
||||
question_prompt = f"In your answer, please cite any claims you make back to each source " \
|
||||
if token_count + block_tc > int(NUM_TOKENS * CONTEXT_FRACTION):
|
||||
main_prompt += cap(block_str, int(NUM_TOKENS * CONTEXT_FRACTION) - token_count)
|
||||
break
|
||||
else:
|
||||
main_prompt += block_str
|
||||
token_count += block_tc
|
||||
|
||||
main_prompt = main_prompt.strip() + "\n\n\n"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
main_prompt += f"In your answer, please cite any claims you make back to each source " \
|
||||
f"using the format: [a], [b], etc. If you use multiple sources to make a claim " \
|
||||
f"cite all of them. For example: \"AGI is concerning [c, d, e].\"\n\nQ: " + query
|
||||
|
||||
instruction_context_query_prompt = f"{instruction_context_query_prompt}\n\n{context_prompt}\n\n{question_prompt}"
|
||||
prompt.append({"role": "user", "content": main_prompt})
|
||||
|
||||
total_tokens += len(enc.encode(past_user_queries))
|
||||
total_tokens += len(enc.encode(history[-2]["content"])) if (len(history) >= 2) else 0
|
||||
total_tokens += len(enc.encode(history[-1]["content"])) if (len(history) >= 1) else 0
|
||||
total_tokens += len(enc.encode(instruction_context_query_prompt))
|
||||
|
||||
# If the prompt is too long, truncate the last answer
|
||||
if total_tokens > MAX_TOKEN_LEN_PROMPT - TRUNCATE_HISTORY_LEN:
|
||||
tokens_left = MAX_TOKEN_LEN_PROMPT - total_tokens
|
||||
print(f"WARNING: Prompt is too long! Prompt length: {total_tokens} tokens")
|
||||
last_assistant_reply_trunctated = limit_tokens(prompt[-1]["content"], tokens_left)
|
||||
prompt[-1]["content"] = f"{last_assistant_reply_trunctated}"
|
||||
|
||||
prompt.append({"role": "system", "content": system_prompt})
|
||||
prompt.append({"role": "user", "content": past_user_queries})
|
||||
prompt.extend(history[-2:])
|
||||
prompt.append({"role": "user", "content": instruction_context_query_prompt})
|
||||
|
||||
return prompt, MAX_TOKEN_LEN_PROMPT - (total_tokens + 50) # add 50 tokens for safety
|
||||
return prompt
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
def normal_completion(prompt: List[Dict[str, str]], max_tokens_completion: int) -> str:
|
||||
try:
|
||||
return openai.ChatCompletion.create(
|
||||
model=COMPLETIONS_MODEL,
|
||||
messages=prompt,
|
||||
max_tokens=max_tokens_completion
|
||||
)["choices"][0]["message"]["content"]
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return "I'm sorry, I failed to process your query. Please try again. If the problem persists, please contact the administrator."
|
||||
|
||||
# returns either (True, reply string, embeddings) or (False, error message string, None)
|
||||
def talk_to_robot(dataset_dict, query: str, history: List[Dict[str, str]] = [], k: int = 10):
|
||||
def talk_to_robot(dataset_dict, query: str, history: List[Dict[str, str]], k: int = 10):
|
||||
|
||||
|
||||
# 1. Find the most relevant blocks from the Alignment Research Dataset
|
||||
top_k_blocks: List[Block] = get_top_k_blocks(dataset_dict, query, k)
|
||||
|
||||
# 2. Generate a prompt for the ChatCompletions API
|
||||
prompt, max_tokens_completion = construct_prompt(query, history, top_k_blocks)
|
||||
|
||||
# print(" ------------------------------ prompt: -----------------------------")
|
||||
# for message in prompt:
|
||||
# print(f"{message['role']}: {message['content']}\n\n")
|
||||
|
||||
# if we were to error out, return something like this
|
||||
# return (False, "Example error message", None)
|
||||
|
||||
# 3. Answer the user query
|
||||
return (True, normal_completion(prompt, max_tokens_completion), top_k_blocks)
|
||||
# 2. Generate a prompt
|
||||
prompt = construct_prompt(query, history, top_k_blocks)
|
||||
print('\n' * 10)
|
||||
print(" ------------------------------ prompt: -----------------------------")
|
||||
for message in prompt:
|
||||
print(f"----------- {message['role']}: ------------------")
|
||||
print(message['content'])
|
||||
|
||||
print('\n' * 10)
|
||||
|
||||
|
||||
|
||||
# 3. Count number of tokens left for completion (-50 for a buffer)
|
||||
max_tokens_completion = NUM_TOKENS - sum([len(ENCODER.encode(message["content"]) + ENCODER.encode(message["role"])) for message in prompt]) - 50
|
||||
|
||||
|
||||
# 4. Answer the user query
|
||||
try:
|
||||
return (True, openai.ChatCompletion.create(
|
||||
model=COMPLETIONS_MODEL,
|
||||
messages=prompt,
|
||||
max_tokens=max_tokens_completion
|
||||
)["choices"][0]["message"]["content"], top_k_blocks)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return (False, "Error: " + str(e), None)
|
||||
|
||||
|
||||
+2
-2
@@ -75,7 +75,7 @@ def get_top_k_blocks(data, user_query: str, k: int = 10) -> List[Block]:
|
||||
blocks = [Block(*block) for block in top_k_metadata_and_text]
|
||||
|
||||
# for all blocks that are "the same" (same title, author, date, url, tags),
|
||||
# combine their text with "\n\n.....\n\n" in between. Return them in order such
|
||||
# combine their text with "....." in between. Return them in order such
|
||||
# that the combined block has the minimum index of the blocks combined.
|
||||
|
||||
key = lambda bi: (bi[0].title or "", bi[0].author or "", bi[0].date or "", bi[0].url or "", bi[0].tags or "")
|
||||
@@ -91,7 +91,7 @@ def get_top_k_blocks(data, user_query: str, k: int = 10) -> List[Block]:
|
||||
|
||||
group = group[:3] # limit to a max of 3 blocks from any one source
|
||||
|
||||
text = "\n\n\n.....\n\n\n".join([block[0].text for block in group])
|
||||
text = "\n.....\n".join([block[0].text for block in group])
|
||||
|
||||
min_index = min([block[1] for block in group])
|
||||
|
||||
|
||||
+3
-1
@@ -48,9 +48,11 @@ def semantic():
|
||||
@app.route('/chat', methods=['POST'])
|
||||
@cross_origin()
|
||||
def chat():
|
||||
|
||||
query = request.json['query']
|
||||
history = request.json['history']
|
||||
|
||||
is_valid, response, context = talk_to_robot(dataset_dict, query)
|
||||
is_valid, response, context = talk_to_robot(dataset_dict, query, history)
|
||||
|
||||
if is_valid:
|
||||
return jsonify({'response': response, 'citations': [{'title': block.title, 'author': block.author, 'date': block.date, 'url': block.url} for block in context]})
|
||||
|
||||
Reference in New Issue
Block a user