wassname/stampy-chat
1
0
Fork 0
mirror of https://github.com/wassname/stampy-chat.git synced 2026-09-09 11:36:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #108 from StampyAI/fixes

Browse Source 
Fixes
This commit is contained in:
Daniel O'Connell
2023-09-30 19:02:29 +02:00
committed by GitHub
parent d091bdcc50 42108e8dfc
commit bd8da81f9e
7 changed files with 74 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/src/stampy_chat/chat.py
+7 -4
View File
@@ -226,11 +226,14 @@ def talk_to_robot_internal(index, query: str, mode: str, history: Prompt, sessio
logger.interaction(session_id, query, response, history, prompt, top_k_blocks)
# yield done state, possibly with followup questions
fin_json = {'state': 'done'}
yield {"state": "loading", "phase": "followups"}
# yield optional followups
followups = multisearch_authored([query, response])
for i, followup in enumerate(followups):
fin_json[f'followup_{i}'] = asdict(followup)
if followups:
yield {'state': 'followups', 'followups': list(map(asdict, followups))}
# yield done state
fin_json = {'state': 'done'}
yield fin_json
except Exception as e:
api/src/stampy_chat/get_blocks.py
+17 -12
View File
@@ -1,13 +1,12 @@
import dataclasses
import datetime
import itertools
import numpy as np
import openai
import regex as re
import requests
import time
from itertools import groupby
from typing import Iterable, List, Tuple
from typing import Iterable, List
from stampy_chat.env import PINECONE_NAMESPACE, REMOTE_CHAT_INSTANCE, EMBEDDING_MODEL
from stampy_chat import logging
@@ -54,12 +53,12 @@ def parse_block(match) -> Block:
date = metadata.get('date_published') or metadata.get('date')
if isinstance(date, datetime.date):
date = date.isoformat()
elif isinstance(date, datetime.datetime):
if isinstance(date, datetime.datetime):
date = date.date().isoformat()
elif isinstance(date, datetime.date):
date = date.isoformat()
elif isinstance(date, (int, float)):
date = datetime.datetime.fromtimestamp(date).isoformat()
date = datetime.datetime.fromtimestamp(date).date().isoformat()
authors = metadata.get('authors')
if not authors and metadata.get('author'):
@@ -82,16 +81,22 @@ def join_blocks(blocks: Iterable[Block]) -> List[Block]:
# that the combined block has the minimum index of the blocks combined.
def to_tuple(block):
return (block.id, block.title or "", block.authors or [], block.date or "", block.url or "", block.tags or "")
return (block.title or "", block.authors or [], block.date or "", block.url or "", block.tags or "")
def merge_texts(blocks):
return "\n.....\n".join(sorted(block.text for block in blocks))
unified_blocks = [
Block(*key, merge_texts(group))
for key, group in groupby(blocks, key=to_tuple)
]
return sorted(unified_blocks, key=to_tuple)
# There are sometimes duplicates in the dataset, but which have different ids, so the id
# is ignored when making sorting the blocks.
def make_block(key, group):
group = list(group)
# Just use the id of the first item - it doesn't matter that much in this case, as the other data points
# will be the same
return Block(group[0].id, *key, merge_texts(group))
blocks = sorted(blocks, key=to_tuple)
blocks = [make_block(key, group) for key, group in groupby(blocks, key=to_tuple)]
return blocks
# Get the k blocks most semantically similar to the query using Pinecone.
api/tests/stampy_chat/test_chat.py
+9 -8
View File
@@ -268,7 +268,7 @@ def test_talk_to_robot_internal(history, context):
with patch('stampy_chat.chat.get_top_k_blocks', return_value=context):
with patch('stampy_chat.chat.multisearch_authored', return_value=followups):
with patch('openai.ChatCompletion.create', return_value=chunks):
assert list(talk_to_robot_internal("index", "what is this about?", "default", history)) == [
assert list(talk_to_robot_internal("index", "what is this about?", "default", history, 'session id')) == [
{'phase': 'semantic', 'state': 'loading'},
{'citations': [], 'phase': 'semantic', 'state': 'loading'},
{'phase': 'prompt', 'state': 'loading'},
@@ -277,12 +277,13 @@ def test_talk_to_robot_internal(history, context):
{'content': 'response 2', 'state': 'streaming'},
{'content': 'response 3', 'state': 'streaming'},
{'content': 'response 4', 'state': 'streaming'},
{
'followup_0': {'pageid': '1', 'score': 0.231, 'text': 'followup 1'},
'followup_1': {'pageid': '2', 'score': 0.231, 'text': 'followup 2'},
'followup_2': {'pageid': '3', 'score': 0.231, 'text': 'followup 3'},
'state': 'done'
},
{'state': 'loading', 'phase': 'followups'},
{'state': 'followups', 'followups': [
{'pageid': '1', 'score': 0.231, 'text': 'followup 1'},
{'pageid': '2', 'score': 0.231, 'text': 'followup 2'},
{'pageid': '3', 'score': 0.231, 'text': 'followup 3'},
]},
{'state': 'done'},
]
@@ -297,7 +298,7 @@ def test_talk_to_robot_internal_error(history, context):
]
with patch('stampy_chat.chat.get_top_k_blocks', return_value=context):
with patch('openai.ChatCompletion.create', return_value=chunks):
assert list(talk_to_robot_internal("index", "what is this about?", "default", history)) == [
assert list(talk_to_robot_internal("index", "what is this about?", "default", history, 'session id')) == [
{'phase': 'semantic', 'state': 'loading'},
{'citations': [], 'phase': 'semantic', 'state': 'loading'},
{'phase': 'prompt', 'state': 'loading'},
api/tests/stampy_chat/test_get_blocks.py
+23 -9
View File
@@ -8,22 +8,22 @@ from stampy_chat.get_blocks import Block, get_top_k_blocks, parse_block, join_bl
({}, {}),
# Check dates
({'date_published': '2023-01-02T03:04:05'}, {'date': '2023-01-02T03:04:05'}),
({'date_published': '2023-01-01T03:04:05'}, {'date': '2023-01-01T03:04:05'}),
(
{'date_published': datetime.fromisoformat('2023-01-02T03:04:05')},
{'date': '2023-01-02T03:04:05'}
),
(
{'date_published': datetime.fromisoformat('2023-01-02T03:04:05').date()},
{'date': '2023-01-02'}
),
(
{'date_published': datetime.fromisoformat('2023-01-02T03:04:05').timestamp()},
{'date': '2023-01-02T03:04:05'}
{'date_published': datetime.fromisoformat('2023-01-03T03:04:05').date()},
{'date': '2023-01-03'}
),
(
{'date_published': int(datetime.fromisoformat('2023-01-02T03:04:05').timestamp())},
{'date': '2023-01-02T03:04:05'}
{'date_published': datetime.fromisoformat('2023-01-04T03:04:05').timestamp()},
{'date': '2023-01-04'}
),
(
{'date_published': int(datetime.fromisoformat('2023-01-05T03:04:05').timestamp())},
{'date': '2023-01-05'}
),
# Check authors
@@ -84,6 +84,20 @@ def test_parse_block(match_override, block_override):
Block('id3', 'title3', ['author3'], 'date3', 'url3', 'tags3', 'text3'),
]
),
(
[
Block('id1', 'title1', ['author1'], 'date1', 'url1', 'tags1', 'text1-1'),
Block('id3', 'title3', ['author3'], 'date3', 'url3', 'tags3', 'text3'),
Block('id1', 'title1', ['author1'], 'date1', 'url1', 'tags1', 'text1-2'),
Block('id2', 'title2', ['author2'], 'date2', 'url2', 'tags2', 'text2'),
Block('id1', 'title1', ['author1'], 'date1', 'url1', 'tags1', 'text1-3'),
],
[
Block('id1', 'title1', ['author1'], 'date1', 'url1', 'tags1', 'text1-1\n.....\ntext1-2\n.....\ntext1-3'),
Block('id2', 'title2', ['author2'], 'date2', 'url2', 'tags2', 'text2'),
Block('id3', 'title3', ['author3'], 'date3', 'url3', 'tags3', 'text3'),
]
),
])
def test_join_blocks(blocks, expected):
assert list(join_blocks(blocks)) == expected
web/src/components/entry.tsx
+3 -5
View File
@@ -9,14 +9,12 @@ import { ShowAssistantEntry } from "./assistant";
import { GlossarySpan } from "./glossary";
import Image from "next/image";
import logo from "../logo.svg";
import TextareaAutosize from 'react-textarea-autosize';
export const User = ({ entry }: { entry: UserEntry }) => {
return (
<li>
<p className="border border-gray-300 px-1 text-right">
{" "}
{entry.content}{" "}
</p>
<li className="flex mt-1 mb-2">
<TextareaAutosize className="border border-gray-300 px-1 flex-1 resize-none" value={entry.content} />
</li>
);
};
web/src/hooks/useSearch.ts
+9 -9
View File
@@ -12,7 +12,7 @@ import type {
const MAX_FOLLOWUPS = 4;
const DATA_HEADER = "data: "
const EVENT_END_HEADER = "event: close\n"
const EVENT_END_HEADER = "event: close"
type HistoryEntry = {
role: "error" | "stampy" | "assistant" | "user";
@@ -29,13 +29,13 @@ export async function* iterateData(res: Response) {
if (done) return;
const chunk = new TextDecoder("utf-8").decode(value);
if (chunk.startsWith(EVENT_END_HEADER)) return;
for (const line of chunk.split("\n")) {
// Most times, it seems that a single read() call will be one SSE "message",
// but I'll do the proper aggregation spec thing in case that's not always true.
if (line.startsWith(DATA_HEADER)) {
if (line.startsWith(EVENT_END_HEADER)) {
return;
} else if (line.startsWith(DATA_HEADER)) {
message += line.slice(DATA_HEADER.length);
// Fixes #43
} else if (line !== "") {
@@ -83,12 +83,12 @@ export const extractAnswer = async (
setCurrent({ phase: "streaming", ...result });
break;
case "followups":
// add any potential followup questions
followups = data.followups.map((value) => value as Followup);
break;
case "done":
// add any potential followup questions
const followups = Object.entries(data)
.filter(([key]) => key.startsWith("followup_"))
.map(([k, value]) => value as Followup);
return { result, followups };
break;
case "error":
throw data.error;
}
web/src/pages/index.tsx
+6
View File
@@ -118,6 +118,12 @@ const Home: NextPage = () => {
case "streaming":
last_entry = <ShowAssistantEntry entry={current} />;
break;
case "followups":
last_entry = <>
<ShowAssistantEntry entry={current} />
<p>Checking for followups...</p>
</>;
break;
}
return (