diff --git a/api/src/stampy_chat/chat.py b/api/src/stampy_chat/chat.py
index 3a417e6..96e67db 100644
--- a/api/src/stampy_chat/chat.py
+++ b/api/src/stampy_chat/chat.py
@@ -226,11 +226,14 @@ def talk_to_robot_internal(index, query: str, mode: str, history: Prompt, sessio
logger.interaction(session_id, query, response, history, prompt, top_k_blocks)
- # yield done state, possibly with followup questions
- fin_json = {'state': 'done'}
+ yield {"state": "loading", "phase": "followups"}
+ # yield optional followups
followups = multisearch_authored([query, response])
- for i, followup in enumerate(followups):
- fin_json[f'followup_{i}'] = asdict(followup)
+ if followups:
+ yield {'state': 'followups', 'followups': list(map(asdict, followups))}
+
+ # yield done state
+ fin_json = {'state': 'done'}
yield fin_json
except Exception as e:
diff --git a/api/src/stampy_chat/get_blocks.py b/api/src/stampy_chat/get_blocks.py
index 91fe738..9b59956 100644
--- a/api/src/stampy_chat/get_blocks.py
+++ b/api/src/stampy_chat/get_blocks.py
@@ -1,13 +1,12 @@
import dataclasses
import datetime
-import itertools
import numpy as np
import openai
import regex as re
import requests
import time
from itertools import groupby
-from typing import Iterable, List, Tuple
+from typing import Iterable, List
from stampy_chat.env import PINECONE_NAMESPACE, REMOTE_CHAT_INSTANCE, EMBEDDING_MODEL
from stampy_chat import logging
@@ -54,12 +53,12 @@ def parse_block(match) -> Block:
date = metadata.get('date_published') or metadata.get('date')
- if isinstance(date, datetime.date):
- date = date.isoformat()
- elif isinstance(date, datetime.datetime):
+ if isinstance(date, datetime.datetime):
date = date.date().isoformat()
+ elif isinstance(date, datetime.date):
+ date = date.isoformat()
elif isinstance(date, (int, float)):
- date = datetime.datetime.fromtimestamp(date).isoformat()
+ date = datetime.datetime.fromtimestamp(date).date().isoformat()
authors = metadata.get('authors')
if not authors and metadata.get('author'):
@@ -82,16 +81,22 @@ def join_blocks(blocks: Iterable[Block]) -> List[Block]:
# that the combined block has the minimum index of the blocks combined.
def to_tuple(block):
- return (block.id, block.title or "", block.authors or [], block.date or "", block.url or "", block.tags or "")
+ return (block.title or "", block.authors or [], block.date or "", block.url or "", block.tags or "")
def merge_texts(blocks):
return "\n.....\n".join(sorted(block.text for block in blocks))
- unified_blocks = [
- Block(*key, merge_texts(group))
- for key, group in groupby(blocks, key=to_tuple)
- ]
- return sorted(unified_blocks, key=to_tuple)
+ # There are sometimes duplicates in the dataset, but which have different ids, so the id
+ # is ignored when making sorting the blocks.
+ def make_block(key, group):
+ group = list(group)
+ # Just use the id of the first item - it doesn't matter that much in this case, as the other data points
+ # will be the same
+ return Block(group[0].id, *key, merge_texts(group))
+
+ blocks = sorted(blocks, key=to_tuple)
+ blocks = [make_block(key, group) for key, group in groupby(blocks, key=to_tuple)]
+ return blocks
# Get the k blocks most semantically similar to the query using Pinecone.
diff --git a/api/tests/stampy_chat/test_chat.py b/api/tests/stampy_chat/test_chat.py
index 38f0a32..b1b64f5 100644
--- a/api/tests/stampy_chat/test_chat.py
+++ b/api/tests/stampy_chat/test_chat.py
@@ -268,7 +268,7 @@ def test_talk_to_robot_internal(history, context):
with patch('stampy_chat.chat.get_top_k_blocks', return_value=context):
with patch('stampy_chat.chat.multisearch_authored', return_value=followups):
with patch('openai.ChatCompletion.create', return_value=chunks):
- assert list(talk_to_robot_internal("index", "what is this about?", "default", history)) == [
+ assert list(talk_to_robot_internal("index", "what is this about?", "default", history, 'session id')) == [
{'phase': 'semantic', 'state': 'loading'},
{'citations': [], 'phase': 'semantic', 'state': 'loading'},
{'phase': 'prompt', 'state': 'loading'},
@@ -277,12 +277,13 @@ def test_talk_to_robot_internal(history, context):
{'content': 'response 2', 'state': 'streaming'},
{'content': 'response 3', 'state': 'streaming'},
{'content': 'response 4', 'state': 'streaming'},
- {
- 'followup_0': {'pageid': '1', 'score': 0.231, 'text': 'followup 1'},
- 'followup_1': {'pageid': '2', 'score': 0.231, 'text': 'followup 2'},
- 'followup_2': {'pageid': '3', 'score': 0.231, 'text': 'followup 3'},
- 'state': 'done'
- },
+ {'state': 'loading', 'phase': 'followups'},
+ {'state': 'followups', 'followups': [
+ {'pageid': '1', 'score': 0.231, 'text': 'followup 1'},
+ {'pageid': '2', 'score': 0.231, 'text': 'followup 2'},
+ {'pageid': '3', 'score': 0.231, 'text': 'followup 3'},
+ ]},
+ {'state': 'done'},
]
@@ -297,7 +298,7 @@ def test_talk_to_robot_internal_error(history, context):
]
with patch('stampy_chat.chat.get_top_k_blocks', return_value=context):
with patch('openai.ChatCompletion.create', return_value=chunks):
- assert list(talk_to_robot_internal("index", "what is this about?", "default", history)) == [
+ assert list(talk_to_robot_internal("index", "what is this about?", "default", history, 'session id')) == [
{'phase': 'semantic', 'state': 'loading'},
{'citations': [], 'phase': 'semantic', 'state': 'loading'},
{'phase': 'prompt', 'state': 'loading'},
diff --git a/api/tests/stampy_chat/test_get_blocks.py b/api/tests/stampy_chat/test_get_blocks.py
index 842f60e..4dbc917 100644
--- a/api/tests/stampy_chat/test_get_blocks.py
+++ b/api/tests/stampy_chat/test_get_blocks.py
@@ -8,22 +8,22 @@ from stampy_chat.get_blocks import Block, get_top_k_blocks, parse_block, join_bl
({}, {}),
# Check dates
- ({'date_published': '2023-01-02T03:04:05'}, {'date': '2023-01-02T03:04:05'}),
+ ({'date_published': '2023-01-01T03:04:05'}, {'date': '2023-01-01T03:04:05'}),
(
{'date_published': datetime.fromisoformat('2023-01-02T03:04:05')},
- {'date': '2023-01-02T03:04:05'}
- ),
- (
- {'date_published': datetime.fromisoformat('2023-01-02T03:04:05').date()},
{'date': '2023-01-02'}
),
(
- {'date_published': datetime.fromisoformat('2023-01-02T03:04:05').timestamp()},
- {'date': '2023-01-02T03:04:05'}
+ {'date_published': datetime.fromisoformat('2023-01-03T03:04:05').date()},
+ {'date': '2023-01-03'}
),
(
- {'date_published': int(datetime.fromisoformat('2023-01-02T03:04:05').timestamp())},
- {'date': '2023-01-02T03:04:05'}
+ {'date_published': datetime.fromisoformat('2023-01-04T03:04:05').timestamp()},
+ {'date': '2023-01-04'}
+ ),
+ (
+ {'date_published': int(datetime.fromisoformat('2023-01-05T03:04:05').timestamp())},
+ {'date': '2023-01-05'}
),
# Check authors
@@ -84,6 +84,20 @@ def test_parse_block(match_override, block_override):
Block('id3', 'title3', ['author3'], 'date3', 'url3', 'tags3', 'text3'),
]
),
+ (
+ [
+ Block('id1', 'title1', ['author1'], 'date1', 'url1', 'tags1', 'text1-1'),
+ Block('id3', 'title3', ['author3'], 'date3', 'url3', 'tags3', 'text3'),
+ Block('id1', 'title1', ['author1'], 'date1', 'url1', 'tags1', 'text1-2'),
+ Block('id2', 'title2', ['author2'], 'date2', 'url2', 'tags2', 'text2'),
+ Block('id1', 'title1', ['author1'], 'date1', 'url1', 'tags1', 'text1-3'),
+ ],
+ [
+ Block('id1', 'title1', ['author1'], 'date1', 'url1', 'tags1', 'text1-1\n.....\ntext1-2\n.....\ntext1-3'),
+ Block('id2', 'title2', ['author2'], 'date2', 'url2', 'tags2', 'text2'),
+ Block('id3', 'title3', ['author3'], 'date3', 'url3', 'tags3', 'text3'),
+ ]
+ ),
])
def test_join_blocks(blocks, expected):
assert list(join_blocks(blocks)) == expected
diff --git a/web/src/components/entry.tsx b/web/src/components/entry.tsx
index d9316b6..10e8eb3 100644
--- a/web/src/components/entry.tsx
+++ b/web/src/components/entry.tsx
@@ -9,14 +9,12 @@ import { ShowAssistantEntry } from "./assistant";
import { GlossarySpan } from "./glossary";
import Image from "next/image";
import logo from "../logo.svg";
+import TextareaAutosize from 'react-textarea-autosize';
export const User = ({ entry }: { entry: UserEntry }) => {
return (
-
-
- {" "}
- {entry.content}{" "}
-
+
+
);
};
diff --git a/web/src/hooks/useSearch.ts b/web/src/hooks/useSearch.ts
index 44e48ee..919d62d 100644
--- a/web/src/hooks/useSearch.ts
+++ b/web/src/hooks/useSearch.ts
@@ -12,7 +12,7 @@ import type {
const MAX_FOLLOWUPS = 4;
const DATA_HEADER = "data: "
-const EVENT_END_HEADER = "event: close\n"
+const EVENT_END_HEADER = "event: close"
type HistoryEntry = {
role: "error" | "stampy" | "assistant" | "user";
@@ -29,13 +29,13 @@ export async function* iterateData(res: Response) {
if (done) return;
const chunk = new TextDecoder("utf-8").decode(value);
- if (chunk.startsWith(EVENT_END_HEADER)) return;
-
for (const line of chunk.split("\n")) {
// Most times, it seems that a single read() call will be one SSE "message",
// but I'll do the proper aggregation spec thing in case that's not always true.
- if (line.startsWith(DATA_HEADER)) {
+ if (line.startsWith(EVENT_END_HEADER)) {
+ return;
+ } else if (line.startsWith(DATA_HEADER)) {
message += line.slice(DATA_HEADER.length);
// Fixes #43
} else if (line !== "") {
@@ -83,12 +83,12 @@ export const extractAnswer = async (
setCurrent({ phase: "streaming", ...result });
break;
+ case "followups":
+ // add any potential followup questions
+ followups = data.followups.map((value) => value as Followup);
+ break;
case "done":
- // add any potential followup questions
- const followups = Object.entries(data)
- .filter(([key]) => key.startsWith("followup_"))
- .map(([k, value]) => value as Followup);
- return { result, followups };
+ break;
case "error":
throw data.error;
}
diff --git a/web/src/pages/index.tsx b/web/src/pages/index.tsx
index 5cb65ae..8e01623 100644
--- a/web/src/pages/index.tsx
+++ b/web/src/pages/index.tsx
@@ -118,6 +118,12 @@ const Home: NextPage = () => {
case "streaming":
last_entry = ;
break;
+ case "followups":
+ last_entry = <>
+
+ Checking for followups...
+ ;
+ break;
}
return (