diff --git a/web/src/components/assistant.tsx b/web/src/components/assistant.tsx new file mode 100644 index 0000000..8047d82 --- /dev/null +++ b/web/src/components/assistant.tsx @@ -0,0 +1,50 @@ +import { ProcessText, ShowCitation, ShowInTextCitation } from "./citations"; +import { GlossarySpan } from "./glossary"; +import type { Citation, AssistantEntry } from "../types"; + +export const ShowAssistantEntry: React.FC<{entry: AssistantEntry}> = ({entry}) => { + const in_text_citation_regex = /\[([0-9]+)\]/g; + + let [response, cite_map] = ProcessText(entry.content, entry.base_count); + + // ----------------- create the ordered citation array ----------------- + + const citations = new Map(); + cite_map.forEach((value, key) => { + const index = key.charCodeAt(0) - 'a'.charCodeAt(0); + if (index >= entry.citations.length) { + console.log("invalid citation index: " + index); + } else { + citations.set(value, entry.citations[index]!); + } + }); + + return ( +
+ { // split into paragraphs + response.split("\n").map(paragraph => (

{ + paragraph.split(in_text_citation_regex).map((text, i) => { + if (i % 2 === 0) { + return ; + } + i = parseInt(text) - 1; + if (!citations.has(i)) return `[${text}]`; + const citation = citations.get(i)!; + return ( + + ); + }) + }

)) + } + +
+ ); +}; diff --git a/web/src/components/citations.tsx b/web/src/components/citations.tsx new file mode 100644 index 0000000..2b08aa3 --- /dev/null +++ b/web/src/components/citations.tsx @@ -0,0 +1,103 @@ +import type { Citation } from "../types"; +import { Colours, A } from "./html"; + + +// todo: memoize this if too slow. +export const ProcessText: (text: string, base_count: number) => [string, Map] = (text, base_count) => { + + // ---------------------- normalize citation form ---------------------- + // the general plan here is just to add parsing cases until we can respond + // well to almost everything the LLM emits. We won't ever reach five nines, + // but the domain is one where occasionally failing isn't catastrophic. + + // transform all things that look like [a, b, c] into [a][b][c] + let response = text.replace( + + /\[((?:[a-z]+,\s*)*[a-z]+)\]/g, // identify groups of this form + + (block: string) => block.split(',') + .map((x) => x.trim()) + .join("][") + ) + + // transform all things that look like [(a), (b), (c)] into [(a)][(b)][(c)] + response = response.replace( + + /\[((?:\([a-z]+\),\s*)*\([a-z]+\))\]/g, // identify groups of this form + + (block: string) => block.split(',') + .map((x) => x.trim()) + .join("][") + ) + + // transform all things that look like [(a)] into [a] + response = response.replace( + /\[\(([a-z]+)\)\]/g, + (_match: string, x: string) => `[${x}]` + ) + + // transform all things that look like [ a ] into [a] + response = response.replace( + /\[\s*([a-z]+)\s*\]/g, + (_match: string, x: string) => `[${x}]` + ) + + // -------------- map citations from strings into numbers -------------- + + // figure out what citations are in the response, and map them appropriately + const cite_map = new Map(); + let cite_count = 0; + + // scan a regex for [x] over the response. If x isn't in the map, add it. + // (note: we're actually doing this twice - once on parsing, once on render. + // if that looks like a problem, we could swap from strings to custom ropes). + const regex = /\[([a-z]+)\]/g; + let match; + let response_copy = "" + while ((match = regex.exec(response)) !== null) { + if (!cite_map.has(match[1]!)) { + cite_map.set(match[1]!, base_count + cite_count++); + } + // replace [x] with [i] + response_copy += response.slice(response_copy.length, match.index) + `[${cite_map.get(match[1]!)! + 1}]`; + } + + response = response_copy + response.slice(response_copy.length); + + return [response, cite_map] +} + +export const ShowCitation: React.FC<{citation: Citation, i: number}> = ({citation, i}) => { + + var c_str = citation.title; + + if (citation.authors && citation.authors.length > 0) + c_str += " - " + citation.authors.join(', '); + if (citation.date && citation.date !== "") + c_str += " - " + citation.date; + + // if we don't have a url, link to a duckduckgo search for the title instead + const url = citation.url && citation.url !== "" + ? citation.url + : `https://duckduckgo.com/?q=${encodeURIComponent(citation.title)}`; + + return ( + + [{i + 1}] +

{c_str}

+ + ); +}; + +export const ShowInTextCitation: React.FC<{citation: Citation, i: number}> = ({citation, i}) => { + const url = citation.url && citation.url !== "" + ? citation.url + : `https://duckduckgo.com/?q=${encodeURIComponent(citation.title)}`; + return ( + + [{i + 1}] + + ); +}; diff --git a/web/src/components/controls.tsx b/web/src/components/controls.tsx new file mode 100644 index 0000000..d7fc17c --- /dev/null +++ b/web/src/components/controls.tsx @@ -0,0 +1,57 @@ +export type Mode = "rookie" | "concise" | "default"; + +export const Controls = ({ + mode, + setMode, +}: { + mode: [Mode, boolean]; + setMode: (m: any) => void; +}) => { + { + /* three buttons for the three modes, place far right, 1rem between each */ + } + return ( +
+ + + +
+ ); +}; diff --git a/web/src/glossary.tsx b/web/src/components/glossary.tsx similarity index 100% rename from web/src/glossary.tsx rename to web/src/components/glossary.tsx diff --git a/web/src/header.tsx b/web/src/components/header.tsx similarity index 95% rename from web/src/header.tsx rename to web/src/components/header.tsx index 768b3a6..79c6196 100644 --- a/web/src/header.tsx +++ b/web/src/components/header.tsx @@ -1,7 +1,7 @@ import React from "react"; import Link from "next/link"; import Image from 'next/image'; -import logo from "./logo.svg" +import logo from "../logo.svg" const Header: React.FC<{page: "index" | "semantic"}> = ({page}) => { const sidebar = page === "index" ? ( diff --git a/web/src/components/html.tsx b/web/src/components/html.tsx new file mode 100644 index 0000000..4bc5b01 --- /dev/null +++ b/web/src/components/html.tsx @@ -0,0 +1,31 @@ +// const Colours = ["blue", "cyan", "teal", "green", "amber"].map( +// colour => `bg-${colour}-100 border-${colour}-300 text-${colour}-800` +// ); +// this would be nice, but Tailwind needs te actual string of the class to be in +// the source file for it to be included in the build + +export const Colours = [ + "bg-red-100 border-red-300 text-red-800", + "bg-amber-100 border-amber-300 text-amber-800", + "bg-orange-100 border-orange-300 text-orange-800", + "bg-lime-100 border-lime-300 text-lime-800", + "bg-green-100 border-green-300 text-green-800", + "bg-cyan-100 border-cyan-300 text-cyan-800", + "bg-blue-100 border-blue-300 text-blue-800", + "bg-violet-100 border-violet-300 text-violet-800", + "bg-pink-100 border-pink-300 text-pink-800", +]; + + +export const A: React.FC<{href: string, className?: string, children: React.ReactNode}> = ({href, className, children}) => { + // link element that only populates the href field if the contents are there + return href && href !== "" ? ( + + {children} + + ) : ( + + {children} + + ); +} diff --git a/web/src/searchbox.tsx b/web/src/components/searchbox.tsx similarity index 97% rename from web/src/searchbox.tsx rename to web/src/components/searchbox.tsx index 9af7b35..4b3c433 100644 --- a/web/src/searchbox.tsx +++ b/web/src/components/searchbox.tsx @@ -1,13 +1,9 @@ import React from "react"; import { useState, useEffect } from "react"; +import type { Followup } from "../types"; import TextareaAutosize from 'react-textarea-autosize'; import dynamic from 'next/dynamic' - -export type Followup = { - text: string; - pageid: string; - score: number; -} +import type { Followup } from "../types"; // initial questions to fill the search box with. export const initialQuestions: string[] = [ @@ -120,4 +116,3 @@ const SearchBoxInternal: React.FC<{search: ( export const SearchBox = dynamic(() => Promise.resolve(SearchBoxInternal), { ssr: false, }); - diff --git a/web/src/pages/_app.tsx b/web/src/pages/_app.tsx index 6e1fa20..792401c 100644 --- a/web/src/pages/_app.tsx +++ b/web/src/pages/_app.tsx @@ -3,7 +3,7 @@ import { useEffect, useState } from "react"; import "~/styles/globals.css"; -import { Glossary, GlossaryContext } from "../glossary"; +import { Glossary, GlossaryContext } from "../components/glossary"; const MyApp: AppType = ({ Component, pageProps }) => { const [glossary, setGlossary] = useState<{ g: Glossary, r: RegExp } | null>(null); diff --git a/web/src/pages/index.tsx b/web/src/pages/index.tsx index d3a37f2..2a1699a 100644 --- a/web/src/pages/index.tsx +++ b/web/src/pages/index.tsx @@ -5,235 +5,18 @@ import { useState, useEffect } from "react"; import Link from "next/link"; import Image from 'next/image'; -import Header from "../header"; -import { SearchBox, Followup } from "../searchbox"; +import Header from "../components/header"; import logo from "../logo.svg" -import { GlossarySpan } from "../glossary"; import { API_URL } from "../settings" - -type Citation = { - title: string; - authors: string[]; - date: string; - url: string; -} - - -type Entry = UserEntry | AssistantEntry | ErrorMessage | StampyMessage; - -type UserEntry = { - role: "user"; - content: string; -} - -type AssistantEntry = { - role: "assistant"; - content: string; - citations: Citation[]; - base_count: number; // the number to start counting citations at -} - -type ErrorMessage = { - role: "error"; - content: string; -} - -type StampyMessage = { - role: "stampy"; - content: string; - url: string; -} +import type { Citation, Entry, UserEntry, AssistantEntry, ErrorMessage, StampyMessage } from "../types"; +import { SearchBox, Followup } from "../components/searchbox"; +import { GlossarySpan } from "../components/glossary"; +import { Controls, Mode } from "../components/controls"; +import { ShowAssistantEntry } from "../components/assistant"; +import { ProcessText } from "../components/citations"; const MAX_FOLLOWUPS = 4; -// const Colours = ["blue", "cyan", "teal", "green", "amber"].map( -// colour => `bg-${colour}-100 border-${colour}-300 text-${colour}-800` -// ); -// this would be nice, but Tailwind needs te actual string of the class to be in -// the source file for it to be included in the build - -const Colours = [ - "bg-red-100 border-red-300 text-red-800", - "bg-amber-100 border-amber-300 text-amber-800", - "bg-orange-100 border-orange-300 text-orange-800", - "bg-lime-100 border-lime-300 text-lime-800", - "bg-green-100 border-green-300 text-green-800", - "bg-cyan-100 border-cyan-300 text-cyan-800", - "bg-blue-100 border-blue-300 text-blue-800", - "bg-violet-100 border-violet-300 text-violet-800", - "bg-pink-100 border-pink-300 text-pink-800", -]; - -const ShowCitation: React.FC<{citation: Citation, i: number}> = ({citation, i}) => { - - var c_str = citation.title; - - if (citation.authors && citation.authors.length > 0) - c_str += " - " + citation.authors.join(', '); - if (citation.date && citation.date !== "") - c_str += " - " + citation.date; - - // if we don't have a url, link to a duckduckgo search for the title instead - const url = citation.url && citation.url !== "" - ? citation.url - : `https://duckduckgo.com/?q=${encodeURIComponent(citation.title)}`; - - return ( - - [{i + 1}] -

{c_str}

- - ); -}; - -const ShowInTextCitation: React.FC<{citation: Citation, i: number}> = ({citation, i}) => { - const url = citation.url && citation.url !== "" - ? citation.url - : `https://duckduckgo.com/?q=${encodeURIComponent(citation.title)}`; - return ( - - [{i + 1}] - - ); -}; - -const A: React.FC<{href: string, className?: string, children: React.ReactNode}> = ({href, className, children}) => { - // link element that only populates the href field if the contents are there - return href && href !== "" ? ( - - {children} - - ) : ( - - {children} - - ); -} - - - - - -// todo: memoize this if too slow. -const ProcessText: (text: string, base_count: number) => [string, Map] = (text, base_count) => { - - // ---------------------- normalize citation form ---------------------- - // the general plan here is just to add parsing cases until we can respond - // well to almost everything the LLM emits. We won't ever reach five nines, - // but the domain is one where occasionally failing isn't catastrophic. - - // transform all things that look like [a, b, c] into [a][b][c] - let response = text.replace( - - /\[((?:[a-z]+,\s*)*[a-z]+)\]/g, // identify groups of this form - - (block: string) => block.split(',') - .map((x) => x.trim()) - .join("][") - ) - - // transform all things that look like [(a), (b), (c)] into [(a)][(b)][(c)] - response = response.replace( - - /\[((?:\([a-z]+\),\s*)*\([a-z]+\))\]/g, // identify groups of this form - - (block: string) => block.split(',') - .map((x) => x.trim()) - .join("][") - ) - - // transform all things that look like [(a)] into [a] - response = response.replace( - /\[\(([a-z]+)\)\]/g, - (_match: string, x: string) => `[${x}]` - ) - - // transform all things that look like [ a ] into [a] - response = response.replace( - /\[\s*([a-z]+)\s*\]/g, - (_match: string, x: string) => `[${x}]` - ) - - // -------------- map citations from strings into numbers -------------- - - // figure out what citations are in the response, and map them appropriately - const cite_map = new Map(); - let cite_count = 0; - - // scan a regex for [x] over the response. If x isn't in the map, add it. - // (note: we're actually doing this twice - once on parsing, once on render. - // if that looks like a problem, we could swap from strings to custom ropes). - const regex = /\[([a-z]+)\]/g; - let match; - let response_copy = "" - while ((match = regex.exec(response)) !== null) { - if (!cite_map.has(match[1]!)) { - cite_map.set(match[1]!, base_count + cite_count++); - } - // replace [x] with [i] - response_copy += response.slice(response_copy.length, match.index) + `[${cite_map.get(match[1]!)! + 1}]`; - } - - response = response_copy + response.slice(response_copy.length); - - return [response, cite_map] -} - - -const ShowAssistantEntry: React.FC<{entry: AssistantEntry}> = ({entry}) => { - const in_text_citation_regex = /\[([0-9]+)\]/g; - - let [response, cite_map] = ProcessText(entry.content, entry.base_count); - - // ----------------- create the ordered citation array ----------------- - - const citations = new Map(); - cite_map.forEach((value, key) => { - const index = key.charCodeAt(0) - 'a'.charCodeAt(0); - if (index >= entry.citations.length) { - console.log("invalid citation index: " + index); - } else { - citations.set(value, entry.citations[index]!); - } - }); - - return ( -
- { // split into paragraphs - response.split("\n").map(paragraph => (

{ - paragraph.split(in_text_citation_regex).map((text, i) => { - if (i % 2 === 0) { - return ; - } - i = parseInt(text) - 1; - if (!citations.has(i)) return `[${text}]`; - const citation = citations.get(i)!; - return ( - - ); - }) - }

)) - } -
    - { // show citations - Array.from(citations.entries()).map(([i, citation]) => ( -
  • - -
    • - )) - } -
-
- ); -}; - - - - - - type State = { state: "idle"; } | { @@ -499,34 +282,7 @@ const Home: NextPage = () => {
- {/* three buttons for the three modes, place far right, 1rem between each */} -
- - // - - // - -
- +

WARNING: This is a very early prototype. Feedback welcomed.

diff --git a/web/src/pages/semantic.tsx b/web/src/pages/semantic.tsx index 23c6f1c..755399b 100644 --- a/web/src/pages/semantic.tsx +++ b/web/src/pages/semantic.tsx @@ -1,8 +1,8 @@ import { type NextPage } from "next"; import React from "react"; import Head from "next/head"; -import Header from "../header"; -import { SearchBox, Followup } from "../searchbox"; +import Header from "../components/header"; +import { SearchBox, Followup } from "../components/searchbox"; import { useState } from "react"; import { API_URL } from "../settings" diff --git a/web/src/types.ts b/web/src/types.ts new file mode 100644 index 0000000..98ecaa6 --- /dev/null +++ b/web/src/types.ts @@ -0,0 +1,37 @@ +export type Citation = { + title: string; + authors: string[]; + date: string; + url: string; +} + +export type Followup = { + text: string; + pageid: string; + score: number; +} + +export type Entry = UserEntry | AssistantEntry | ErrorMessage | StampyMessage; + +export type UserEntry = { + role: "user"; + content: string; +} + +export type AssistantEntry = { + role: "assistant"; + content: string; + citations: Citation[]; + base_count: number; // the number to start counting citations at +} + +export type ErrorMessage = { + role: "error"; + content: string; +} + +export type StampyMessage = { + role: "stampy"; + content: string; + url: string; +}