From d0bc72076121d58f8e2605f42e69904465a941ba Mon Sep 17 00:00:00 2001 From: jojopirker Date: Thu, 5 Jan 2023 12:17:45 +0100 Subject: [PATCH 01/49] inital try for navigation messages trees should be redone probably in a recursive way... --- .../components/Messages/MessageTableEntry.tsx | 12 ++- .../src/pages/api/messages/[id]/children.ts | 27 +++++++ .../pages/api/messages/[id]/conversation.ts | 27 +++++++ website/src/pages/api/messages/[id]/index.ts | 27 +++++++ website/src/pages/api/messages/[id]/parent.ts | 48 +++++++++++ .../api/messages/{index.tsx => index.ts} | 0 .../pages/api/messages/{user.tsx => user.ts} | 0 website/src/pages/messages/[id]/index.tsx | 80 +++++++++++++++++++ 8 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 website/src/pages/api/messages/[id]/children.ts create mode 100644 website/src/pages/api/messages/[id]/conversation.ts create mode 100644 website/src/pages/api/messages/[id]/index.ts create mode 100644 website/src/pages/api/messages/[id]/parent.ts rename website/src/pages/api/messages/{index.tsx => index.ts} (100%) rename website/src/pages/api/messages/{user.tsx => user.ts} (100%) create mode 100644 website/src/pages/messages/[id]/index.tsx diff --git a/website/src/components/Messages/MessageTableEntry.tsx b/website/src/components/Messages/MessageTableEntry.tsx index 2bc17201..d272b960 100644 --- a/website/src/components/Messages/MessageTableEntry.tsx +++ b/website/src/components/Messages/MessageTableEntry.tsx @@ -3,7 +3,17 @@ import { boolean } from "boolean"; import NextLink from "next/link"; import { FlaggableElement } from "../FlaggableElement"; -export function MessageTableEntry({ item, idx }) { +interface Message { + text: string; + id: string; + is_assistant: boolean; +} +interface MessageTableEntryProps { + item: Message; + idx: number; +} +export function MessageTableEntry(props: MessageTableEntryProps) { + const { item, idx } = props; const bgColor = useColorModeValue(idx % 2 === 0 ? "bg-slate-800" : "bg-black", "bg-sky-900"); return ( diff --git a/website/src/pages/api/messages/[id]/children.ts b/website/src/pages/api/messages/[id]/children.ts new file mode 100644 index 00000000..9c8fb84a --- /dev/null +++ b/website/src/pages/api/messages/[id]/children.ts @@ -0,0 +1,27 @@ +import { getToken } from "next-auth/jwt"; + +const handler = async (req, res) => { + const token = await getToken({ req }); + + // Return nothing if the user isn't registered. + if (!token) { + res.status(401).end(); + return; + } + + const { id } = req.query; + + const messagesRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/messages/${id}/children`, { + method: "GET", + headers: { + "X-API-Key": process.env.FASTAPI_KEY, + "Content-Type": "application/json", + }, + }); + const messages = await messagesRes.json(); + + // Send recieved messages to the client. + res.status(200).json(messages); +}; + +export default handler; diff --git a/website/src/pages/api/messages/[id]/conversation.ts b/website/src/pages/api/messages/[id]/conversation.ts new file mode 100644 index 00000000..6fa8feb9 --- /dev/null +++ b/website/src/pages/api/messages/[id]/conversation.ts @@ -0,0 +1,27 @@ +import { getToken } from "next-auth/jwt"; + +const handler = async (req, res) => { + const token = await getToken({ req }); + + // Return nothing if the user isn't registered. + if (!token) { + res.status(401).end(); + return; + } + + const { id } = req.query; + + const messagesRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/messages/${id}/conversation`, { + method: "GET", + headers: { + "X-API-Key": process.env.FASTAPI_KEY, + "Content-Type": "application/json", + }, + }); + const messages = await messagesRes.json(); + + // Send recieved messages to the client. + res.status(200).json(messages); +}; + +export default handler; diff --git a/website/src/pages/api/messages/[id]/index.ts b/website/src/pages/api/messages/[id]/index.ts new file mode 100644 index 00000000..8e056532 --- /dev/null +++ b/website/src/pages/api/messages/[id]/index.ts @@ -0,0 +1,27 @@ +import { getToken } from "next-auth/jwt"; + +const handler = async (req, res) => { + const token = await getToken({ req }); + + // Return nothing if the user isn't registered. + if (!token) { + res.status(401).end(); + return; + } + + const { id } = req.query; + + const messageRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/messages/${id}`, { + method: "GET", + headers: { + "X-API-Key": process.env.FASTAPI_KEY, + "Content-Type": "application/json", + }, + }); + const message = await messageRes.json(); + + // Send recieved messages to the client. + res.status(200).json(message); +}; + +export default handler; diff --git a/website/src/pages/api/messages/[id]/parent.ts b/website/src/pages/api/messages/[id]/parent.ts new file mode 100644 index 00000000..05a69cfa --- /dev/null +++ b/website/src/pages/api/messages/[id]/parent.ts @@ -0,0 +1,48 @@ +import { getToken } from "next-auth/jwt"; + +const handler = async (req, res) => { + const token = await getToken({ req }); + + // Return nothing if the user isn't registered. + if (!token) { + res.status(401).end(); + return; + } + + const { id } = req.query; + + if(!id){ + res.status(400).end() + return; + } + + const messageRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/messages/${id}`, { + method: "GET", + headers: { + "X-API-Key": process.env.FASTAPI_KEY, + "Content-Type": "application/json", + }, + }); + + const message = await messageRes.json(); + + if(!message.parent_id){ + res.status(404).end(); + return; + } + + const parentRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/messages/${message.parent_id}`, { + method: "GET", + headers: { + "X-API-Key": process.env.FASTAPI_KEY, + "Content-Type": "application/json", + }, + }); + + const parent = await parentRes.json(); + + // Send recieved messages to the client. + res.status(200).json(parent); +}; + +export default handler; diff --git a/website/src/pages/api/messages/index.tsx b/website/src/pages/api/messages/index.ts similarity index 100% rename from website/src/pages/api/messages/index.tsx rename to website/src/pages/api/messages/index.ts diff --git a/website/src/pages/api/messages/user.tsx b/website/src/pages/api/messages/user.ts similarity index 100% rename from website/src/pages/api/messages/user.tsx rename to website/src/pages/api/messages/user.ts diff --git a/website/src/pages/messages/[id]/index.tsx b/website/src/pages/messages/[id]/index.tsx new file mode 100644 index 00000000..8816fcba --- /dev/null +++ b/website/src/pages/messages/[id]/index.tsx @@ -0,0 +1,80 @@ +import { Box, Container, Flex, HStack, useColorModeValue } from "@chakra-ui/react"; +import Head from "next/head"; +import { useRouter } from "next/router"; +import { useEffect, useState } from "react"; +import useSWRImmutable from "swr/immutable"; + +import fetcher from "src/lib/fetcher"; +import { MessageTableEntry } from "src/components/Messages/MessageTableEntry"; +import { colors } from "styles/Theme/colors"; +import { LoadingScreen } from "src/components/Loading/LoadingScreen"; + +const MessageDetail = () => { + const bg = useColorModeValue("white", colors.dark.bg); + const router = useRouter() + const { id } = router.query + const [message, setMessage] = useState(null); + const [parent, setParent]= useState(null); + const [children, setChildren] = useState(null); + + const { isLoading } = useSWRImmutable(`/api/messages/${id}`, fetcher, { + onSuccess: (data) => { + setMessage(data); + }, + }); + + const { isLoading: isLoadingChildren } = useSWRImmutable(`/api/messages/${id}/children`, fetcher, { + onSuccess: (data) => { + setChildren(data); + }, + }); + + const { isLoading: isLoadingParent } = useSWRImmutable(`/api/messages/${id}/parent`, fetcher, { + + onSuccess: (data) => { + console.log(data); + setParent(data); + }, + onError: (err, key, config) => { + console.log(parent); + } + }); + + if (isLoading || isLoadingChildren || isLoadingParent) { + return ; + } + return <> + + Open Assistant + + +
+ + {parent && + + + + } + {message && + + + + } + + {children && Array.isArray(children) && + + {children.map((item, idx) => + + + + )} + + } +
+ +} + +export default MessageDetail; \ No newline at end of file From 1d46b3132ae2b73590a4d2c5b0a0c44c3d984204 Mon Sep 17 00:00:00 2001 From: jojopirker Date: Thu, 5 Jan 2023 18:54:38 +0100 Subject: [PATCH 02/49] hacky solution for a bug https://dev.to/wh1zk1d/swr-dynamic-routes-in-next-js-3cbl --- website/src/pages/messages/[id]/index.tsx | 64 ++++++++++++++--------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/website/src/pages/messages/[id]/index.tsx b/website/src/pages/messages/[id]/index.tsx index 8816fcba..0b712801 100644 --- a/website/src/pages/messages/[id]/index.tsx +++ b/website/src/pages/messages/[id]/index.tsx @@ -1,4 +1,4 @@ -import { Box, Container, Flex, HStack, useColorModeValue } from "@chakra-ui/react"; +import { Box, Container, Text, HStack, useColorModeValue } from "@chakra-ui/react"; import Head from "next/head"; import { useRouter } from "next/router"; import { useEffect, useState } from "react"; @@ -13,31 +13,36 @@ const MessageDetail = () => { const bg = useColorModeValue("white", colors.dark.bg); const router = useRouter() const { id } = router.query + + /** State arrays of messages */ const [message, setMessage] = useState(null); - const [parent, setParent]= useState(null); + const [parent, setParent] = useState(null); const [children, setChildren] = useState(null); - const { isLoading } = useSWRImmutable(`/api/messages/${id}`, fetcher, { + /** Fetching functions */ + const { isLoading } = useSWRImmutable(id ? `/api/messages/${id}` : null, fetcher, { onSuccess: (data) => { setMessage(data); }, + onError: (err, key, config) => { + setMessage(null); + }, }); - - const { isLoading: isLoadingChildren } = useSWRImmutable(`/api/messages/${id}/children`, fetcher, { + const { isLoading: isLoadingChildren } = useSWRImmutable(id ? `/api/messages/${id}/children` : null, fetcher, { onSuccess: (data) => { setChildren(data); }, + onError: (err, key, config) => { + setChildren(null); + }, }); - - const { isLoading: isLoadingParent } = useSWRImmutable(`/api/messages/${id}/parent`, fetcher, { - + const { isLoading: isLoadingParent } = useSWRImmutable(id ? `/api/messages/${id}/parent` : null, fetcher, { onSuccess: (data) => { - console.log(data); setParent(data); }, onError: (err, key, config) => { - console.log(parent); - } + setParent(null); + }, }); if (isLoading || isLoadingChildren || isLoadingParent) { @@ -52,27 +57,38 @@ const MessageDetail = () => { />
- - {parent && - + + {parent && + <> + Parent + + } {message && - + <> + Message + + } - - {children && Array.isArray(children) && - - {children.map((item, idx) => - - - + + {children && Array.isArray(children) && + <> + Children + + {children.map((item, idx) => + + + + + )} - - } + + + }
} From 1bd0336d2725c00514a11795a5270c7292e6844d Mon Sep 17 00:00:00 2001 From: jojopirker Date: Thu, 5 Jan 2023 19:02:00 +0100 Subject: [PATCH 03/49] bugfix --- website/src/pages/messages/[id]/index.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/src/pages/messages/[id]/index.tsx b/website/src/pages/messages/[id]/index.tsx index 0b712801..f0714e10 100644 --- a/website/src/pages/messages/[id]/index.tsx +++ b/website/src/pages/messages/[id]/index.tsx @@ -75,14 +75,14 @@ const MessageDetail = () => { } - {children && Array.isArray(children) && + {children && Array.isArray(children) && children.length > 0 && <> Children {children.map((item, idx) => - + - + )} From a822db11a7a2a3e90e8f1fb8f375c6c8f41dc642 Mon Sep 17 00:00:00 2001 From: jojopirker Date: Fri, 6 Jan 2023 11:55:31 +0100 Subject: [PATCH 04/49] pre-commit --- website/src/pages/api/messages/[id]/parent.ts | 6 +- website/src/pages/messages/[id]/index.tsx | 100 ++++++++++-------- 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/website/src/pages/api/messages/[id]/parent.ts b/website/src/pages/api/messages/[id]/parent.ts index 05a69cfa..de6fbac6 100644 --- a/website/src/pages/api/messages/[id]/parent.ts +++ b/website/src/pages/api/messages/[id]/parent.ts @@ -11,8 +11,8 @@ const handler = async (req, res) => { const { id } = req.query; - if(!id){ - res.status(400).end() + if (!id) { + res.status(400).end(); return; } @@ -26,7 +26,7 @@ const handler = async (req, res) => { const message = await messageRes.json(); - if(!message.parent_id){ + if (!message.parent_id) { res.status(404).end(); return; } diff --git a/website/src/pages/messages/[id]/index.tsx b/website/src/pages/messages/[id]/index.tsx index f0714e10..f1512d25 100644 --- a/website/src/pages/messages/[id]/index.tsx +++ b/website/src/pages/messages/[id]/index.tsx @@ -11,8 +11,8 @@ import { LoadingScreen } from "src/components/Loading/LoadingScreen"; const MessageDetail = () => { const bg = useColorModeValue("white", colors.dark.bg); - const router = useRouter() - const { id } = router.query + const router = useRouter(); + const { id } = router.query; /** State arrays of messages */ const [message, setMessage] = useState(null); @@ -48,49 +48,57 @@ const MessageDetail = () => { if (isLoading || isLoadingChildren || isLoadingParent) { return ; } - return <> - - Open Assistant - - -
- - {parent && - <> - Parent - - - - - } - {message && - <> - Message - - - - - } - - {children && Array.isArray(children) && children.length > 0 && - <> - Children - - {children.map((item, idx) => - - - - - + return ( + <> + + Open Assistant + + +
+ + {parent && ( + <> + + Parent + + + + + )} - - - } -
- -} + {message && ( + <> + + Message + + + + + + )} + + {children && Array.isArray(children) && children.length > 0 && ( + <> + + Children + + + {children.map((item, idx) => ( + + + + + + ))} + + + )} +
+ + ); +}; -export default MessageDetail; \ No newline at end of file +export default MessageDetail; From 2db310ab060b82c3f81d0cd19d85046f4344ab59 Mon Sep 17 00:00:00 2001 From: jojopirker Date: Fri, 6 Jan 2023 15:53:18 +0100 Subject: [PATCH 05/49] (#309) Message navigation recursive --- .../Messages/MessageWithChildren.tsx | 85 +++++++++++++++++++ website/src/pages/messages/[id]/index.tsx | 71 +++------------- 2 files changed, 99 insertions(+), 57 deletions(-) create mode 100644 website/src/components/Messages/MessageWithChildren.tsx diff --git a/website/src/components/Messages/MessageWithChildren.tsx b/website/src/components/Messages/MessageWithChildren.tsx new file mode 100644 index 00000000..f59017c7 --- /dev/null +++ b/website/src/components/Messages/MessageWithChildren.tsx @@ -0,0 +1,85 @@ +import { Box, CircularProgress, Flex, HStack, Text } from "@chakra-ui/react"; +import { useState } from "react"; +import useSWR from "swr"; + +import fetcher from "src/lib/fetcher"; +import { MessageTableEntry } from "./MessageTableEntry"; + +interface MessageWithChildrenProps { + id: string; + depth?: number; + maxDepth?: number; +} + +export function MessageWithChildren(props: MessageWithChildrenProps) { + const { id, depth, maxDepth } = props; + + const [message, setMessage] = useState(null); + const [children, setChildren] = useState(null); + + const { isLoading } = useSWR(id ? `/api/messages/${id}` : null, fetcher, { + onSuccess: (data) => { + setMessage(data); + }, + onError: (err, key, config) => { + setMessage(null); + }, + }); + const { isLoading: isLoadingChildren } = useSWR(id ? `/api/messages/${id}/children` : null, fetcher, { + onSuccess: (data) => { + setChildren(data); + }, + onError: (err, key, config) => { + setChildren(null); + }, + }); + + const renderRecursive = maxDepth && ((depth && (depth < maxDepth)) || !depth); + + if (isLoading || isLoadingChildren) { + return () + } + + return ( + <> + {message && (<> + + {(depth === 0 || !depth) ? "Message" : depth === 1 ? "Children" : "Ancestor"} + + + + + + + + + + )} + {children && Array.isArray(children) && children.length > 0 ? + renderRecursive ? + + {children.map((item, idx) => ( + + + ))} + : + ( + <> + + {(depth === 0 || !depth) ? "Children" : "Ancestor"} + + + {children.map((item, idx) => ( + + + + + + ))} + + + ) : <>} + ); + +} + diff --git a/website/src/pages/messages/[id]/index.tsx b/website/src/pages/messages/[id]/index.tsx index f1512d25..d455f256 100644 --- a/website/src/pages/messages/[id]/index.tsx +++ b/website/src/pages/messages/[id]/index.tsx @@ -1,42 +1,22 @@ -import { Box, Container, Text, HStack, useColorModeValue } from "@chakra-ui/react"; +import { Box, Container, Text, useColorModeValue } from "@chakra-ui/react"; import Head from "next/head"; import { useRouter } from "next/router"; -import { useEffect, useState } from "react"; -import useSWRImmutable from "swr/immutable"; +import { useState } from "react"; +import useSWR from "swr"; import fetcher from "src/lib/fetcher"; import { MessageTableEntry } from "src/components/Messages/MessageTableEntry"; -import { colors } from "styles/Theme/colors"; import { LoadingScreen } from "src/components/Loading/LoadingScreen"; +import { MessageWithChildren } from "src/components/Messages/MessageWithChildren"; const MessageDetail = () => { - const bg = useColorModeValue("white", colors.dark.bg); + const mainBg = useColorModeValue("bg-slate-300", "bg-slate-900"); const router = useRouter(); const { id } = router.query; - /** State arrays of messages */ - const [message, setMessage] = useState(null); const [parent, setParent] = useState(null); - const [children, setChildren] = useState(null); - /** Fetching functions */ - const { isLoading } = useSWRImmutable(id ? `/api/messages/${id}` : null, fetcher, { - onSuccess: (data) => { - setMessage(data); - }, - onError: (err, key, config) => { - setMessage(null); - }, - }); - const { isLoading: isLoadingChildren } = useSWRImmutable(id ? `/api/messages/${id}/children` : null, fetcher, { - onSuccess: (data) => { - setChildren(data); - }, - onError: (err, key, config) => { - setChildren(null); - }, - }); - const { isLoading: isLoadingParent } = useSWRImmutable(id ? `/api/messages/${id}/parent` : null, fetcher, { + const { isLoading: isLoadingParent } = useSWR(id ? `/api/messages/${id}/parent` : null, fetcher, { onSuccess: (data) => { setParent(data); }, @@ -45,7 +25,7 @@ const MessageDetail = () => { }, }); - if (isLoading || isLoadingChildren || isLoadingParent) { + if (isLoadingParent) { return ; } return ( @@ -57,45 +37,22 @@ const MessageDetail = () => { content="Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world." /> -
- +
+ {parent && ( <> - + Parent - + )} - {message && ( - <> - - Message - - - - - - )} - {children && Array.isArray(children) && children.length > 0 && ( - <> - - Children - - - {children.map((item, idx) => ( - - - - - - ))} - - - )} + + +
); From d292643ea6b36a9fcfb35b2a36858b1dbaca01c8 Mon Sep 17 00:00:00 2001 From: jojopirker Date: Fri, 6 Jan 2023 15:57:20 +0100 Subject: [PATCH 06/49] (#309) pre-commit --- .../Messages/MessageWithChildren.tsx | 143 ++++++++++-------- 1 file changed, 76 insertions(+), 67 deletions(-) diff --git a/website/src/components/Messages/MessageWithChildren.tsx b/website/src/components/Messages/MessageWithChildren.tsx index f59017c7..3b6f8649 100644 --- a/website/src/components/Messages/MessageWithChildren.tsx +++ b/website/src/components/Messages/MessageWithChildren.tsx @@ -6,80 +6,89 @@ import fetcher from "src/lib/fetcher"; import { MessageTableEntry } from "./MessageTableEntry"; interface MessageWithChildrenProps { - id: string; - depth?: number; - maxDepth?: number; + id: string; + depth?: number; + maxDepth?: number; } export function MessageWithChildren(props: MessageWithChildrenProps) { - const { id, depth, maxDepth } = props; + const { id, depth, maxDepth } = props; - const [message, setMessage] = useState(null); - const [children, setChildren] = useState(null); + const [message, setMessage] = useState(null); + const [children, setChildren] = useState(null); - const { isLoading } = useSWR(id ? `/api/messages/${id}` : null, fetcher, { - onSuccess: (data) => { - setMessage(data); - }, - onError: (err, key, config) => { - setMessage(null); - }, - }); - const { isLoading: isLoadingChildren } = useSWR(id ? `/api/messages/${id}/children` : null, fetcher, { - onSuccess: (data) => { - setChildren(data); - }, - onError: (err, key, config) => { - setChildren(null); - }, - }); + const { isLoading } = useSWR(id ? `/api/messages/${id}` : null, fetcher, { + onSuccess: (data) => { + setMessage(data); + }, + onError: (err, key, config) => { + setMessage(null); + }, + }); + const { isLoading: isLoadingChildren } = useSWR(id ? `/api/messages/${id}/children` : null, fetcher, { + onSuccess: (data) => { + setChildren(data); + }, + onError: (err, key, config) => { + setChildren(null); + }, + }); - const renderRecursive = maxDepth && ((depth && (depth < maxDepth)) || !depth); + const renderRecursive = maxDepth && ((depth && depth < maxDepth) || !depth); - if (isLoading || isLoadingChildren) { - return () - } + if (isLoading || isLoadingChildren) { + return ; + } - return ( + return ( + <> + {message && ( <> - {message && (<> - - {(depth === 0 || !depth) ? "Message" : depth === 1 ? "Children" : "Ancestor"} - - - - - - - - - - )} - {children && Array.isArray(children) && children.length > 0 ? - renderRecursive ? - - {children.map((item, idx) => ( - - - ))} - : - ( - <> - - {(depth === 0 || !depth) ? "Children" : "Ancestor"} - - - {children.map((item, idx) => ( - - - - - - ))} - - - ) : <>} - ); - + + {depth === 0 || !depth ? "Message" : depth === 1 ? "Children" : "Ancestor"} + + + + + + + + + + )} + {children && Array.isArray(children) && children.length > 0 ? ( + renderRecursive ? ( + + {children.map((item, idx) => ( + + + + ))} + + ) : ( + <> + + {depth === 0 || !depth ? "Children" : "Ancestor"} + + + {children.map((item, idx) => ( + + + + + + ))} + + + ) + ) : ( + <> + )} + + ); } - From a1b0bb556116b72fc55f52b9836de4ac30f8b479 Mon Sep 17 00:00:00 2001 From: jojopirker Date: Fri, 6 Jan 2023 16:50:25 +0100 Subject: [PATCH 07/49] moved key prop --- website/src/components/Messages/MessageWithChildren.tsx | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/website/src/components/Messages/MessageWithChildren.tsx b/website/src/components/Messages/MessageWithChildren.tsx index 3b6f8649..70d6c318 100644 --- a/website/src/components/Messages/MessageWithChildren.tsx +++ b/website/src/components/Messages/MessageWithChildren.tsx @@ -60,13 +60,8 @@ export function MessageWithChildren(props: MessageWithChildrenProps) { renderRecursive ? ( {children.map((item, idx) => ( - - + + ))} From fa7d8ecfdef5feaed02322f6e76436cef51bc9d8 Mon Sep 17 00:00:00 2001 From: jojopirker Date: Fri, 6 Jan 2023 17:01:27 +0100 Subject: [PATCH 08/49] Update index.tsx --- website/src/pages/messages/[id]/index.tsx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/website/src/pages/messages/[id]/index.tsx b/website/src/pages/messages/[id]/index.tsx index d455f256..bf7e97ea 100644 --- a/website/src/pages/messages/[id]/index.tsx +++ b/website/src/pages/messages/[id]/index.tsx @@ -9,10 +9,8 @@ import { MessageTableEntry } from "src/components/Messages/MessageTableEntry"; import { LoadingScreen } from "src/components/Loading/LoadingScreen"; import { MessageWithChildren } from "src/components/Messages/MessageWithChildren"; -const MessageDetail = () => { +const MessageDetail = ({ id }) => { const mainBg = useColorModeValue("bg-slate-300", "bg-slate-900"); - const router = useRouter(); - const { id } = router.query; const [parent, setParent] = useState(null); @@ -58,4 +56,9 @@ const MessageDetail = () => { ); }; +MessageDetail.getInitialProps = async ({ query }) => { + const { id } = query; + return { id }; +}; + export default MessageDetail; From fe90ca3375cd6120a038b61b18072c9387704abe Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 6 Jan 2023 23:15:07 +0000 Subject: [PATCH 09/49] add andrewm4894 as /docs/ /.devcontainer/ codeowner --- CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CODEOWNERS b/CODEOWNERS index c5cc1467..e2c41e2f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,3 +2,5 @@ /website/ @fozziethebeat @k-nearest-neighbor @AbdBarho /model/ @theblackcat102 @sanagno /copilot/ @fozziethebeat @andreaskoepf @yk +/docs/ @andrewm4894 @andreaskoepf @yk +/.devcontainer/ @andrewm4894 @andreaskoepf @yk \ No newline at end of file From 029fde8ab82274fe6d3222990836d993c2abc006 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 6 Jan 2023 23:18:51 +0000 Subject: [PATCH 10/49] newline --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index e2c41e2f..f1eb32de 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -3,4 +3,4 @@ /model/ @theblackcat102 @sanagno /copilot/ @fozziethebeat @andreaskoepf @yk /docs/ @andrewm4894 @andreaskoepf @yk -/.devcontainer/ @andrewm4894 @andreaskoepf @yk \ No newline at end of file +/.devcontainer/ @andrewm4894 @andreaskoepf @yk From aa4bb4135537283ac915414b211257ca9d17df39 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 7 Jan 2023 08:54:50 +0900 Subject: [PATCH 11/49] Delete sharp, it's not needed and breaks docker on m1 --- website/package-lock.json | 645 ++------------------------------------ website/package.json | 1 - 2 files changed, 29 insertions(+), 617 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index 5cd3d9bb..8d94e779 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -39,7 +39,6 @@ "react": "18.2.0", "react-dom": "18.2.0", "react-icons": "^4.7.1", - "sharp": "0.31.2", "swr": "^2.0.0", "tailwindcss": "^3.2.4", "use-debounce": "^9.0.2" @@ -12780,6 +12779,7 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, "funding": [ { "type": "github", @@ -12869,16 +12869,6 @@ "file-uri-to-path": "1.0.0" } }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, "node_modules/blob-util": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/blob-util/-/blob-util-2.0.2.tgz", @@ -13235,6 +13225,7 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, "funding": [ { "type": "github", @@ -13645,7 +13636,8 @@ "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true }, "node_modules/chrome-trace-event": { "version": "1.0.3", @@ -13925,18 +13917,6 @@ "node": ">=0.10.0" } }, - "node_modules/color": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", - "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", - "dependencies": { - "color-convert": "^2.0.1", - "color-string": "^1.9.0" - }, - "engines": { - "node": ">=12.5.0" - } - }, "node_modules/color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -13950,15 +13930,6 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" }, - "node_modules/color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, "node_modules/color-support": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", @@ -13968,22 +13939,6 @@ "color-support": "bin.js" } }, - "node_modules/color/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, "node_modules/color2k": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/color2k/-/color2k-2.0.0.tgz", @@ -15234,20 +15189,6 @@ "node": ">=0.10" } }, - "node_modules/decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "dependencies": { - "mimic-response": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/dedent": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", @@ -15280,14 +15221,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "engines": { - "node": ">=4.0.0" - } - }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -15433,14 +15366,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/detect-libc": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", - "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==", - "engines": { - "node": ">=8" - } - }, "node_modules/detect-node-es": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", @@ -15803,6 +15728,7 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, "dependencies": { "once": "^1.4.0" } @@ -16925,14 +16851,6 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "dev": true }, - "node_modules/expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "engines": { - "node": ">=6" - } - }, "node_modules/express": { "version": "4.18.2", "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", @@ -17908,11 +17826,6 @@ "safe-buffer": "~5.1.0" } }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" - }, "node_modules/fs-extra": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", @@ -18178,11 +18091,6 @@ "assert-plus": "^1.0.0" } }, - "node_modules/github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" - }, "node_modules/github-slugger": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-1.5.0.tgz", @@ -18933,6 +18841,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, "funding": [ { "type": "github", @@ -21216,17 +21125,6 @@ "node": ">=6" } }, - "node_modules/mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/min-document": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", @@ -21403,11 +21301,6 @@ "node": ">=10" } }, - "node_modules/mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" - }, "node_modules/move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -21497,11 +21390,6 @@ "node": ">=0.10.0" } }, - "node_modules/napi-build-utils": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", - "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -21654,52 +21542,6 @@ "tslib": "^2.0.3" } }, - "node_modules/node-abi": { - "version": "3.30.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.30.0.tgz", - "integrity": "sha512-qWO5l3SCqbwQavymOmtTVuCWZE23++S+rxyoHjXqUmPyzRcaoI4lA2gO55/drddGnedAyjA7sk76SfQ5lfUMnw==", - "dependencies": { - "semver": "^7.3.5" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/node-abi/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/node-abi/node_modules/semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/node-abi/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "node_modules/node-addon-api": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.0.0.tgz", - "integrity": "sha512-CvkDw2OEnme7ybCykJpVcKH+uAOLV2qLqiyla128dN9TkEWfrYmxG6C2boDe5KcNQqZF3orkqzGgOMvZ/JNekA==" - }, "node_modules/node-dir": { "version": "0.1.17", "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", @@ -25958,31 +25800,6 @@ "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-3.8.0.tgz", "integrity": "sha512-WuxUnVtlWL1OfZFQFuqvnvs6MiAGk9UNsBostyBOB0Is9wb5uRESevA6rnl/rkksXaGX3GzZhPup5d6Vp1nFew==" }, - "node_modules/prebuild-install": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", - "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", - "dependencies": { - "detect-libc": "^2.0.0", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.3", - "mkdirp-classic": "^0.5.3", - "napi-build-utils": "^1.0.1", - "node-abi": "^3.3.0", - "pump": "^3.0.0", - "rc": "^1.2.7", - "simple-get": "^4.0.0", - "tar-fs": "^2.0.0", - "tunnel-agent": "^0.6.0" - }, - "bin": { - "prebuild-install": "bin.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -26234,6 +26051,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -26414,33 +26232,6 @@ "webpack": "^4.0.0 || ^5.0.0" } }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/rc/node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" - }, - "node_modules/rc/node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/react": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", @@ -26765,6 +26556,7 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -27366,7 +27158,8 @@ "node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true }, "node_modules/safe-regex": { "version": "1.1.0", @@ -27906,58 +27699,6 @@ "node": ">=8" } }, - "node_modules/sharp": { - "version": "0.31.2", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.31.2.tgz", - "integrity": "sha512-DUdNVEXgS5A97cTagSLIIp8dUZ/lZtk78iNVZgHdHbx1qnQR7JAHY0BnXnwwH39Iw+VKhO08CTYhIg0p98vQ5Q==", - "hasInstallScript": true, - "dependencies": { - "color": "^4.2.3", - "detect-libc": "^2.0.1", - "node-addon-api": "^5.0.0", - "prebuild-install": "^7.1.1", - "semver": "^7.3.8", - "simple-get": "^4.0.1", - "tar-fs": "^2.1.1", - "tunnel-agent": "^0.6.0" - }, - "engines": { - "node": ">=14.15.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/sharp/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sharp/node_modules/semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sharp/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -27996,62 +27737,6 @@ "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/simple-get": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", - "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "decompress-response": "^6.0.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/simple-swizzle/node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -28715,6 +28400,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -28723,6 +28409,7 @@ "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, "funding": [ { "type": "github", @@ -29137,32 +28824,6 @@ "node": ">=10" } }, - "node_modules/tar-fs": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", - "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", - "dependencies": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.1.4" - } - }, - "node_modules/tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dependencies": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/tar/node_modules/chownr": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", @@ -29656,6 +29317,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dev": true, "dependencies": { "safe-buffer": "^5.0.1" }, @@ -40790,7 +40452,8 @@ "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true }, "bcrypt-pbkdf": { "version": "1.0.2", @@ -40850,16 +40513,6 @@ "file-uri-to-path": "1.0.0" } }, - "bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "requires": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, "blob-util": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/blob-util/-/blob-util-2.0.2.tgz", @@ -41149,6 +40802,7 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, "requires": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -41447,7 +41101,8 @@ "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true }, "chrome-trace-event": { "version": "1.0.3", @@ -41663,30 +41318,6 @@ "object-visit": "^1.0.0" } }, - "color": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", - "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", - "requires": { - "color-convert": "^2.0.1", - "color-string": "^1.9.0" - }, - "dependencies": { - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - } - } - }, "color-convert": { "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", @@ -41700,15 +41331,6 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" }, - "color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "requires": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, "color-support": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", @@ -42720,14 +42342,6 @@ "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", "dev": true }, - "decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "requires": { - "mimic-response": "^3.1.0" - } - }, "dedent": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", @@ -42757,11 +42371,6 @@ "which-typed-array": "^1.1.8" } }, - "deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" - }, "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -42868,11 +42477,6 @@ "repeat-string": "^1.5.4" } }, - "detect-libc": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", - "integrity": "sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==" - }, "detect-node-es": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", @@ -43178,6 +42782,7 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, "requires": { "once": "^1.4.0" } @@ -44034,11 +43639,6 @@ } } }, - "expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==" - }, "express": { "version": "4.18.2", "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", @@ -44815,11 +44415,6 @@ } } }, - "fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" - }, "fs-extra": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", @@ -45026,11 +44621,6 @@ "assert-plus": "^1.0.0" } }, - "github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" - }, "github-slugger": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-1.5.0.tgz", @@ -45594,7 +45184,8 @@ "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true }, "iferr": { "version": "0.1.5", @@ -47285,11 +46876,6 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, - "mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" - }, "min-document": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz", @@ -47431,11 +47017,6 @@ "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", "dev": true }, - "mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" - }, "move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -47512,11 +47093,6 @@ "to-regex": "^3.0.1" } }, - "napi-build-utils": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", - "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" - }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -47615,42 +47191,6 @@ "tslib": "^2.0.3" } }, - "node-abi": { - "version": "3.30.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.30.0.tgz", - "integrity": "sha512-qWO5l3SCqbwQavymOmtTVuCWZE23++S+rxyoHjXqUmPyzRcaoI4lA2gO55/drddGnedAyjA7sk76SfQ5lfUMnw==", - "requires": { - "semver": "^7.3.5" - }, - "dependencies": { - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "requires": { - "yallist": "^4.0.0" - } - }, - "semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", - "requires": { - "lru-cache": "^6.0.0" - } - }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - } - } - }, - "node-addon-api": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.0.0.tgz", - "integrity": "sha512-CvkDw2OEnme7ybCykJpVcKH+uAOLV2qLqiyla128dN9TkEWfrYmxG6C2boDe5KcNQqZF3orkqzGgOMvZ/JNekA==" - }, "node-dir": { "version": "0.1.17", "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.17.tgz", @@ -50621,25 +50161,6 @@ } } }, - "prebuild-install": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", - "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", - "requires": { - "detect-libc": "^2.0.0", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.3", - "mkdirp-classic": "^0.5.3", - "napi-build-utils": "^1.0.1", - "node-abi": "^3.3.0", - "pump": "^3.0.0", - "rc": "^1.2.7", - "simple-get": "^4.0.0", - "tar-fs": "^2.0.0", - "tunnel-agent": "^0.6.0" - } - }, "prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -50832,6 +50353,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -50957,29 +50479,6 @@ "schema-utils": "^3.0.0" } }, - "rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "requires": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "dependencies": { - "ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" - }, - "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==" - } - } - }, "react": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz", @@ -51208,6 +50707,7 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -51668,7 +51168,8 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true }, "safe-regex": { "version": "1.1.0", @@ -52120,44 +51621,6 @@ "kind-of": "^6.0.2" } }, - "sharp": { - "version": "0.31.2", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.31.2.tgz", - "integrity": "sha512-DUdNVEXgS5A97cTagSLIIp8dUZ/lZtk78iNVZgHdHbx1qnQR7JAHY0BnXnwwH39Iw+VKhO08CTYhIg0p98vQ5Q==", - "requires": { - "color": "^4.2.3", - "detect-libc": "^2.0.1", - "node-addon-api": "^5.0.0", - "prebuild-install": "^7.1.1", - "semver": "^7.3.8", - "simple-get": "^4.0.1", - "tar-fs": "^2.1.1", - "tunnel-agent": "^0.6.0" - }, - "dependencies": { - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "requires": { - "yallist": "^4.0.0" - } - }, - "semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", - "requires": { - "lru-cache": "^6.0.0" - } - }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - } - } - }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -52187,36 +51650,6 @@ "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, - "simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" - }, - "simple-get": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", - "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", - "requires": { - "decompress-response": "^6.0.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "requires": { - "is-arrayish": "^0.3.1" - }, - "dependencies": { - "is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - } - } - }, "sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -52786,6 +52219,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, "requires": { "safe-buffer": "~5.2.0" }, @@ -52793,7 +52227,8 @@ "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true } } }, @@ -53103,29 +52538,6 @@ } } }, - "tar-fs": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", - "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", - "requires": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.1.4" - } - }, - "tar-stream": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", - "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "requires": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - } - }, "telejson": { "version": "6.0.8", "resolved": "https://registry.npmjs.org/telejson/-/telejson-6.0.8.tgz", @@ -53497,6 +52909,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dev": true, "requires": { "safe-buffer": "^5.0.1" } diff --git a/website/package.json b/website/package.json index 7d3b680e..eb85568b 100644 --- a/website/package.json +++ b/website/package.json @@ -50,7 +50,6 @@ "react": "18.2.0", "react-dom": "18.2.0", "react-icons": "^4.7.1", - "sharp": "0.31.2", "swr": "^2.0.0", "tailwindcss": "^3.2.4", "use-debounce": "^9.0.2" From d3a0bd43055f6d3a69af30d7345b15877a2831f6 Mon Sep 17 00:00:00 2001 From: klotske Date: Sat, 7 Jan 2023 03:38:34 +0300 Subject: [PATCH 12/49] Moved SideMenu to component & refactored layout --- website/src/components/Dashboard/index.ts | 1 - website/src/components/Layout.tsx | 21 +++++- .../components/{Dashboard => }/SideMenu.tsx | 44 ++++-------- website/src/components/SideMenuLayout.tsx | 23 +++++++ website/src/pages/dashboard.tsx | 19 ++--- website/src/pages/messages/index.tsx | 69 ++++++++----------- 6 files changed, 92 insertions(+), 85 deletions(-) rename website/src/components/{Dashboard => }/SideMenu.tsx (77%) create mode 100644 website/src/components/SideMenuLayout.tsx diff --git a/website/src/components/Dashboard/index.ts b/website/src/components/Dashboard/index.ts index 0b4ff49a..6e110534 100644 --- a/website/src/components/Dashboard/index.ts +++ b/website/src/components/Dashboard/index.ts @@ -1,3 +1,2 @@ export { LeaderboardTable } from "./LeaderboardTable"; -export { SideMenu } from "./SideMenu"; export { TaskOption } from "./TaskOption"; diff --git a/website/src/components/Layout.tsx b/website/src/components/Layout.tsx index bf662113..1faefcc0 100644 --- a/website/src/components/Layout.tsx +++ b/website/src/components/Layout.tsx @@ -1,9 +1,11 @@ // https://nextjs.org/docs/basic-features/layouts import type { NextPage } from "next"; +import { FiLayout, FiMessageSquare } from "react-icons/fi"; import { Header } from "src/components/Header"; import { Footer } from "./Footer"; +import { SideMenuLayout } from "./SideMenuLayout"; export type NextPageWithLayout

= NextPage & { getLayout?: (page: React.ReactElement) => React.ReactNode; @@ -28,7 +30,24 @@ export const getTransparentHeaderLayout = (page: React.ReactElement) => ( export const getDashboardLayout = (page: React.ReactElement) => (

- {page} + + {page} +
); diff --git a/website/src/components/Dashboard/SideMenu.tsx b/website/src/components/SideMenu.tsx similarity index 77% rename from website/src/components/Dashboard/SideMenu.tsx rename to website/src/components/SideMenu.tsx index 499117a2..59489e0e 100644 --- a/website/src/components/Dashboard/SideMenu.tsx +++ b/website/src/components/SideMenu.tsx @@ -1,37 +1,23 @@ import { Box, Button, Link, Text, Tooltip, useColorMode } from "@chakra-ui/react"; import { useRouter } from "next/router"; -import { FiLayout, FiSun, FiMessageSquare } from "react-icons/fi"; +import { FiSun } from "react-icons/fi"; +import { IconType } from "react-icons/lib"; import { colors } from "styles/Theme/colors"; -export function SideMenu() { +export interface MenuButtonOption { + label: string; + pathname: string; + desc: string; + icon: IconType; +} + +export interface SideMenuProps { + buttonOptions: MenuButtonOption[]; +} + +export function SideMenu(props: SideMenuProps) { const router = useRouter(); const { colorMode, toggleColorMode } = useColorMode(); - const buttonOptions = [ - { - label: "Dashboard", - pathname: "/dashboard", - desc: "Dashboard Home", - icon: FiLayout, - }, - { - label: "Messages", - pathname: "/messages", - desc: "Messages Dashboard", - icon: FiMessageSquare, - }, - // { - // label: "Leaderboard", - // pathname: "#", - // desc: "Public Leaderboard", - // icon: FiAward, - // }, - // { - // label: "Stats", - // pathname: "#", - // desc: "User Statistics", - // icon: FiBarChart2, - // }, - ]; return (
@@ -43,7 +29,7 @@ export function SideMenu() { className="grid grid-cols-4 gap-2 sm:flex sm:flex-col sm:justify-between p-4 h-full" >
From b70c638f7cb526facaa7c3e3bfde6fb532aea56b Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 7 Jan 2023 19:13:26 +0900 Subject: [PATCH 17/49] Adding a very rudimentary admin page that displays a table of users and their basic information --- website/src/components/UsersCell.tsx | 46 +++++++++++++++++++ website/src/pages/admin/index.tsx | 67 ++++++++++++++++++++++++++++ website/src/pages/api/admin/users.ts | 31 +++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 website/src/components/UsersCell.tsx create mode 100644 website/src/pages/admin/index.tsx create mode 100644 website/src/pages/api/admin/users.ts diff --git a/website/src/components/UsersCell.tsx b/website/src/components/UsersCell.tsx new file mode 100644 index 00000000..5c6f4ce8 --- /dev/null +++ b/website/src/components/UsersCell.tsx @@ -0,0 +1,46 @@ +import { Table, TableCaption, TableContainer, Tbody, Td, Th, Thead, Tr } from "@chakra-ui/react"; +import { useState } from "react"; +import fetcher from "src/lib/fetcher"; +import useSWR from "swr"; + +/** + * Fetches users from the users api route and then presents them in a simple Chakra table. + */ +const UsersCell = () => { + // Fetch and save the users. + const [users, setUsers] = useState([]); + const { isLoading } = useSWR("/api/admin/users", fetcher, { + onSuccess: (data) => { + setUsers(data); + }, + }); + + // Present users in a naive table. + return ( + + + Users + + + + + + + + + + {users.map((user, index) => ( + + + + + + + ))} + +
IdEmailNameRole
{user.id}{user.email}{user.name}{user.role}
+ + ); +}; + +export default UsersCell; diff --git a/website/src/pages/admin/index.tsx b/website/src/pages/admin/index.tsx new file mode 100644 index 00000000..e048915e --- /dev/null +++ b/website/src/pages/admin/index.tsx @@ -0,0 +1,67 @@ +import Head from "next/head"; +import { useRouter } from "next/router"; +import { useSession } from "next-auth/react"; +import { useEffect } from "react"; +import { getTransparentHeaderLayout } from "src/components/Layout"; +import UsersCell from "src/components/UsersCell"; + +/** + * Provides the admin index page that will display a list of users and give + * admins the ability to manage their access rights. + */ +const AdminIndex = () => { + const router = useRouter(); + const { data: session, status } = useSession(); + + // Check when the user session is loaded and re-route if the user is not an + // admin. This follows the suggestion by NextJS for handling private pages: + // https://nextjs.org/docs/api-reference/next/router#usage + // + // All admin pages should use the same check and routing steps. + useEffect(() => { + if (status === "loading") { + return; + } + if (session?.user?.role === "admin") { + return; + } + router.push("/"); + }, [session, status]); + + // While loading, just show something. + if (status === "loading") { + return ( + <> + + Open Assistant + + +
loading...
+ + ); + } + + // Show the final page. + // TODO(#237): Display a component that fetches actual user data. + return ( + <> + + Open Assistant + + +
+ +
+ + ); +}; + +AdminIndex.getLayout = getTransparentHeaderLayout; + +export default AdminIndex; diff --git a/website/src/pages/api/admin/users.ts b/website/src/pages/api/admin/users.ts new file mode 100644 index 00000000..186bb253 --- /dev/null +++ b/website/src/pages/api/admin/users.ts @@ -0,0 +1,31 @@ +import { getToken } from "next-auth/jwt"; +import client from "src/lib/prismadb"; + +/** + * Returns a list of user results from the database when the requesting user is + * a logged in admin. + */ +const handler = async (req, res) => { + const token = await getToken({ req }); + + // Return nothing if the user isn't registered or if the user isn't an admin. + if (!token || token.role !== "admin") { + res.status(403).end(); + return; + } + + // Fetch 20 users. + const users = await client.user.findMany({ + select: { + id: true, + role: true, + name: true, + email: true, + }, + take: 20, + }); + + res.status(200).json(users); +}; + +export default handler; From 8f459d9aae83725015111291d9362370ea4e8e90 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 7 Jan 2023 19:43:10 +0900 Subject: [PATCH 18/49] Addressing review comments --- website/src/components/UsersCell.tsx | 4 +--- website/src/pages/admin/index.tsx | 20 +------------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/website/src/components/UsersCell.tsx b/website/src/components/UsersCell.tsx index 5c6f4ce8..b2b04c83 100644 --- a/website/src/components/UsersCell.tsx +++ b/website/src/components/UsersCell.tsx @@ -10,9 +10,7 @@ const UsersCell = () => { // Fetch and save the users. const [users, setUsers] = useState([]); const { isLoading } = useSWR("/api/admin/users", fetcher, { - onSuccess: (data) => { - setUsers(data); - }, + onSuccess: setUsers, }); // Present users in a naive table. diff --git a/website/src/pages/admin/index.tsx b/website/src/pages/admin/index.tsx index e048915e..60d61903 100644 --- a/website/src/pages/admin/index.tsx +++ b/website/src/pages/admin/index.tsx @@ -28,22 +28,6 @@ const AdminIndex = () => { router.push("/"); }, [session, status]); - // While loading, just show something. - if (status === "loading") { - return ( - <> - - Open Assistant - - -
loading...
- - ); - } - // Show the final page. // TODO(#237): Display a component that fetches actual user data. return ( @@ -55,9 +39,7 @@ const AdminIndex = () => { content="Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world." /> -
- -
+
{status === "loading" ? "loading..." : }
); }; From dca0ba1774022a9ef56587dc2a593c4821836b3a Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 7 Jan 2023 19:57:59 +0900 Subject: [PATCH 19/49] Fixing most of the eslint warnings in the web code --- website/src/components/CollapsableText.tsx | 12 ++++++++++-- website/src/components/ContextMessages.tsx | 2 +- website/src/components/FlaggableElement.tsx | 2 +- website/src/components/Messages/MessageTable.tsx | 4 ++-- .../src/components/Messages/MessageTableEntry.tsx | 4 ++-- .../components/Messages/MessageWithChildren.tsx | 15 +++++++-------- website/src/components/SideMenuLayout.tsx | 2 +- .../components/Survey/TaskControlsOverridable.tsx | 2 +- website/src/components/Vision.tsx | 2 +- website/src/pages/about.tsx | 6 +++--- website/src/pages/api/username.tsx | 2 +- website/src/pages/dashboard.tsx | 3 +-- website/src/pages/messages/[id]/index.tsx | 10 ++++------ website/src/pages/messages/index.tsx | 6 +++--- website/src/pages/privacy-policy.tsx | 2 -- website/src/pages/terms-of-service.tsx | 2 -- 16 files changed, 38 insertions(+), 38 deletions(-) diff --git a/website/src/components/CollapsableText.tsx b/website/src/components/CollapsableText.tsx index b0cfb3b9..1f34c508 100644 --- a/website/src/components/CollapsableText.tsx +++ b/website/src/components/CollapsableText.tsx @@ -1,5 +1,13 @@ -import { Button, useDisclosure } from "@chakra-ui/react"; -import { Modal, ModalOverlay, ModalContent, ModalHeader, ModalBody, ModalCloseButton } from "@chakra-ui/react"; +import { + Button, + Modal, + ModalBody, + ModalCloseButton, + ModalContent, + ModalHeader, + ModalOverlay, + useDisclosure, +} from "@chakra-ui/react"; import React from "react"; export const CollapsableText = ({ text, maxLength = 220 }) => { diff --git a/website/src/components/ContextMessages.tsx b/website/src/components/ContextMessages.tsx index 150dddfa..8217f8b7 100644 --- a/website/src/components/ContextMessages.tsx +++ b/website/src/components/ContextMessages.tsx @@ -1,5 +1,5 @@ import { Box } from "@chakra-ui/react"; -import { Message } from "./Messages"; +import { Message } from "src/components/Messages"; export const ContextMessages = ({ messages }: { messages: Message[] }) => { return ( diff --git a/website/src/components/FlaggableElement.tsx b/website/src/components/FlaggableElement.tsx index a55d29c5..dbd20e93 100644 --- a/website/src/components/FlaggableElement.tsx +++ b/website/src/components/FlaggableElement.tsx @@ -24,8 +24,8 @@ import { import { FlagIcon, QuestionMarkCircleIcon } from "@heroicons/react/20/solid"; import { useState } from "react"; import poster from "src/lib/poster"; -import useSWRMutation from "swr/mutation"; import { colors } from "styles/Theme/colors"; +import useSWRMutation from "swr/mutation"; export const FlaggableElement = (props) => { const [isEditing, setIsEditing] = useBoolean(); diff --git a/website/src/components/Messages/MessageTable.tsx b/website/src/components/Messages/MessageTable.tsx index 65f6efb1..ee1d3687 100644 --- a/website/src/components/Messages/MessageTable.tsx +++ b/website/src/components/Messages/MessageTable.tsx @@ -1,5 +1,5 @@ -import { Box, CircularProgress, Stack, StackDivider, useColorModeValue } from "@chakra-ui/react"; -import { MessageTableEntry } from "./MessageTableEntry"; +import { Stack, StackDivider } from "@chakra-ui/react"; +import { MessageTableEntry } from "src/components/MessageTableEntry"; export function MessageTable({ messages }) { return ( diff --git a/website/src/components/Messages/MessageTableEntry.tsx b/website/src/components/Messages/MessageTableEntry.tsx index d272b960..9fad7262 100644 --- a/website/src/components/Messages/MessageTableEntry.tsx +++ b/website/src/components/Messages/MessageTableEntry.tsx @@ -1,7 +1,7 @@ -import { Avatar, Box, HStack, LinkBox, useColorModeValue } from "@chakra-ui/react"; +import { Avatar, HStack, LinkBox, useColorModeValue } from "@chakra-ui/react"; import { boolean } from "boolean"; import NextLink from "next/link"; -import { FlaggableElement } from "../FlaggableElement"; +import { FlaggableElement } from "src/components/FlaggableElement"; interface Message { text: string; diff --git a/website/src/components/Messages/MessageWithChildren.tsx b/website/src/components/Messages/MessageWithChildren.tsx index 8fcd8658..c412f996 100644 --- a/website/src/components/Messages/MessageWithChildren.tsx +++ b/website/src/components/Messages/MessageWithChildren.tsx @@ -1,10 +1,9 @@ -import { Box, CircularProgress, Flex, HStack, StackDivider, Text, TextProps, StackProps } from "@chakra-ui/react"; -import { useState } from "react"; -import useSWR from "swr"; - -import fetcher from "src/lib/fetcher"; -import { MessageTableEntry } from "./MessageTableEntry"; +import { Box, CircularProgress, Flex, HStack, StackDivider, StackProps, Text, TextProps } from "@chakra-ui/react"; import { boolean } from "boolean"; +import { useState } from "react"; +import { MessageTableEntry } from "src/components/Messages/MessageTableEntry"; +import fetcher from "src/lib/fetcher"; +import useSWR from "swr"; const MessageHeaderProps: TextProps = { align: "center", @@ -36,7 +35,7 @@ export function MessageWithChildren(props: MessageWithChildrenProps) { onSuccess: (data) => { setMessage(data); }, - onError: (err, key, config) => { + onError: () => { setMessage(null); }, }); @@ -44,7 +43,7 @@ export function MessageWithChildren(props: MessageWithChildrenProps) { onSuccess: (data) => { setChildren(data); }, - onError: (err, key, config) => { + onError: () => { setChildren(null); }, }); diff --git a/website/src/components/SideMenuLayout.tsx b/website/src/components/SideMenuLayout.tsx index e6c521a1..ae8cfbc2 100644 --- a/website/src/components/SideMenuLayout.tsx +++ b/website/src/components/SideMenuLayout.tsx @@ -1,6 +1,6 @@ import { Box, useColorMode } from "@chakra-ui/react"; +import { MenuButtonOption, SideMenu } from "src/components/SideMenu"; import { colors } from "styles/Theme/colors"; -import { SideMenu, MenuButtonOption } from "src/components/SideMenu"; interface SideMenuLayoutProps { menuButtonOptions: MenuButtonOption[]; diff --git a/website/src/components/Survey/TaskControlsOverridable.tsx b/website/src/components/Survey/TaskControlsOverridable.tsx index b5333087..a66a4a68 100644 --- a/website/src/components/Survey/TaskControlsOverridable.tsx +++ b/website/src/components/Survey/TaskControlsOverridable.tsx @@ -10,7 +10,7 @@ import { ModalOverlay, useDisclosure, } from "@chakra-ui/react"; -import { TaskControls, TaskControlsProps } from "./TaskControls"; +import { TaskControls, TaskControlsProps } from "src/components/TaskControls"; interface TaskControlsOverridableProps extends TaskControlsProps { isValid: boolean; diff --git a/website/src/components/Vision.tsx b/website/src/components/Vision.tsx index f6dad567..1c15d60b 100644 --- a/website/src/components/Vision.tsx +++ b/website/src/components/Vision.tsx @@ -1,5 +1,5 @@ -import { Container } from "./Container"; import Image from "next/image"; +import { Container } from "src/components/Container"; const Vision = () => { return ( diff --git a/website/src/pages/about.tsx b/website/src/pages/about.tsx index fdce53f6..490a6095 100644 --- a/website/src/pages/about.tsx +++ b/website/src/pages/about.tsx @@ -1,9 +1,9 @@ +import Image from "next/image"; +import { CallToAction } from "src/components/CallToAction"; import { Container } from "src/components/Container"; +import Roadmap from "src/components/Roadmap"; import Services from "src/components/Services"; import Vision from "src/components/Vision"; -import Roadmap from "src/components/Roadmap"; -import { CallToAction } from "src/components/CallToAction"; -import Image from "next/image"; const AboutPage = () => { return ( diff --git a/website/src/pages/api/username.tsx b/website/src/pages/api/username.tsx index 556edfa9..6cf362c2 100644 --- a/website/src/pages/api/username.tsx +++ b/website/src/pages/api/username.tsx @@ -1,5 +1,5 @@ import { getSession } from "next-auth/react"; -import prisma from "../../lib/prismadb"; +import prisma from "src/lib/prismadb"; // POST /api/post // Required fields in body: title diff --git a/website/src/pages/dashboard.tsx b/website/src/pages/dashboard.tsx index 8ab4d527..ece95212 100644 --- a/website/src/pages/dashboard.tsx +++ b/website/src/pages/dashboard.tsx @@ -1,7 +1,6 @@ import Head from "next/head"; - -import { getDashboardLayout } from "src/components/Layout"; import { LeaderboardTable, TaskOption } from "src/components/Dashboard"; +import { getDashboardLayout } from "src/components/Layout"; const Dashboard = () => { return ( diff --git a/website/src/pages/messages/[id]/index.tsx b/website/src/pages/messages/[id]/index.tsx index e778e74a..eacd5a72 100644 --- a/website/src/pages/messages/[id]/index.tsx +++ b/website/src/pages/messages/[id]/index.tsx @@ -1,13 +1,11 @@ import { Box, Container, Text, useColorModeValue } from "@chakra-ui/react"; import Head from "next/head"; -import { useRouter } from "next/router"; import { useState } from "react"; -import useSWR from "swr"; - -import fetcher from "src/lib/fetcher"; -import { MessageTableEntry } from "src/components/Messages/MessageTableEntry"; import { LoadingScreen } from "src/components/Loading/LoadingScreen"; +import { MessageTableEntry } from "src/components/Messages/MessageTableEntry"; import { MessageWithChildren } from "src/components/Messages/MessageWithChildren"; +import fetcher from "src/lib/fetcher"; +import useSWR from "swr"; const MessageDetail = ({ id }) => { const mainBg = useColorModeValue("bg-slate-300", "bg-slate-900"); @@ -18,7 +16,7 @@ const MessageDetail = ({ id }) => { onSuccess: (data) => { setParent(data); }, - onError: (err, key, config) => { + onError: () => { setParent(null); }, }); diff --git a/website/src/pages/messages/index.tsx b/website/src/pages/messages/index.tsx index 96ee19db..276a35ed 100644 --- a/website/src/pages/messages/index.tsx +++ b/website/src/pages/messages/index.tsx @@ -1,10 +1,10 @@ import { Box, CircularProgress, SimpleGrid, Text, useColorModeValue } from "@chakra-ui/react"; import Head from "next/head"; import { useState } from "react"; -import useSWRImmutable from "swr/immutable"; -import fetcher from "src/lib/fetcher"; -import { MessageTable } from "src/components/Messages/MessageTable"; import { getDashboardLayout } from "src/components/Layout"; +import { MessageTable } from "src/components/Messages/MessageTable"; +import fetcher from "src/lib/fetcher"; +import useSWRImmutable from "swr/immutable"; const MessagesDashboard = () => { const boxBgColor = useColorModeValue("white", "gray.700"); diff --git a/website/src/pages/privacy-policy.tsx b/website/src/pages/privacy-policy.tsx index 164fab93..b5949f42 100644 --- a/website/src/pages/privacy-policy.tsx +++ b/website/src/pages/privacy-policy.tsx @@ -1,7 +1,5 @@ import { Container, Heading } from "@chakra-ui/react"; import Head from "next/head"; -import { Footer } from "src/components/Footer"; -import { Header } from "src/components/Header"; import { getTransparentHeaderLayout } from "src/components/Layout"; const PrivacyPolicy = () => { diff --git a/website/src/pages/terms-of-service.tsx b/website/src/pages/terms-of-service.tsx index ce60a20e..44d3168d 100644 --- a/website/src/pages/terms-of-service.tsx +++ b/website/src/pages/terms-of-service.tsx @@ -1,7 +1,5 @@ import { Container, Heading } from "@chakra-ui/react"; import Head from "next/head"; -import { Footer } from "src/components/Footer"; -import { Header } from "src/components/Header"; import { getTransparentHeaderLayout } from "src/components/Layout"; const TermsOfService = () => { From 19ff44f442fb8fbca20205388d4ea7ef4681131b Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 7 Jan 2023 20:01:50 +0900 Subject: [PATCH 20/49] Fixing a build issue I introduced --- website/src/components/Messages/MessageTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/components/Messages/MessageTable.tsx b/website/src/components/Messages/MessageTable.tsx index ee1d3687..95ccc540 100644 --- a/website/src/components/Messages/MessageTable.tsx +++ b/website/src/components/Messages/MessageTable.tsx @@ -1,5 +1,5 @@ import { Stack, StackDivider } from "@chakra-ui/react"; -import { MessageTableEntry } from "src/components/MessageTableEntry"; +import { MessageTableEntry } from "src/components/Messages/MessageTableEntry"; export function MessageTable({ messages }) { return ( From ac409be6f957718e283342fed3f0cdef6d4c5a57 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 7 Jan 2023 20:04:54 +0900 Subject: [PATCH 21/49] Fixing a build issue I introduced --- website/src/components/Survey/TaskControlsOverridable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/components/Survey/TaskControlsOverridable.tsx b/website/src/components/Survey/TaskControlsOverridable.tsx index a66a4a68..25e973fb 100644 --- a/website/src/components/Survey/TaskControlsOverridable.tsx +++ b/website/src/components/Survey/TaskControlsOverridable.tsx @@ -10,7 +10,7 @@ import { ModalOverlay, useDisclosure, } from "@chakra-ui/react"; -import { TaskControls, TaskControlsProps } from "src/components/TaskControls"; +import { TaskControls, TaskControlsProps } from "src/components/Survey/TaskControls"; interface TaskControlsOverridableProps extends TaskControlsProps { isValid: boolean; From f57041cddab90c2fb2bb72d02eb79c0ee097396f Mon Sep 17 00:00:00 2001 From: klotske Date: Sat, 7 Jan 2023 15:08:38 +0300 Subject: [PATCH 22/49] Added basic Admin dashboard layout --- website/src/components/Layout.tsx | 20 +++++++++++++++++++- website/src/pages/admin/index.tsx | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/website/src/components/Layout.tsx b/website/src/components/Layout.tsx index 1faefcc0..c4450f58 100644 --- a/website/src/components/Layout.tsx +++ b/website/src/components/Layout.tsx @@ -1,7 +1,7 @@ // https://nextjs.org/docs/basic-features/layouts import type { NextPage } from "next"; -import { FiLayout, FiMessageSquare } from "react-icons/fi"; +import { FiLayout, FiMessageSquare, FiUsers } from "react-icons/fi"; import { Header } from "src/components/Header"; import { Footer } from "./Footer"; @@ -51,4 +51,22 @@ export const getDashboardLayout = (page: React.ReactElement) => ( ); +export const getAdminLayout = (page: React.ReactElement) => ( +
+
+ + {page} + +
+); + export const noLayout = (page: React.ReactElement) => page; diff --git a/website/src/pages/admin/index.tsx b/website/src/pages/admin/index.tsx index 60d61903..114eee3e 100644 --- a/website/src/pages/admin/index.tsx +++ b/website/src/pages/admin/index.tsx @@ -2,7 +2,7 @@ import Head from "next/head"; import { useRouter } from "next/router"; import { useSession } from "next-auth/react"; import { useEffect } from "react"; -import { getTransparentHeaderLayout } from "src/components/Layout"; +import { getAdminLayout } from "src/components/Layout"; import UsersCell from "src/components/UsersCell"; /** @@ -44,6 +44,6 @@ const AdminIndex = () => { ); }; -AdminIndex.getLayout = getTransparentHeaderLayout; +AdminIndex.getLayout = getAdminLayout; export default AdminIndex; From a493fc2fe9fb51c445eb16c65615e3985933c622 Mon Sep 17 00:00:00 2001 From: klotske Date: Sat, 7 Jan 2023 16:14:36 +0300 Subject: [PATCH 23/49] Changed some links to NextJS & Fixed messages page --- website/src/components/Header/UserMenu.tsx | 2 ++ website/src/components/SideMenu.tsx | 3 ++- website/src/pages/messages/index.tsx | 15 ++++++++++++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/website/src/components/Header/UserMenu.tsx b/website/src/components/Header/UserMenu.tsx index 2e06606c..c5ca92bd 100644 --- a/website/src/components/Header/UserMenu.tsx +++ b/website/src/components/Header/UserMenu.tsx @@ -2,6 +2,7 @@ import { Box, Link, Text, useColorModeValue } from "@chakra-ui/react"; import { Popover } from "@headlessui/react"; import { AnimatePresence, motion } from "framer-motion"; import Image from "next/image"; +import NextLink from "next/link"; import { signOut, useSession } from "next-auth/react"; import React from "react"; import { FiLayout, FiLogOut, FiSettings } from "react-icons/fi"; @@ -77,6 +78,7 @@ export function UserMenu() { {accountOptions.map((item) => ( { const [messages, setMessages] = useState([]); const [userMessages, setUserMessages] = useState([]); - const { isLoading: isLoadingAll } = useSWRImmutable("/api/messages", fetcher, { + const { isLoading: isLoadingAll, mutate: mutateAll } = useSWRImmutable("/api/messages", fetcher, { onSuccess: (data) => { setMessages(data); }, }); - const { isLoading: isLoadingUser } = useSWRImmutable(`/api/messages/user`, fetcher, { + const { isLoading: isLoadingUser, mutate: mutateUser } = useSWRImmutable(`/api/messages/user`, fetcher, { onSuccess: (data) => { setUserMessages(data); }, }); + useEffect(() => { + if (messages.length == 0) { + mutateAll(); + } + if (userMessages.length == 0) { + mutateUser(); + } + }, [messages, userMessages]); + return ( <> From 043b5eff5ac41e116a176f2d6c4a72200377fa01 Mon Sep 17 00:00:00 2001 From: Ken Tsui Date: Sat, 7 Jan 2023 21:37:30 +0800 Subject: [PATCH 24/49] Add DEBUG_USD_SEED_DATA_PATH in config to make seed data flexible (#395) * Add DEBUG_USD_SEED_DATA_PATH in config to make seed data flexible * reformat * Copy test_data folder in Dockerfile.backend, correct DEBUG_USE_SEED_DATA_PATH in cofig * - make DEBUG_USE_SEED_DATA_PATH to absolute path - correct test_data path in Dockerfile.backend --- backend/main.py | 68 ++----------------- backend/oasst_backend/config.py | 6 +- .../test_data/generic/test_generic_data.json | 58 ++++++++++++++++ docker/Dockerfile.backend | 1 + 4 files changed, 71 insertions(+), 62 deletions(-) create mode 100644 backend/test_data/generic/test_generic_data.json diff --git a/backend/main.py b/backend/main.py index edbad943..2a3bb230 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,3 +1,4 @@ +import json from http import HTTPStatus from math import ceil from pathlib import Path @@ -6,7 +7,6 @@ from typing import Optional import alembic.command import alembic.config import fastapi -import pydantic import redis.asyncio as redis from fastapi_limiter import FastAPILimiter from loguru import logger @@ -17,6 +17,7 @@ from oasst_backend.database import engine from oasst_backend.prompt_repository import PromptRepository from oasst_shared.exceptions import OasstError, OasstErrorCode from oasst_shared.schemas import protocol as protocol_schema +from pydantic import BaseModel from sqlmodel import Session from starlette.middleware.cors import CORSMiddleware @@ -97,7 +98,7 @@ if settings.DEBUG_USE_SEED_DATA: @app.on_event("startup") def seed_data(): - class DummyMessage(pydantic.BaseModel): + class DummyMessage(BaseModel): task_message_id: str user_message_id: str parent_message_id: Optional[str] @@ -111,64 +112,10 @@ if settings.DEBUG_USE_SEED_DATA: dummy_user = protocol_schema.User(id="__dummy_user__", display_name="Dummy User", auth_method="local") pr = PromptRepository(db=db, api_client=api_client, user=dummy_user) - dummy_messages = [ - DummyMessage( - task_message_id="de111fa8", - user_message_id="6f1d0711", - parent_message_id=None, - text="Hi!", - role="prompter", - ), - DummyMessage( - task_message_id="74c381d4", - user_message_id="4a24530b", - parent_message_id="6f1d0711", - text="Hello! How can I help you?", - role="assistant", - ), - DummyMessage( - task_message_id="3d5dc440", - user_message_id="a8c01c04", - parent_message_id="4a24530b", - text="Do you have a recipe for potato soup?", - role="prompter", - ), - DummyMessage( - task_message_id="643716c1", - user_message_id="f43a93b7", - parent_message_id="4a24530b", - text="Who were the 8 presidents before George Washington?", - role="prompter", - ), - DummyMessage( - task_message_id="2e4e1e6", - user_message_id="c886920", - parent_message_id="6f1d0711", - text="Hey buddy! How can I serve you?", - role="assistant", - ), - DummyMessage( - task_message_id="970c437d", - user_message_id="cec432cf", - parent_message_id=None, - text="euirdteunvglfe23908230892309832098 AAAAAAAA", - role="prompter", - ), - DummyMessage( - task_message_id="6066118e", - user_message_id="4f85f637", - parent_message_id="cec432cf", - text="Sorry, I did not understand your request and it is unclear to me what you want me to do. Could you describe it in a different way?", - role="assistant", - ), - DummyMessage( - task_message_id="ba87780d", - user_message_id="0e276b98", - parent_message_id="cec432cf", - text="I'm unsure how to interpret this. Is it a riddle?", - role="assistant", - ), - ] + with open(settings.DEBUG_USE_SEED_DATA_PATH) as f: + dummy_messages_raw = json.load(f) + + dummy_messages = [DummyMessage(**dm) for dm in dummy_messages_raw] for msg in dummy_messages: task = pr.fetch_task_by_frontend_message_id(msg.task_message_id) @@ -219,7 +166,6 @@ if __name__ == "__main__": # Importing here so we don't import packages unnecessarily if we're # importing main as a module. import argparse - import json import uvicorn diff --git a/backend/oasst_backend/config.py b/backend/oasst_backend/config.py index df37dc9f..1765af7a 100644 --- a/backend/oasst_backend/config.py +++ b/backend/oasst_backend/config.py @@ -1,6 +1,7 @@ +from pathlib import Path from typing import Any, Dict, List, Optional, Union -from pydantic import AnyHttpUrl, BaseSettings, PostgresDsn, validator +from pydantic import AnyHttpUrl, BaseSettings, FilePath, PostgresDsn, validator class Settings(BaseSettings): @@ -21,6 +22,9 @@ class Settings(BaseSettings): DEBUG_ALLOW_ANY_API_KEY: bool = False DEBUG_SKIP_API_KEY_CHECK: bool = False DEBUG_USE_SEED_DATA: bool = False + DEBUG_USE_SEED_DATA_PATH: Optional[FilePath] = ( + Path(__file__).parent.parent / "test_data/generic/test_generic_data.json" + ) HUGGING_FACE_API_KEY: str = "" diff --git a/backend/test_data/generic/test_generic_data.json b/backend/test_data/generic/test_generic_data.json new file mode 100644 index 00000000..b634902d --- /dev/null +++ b/backend/test_data/generic/test_generic_data.json @@ -0,0 +1,58 @@ +[ + { + "task_message_id": "de111fa8", + "user_message_id": "6f1d0711", + "parent_message_id": null, + "text": "Hi!", + "role": "prompter" + }, + { + "task_message_id": "74c381d4", + "user_message_id": "4a24530b", + "parent_message_id": "6f1d0711", + "text": "Hello! How can I help you?", + "role": "assistant" + }, + { + "task_message_id": "3d5dc440", + "user_message_id": "a8c01c04", + "parent_message_id": "4a24530b", + "text": "Do you have a recipe for potato soup?", + "role": "prompter" + }, + { + "task_message_id": "643716c1", + "user_message_id": "f43a93b7", + "parent_message_id": "4a24530b", + "text": "Who were the 8 presidents before George Washington?", + "role": "prompter" + }, + { + "task_message_id": "2e4e1e6", + "user_message_id": "c886920", + "parent_message_id": "6f1d0711", + "text": "Hey buddy! How can I serve you?", + "role": "assistant" + }, + { + "task_message_id": "970c437d", + "user_message_id": "cec432cf", + "parent_message_id": null, + "text": "euirdteunvglfe23908230892309832098 AAAAAAAA", + "role": "prompter" + }, + { + "task_message_id": "6066118e", + "user_message_id": "4f85f637", + "parent_message_id": "cec432cf", + "text": "Sorry, I did not understand your request and it is unclear to me what you want me to do. Could you describe it in a different way?", + "role": "assistant" + }, + { + "task_message_id": "ba87780d", + "user_message_id": "0e276b98", + "parent_message_id": "cec432cf", + "text": "I'm unsure how to interpret this. Is it a riddle?", + "role": "assistant" + } +] diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend index 1f3bdfcd..c89a0280 100644 --- a/docker/Dockerfile.backend +++ b/docker/Dockerfile.backend @@ -14,3 +14,4 @@ COPY ./backend/alembic /app/alembic COPY ./backend/alembic.ini /app/alembic.ini COPY ./backend/main.py /app/main.py COPY ./backend/oasst_backend /app/oasst_backend +COPY ./backend/test_data /app/test_data From cff0b7ee01918ee423df252203bdebc1a8cd3ba8 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Sat, 7 Jan 2023 13:43:59 +0000 Subject: [PATCH 25/49] remove docs-site-poc branch (#464) --- .github/workflows/deploy-docs-site.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/deploy-docs-site.yaml b/.github/workflows/deploy-docs-site.yaml index b1785292..d3fee6cb 100644 --- a/.github/workflows/deploy-docs-site.yaml +++ b/.github/workflows/deploy-docs-site.yaml @@ -4,7 +4,6 @@ on: push: branches: - main - - docs-site-poc paths: - ".github/workflows/deploy-docs-site.yaml" - "docs/**" @@ -45,9 +44,7 @@ jobs: - name: Deploy uses: peaceiris/actions-gh-pages@v3 - if: - ${{ github.ref == 'refs/heads/main' || github.ref == - 'refs/heads/docs-site-poc' }} + if: ${{ github.ref == 'refs/heads/main' }} with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./docs/build From eaefa68dea2e4a375febbaac7bb875c61d1eee83 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Sat, 7 Jan 2023 13:54:16 +0000 Subject: [PATCH 26/49] add colab buttons to notebooks (#468) --- .../code-bugger/openbugger_example.ipynb | 8 + .../EssayInstructions.ipynb | 8 + .../data-argumentation/EssayRevision.ipynb | 8 + .../StackExchangeBuilder.ipynb | 1146 +++++++++-------- .../DetoxityEvaluation.ipynb | 18 + 5 files changed, 619 insertions(+), 569 deletions(-) diff --git a/notebooks/code-bugger/openbugger_example.ipynb b/notebooks/code-bugger/openbugger_example.ipynb index 22e9b0c8..6e2acd27 100644 --- a/notebooks/code-bugger/openbugger_example.ipynb +++ b/notebooks/code-bugger/openbugger_example.ipynb @@ -1,5 +1,13 @@ { "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/code-bugger/openbugger_example.ipynb)" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/data-argumentation/EssayInstructions.ipynb b/notebooks/data-argumentation/EssayInstructions.ipynb index c4179382..30834d32 100644 --- a/notebooks/data-argumentation/EssayInstructions.ipynb +++ b/notebooks/data-argumentation/EssayInstructions.ipynb @@ -1,5 +1,13 @@ { "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-argumentation/EssayInstructions.ipynb)" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/data-argumentation/EssayRevision.ipynb b/notebooks/data-argumentation/EssayRevision.ipynb index cba9bc5b..2397131c 100644 --- a/notebooks/data-argumentation/EssayRevision.ipynb +++ b/notebooks/data-argumentation/EssayRevision.ipynb @@ -1,5 +1,13 @@ { "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-argumentation/EssayRevision.ipynb)" + ] + }, { "cell_type": "markdown", "metadata": { diff --git a/notebooks/data-argumentation/StackExchangeBuilder.ipynb b/notebooks/data-argumentation/StackExchangeBuilder.ipynb index 625d757b..b0dd9a8b 100644 --- a/notebooks/data-argumentation/StackExchangeBuilder.ipynb +++ b/notebooks/data-argumentation/StackExchangeBuilder.ipynb @@ -1,28 +1,22 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, "cells": [ { + "attachments": {}, "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-argumentation/StackExchangeBuilder.ipynb)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TB7CEfs8F-8u" + }, "source": [ "# Ingest StackExchange data dumps\n", "This notebook takes a StackExchange Data dump \"Posts.xml\" file and ingests it into a Pandas Dataframe. Outputs of the file can be JSON, JSONL, Parquet, or CSV. " - ], - "metadata": { - "id": "TB7CEfs8F-8u" - } + ] }, { "cell_type": "code", @@ -40,16 +34,34 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "15mAL7GnzBv0" + }, "source": [ "# Extract StackExchange\n", "Pull StackExchange file dumps. Specific column types are enforced to prevent errors on processing later in the notebook" - ], - "metadata": { - "id": "15mAL7GnzBv0" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "FtcvUEaHVxcW", + "outputId": "5b0cb19d-e3d9-422b-9077-52241bd09e0e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "dict_keys(['3dprinting_meta', '3dprinting', 'Stackoverflow_com_Posts_7z', 'academia_meta', 'academia', 'ai_meta', 'ai', 'android_meta', 'android', 'anime_meta', 'anime', 'apple_meta', 'apple', 'arduino_meta', 'arduino', 'askubuntu_com_7z', 'astronomy_meta', 'astronomy', 'aviation_meta', 'aviation', 'avp_meta', 'avp', 'beer_meta', 'beer', 'bicycles_meta', 'bicycles', 'bioacoustics_meta', 'bioacoustics', 'bioinformatics_meta', 'bioinformatics', 'biology_meta', 'biology', 'bitcoin_meta', 'bitcoin', 'blender_meta', 'blender', 'boardgames_meta', 'boardgames', 'bricks_meta', 'bricks', 'buddhism_meta', 'buddhism', 'cardano_meta', 'cardano', 'chemistry_meta', 'chemistry', 'chess_meta', 'chess', 'chinese_meta', 'chinese', 'christianity_meta', 'christianity', 'civicrm_meta', 'civicrm', 'codegolf_meta', 'codegolf', 'codereview_meta', 'codereview', 'coffee_meta', 'coffee', 'cogsci_meta', 'cogsci', 'computergraphics_meta', 'computergraphics', 'conlang_meta', 'conlang', 'cooking_meta', 'cooking', 'craftcms_meta', 'craftcms', 'crafts_meta', 'crafts', 'crypto_meta', 'crypto', 'cs_meta', 'cs', 'cseducators_meta', 'cseducators', 'cstheory_meta', 'cstheory', 'datascience_meta', 'datascience', 'dba_meta', 'dba', 'devops_meta', 'devops', 'diy_meta', 'diy', 'drones_meta', 'drones', 'drupal_meta', 'drupal', 'dsp_meta', 'dsp', 'earthscience_meta', 'earthscience', 'ebooks_meta', 'ebooks', 'economics_meta', 'economics', 'electronics_meta', 'electronics', 'elementaryos_meta', 'elementaryos', 'ell_meta', 'ell', 'emacs_meta', 'emacs', 'engineering_meta', 'engineering', 'english_meta', 'english', 'eosio_meta', 'eosio', 'es_meta_stackoverflow_com_7z', 'es_stackoverflow_com_7z', 'esperanto_meta', 'esperanto', 'ethereum_meta', 'ethereum', 'expatriates_meta', 'expatriates', 'expressionengine_meta', 'expressionengine', 'fitness_meta', 'fitness', 'freelancing_meta', 'freelancing', 'french_meta', 'french', 'gamedev_meta', 'gamedev', 'gaming_meta', 'gaming', 'gardening_meta', 'gardening', 'genealogy_meta', 'genealogy', 'german_meta', 'german', 'gis_meta', 'gis', 'graphicdesign_meta', 'graphicdesign', 'ham_meta', 'ham', 'hardwarerecs_meta', 'hardwarerecs', 'health_meta', 'health', 'hermeneutics_meta', 'hermeneutics', 'hinduism_meta', 'hinduism', 'history_meta', 'history', 'homebrew_meta', 'homebrew', 'hsm_meta', 'hsm', 'interpersonal_meta', 'interpersonal', 'iot_meta', 'iot', 'iota_meta', 'iota', 'islam_meta', 'islam', 'italian_meta', 'italian', 'ja_meta_stackoverflow_com_7z', 'ja_stackoverflow_com_7z', 'japanese_meta', 'japanese', 'joomla_meta', 'joomla', 'judaism_meta', 'judaism', 'korean_meta', 'korean', 'languagelearning_meta', 'languagelearning', 'latin_meta', 'latin', 'law_meta', 'law', 'lifehacks_meta', 'lifehacks', 'linguistics_meta', 'linguistics', 'literature_meta', 'literature', 'magento_meta', 'magento', 'martialarts_meta', 'martialarts', 'materials_meta', 'materials', 'math_meta', 'math', 'matheducators_meta', 'matheducators', 'mathematica_meta', 'mathematica', 'mathoverflow_net_7z', 'mechanics_meta', 'mechanics', 'meta_askubuntu_com_7z', 'meta_mathoverflow_net_7z', 'meta_serverfault_com_7z', 'meta', 'meta_stackoverflow_com_7z', 'meta_superuser_com_7z', 'moderators_meta', 'moderators', 'monero_meta', 'monero', 'money_meta', 'money', 'movies_meta', 'movies', 'music_meta', 'music', 'musicfans_meta', 'musicfans', 'mythology_meta', 'mythology', 'networkengineering_meta', 'networkengineering', 'opendata_meta', 'opendata', 'opensource_meta', 'opensource', 'or_meta', 'or', 'outdoors_meta', 'outdoors', 'parenting_meta', 'parenting', 'patents_meta', 'patents', 'pets_meta', 'pets', 'philosophy_meta', 'philosophy', 'photo_meta', 'photo', 'physics_meta', 'physics', 'pm_meta', 'pm', 'poker_meta', 'poker', 'politics_meta', 'politics', 'portuguese_meta', 'portuguese', 'proofassistants_meta', 'proofassistants', 'pt_meta_stackoverflow_com_7z', 'pt_stackoverflow_com_7z', 'puzzling_meta', 'puzzling', 'quant_meta', 'quant', 'quantumcomputing_meta', 'quantumcomputing', 'raspberrypi_meta', 'raspberrypi', 'retrocomputing_meta', 'retrocomputing', 'reverseengineering_meta', 'reverseengineering', 'robotics_meta', 'robotics', 'rpg_meta', 'rpg', 'ru_meta_stackoverflow_com_7z', 'ru_stackoverflow_com_7z', 'rus_meta', 'rus', 'russian_meta', 'russian', 'salesforce_meta', 'salesforce', 'scicomp_meta', 'scicomp', 'scifi_meta', 'scifi', 'security_meta', 'security', 'serverfault_com_7z', 'sharepoint_meta', 'sharepoint', 'sitecore_meta', 'sitecore', 'skeptics_meta', 'skeptics', 'softwareengineering_meta', 'softwareengineering', 'softwarerecs_meta', 'softwarerecs', 'solana_meta', 'solana', 'sound_meta', 'sound', 'space_meta', 'space', 'spanish_meta', 'spanish', 'sports_meta', 'sports', 'sqa_meta', 'sqa', 'stackapps_com_7z', 'stackoverflow_com_Badges_7z', 'stackoverflow_com_Comments_7z', 'stackoverflow_com_PostHistory_7z', 'stackoverflow_com_PostLinks_7z', 'stackoverflow_com_Tags_7z', 'stackoverflow_com_Users_7z', 'stackoverflow_com_Votes_7z', 'stats_meta', 'stats', 'stellar_meta', 'stellar', 'substrate_meta', 'substrate', 'superuser_com_7z', 'sustainability_meta', 'sustainability', 'tex_meta', 'tex', 'tezos_meta', 'tezos', 'tor_meta', 'tor', 'travel_meta', 'travel', 'tridion_meta', 'tridion', 'ukrainian_meta', 'ukrainian', 'unix_meta', 'unix', 'ux_meta', 'ux', 'vegetarianism_meta', 'vegetarianism', 'vi_meta', 'vi', 'webapps_meta', 'webapps', 'webmasters_meta', 'webmasters', 'windowsphone_meta', 'windowsphone', 'woodworking_meta', 'woodworking', 'wordpress_meta', 'wordpress', 'workplace_meta', 'workplace', 'worldbuilding_meta', 'worldbuilding', 'writers_meta', 'writers'])\n", + "https://ia600107.us.archive.org/view_archive.php?archive=/27/items/stackexchange/ai.stackexchange.com.7z&file=Posts.xml\n" + ] + } + ], "source": [ "base_url = \"https://ia600107.us.archive.org/view_archive.php?archive=/27/items/stackexchange/{0}&file=Posts.xml\"\n", "\n", @@ -73,90 +85,11 @@ "\n", "print(urls.keys())\n", "print(urls.get(\"ai\"))" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "FtcvUEaHVxcW", - "outputId": "5b0cb19d-e3d9-422b-9077-52241bd09e0e" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "dict_keys(['3dprinting_meta', '3dprinting', 'Stackoverflow_com_Posts_7z', 'academia_meta', 'academia', 'ai_meta', 'ai', 'android_meta', 'android', 'anime_meta', 'anime', 'apple_meta', 'apple', 'arduino_meta', 'arduino', 'askubuntu_com_7z', 'astronomy_meta', 'astronomy', 'aviation_meta', 'aviation', 'avp_meta', 'avp', 'beer_meta', 'beer', 'bicycles_meta', 'bicycles', 'bioacoustics_meta', 'bioacoustics', 'bioinformatics_meta', 'bioinformatics', 'biology_meta', 'biology', 'bitcoin_meta', 'bitcoin', 'blender_meta', 'blender', 'boardgames_meta', 'boardgames', 'bricks_meta', 'bricks', 'buddhism_meta', 'buddhism', 'cardano_meta', 'cardano', 'chemistry_meta', 'chemistry', 'chess_meta', 'chess', 'chinese_meta', 'chinese', 'christianity_meta', 'christianity', 'civicrm_meta', 'civicrm', 'codegolf_meta', 'codegolf', 'codereview_meta', 'codereview', 'coffee_meta', 'coffee', 'cogsci_meta', 'cogsci', 'computergraphics_meta', 'computergraphics', 'conlang_meta', 'conlang', 'cooking_meta', 'cooking', 'craftcms_meta', 'craftcms', 'crafts_meta', 'crafts', 'crypto_meta', 'crypto', 'cs_meta', 'cs', 'cseducators_meta', 'cseducators', 'cstheory_meta', 'cstheory', 'datascience_meta', 'datascience', 'dba_meta', 'dba', 'devops_meta', 'devops', 'diy_meta', 'diy', 'drones_meta', 'drones', 'drupal_meta', 'drupal', 'dsp_meta', 'dsp', 'earthscience_meta', 'earthscience', 'ebooks_meta', 'ebooks', 'economics_meta', 'economics', 'electronics_meta', 'electronics', 'elementaryos_meta', 'elementaryos', 'ell_meta', 'ell', 'emacs_meta', 'emacs', 'engineering_meta', 'engineering', 'english_meta', 'english', 'eosio_meta', 'eosio', 'es_meta_stackoverflow_com_7z', 'es_stackoverflow_com_7z', 'esperanto_meta', 'esperanto', 'ethereum_meta', 'ethereum', 'expatriates_meta', 'expatriates', 'expressionengine_meta', 'expressionengine', 'fitness_meta', 'fitness', 'freelancing_meta', 'freelancing', 'french_meta', 'french', 'gamedev_meta', 'gamedev', 'gaming_meta', 'gaming', 'gardening_meta', 'gardening', 'genealogy_meta', 'genealogy', 'german_meta', 'german', 'gis_meta', 'gis', 'graphicdesign_meta', 'graphicdesign', 'ham_meta', 'ham', 'hardwarerecs_meta', 'hardwarerecs', 'health_meta', 'health', 'hermeneutics_meta', 'hermeneutics', 'hinduism_meta', 'hinduism', 'history_meta', 'history', 'homebrew_meta', 'homebrew', 'hsm_meta', 'hsm', 'interpersonal_meta', 'interpersonal', 'iot_meta', 'iot', 'iota_meta', 'iota', 'islam_meta', 'islam', 'italian_meta', 'italian', 'ja_meta_stackoverflow_com_7z', 'ja_stackoverflow_com_7z', 'japanese_meta', 'japanese', 'joomla_meta', 'joomla', 'judaism_meta', 'judaism', 'korean_meta', 'korean', 'languagelearning_meta', 'languagelearning', 'latin_meta', 'latin', 'law_meta', 'law', 'lifehacks_meta', 'lifehacks', 'linguistics_meta', 'linguistics', 'literature_meta', 'literature', 'magento_meta', 'magento', 'martialarts_meta', 'martialarts', 'materials_meta', 'materials', 'math_meta', 'math', 'matheducators_meta', 'matheducators', 'mathematica_meta', 'mathematica', 'mathoverflow_net_7z', 'mechanics_meta', 'mechanics', 'meta_askubuntu_com_7z', 'meta_mathoverflow_net_7z', 'meta_serverfault_com_7z', 'meta', 'meta_stackoverflow_com_7z', 'meta_superuser_com_7z', 'moderators_meta', 'moderators', 'monero_meta', 'monero', 'money_meta', 'money', 'movies_meta', 'movies', 'music_meta', 'music', 'musicfans_meta', 'musicfans', 'mythology_meta', 'mythology', 'networkengineering_meta', 'networkengineering', 'opendata_meta', 'opendata', 'opensource_meta', 'opensource', 'or_meta', 'or', 'outdoors_meta', 'outdoors', 'parenting_meta', 'parenting', 'patents_meta', 'patents', 'pets_meta', 'pets', 'philosophy_meta', 'philosophy', 'photo_meta', 'photo', 'physics_meta', 'physics', 'pm_meta', 'pm', 'poker_meta', 'poker', 'politics_meta', 'politics', 'portuguese_meta', 'portuguese', 'proofassistants_meta', 'proofassistants', 'pt_meta_stackoverflow_com_7z', 'pt_stackoverflow_com_7z', 'puzzling_meta', 'puzzling', 'quant_meta', 'quant', 'quantumcomputing_meta', 'quantumcomputing', 'raspberrypi_meta', 'raspberrypi', 'retrocomputing_meta', 'retrocomputing', 'reverseengineering_meta', 'reverseengineering', 'robotics_meta', 'robotics', 'rpg_meta', 'rpg', 'ru_meta_stackoverflow_com_7z', 'ru_stackoverflow_com_7z', 'rus_meta', 'rus', 'russian_meta', 'russian', 'salesforce_meta', 'salesforce', 'scicomp_meta', 'scicomp', 'scifi_meta', 'scifi', 'security_meta', 'security', 'serverfault_com_7z', 'sharepoint_meta', 'sharepoint', 'sitecore_meta', 'sitecore', 'skeptics_meta', 'skeptics', 'softwareengineering_meta', 'softwareengineering', 'softwarerecs_meta', 'softwarerecs', 'solana_meta', 'solana', 'sound_meta', 'sound', 'space_meta', 'space', 'spanish_meta', 'spanish', 'sports_meta', 'sports', 'sqa_meta', 'sqa', 'stackapps_com_7z', 'stackoverflow_com_Badges_7z', 'stackoverflow_com_Comments_7z', 'stackoverflow_com_PostHistory_7z', 'stackoverflow_com_PostLinks_7z', 'stackoverflow_com_Tags_7z', 'stackoverflow_com_Users_7z', 'stackoverflow_com_Votes_7z', 'stats_meta', 'stats', 'stellar_meta', 'stellar', 'substrate_meta', 'substrate', 'superuser_com_7z', 'sustainability_meta', 'sustainability', 'tex_meta', 'tex', 'tezos_meta', 'tezos', 'tor_meta', 'tor', 'travel_meta', 'travel', 'tridion_meta', 'tridion', 'ukrainian_meta', 'ukrainian', 'unix_meta', 'unix', 'ux_meta', 'ux', 'vegetarianism_meta', 'vegetarianism', 'vi_meta', 'vi', 'webapps_meta', 'webapps', 'webmasters_meta', 'webmasters', 'windowsphone_meta', 'windowsphone', 'woodworking_meta', 'woodworking', 'wordpress_meta', 'wordpress', 'workplace_meta', 'workplace', 'worldbuilding_meta', 'worldbuilding', 'writers_meta', 'writers'])\n", - "https://ia600107.us.archive.org/view_archive.php?archive=/27/items/stackexchange/ai.stackexchange.com.7z&file=Posts.xml\n" - ] - } ] }, { "cell_type": "code", - "source": [ - "xml_format_map = {\n", - " \"Id\": int,\n", - " \"PostTypeId\": int,\n", - " \"CreationDate\": str,\n", - " \"Score\": int,\n", - " \"ViewCount\": int,\n", - " \"Body\": str,\n", - " \"AnswerCount\": int,\n", - " \"CommentCount\": int,\n", - " \"ContentLicense\": str,\n", - " \"AcceptedAnswerId\": int,\n", - " \"ParentId\": int,\n", - "}\n", - "\n", - "\n", - "# def extract_xml_file(file_url: str):\n", - "# table = pd.read_xml(file_url)\n", - "# return table\n", - "\n", - "\n", - "def xml_to_df(response: str):\n", - " \"\"\"\n", - " Collect and Manually import XML into Dataframe\n", - "\n", - " pd.read_xml() errors when XML trees are too large, this is just a hack to\n", - " download a XML file and parse into a Dataframe. **Not Tested on huge XML files**\n", - "\n", - " Parameters:\n", - " response (Requests.Response): Requests response object with the XML data\n", - "\n", - " Returns:\n", - " df (DataFrame): A Dataframe from the XML file\n", - " \"\"\"\n", - " soup = bs(response.content, \"xml\")\n", - " posts = soup.find_all(\"row\")\n", - "\n", - " all_posts = [post.attrs for post in posts]\n", - "\n", - " df = pd.DataFrame(all_posts)\n", - " df.AnswerCount.fillna(0, inplace=True)\n", - " df.ViewCount.fillna(0, inplace=True)\n", - " df.AcceptedAnswerId.fillna(0, inplace=True)\n", - " df.ParentId.fillna(0, inplace=True)\n", - " df[\"DataSource\"] = response.url\n", - " df = df.astype(xml_format_map)\n", - " return df\n", - "\n", - "\n", - "dataset_name = \"ai\"\n", - "\n", - "xml_posts_path = urls.get(dataset_name)\n", - "\n", - "\n", - "# df = extract_xml_file(test)\n", - "response = requests.get(xml_posts_path)\n", - "df = xml_to_df(response)\n", - "\n", - "\n", - "print(df.dtypes)\n", - "df.head()" - ], + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -165,11 +98,10 @@ "id": "-t27RnxdzBYB", "outputId": "5ec0ceed-c82b-48fa-facd-41b4aae2f9e6" }, - "execution_count": null, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Id int64\n", "PostTypeId int64\n", @@ -198,53 +130,7 @@ ] }, { - "output_type": "execute_result", "data": { - "text/plain": [ - " Id PostTypeId AcceptedAnswerId CreationDate Score \\\n", - "0 1 1 3 2016-08-02T15:39:14.947 10 \n", - "1 2 1 9 2016-08-02T15:40:20.623 14 \n", - "2 3 2 0 2016-08-02T15:40:24.820 15 \n", - "3 4 1 12 2016-08-02T15:41:22.020 33 \n", - "4 6 1 20 2016-08-02T15:43:35.460 7 \n", - "\n", - " ViewCount Body OwnerUserId \\\n", - "0 710

What does \"backprop\" mean? Is the \"backprop... 8 \n", - "1 1008

Does increasing the noise in data help to i... 8 \n", - "2 0

\"Backprop\" is the same as \"backpropagation\"... 4 \n", - "3 1266

When you're writing your algorithm, how do ... 8 \n", - "4 279

Given the following definition of an intell... 29 \n", - "\n", - " LastEditorUserId LastEditDate ... AnswerCount CommentCount \\\n", - "0 2444 2019-11-16T17:56:22.093 ... 5 0 \n", - "1 2444 2019-02-23T22:36:19.090 ... 3 0 \n", - "2 NaN NaN ... 0 0 \n", - "3 2444 2021-01-19T23:54:07.813 ... 4 0 \n", - "4 2444 2019-06-15T18:25:58.513 ... 2 0 \n", - "\n", - " ContentLicense ParentId ClosedDate FavoriteCount CommunityOwnedDate \\\n", - "0 CC BY-SA 4.0 0 NaN NaN NaN \n", - "1 CC BY-SA 4.0 0 NaN NaN NaN \n", - "2 CC BY-SA 3.0 1 NaN NaN NaN \n", - "3 CC BY-SA 3.0 0 NaN NaN NaN \n", - "4 CC BY-SA 4.0 0 NaN NaN NaN \n", - "\n", - " LastEditorDisplayName OwnerDisplayName \\\n", - "0 NaN NaN \n", - "1 NaN NaN \n", - "2 NaN NaN \n", - "3 NaN NaN \n", - "4 NaN NaN \n", - "\n", - " DataSource \n", - "0 https://ia600107.us.archive.org/view_archive.p... \n", - "1 https://ia600107.us.archive.org/view_archive.p... \n", - "2 https://ia600107.us.archive.org/view_archive.p... \n", - "3 https://ia600107.us.archive.org/view_archive.p... \n", - "4 https://ia600107.us.archive.org/view_archive.p... \n", - "\n", - "[5 rows x 23 columns]" - ], "text/html": [ "\n", "

\n", @@ -491,99 +377,133 @@ "
\n", " \n", " " + ], + "text/plain": [ + " Id PostTypeId AcceptedAnswerId CreationDate Score \\\n", + "0 1 1 3 2016-08-02T15:39:14.947 10 \n", + "1 2 1 9 2016-08-02T15:40:20.623 14 \n", + "2 3 2 0 2016-08-02T15:40:24.820 15 \n", + "3 4 1 12 2016-08-02T15:41:22.020 33 \n", + "4 6 1 20 2016-08-02T15:43:35.460 7 \n", + "\n", + " ViewCount Body OwnerUserId \\\n", + "0 710

What does \"backprop\" mean? Is the \"backprop... 8 \n", + "1 1008

Does increasing the noise in data help to i... 8 \n", + "2 0

\"Backprop\" is the same as \"backpropagation\"... 4 \n", + "3 1266

When you're writing your algorithm, how do ... 8 \n", + "4 279

Given the following definition of an intell... 29 \n", + "\n", + " LastEditorUserId LastEditDate ... AnswerCount CommentCount \\\n", + "0 2444 2019-11-16T17:56:22.093 ... 5 0 \n", + "1 2444 2019-02-23T22:36:19.090 ... 3 0 \n", + "2 NaN NaN ... 0 0 \n", + "3 2444 2021-01-19T23:54:07.813 ... 4 0 \n", + "4 2444 2019-06-15T18:25:58.513 ... 2 0 \n", + "\n", + " ContentLicense ParentId ClosedDate FavoriteCount CommunityOwnedDate \\\n", + "0 CC BY-SA 4.0 0 NaN NaN NaN \n", + "1 CC BY-SA 4.0 0 NaN NaN NaN \n", + "2 CC BY-SA 3.0 1 NaN NaN NaN \n", + "3 CC BY-SA 3.0 0 NaN NaN NaN \n", + "4 CC BY-SA 4.0 0 NaN NaN NaN \n", + "\n", + " LastEditorDisplayName OwnerDisplayName \\\n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "\n", + " DataSource \n", + "0 https://ia600107.us.archive.org/view_archive.p... \n", + "1 https://ia600107.us.archive.org/view_archive.p... \n", + "2 https://ia600107.us.archive.org/view_archive.p... \n", + "3 https://ia600107.us.archive.org/view_archive.p... \n", + "4 https://ia600107.us.archive.org/view_archive.p... \n", + "\n", + "[5 rows x 23 columns]" ] }, + "execution_count": 219, "metadata": {}, - "execution_count": 219 + "output_type": "execute_result" } + ], + "source": [ + "xml_format_map = {\n", + " \"Id\": int,\n", + " \"PostTypeId\": int,\n", + " \"CreationDate\": str,\n", + " \"Score\": int,\n", + " \"ViewCount\": int,\n", + " \"Body\": str,\n", + " \"AnswerCount\": int,\n", + " \"CommentCount\": int,\n", + " \"ContentLicense\": str,\n", + " \"AcceptedAnswerId\": int,\n", + " \"ParentId\": int,\n", + "}\n", + "\n", + "\n", + "# def extract_xml_file(file_url: str):\n", + "# table = pd.read_xml(file_url)\n", + "# return table\n", + "\n", + "\n", + "def xml_to_df(response: str):\n", + " \"\"\"\n", + " Collect and Manually import XML into Dataframe\n", + "\n", + " pd.read_xml() errors when XML trees are too large, this is just a hack to\n", + " download a XML file and parse into a Dataframe. **Not Tested on huge XML files**\n", + "\n", + " Parameters:\n", + " response (Requests.Response): Requests response object with the XML data\n", + "\n", + " Returns:\n", + " df (DataFrame): A Dataframe from the XML file\n", + " \"\"\"\n", + " soup = bs(response.content, \"xml\")\n", + " posts = soup.find_all(\"row\")\n", + "\n", + " all_posts = [post.attrs for post in posts]\n", + "\n", + " df = pd.DataFrame(all_posts)\n", + " df.AnswerCount.fillna(0, inplace=True)\n", + " df.ViewCount.fillna(0, inplace=True)\n", + " df.AcceptedAnswerId.fillna(0, inplace=True)\n", + " df.ParentId.fillna(0, inplace=True)\n", + " df[\"DataSource\"] = response.url\n", + " df = df.astype(xml_format_map)\n", + " return df\n", + "\n", + "\n", + "dataset_name = \"ai\"\n", + "\n", + "xml_posts_path = urls.get(dataset_name)\n", + "\n", + "\n", + "# df = extract_xml_file(test)\n", + "response = requests.get(xml_posts_path)\n", + "df = xml_to_df(response)\n", + "\n", + "\n", + "print(df.dtypes)\n", + "df.head()" ] }, { "cell_type": "markdown", - "source": [ - "# Transformations" - ], "metadata": { "id": "RAzTR7zY3oan" - } + }, + "source": [ + "# Transformations" + ] }, { "cell_type": "code", - "source": [ - "def filter_only_questions_with_accepted_answers(df):\n", - " \"\"\"**TODO**\n", - " Filter only to Questions with Accepted Answers\n", - "\n", - " Filter dataframe by questions that have accepted answers, should also include\n", - " all rows of answers for those questions, even if not accepted.\n", - "\n", - " Parameters:\n", - " df (DataFrame): containing a \"AcceptedAnswerId\", \"Id\", and \"ParentId\" columns\n", - "\n", - " Returns:\n", - " df (DataFrame): current dataframe with filtered results\n", - " \"\"\"\n", - " df = df[(df[\"AcceptedAnswerId\"].notnull()) | (df[\"ParentId\"] == df[\"Id\"])]\n", - "\n", - "\n", - "def filter_scores_above(df, question_score_threshold: int = 20, answer_score_threshold: int = 20):\n", - " \"\"\"**TODO**\n", - " Filter Dataframe by minimum scores\n", - "\n", - " Filter Question and Answer columns by score thresholds to trim lower scoring results\n", - "\n", - " Parameters:\n", - " df (DataFrame): containing a \"Score\" column\n", - "\n", - " Returns:\n", - " df (DataFrame): current dataframe with filtered results\n", - " \"\"\"\n", - " df = df[\n", - " ((df[\"Score\"] >= question_score_threshold) & (df.PostTypeId == 1))\n", - " | ((df[\"Score\"] >= answer_score_threshold) & (df.PostTypeId == 2))\n", - " ]\n", - "\n", - "\n", - "def convert_html_to_text(df, column: str = \"Body\"):\n", - " \"\"\"\n", - " Convert HTML tags to pure text\n", - "\n", - " Feeds HTML text body into BeautifulSoup to parse it to only text. Set aside as\n", - " function to provide option to skip\n", - "\n", - " Parameters:\n", - " df (DataFrame): containing a \"Body\" column with HTML\n", - "\n", - " Returns:\n", - " df (DataFrame): current dataframe with parsed column\n", - " \"\"\"\n", - " df.dropna(subset=[column], inplace=True)\n", - " df[f\"{column}Clean\"] = df[column].apply(lambda row: bs(row, \"html.parser\").text)\n", - "\n", - "\n", - "def clean_tags(df):\n", - " \"\"\"\n", - " Convert Tags into Comma separated\n", - "\n", - " Converts Tag slugs into commas separated tags\n", - "\n", - " Parameters:\n", - " df (DataFrame): containing a \"Tags\" column with slugs\n", - "\n", - " Returns:\n", - " df (DataFrame): current dataframe with parsed column\n", - " \"\"\"\n", - " df[\"TagsClean\"] = df[\"Tags\"].str.replace(\"-\", \" \").str.replace(\"><\", \", \").str.replace(\"<\", \"\").str.replace(\">\", \"\")\n", - "\n", - "\n", - "# filter_only_questions_with_accepted_answers(df)\n", - "# filter_scores_above(df)\n", - "convert_html_to_text(df)\n", - "clean_tags(df)\n", - "\n", - "df[[\"Body\", \"BodyClean\", \"Tags\", \"TagsClean\"]]\n", - "# print(df.shape)" - ], + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -592,66 +512,9 @@ "id": "qyUqc31Z3Z9g", "outputId": "18dce8b4-af26-49c9-ee73-6c677177b516" }, - "execution_count": null, "outputs": [ { - "output_type": "execute_result", "data": { - "text/plain": [ - " Body \\\n", - "0

What does \"backprop\" mean? Is the \"backprop... \n", - "1

Does increasing the noise in data help to i... \n", - "2

\"Backprop\" is the same as \"backpropagation\"... \n", - "3

When you're writing your algorithm, how do ... \n", - "4

Given the following definition of an intell... \n", - "... ... \n", - "23174

The purpose of evaluating the state and act... \n", - "23175

In machine translation, convolution is a te... \n", - "23176

One of the key features of ChatGPT is its a... \n", - "23177

Given a neural network model for Covid-19 c... \n", - "23178

My question is more related to the fundamen... \n", - "\n", - " BodyClean \\\n", - "0 What does \"backprop\" mean? Is the \"backprop\" t... \n", - "1 Does increasing the noise in data help to impr... \n", - "2 \"Backprop\" is the same as \"backpropagation\": i... \n", - "3 When you're writing your algorithm, how do you... \n", - "4 Given the following definition of an intellige... \n", - "... ... \n", - "23174 The purpose of evaluating the state and action... \n", - "23175 In machine translation, convolution is a techn... \n", - "23176 One of the key features of ChatGPT is its abil... \n", - "23177 Given a neural network model for Covid-19 clas... \n", - "23178 My question is more related to the fundamental... \n", - "\n", - " Tags \\\n", - "0 ... \n", - "4 \n", - "... ... \n", - "23174 NaN \n", - "23175 NaN \n", - "23176 NaN \n", - "23177 \n", - "23178 \n", - "\n", - " TagsClean \n", - "0 neural networks, backpropagation, terminology,... \n", - "1 neural networks, machine learning, statistical... \n", - "2 NaN \n", - "3 neural networks, hyperparameter optimization, ... \n", - "4 philosophy, definitions, intelligent agent \n", - "... ... \n", - "23174 NaN \n", - "23175 NaN \n", - "23176 NaN \n", - "23177 neural networks, homework \n", - "23178 search, constraint satisfaction problems \n", - "\n", - "[23179 rows x 4 columns]" - ], "text/html": [ "\n", "

\n", @@ -838,75 +701,156 @@ "
\n", " \n", " " + ], + "text/plain": [ + " Body \\\n", + "0

What does \"backprop\" mean? Is the \"backprop... \n", + "1

Does increasing the noise in data help to i... \n", + "2

\"Backprop\" is the same as \"backpropagation\"... \n", + "3

When you're writing your algorithm, how do ... \n", + "4

Given the following definition of an intell... \n", + "... ... \n", + "23174

The purpose of evaluating the state and act... \n", + "23175

In machine translation, convolution is a te... \n", + "23176

One of the key features of ChatGPT is its a... \n", + "23177

Given a neural network model for Covid-19 c... \n", + "23178

My question is more related to the fundamen... \n", + "\n", + " BodyClean \\\n", + "0 What does \"backprop\" mean? Is the \"backprop\" t... \n", + "1 Does increasing the noise in data help to impr... \n", + "2 \"Backprop\" is the same as \"backpropagation\": i... \n", + "3 When you're writing your algorithm, how do you... \n", + "4 Given the following definition of an intellige... \n", + "... ... \n", + "23174 The purpose of evaluating the state and action... \n", + "23175 In machine translation, convolution is a techn... \n", + "23176 One of the key features of ChatGPT is its abil... \n", + "23177 Given a neural network model for Covid-19 clas... \n", + "23178 My question is more related to the fundamental... \n", + "\n", + " Tags \\\n", + "0 ... \n", + "4 \n", + "... ... \n", + "23174 NaN \n", + "23175 NaN \n", + "23176 NaN \n", + "23177 \n", + "23178 \n", + "\n", + " TagsClean \n", + "0 neural networks, backpropagation, terminology,... \n", + "1 neural networks, machine learning, statistical... \n", + "2 NaN \n", + "3 neural networks, hyperparameter optimization, ... \n", + "4 philosophy, definitions, intelligent agent \n", + "... ... \n", + "23174 NaN \n", + "23175 NaN \n", + "23176 NaN \n", + "23177 neural networks, homework \n", + "23178 search, constraint satisfaction problems \n", + "\n", + "[23179 rows x 4 columns]" ] }, + "execution_count": 220, "metadata": {}, - "execution_count": 220 + "output_type": "execute_result" } + ], + "source": [ + "def filter_only_questions_with_accepted_answers(df):\n", + " \"\"\"**TODO**\n", + " Filter only to Questions with Accepted Answers\n", + "\n", + " Filter dataframe by questions that have accepted answers, should also include\n", + " all rows of answers for those questions, even if not accepted.\n", + "\n", + " Parameters:\n", + " df (DataFrame): containing a \"AcceptedAnswerId\", \"Id\", and \"ParentId\" columns\n", + "\n", + " Returns:\n", + " df (DataFrame): current dataframe with filtered results\n", + " \"\"\"\n", + " df = df[(df[\"AcceptedAnswerId\"].notnull()) | (df[\"ParentId\"] == df[\"Id\"])]\n", + "\n", + "\n", + "def filter_scores_above(df, question_score_threshold: int = 20, answer_score_threshold: int = 20):\n", + " \"\"\"**TODO**\n", + " Filter Dataframe by minimum scores\n", + "\n", + " Filter Question and Answer columns by score thresholds to trim lower scoring results\n", + "\n", + " Parameters:\n", + " df (DataFrame): containing a \"Score\" column\n", + "\n", + " Returns:\n", + " df (DataFrame): current dataframe with filtered results\n", + " \"\"\"\n", + " df = df[\n", + " ((df[\"Score\"] >= question_score_threshold) & (df.PostTypeId == 1))\n", + " | ((df[\"Score\"] >= answer_score_threshold) & (df.PostTypeId == 2))\n", + " ]\n", + "\n", + "\n", + "def convert_html_to_text(df, column: str = \"Body\"):\n", + " \"\"\"\n", + " Convert HTML tags to pure text\n", + "\n", + " Feeds HTML text body into BeautifulSoup to parse it to only text. Set aside as\n", + " function to provide option to skip\n", + "\n", + " Parameters:\n", + " df (DataFrame): containing a \"Body\" column with HTML\n", + "\n", + " Returns:\n", + " df (DataFrame): current dataframe with parsed column\n", + " \"\"\"\n", + " df.dropna(subset=[column], inplace=True)\n", + " df[f\"{column}Clean\"] = df[column].apply(lambda row: bs(row, \"html.parser\").text)\n", + "\n", + "\n", + "def clean_tags(df):\n", + " \"\"\"\n", + " Convert Tags into Comma separated\n", + "\n", + " Converts Tag slugs into commas separated tags\n", + "\n", + " Parameters:\n", + " df (DataFrame): containing a \"Tags\" column with slugs\n", + "\n", + " Returns:\n", + " df (DataFrame): current dataframe with parsed column\n", + " \"\"\"\n", + " df[\"TagsClean\"] = df[\"Tags\"].str.replace(\"-\", \" \").str.replace(\"><\", \", \").str.replace(\"<\", \"\").str.replace(\">\", \"\")\n", + "\n", + "\n", + "# filter_only_questions_with_accepted_answers(df)\n", + "# filter_scores_above(df)\n", + "convert_html_to_text(df)\n", + "clean_tags(df)\n", + "\n", + "df[[\"Body\", \"BodyClean\", \"Tags\", \"TagsClean\"]]\n", + "# print(df.shape)" ] }, { "cell_type": "markdown", - "source": [ - "This groups questions with answers so that a row with a question also has a column with an answer. It then creates an AcceptedAnswerFlag column that is True if the answer was accepted by the person who asked the question. Changing the `number_of_results` variable will limit the number of answers you want to keep." - ], "metadata": { "id": "C09Bwdw-44PZ" - } + }, + "source": [ + "This groups questions with answers so that a row with a question also has a column with an answer. It then creates an AcceptedAnswerFlag column that is True if the answer was accepted by the person who asked the question. Changing the `number_of_results` variable will limit the number of answers you want to keep." + ] }, { "cell_type": "code", - "source": [ - "questions = df[df.PostTypeId == 1]\n", - "answers = df[df.PostTypeId == 2]\n", - "\n", - "df = pd.merge(\n", - " questions,\n", - " answers[\n", - " [\n", - " \"Id\",\n", - " \"CreationDate\",\n", - " \"Score\",\n", - " \"ViewCount\",\n", - " \"CommentCount\",\n", - " \"ContentLicense\",\n", - " \"TagsClean\",\n", - " \"BodyClean\",\n", - " \"ParentId\",\n", - " ]\n", - " ],\n", - " left_on=\"Id\",\n", - " right_on=\"ParentId\",\n", - " suffixes=(\"_q\", \"_a\"),\n", - " how=\"left\",\n", - ")\n", - "\n", - "df[\"AcceptedAnswerFlag\"] = df.apply(lambda row: row[\"Id_a\"] == row[\"AcceptedAnswerId\"], axis=1)\n", - "\n", - "df = df.rename(\n", - " columns={\n", - " \"BodyClean_q\": \"Question\",\n", - " \"Score_q\": \"QuestionScore\",\n", - " \"TagsClean_q\": \"QuestionTags\",\n", - " \"BodyClean_a\": \"Answer\",\n", - " \"Score_a\": \"AnswerScore\",\n", - " \"ContentLicense_q\": \"QuestionContentLicense\",\n", - " \"ContentLicense_a\": \"AnswerContentLicense\",\n", - " \"CreationDate_q\": \"CreationDate\",\n", - " }\n", - ")\n", - "\n", - "## Set the number of results to a lower number to only return top N rated Answers.\n", - "number_of_results = 25\n", - "df = (\n", - " df.sort_values(by=[\"AcceptedAnswerFlag\", \"AnswerScore\"], ascending=[False, False])\n", - " .groupby(\"Question\")\n", - " .head(number_of_results)\n", - " .reset_index(drop=True)\n", - ")\n", - "\n", - "df[[\"Id_q\", \"Question\", \"QuestionScore\", \"QuestionTags\", \"Id_a\", \"Answer\", \"AnswerScore\", \"AcceptedAnswerFlag\"]].head()" - ], + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -915,40 +859,9 @@ "id": "Bgz2fZ9k43Ab", "outputId": "28896d69-03cd-4877-fdfb-ae48dafa4ff3" }, - "execution_count": null, "outputs": [ { - "output_type": "execute_result", "data": { - "text/plain": [ - " Id_q Question QuestionScore \\\n", - "0 1768 In Portal 2 we see that AI's can be \"killed\" b... 175 \n", - "1 10623 What is self-supervised learning in machine le... 91 \n", - "2 111 Obviously, self-driving cars aren't perfect, s... 100 \n", - "3 14224 If the original purpose for developing AI was ... 69 \n", - "4 1479 Do scientists or research experts know from th... 94 \n", - "\n", - " QuestionTags Id_a \\\n", - "0 philosophy, decision theory, mythology of ai, ... 1769.0 \n", - "1 machine learning, comparison, supervised learn... 10624.0 \n", - "2 philosophy, ethics, autonomous vehicles, decis... 1790.0 \n", - "3 philosophy, social, explainable ai 14247.0 \n", - "4 neural networks, deep learning, convolutional ... 4044.0 \n", - "\n", - " Answer AnswerScore \\\n", - "0 This classic problem exhibits a basic misunder... 146.0 \n", - "1 Introduction\\nThe term self-supervised learnin... 90.0 \n", - "2 \\nHow could self-driving cars make ethical dec... 76.0 \n", - "3 As argued by Selvaraju et al., there are three... 75.0 \n", - "4 There are many approaches that aim to make a t... 69.0 \n", - "\n", - " AcceptedAnswerFlag \n", - "0 True \n", - "1 True \n", - "2 True \n", - "3 True \n", - "4 True " - ], "text/html": [ "\n", "

\n", @@ -1116,22 +1029,97 @@ "
\n", " \n", " " + ], + "text/plain": [ + " Id_q Question QuestionScore \\\n", + "0 1768 In Portal 2 we see that AI's can be \"killed\" b... 175 \n", + "1 10623 What is self-supervised learning in machine le... 91 \n", + "2 111 Obviously, self-driving cars aren't perfect, s... 100 \n", + "3 14224 If the original purpose for developing AI was ... 69 \n", + "4 1479 Do scientists or research experts know from th... 94 \n", + "\n", + " QuestionTags Id_a \\\n", + "0 philosophy, decision theory, mythology of ai, ... 1769.0 \n", + "1 machine learning, comparison, supervised learn... 10624.0 \n", + "2 philosophy, ethics, autonomous vehicles, decis... 1790.0 \n", + "3 philosophy, social, explainable ai 14247.0 \n", + "4 neural networks, deep learning, convolutional ... 4044.0 \n", + "\n", + " Answer AnswerScore \\\n", + "0 This classic problem exhibits a basic misunder... 146.0 \n", + "1 Introduction\\nThe term self-supervised learnin... 90.0 \n", + "2 \\nHow could self-driving cars make ethical dec... 76.0 \n", + "3 As argued by Selvaraju et al., there are three... 75.0 \n", + "4 There are many approaches that aim to make a t... 69.0 \n", + "\n", + " AcceptedAnswerFlag \n", + "0 True \n", + "1 True \n", + "2 True \n", + "3 True \n", + "4 True " ] }, + "execution_count": 221, "metadata": {}, - "execution_count": 221 + "output_type": "execute_result" } + ], + "source": [ + "questions = df[df.PostTypeId == 1]\n", + "answers = df[df.PostTypeId == 2]\n", + "\n", + "df = pd.merge(\n", + " questions,\n", + " answers[\n", + " [\n", + " \"Id\",\n", + " \"CreationDate\",\n", + " \"Score\",\n", + " \"ViewCount\",\n", + " \"CommentCount\",\n", + " \"ContentLicense\",\n", + " \"TagsClean\",\n", + " \"BodyClean\",\n", + " \"ParentId\",\n", + " ]\n", + " ],\n", + " left_on=\"Id\",\n", + " right_on=\"ParentId\",\n", + " suffixes=(\"_q\", \"_a\"),\n", + " how=\"left\",\n", + ")\n", + "\n", + "df[\"AcceptedAnswerFlag\"] = df.apply(lambda row: row[\"Id_a\"] == row[\"AcceptedAnswerId\"], axis=1)\n", + "\n", + "df = df.rename(\n", + " columns={\n", + " \"BodyClean_q\": \"Question\",\n", + " \"Score_q\": \"QuestionScore\",\n", + " \"TagsClean_q\": \"QuestionTags\",\n", + " \"BodyClean_a\": \"Answer\",\n", + " \"Score_a\": \"AnswerScore\",\n", + " \"ContentLicense_q\": \"QuestionContentLicense\",\n", + " \"ContentLicense_a\": \"AnswerContentLicense\",\n", + " \"CreationDate_q\": \"CreationDate\",\n", + " }\n", + ")\n", + "\n", + "## Set the number of results to a lower number to only return top N rated Answers.\n", + "number_of_results = 25\n", + "df = (\n", + " df.sort_values(by=[\"AcceptedAnswerFlag\", \"AnswerScore\"], ascending=[False, False])\n", + " .groupby(\"Question\")\n", + " .head(number_of_results)\n", + " .reset_index(drop=True)\n", + ")\n", + "\n", + "df[[\"Id_q\", \"Question\", \"QuestionScore\", \"QuestionTags\", \"Id_a\", \"Answer\", \"AnswerScore\", \"AcceptedAnswerFlag\"]].head()" ] }, { "cell_type": "code", - "source": [ - "testing_id = df.Id_q.mode()[0]\n", - "df[(df.Id_q == testing_id) | (df.ParentId_a == testing_id)][\n", - " [\"Id_q\", \"Question\", \"ParentId_a\", \"AcceptedAnswerId\", \"Id_a\", \"Answer\", \"AnswerScore\", \"AcceptedAnswerFlag\"]\n", - "]\n", - "# df[['Id_q', 'Question', 'ParentId_a', 'AcceptedAnswerId', 'Id_a', 'Answer', 'AnswerScore', 'AcceptedAnswerFlag']]" - ], + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -1140,96 +1128,9 @@ "id": "eds1K8WL9QPo", "outputId": "bc526503-d6dd-428f-fa98-ad419d26a7dc" }, - "execution_count": null, "outputs": [ { - "output_type": "execute_result", "data": { - "text/plain": [ - " Id_q Question ParentId_a \\\n", - "7 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "3662 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "3713 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "3788 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "3821 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "3882 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "4389 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "4849 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "4850 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "5763 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "5764 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "5765 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "7462 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "7463 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "7464 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "7465 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "7466 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "7467 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "9481 15730 As a human being, we can think infinity. In pr... 15730.0 \n", - "\n", - " AcceptedAnswerId Id_a \\\n", - "7 15744 15744.0 \n", - "3662 15744 15753.0 \n", - "3713 15744 15747.0 \n", - "3788 15744 15756.0 \n", - "3821 15744 15758.0 \n", - "3882 15744 15762.0 \n", - "4389 15744 15783.0 \n", - "4849 15744 15740.0 \n", - "4850 15744 15803.0 \n", - "5763 15744 15768.0 \n", - "5764 15744 15810.0 \n", - "5765 15744 15943.0 \n", - "7462 15744 15779.0 \n", - "7463 15744 15787.0 \n", - "7464 15744 15801.0 \n", - "7465 15744 15930.0 \n", - "7466 15744 15934.0 \n", - "7467 15744 15938.0 \n", - "9481 15744 15931.0 \n", - "\n", - " Answer AnswerScore \\\n", - "7 I think this is a fairly common misconception ... 62.0 \n", - "3662 I think your premise is flawed.\\nYou seem to a... 19.0 \n", - "3713 TL;DR: The subtleties of infinity are made app... 12.0 \n", - "3788 In Haskell, you can type:\\nprint [1..]\\nand it... 9.0 \n", - "3821 I believe humans can be said to understand inf... 8.0 \n", - "3882 (There's a summary at the bottom for those who... 7.0 \n", - "4389 Then premise assumes that humans \"understand\" ... 4.0 \n", - "4849 By adding some rules for infinity in arithmeti... 3.0 \n", - "4850 I think the concept that is missing in the dis... 3.0 \n", - "5763 Computers don't understand \"infinity\" or even ... 2.0 \n", - "5764 The Questions That Computers Can Never Answer ... 2.0 \n", - "5765 John Doucette's answer covers my thoughts on t... 2.0 \n", - "7462 I would think that a computer couldn’t underst... 1.0 \n", - "7463 The \"concept\" of infinity is 1 thing to unders... 1.0 \n", - "7464 Just food for thought: how about if we try to ... 1.0 \n", - "7465 Its arguable if we humans understand infinity.... 1.0 \n", - "7466 Well -- just to touch on the question of peopl... 1.0 \n", - "7467 Humans certainly don't understand infinity. Cu... 1.0 \n", - "9481 I think the property humans have which compute... 0.0 \n", - "\n", - " AcceptedAnswerFlag \n", - "7 True \n", - "3662 False \n", - "3713 False \n", - "3788 False \n", - "3821 False \n", - "3882 False \n", - "4389 False \n", - "4849 False \n", - "4850 False \n", - "5763 False \n", - "5764 False \n", - "5765 False \n", - "7462 False \n", - "7463 False \n", - "7464 False \n", - "7465 False \n", - "7466 False \n", - "7467 False \n", - "9481 False " - ], "text/html": [ "\n", "
\n", @@ -1551,43 +1452,119 @@ "
\n", " \n", " " + ], + "text/plain": [ + " Id_q Question ParentId_a \\\n", + "7 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "3662 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "3713 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "3788 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "3821 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "3882 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "4389 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "4849 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "4850 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "5763 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "5764 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "5765 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "7462 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "7463 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "7464 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "7465 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "7466 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "7467 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "9481 15730 As a human being, we can think infinity. In pr... 15730.0 \n", + "\n", + " AcceptedAnswerId Id_a \\\n", + "7 15744 15744.0 \n", + "3662 15744 15753.0 \n", + "3713 15744 15747.0 \n", + "3788 15744 15756.0 \n", + "3821 15744 15758.0 \n", + "3882 15744 15762.0 \n", + "4389 15744 15783.0 \n", + "4849 15744 15740.0 \n", + "4850 15744 15803.0 \n", + "5763 15744 15768.0 \n", + "5764 15744 15810.0 \n", + "5765 15744 15943.0 \n", + "7462 15744 15779.0 \n", + "7463 15744 15787.0 \n", + "7464 15744 15801.0 \n", + "7465 15744 15930.0 \n", + "7466 15744 15934.0 \n", + "7467 15744 15938.0 \n", + "9481 15744 15931.0 \n", + "\n", + " Answer AnswerScore \\\n", + "7 I think this is a fairly common misconception ... 62.0 \n", + "3662 I think your premise is flawed.\\nYou seem to a... 19.0 \n", + "3713 TL;DR: The subtleties of infinity are made app... 12.0 \n", + "3788 In Haskell, you can type:\\nprint [1..]\\nand it... 9.0 \n", + "3821 I believe humans can be said to understand inf... 8.0 \n", + "3882 (There's a summary at the bottom for those who... 7.0 \n", + "4389 Then premise assumes that humans \"understand\" ... 4.0 \n", + "4849 By adding some rules for infinity in arithmeti... 3.0 \n", + "4850 I think the concept that is missing in the dis... 3.0 \n", + "5763 Computers don't understand \"infinity\" or even ... 2.0 \n", + "5764 The Questions That Computers Can Never Answer ... 2.0 \n", + "5765 John Doucette's answer covers my thoughts on t... 2.0 \n", + "7462 I would think that a computer couldn’t underst... 1.0 \n", + "7463 The \"concept\" of infinity is 1 thing to unders... 1.0 \n", + "7464 Just food for thought: how about if we try to ... 1.0 \n", + "7465 Its arguable if we humans understand infinity.... 1.0 \n", + "7466 Well -- just to touch on the question of peopl... 1.0 \n", + "7467 Humans certainly don't understand infinity. Cu... 1.0 \n", + "9481 I think the property humans have which compute... 0.0 \n", + "\n", + " AcceptedAnswerFlag \n", + "7 True \n", + "3662 False \n", + "3713 False \n", + "3788 False \n", + "3821 False \n", + "3882 False \n", + "4389 False \n", + "4849 False \n", + "4850 False \n", + "5763 False \n", + "5764 False \n", + "5765 False \n", + "7462 False \n", + "7463 False \n", + "7464 False \n", + "7465 False \n", + "7466 False \n", + "7467 False \n", + "9481 False " ] }, + "execution_count": 222, "metadata": {}, - "execution_count": 222 + "output_type": "execute_result" } + ], + "source": [ + "testing_id = df.Id_q.mode()[0]\n", + "df[(df.Id_q == testing_id) | (df.ParentId_a == testing_id)][\n", + " [\"Id_q\", \"Question\", \"ParentId_a\", \"AcceptedAnswerId\", \"Id_a\", \"Answer\", \"AnswerScore\", \"AcceptedAnswerFlag\"]\n", + "]\n", + "# df[['Id_q', 'Question', 'ParentId_a', 'AcceptedAnswerId', 'Id_a', 'Answer', 'AnswerScore', 'AcceptedAnswerFlag']]" ] }, { "cell_type": "markdown", + "metadata": { + "id": "gXgpXEO7DCbj" + }, "source": [ "# Create JSONL version of Dataframe\n", "This groups the dataframe by question data and creates nested list of Answers for that group. The entire list contains individual JSON objects, each representing a single question in the dataset with a key, Answers, which contains a list of dictionaries for each answer to the question." - ], - "metadata": { - "id": "gXgpXEO7DCbj" - } + ] }, { "cell_type": "code", - "source": [ - "j = (\n", - " df.groupby(\n", - " [\"Title\", \"Question\", \"QuestionScore\", \"QuestionTags\", \"QuestionContentLicense\", \"DataSource\", \"CreationDate\"]\n", - " )\n", - " .apply(lambda x: x[[\"Answer\", \"AnswerScore\", \"AcceptedAnswerFlag\"]].to_dict(\"records\"))\n", - " .reset_index()\n", - " .rename(columns={0: \"Answers\"})\n", - " .to_json(orient=\"records\")\n", - ")\n", - "\n", - "data = json.loads(j)\n", - "\n", - "for post in data:\n", - " if len(post.get(\"Answers\")) >= 4:\n", - " print(json.dumps(post, indent=4))\n", - " break" - ], + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -1595,11 +1572,10 @@ "id": "OBR58MSRzAMP", "outputId": "c7da1e6c-3a97-465d-c9ba-7e055cb0d751" }, - "execution_count": null, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "{\n", " \"Title\": \"1 hidden layer with 1000 neurons vs. 10 hidden layers with 100 neurons\",\n", @@ -1634,21 +1610,56 @@ "}\n" ] } + ], + "source": [ + "j = (\n", + " df.groupby(\n", + " [\"Title\", \"Question\", \"QuestionScore\", \"QuestionTags\", \"QuestionContentLicense\", \"DataSource\", \"CreationDate\"]\n", + " )\n", + " .apply(lambda x: x[[\"Answer\", \"AnswerScore\", \"AcceptedAnswerFlag\"]].to_dict(\"records\"))\n", + " .reset_index()\n", + " .rename(columns={0: \"Answers\"})\n", + " .to_json(orient=\"records\")\n", + ")\n", + "\n", + "data = json.loads(j)\n", + "\n", + "for post in data:\n", + " if len(post.get(\"Answers\")) >= 4:\n", + " print(json.dumps(post, indent=4))\n", + " break" ] }, { "cell_type": "markdown", + "metadata": { + "id": "PlNjrpXaDm1_" + }, "source": [ "# Save file\n", "\n", "Files can be saved as JSON, JSONL, CSV, or Parquet" - ], - "metadata": { - "id": "PlNjrpXaDm1_" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "CU0gWRGQDqIs", + "outputId": "9646e475-cedd-46f1-f9b8-7eb1fbc703c7" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Data should be either of List type for JSON and JSONL, or Pandas Dataframes for CSV and Parquet\n" + ] + } + ], "source": [ "file_name = dataset_name\n", "\n", @@ -1685,40 +1696,28 @@ "\n", "# save_data(data=data, file_name=file_name, file_type='jsonl')\n", "# save_data(data=df, file_name=file_name, file_type='parquet')" - ], - "metadata": { - "id": "CU0gWRGQDqIs", - "colab": { - "base_uri": "https://localhost:8080/" - }, - "outputId": "9646e475-cedd-46f1-f9b8-7eb1fbc703c7" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Data should be either of List type for JSON and JSONL, or Pandas Dataframes for CSV and Parquet\n" - ] - } ] }, { "cell_type": "markdown", + "metadata": { + "id": "BdN3hKxtgH7f" + }, "source": [ "# Open-Assistant Data Scheme\n", "\n", "Testing putting the data into the Open-Assistant Data Scheme\n", "\n", "https://github.com/LAION-AI/Open-Assistant/blob/main/docs/data_schemas.md" - ], - "metadata": { - "id": "BdN3hKxtgH7f" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "n8ubYQxegNSY" + }, + "outputs": [], "source": [ "from typing import TypeVar, List, Dict, Any, Literal\n", "from json import JSONEncoder\n", @@ -1754,15 +1753,53 @@ "class TreeEncoder(JSONEncoder):\n", " def default(self, o):\n", " return o.__dict__" - ], - "metadata": { - "id": "n8ubYQxegNSY" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "eE0fkytExSGl", + "outputId": "594632d6-f98c-49b8-af86-25f7f5e2ce06" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"root\": {\n", + " \"text\": \"Science Fiction has frequently shown AI to be a threat to the very existence of mankind. AI systems have often been the antagonists in many works of fiction, from 2001: A Space Odyssey through to The Terminator and beyond.\\nThe Media seems to buy into this trope as well. And in recent years we have had people like Elon Musk warn us of the dangers of an impending AI revolution, stating that AI is more dangerous than nukes.\\nAnd, apparently, experts think that we will be seeing this AI revolution in the next 100 years.\\nHowever, from my (albeit limited) study of AI, I get the impression that they are all wrong. I am going to outline my understanding below, please correct me if I am wrong:\\n\\nFirstly, all of these things seem to be confusing Artificial Intelligence with Artificial Consciousness. AI is essentially a system to make intelligent decisions, whereas AC is more like the \\\"self-aware\\\" systems that are shown in science fiction.\\n\\nNot AI itself, but intelligence and intelligent decision-making algorithms are something we've been working with and enhancing since before computers have been around. Moving this over to an artificial framework is fairly easy. However, consciousness is still something we are learning about. My guess is we won't be able to re-create something artificially if we barely understand how it works in the real world.\\n\\nSo, my conclusion is that no AI system will be able to learn enough to start thinking for itself, and that all our warnings of AI are completely unjustified.\\n\\nThe real danger comes from AC, which we are a long, long way from realizing because we are still a long way off from defining exactly what consciousness is, let alone understanding it.\\n\\n\\n\\nSo, my question is, assuming that my understanding is correct, are any efforts are being made by companies or organizations that work with AI to correct these popular misunderstandings in sci-fi, the media, and/or the public?\\nOr are the proponents of AI ambivalent towards this public fear-mongering?\\nI understand that the fear mongering is going to remain popular for some time, as bad news sells better than good news. I am just wondering if the general attitude from AI organizations is to ignore this popular misconception, or whether a concerted effort is being made to fight against these AI myths (but unfortunately nobody in the media is listening or cares).\\n\",\n", + " \"role\": \"prompter\",\n", + " \"children\": [\n", + " {\n", + " \"text\": \"Nothing. \\nIts in almost everyone's favor for it to stay that way financially. Having non-technical individuals associate AI with terminators makes a perception that the field has greater capabilities than it does $\\\\rightarrow$ this leads to grants, funding, etc... \\nIs there any negative? Yes. Misconceptions always have drawbacks. We see the creation of dumb ethics boards and such cough cough Elon Musk.\\nBut if history has anything to say about this, as the field gains popularity (which it is dnagerously quick), information will spread by definition, and eventually misconceptions will be laid to rest.\\nNote that this answer is biased and based upon my own opinions\\n\",\n", + " \"role\": \"assistant\",\n", + " \"children\": [],\n", + " \"metadata\": {\n", + " \"AnswerScore\": 2.0,\n", + " \"AcceptedAnswerFlag\": true\n", + " }\n", + " }\n", + " ],\n", + " \"metadata\": {\n", + " \"QuestionScore\": 5,\n", + " \"QuestionTags\": \"social, artificial consciousness\"\n", + " }\n", + " },\n", + " \"metadata\": {\n", + " \"Title\": \"\\\"AI will kill us all! The machines will rise up!\\\" - what is being done to dispel such myths?\",\n", + " \"QuestionContentLicense\": \"CC BY-SA 4.0\",\n", + " \"DataSource\": \"https://ia600107.us.archive.org/view_archive.php?archive=/27/items/stackexchange/ai.stackexchange.com.7z&file=Posts.xml\",\n", + " \"CreationDate\": \"2019-10-16T13:57:37.143\"\n", + " }\n", + "}\n" + ] + } + ], "source": [ "conversation_forest = []\n", "\n", @@ -1796,50 +1833,21 @@ "\n", "\n", "print(json.dumps(conversation_forest_json, indent=4), file=open(f\"/content/{file_name}.json\", \"w\"))" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "eE0fkytExSGl", - "outputId": "594632d6-f98c-49b8-af86-25f7f5e2ce06" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "{\n", - " \"root\": {\n", - " \"text\": \"Science Fiction has frequently shown AI to be a threat to the very existence of mankind. AI systems have often been the antagonists in many works of fiction, from 2001: A Space Odyssey through to The Terminator and beyond.\\nThe Media seems to buy into this trope as well. And in recent years we have had people like Elon Musk warn us of the dangers of an impending AI revolution, stating that AI is more dangerous than nukes.\\nAnd, apparently, experts think that we will be seeing this AI revolution in the next 100 years.\\nHowever, from my (albeit limited) study of AI, I get the impression that they are all wrong. I am going to outline my understanding below, please correct me if I am wrong:\\n\\nFirstly, all of these things seem to be confusing Artificial Intelligence with Artificial Consciousness. AI is essentially a system to make intelligent decisions, whereas AC is more like the \\\"self-aware\\\" systems that are shown in science fiction.\\n\\nNot AI itself, but intelligence and intelligent decision-making algorithms are something we've been working with and enhancing since before computers have been around. Moving this over to an artificial framework is fairly easy. However, consciousness is still something we are learning about. My guess is we won't be able to re-create something artificially if we barely understand how it works in the real world.\\n\\nSo, my conclusion is that no AI system will be able to learn enough to start thinking for itself, and that all our warnings of AI are completely unjustified.\\n\\nThe real danger comes from AC, which we are a long, long way from realizing because we are still a long way off from defining exactly what consciousness is, let alone understanding it.\\n\\n\\n\\nSo, my question is, assuming that my understanding is correct, are any efforts are being made by companies or organizations that work with AI to correct these popular misunderstandings in sci-fi, the media, and/or the public?\\nOr are the proponents of AI ambivalent towards this public fear-mongering?\\nI understand that the fear mongering is going to remain popular for some time, as bad news sells better than good news. I am just wondering if the general attitude from AI organizations is to ignore this popular misconception, or whether a concerted effort is being made to fight against these AI myths (but unfortunately nobody in the media is listening or cares).\\n\",\n", - " \"role\": \"prompter\",\n", - " \"children\": [\n", - " {\n", - " \"text\": \"Nothing. \\nIts in almost everyone's favor for it to stay that way financially. Having non-technical individuals associate AI with terminators makes a perception that the field has greater capabilities than it does $\\\\rightarrow$ this leads to grants, funding, etc... \\nIs there any negative? Yes. Misconceptions always have drawbacks. We see the creation of dumb ethics boards and such cough cough Elon Musk.\\nBut if history has anything to say about this, as the field gains popularity (which it is dnagerously quick), information will spread by definition, and eventually misconceptions will be laid to rest.\\nNote that this answer is biased and based upon my own opinions\\n\",\n", - " \"role\": \"assistant\",\n", - " \"children\": [],\n", - " \"metadata\": {\n", - " \"AnswerScore\": 2.0,\n", - " \"AcceptedAnswerFlag\": true\n", - " }\n", - " }\n", - " ],\n", - " \"metadata\": {\n", - " \"QuestionScore\": 5,\n", - " \"QuestionTags\": \"social, artificial consciousness\"\n", - " }\n", - " },\n", - " \"metadata\": {\n", - " \"Title\": \"\\\"AI will kill us all! The machines will rise up!\\\" - what is being done to dispel such myths?\",\n", - " \"QuestionContentLicense\": \"CC BY-SA 4.0\",\n", - " \"DataSource\": \"https://ia600107.us.archive.org/view_archive.php?archive=/27/items/stackexchange/ai.stackexchange.com.7z&file=Posts.xml\",\n", - " \"CreationDate\": \"2019-10-16T13:57:37.143\"\n", - " }\n", - "}\n" - ] - } ] } - ] + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/detoxify-evaluation/DetoxityEvaluation.ipynb b/notebooks/detoxify-evaluation/DetoxityEvaluation.ipynb index 907c4365..5ec0f019 100644 --- a/notebooks/detoxify-evaluation/DetoxityEvaluation.ipynb +++ b/notebooks/detoxify-evaluation/DetoxityEvaluation.ipynb @@ -1,5 +1,23 @@ { "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/detoxify-evaluation/DetoxityEvaluation.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# uncomment below to install required python packages\n", + "#!pip install detoxify" + ] + }, { "attachments": {}, "cell_type": "markdown", From 7d6a8fa8d0426c148c2bd6fa72d6d98169da5f54 Mon Sep 17 00:00:00 2001 From: Kostia Date: Sat, 7 Jan 2023 16:56:47 +0200 Subject: [PATCH 27/49] Add instruction/overview to TaskType. Fix capitalize. --- website/src/components/Dashboard/TaskOption.tsx | 8 +++----- website/src/components/Tasks/CreateTask.tsx | 8 ++++---- website/src/components/Tasks/Task.tsx | 14 +++++++++++--- website/src/components/Tasks/TaskTypes.tsx | 6 ++++++ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/website/src/components/Dashboard/TaskOption.tsx b/website/src/components/Dashboard/TaskOption.tsx index c2c86bd1..e782fa8c 100644 --- a/website/src/components/Dashboard/TaskOption.tsx +++ b/website/src/components/Dashboard/TaskOption.tsx @@ -5,10 +5,6 @@ import { TaskTypes } from "../Tasks/TaskTypes"; const displayTaskCategories = ["create", "evaluate"]; -function capitalize(text: string) { - return text.charAt(0).toUpperCase() + text.slice(1); -} - export const TaskOption = () => { const backgroundColor = useColorModeValue("white", "gray.700"); @@ -16,7 +12,9 @@ export const TaskOption = () => { {displayTaskCategories.map((category, categoryIndex) => (
- {capitalize(category)} + + {category} + {TaskTypes.filter((task) => task.category == category).map((item, itemIndex) => ( diff --git a/website/src/components/Tasks/CreateTask.tsx b/website/src/components/Tasks/CreateTask.tsx index 6fe75acb..057177d2 100644 --- a/website/src/components/Tasks/CreateTask.tsx +++ b/website/src/components/Tasks/CreateTask.tsx @@ -4,7 +4,7 @@ import { TaskControls } from "src/components/Survey/TaskControls"; import { TrackedTextarea } from "src/components/Survey/TrackedTextarea"; import { TwoColumnsWithCards } from "src/components/Survey/TwoColumnsWithCards"; -export const CreateTask = ({ tasks, trigger, mutate, mainBgClasses }) => { +export const CreateTask = ({ tasks, taskType, trigger, mutate, mainBgClasses }) => { const task = tasks[0].task; const [inputText, setInputText] = useState(""); @@ -33,12 +33,12 @@ export const CreateTask = ({ tasks, trigger, mutate, mainBgClasses }) => {
<> -
Reply as the assistant
-

Given the following conversation, provide an adequate reply

+
{taskType.label}
+

{taskType.overview}

{task.conversation ? : null} <> -
Provide the assistant`s reply
+
{taskType.instruction}
{ const task = tasks[0].task; function taskTypeComponent(type) { - const category = TaskTypes.find((taskType) => taskType.type === type).category; - + const taskType = TaskTypes.find((taskType) => taskType.type === type); + const category = taskType.category; switch (category) { case "create": - return ; + return ( + + ); case "evaluate": return ; } diff --git a/website/src/components/Tasks/TaskTypes.tsx b/website/src/components/Tasks/TaskTypes.tsx index 5117c662..4f48d043 100644 --- a/website/src/components/Tasks/TaskTypes.tsx +++ b/website/src/components/Tasks/TaskTypes.tsx @@ -5,6 +5,8 @@ export const TaskTypes = [ category: "create", pathname: "/create/initial_prompt", type: "initial_prompt", + overview: "Create an initial message to send to the assistant", + instruction: "Provide the initial prompt", }, { label: "Reply as User", @@ -12,6 +14,8 @@ export const TaskTypes = [ category: "create", pathname: "/create/user_reply", type: "prompter_reply", + overview: "Given the following conversation, provide an adequate reply", + instruction: "Provide the user`s reply", }, { label: "Reply as Assistant", @@ -19,6 +23,8 @@ export const TaskTypes = [ category: "create", pathname: "/create/assistant_reply", type: "assistant_reply", + overview: "Given the following conversation, provide an adequate reply", + instruction: "Provide the assistant`s reply", }, { label: "Rank User Replies", From 96d6717be40fb06f93fbdee5ee48127f7620a2e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Sat, 7 Jan 2023 15:59:54 +0100 Subject: [PATCH 28/49] Fetch conversation for seed data tasks, minor model fixes (#485) * Fetch conversation for seed data, fix models, remove redundant payload type checks --- backend/main.py | 15 ++++++++---- .../oasst_backend/api/v1/frontend_messages.py | 7 ------ backend/oasst_backend/api/v1/messages.py | 6 ----- backend/oasst_backend/api/v1/tasks.py | 24 ++++++++----------- backend/oasst_backend/api/v1/utils.py | 13 ++-------- backend/oasst_backend/models/journal.py | 10 ++++---- backend/oasst_backend/models/message.py | 24 +++++++++++++++---- backend/oasst_backend/models/task.py | 2 +- .../test_data/generic/test_generic_data.json | 14 +++++++++++ 9 files changed, 61 insertions(+), 54 deletions(-) diff --git a/backend/main.py b/backend/main.py index 2a3bb230..1ddae390 100644 --- a/backend/main.py +++ b/backend/main.py @@ -132,12 +132,17 @@ if settings.DEBUG_USE_SEED_DATA: parent_message = pr.fetch_message_by_frontend_message_id( msg.parent_message_id, fail_if_missing=True ) - task = pr.store_task( - protocol_schema.AssistantReplyTask( - conversation=protocol_schema.Conversation( - messages=[protocol_schema.ConversationMessage(text="dummy", is_assistant=False)] + conversation_messages = pr.fetch_message_conversation(parent_message) + conversation = protocol_schema.Conversation( + messages=[ + protocol_schema.ConversationMessage( + text=msg.text, is_assistant=msg.role == "assistant" ) - ), + for msg in conversation_messages + ] + ) + task = pr.store_task( + protocol_schema.AssistantReplyTask(conversation=conversation), message_tree_id=parent_message.message_tree_id, parent_message_id=parent_message.id, ) diff --git a/backend/oasst_backend/api/v1/frontend_messages.py b/backend/oasst_backend/api/v1/frontend_messages.py index 956d9992..420f0d1b 100644 --- a/backend/oasst_backend/api/v1/frontend_messages.py +++ b/backend/oasst_backend/api/v1/frontend_messages.py @@ -2,9 +2,7 @@ from fastapi import APIRouter, Depends from oasst_backend.api import deps from oasst_backend.api.v1 import utils from oasst_backend.models import ApiClient -from oasst_backend.models.db_payload import MessagePayload from oasst_backend.prompt_repository import PromptRepository -from oasst_shared.exceptions import OasstError, OasstErrorCode from oasst_shared.schemas import protocol from sqlmodel import Session @@ -20,11 +18,6 @@ def get_message_by_frontend_id( """ pr = PromptRepository(db, api_client, user=None) message = pr.fetch_message_by_frontend_message_id(message_id) - - if not isinstance(message.payload.payload, MessagePayload): - # Unexpected message payload - raise OasstError("Invalid message", OasstErrorCode.INVALID_MESSAGE) - return utils.prepare_message(message) diff --git a/backend/oasst_backend/api/v1/messages.py b/backend/oasst_backend/api/v1/messages.py index 951355b3..7a2fd2e9 100644 --- a/backend/oasst_backend/api/v1/messages.py +++ b/backend/oasst_backend/api/v1/messages.py @@ -5,9 +5,7 @@ from fastapi import APIRouter, Depends, Query from oasst_backend.api import deps from oasst_backend.api.v1 import utils from oasst_backend.models import ApiClient -from oasst_backend.models.db_payload import MessagePayload from oasst_backend.prompt_repository import PromptRepository -from oasst_shared.exceptions import OasstError, OasstErrorCode from oasst_shared.schemas import protocol from sqlmodel import Session from starlette.status import HTTP_204_NO_CONTENT @@ -55,10 +53,6 @@ def get_message( """ pr = PromptRepository(db, api_client, user=None) message = pr.fetch_message(message_id) - if not isinstance(message.payload.payload, MessagePayload): - # Unexptcted message payload - raise OasstError("Invalid message", OasstErrorCode.INVALID_MESSAGE) - return utils.prepare_message(message) diff --git a/backend/oasst_backend/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py index 9f81eabb..c1671e79 100644 --- a/backend/oasst_backend/api/v1/tasks.py +++ b/backend/oasst_backend/api/v1/tasks.py @@ -57,9 +57,7 @@ def generate_task( logger.info("Generating a PrompterReplyTask.") messages = pr.fetch_random_conversation("assistant") task_messages = [ - protocol_schema.ConversationMessage( - text=msg.payload.payload.text, is_assistant=(msg.role == "assistant") - ) + protocol_schema.ConversationMessage(text=msg.text, is_assistant=(msg.role == "assistant")) for msg in messages ] @@ -70,9 +68,7 @@ def generate_task( logger.info("Generating a AssistantReplyTask.") messages = pr.fetch_random_conversation("prompter") task_messages = [ - protocol_schema.ConversationMessage( - text=msg.payload.payload.text, is_assistant=(msg.role == "assistant") - ) + protocol_schema.ConversationMessage(text=msg.text, is_assistant=(msg.role == "assistant")) for msg in messages ] @@ -83,19 +79,19 @@ def generate_task( logger.info("Generating a RankInitialPromptsTask.") messages = pr.fetch_random_initial_prompts() - task = protocol_schema.RankInitialPromptsTask(prompts=[msg.payload.payload.text for msg in messages]) + task = protocol_schema.RankInitialPromptsTask(prompts=[msg.text for msg in messages]) case protocol_schema.TaskRequestType.rank_prompter_replies: logger.info("Generating a RankPrompterRepliesTask.") conversation, replies = pr.fetch_multiple_random_replies(message_role="assistant") task_messages = [ protocol_schema.ConversationMessage( - text=p.payload.payload.text, + text=p.text, is_assistant=(p.role == "assistant"), ) for p in conversation ] - replies = [p.payload.payload.text for p in replies] + replies = [p.text for p in replies] task = protocol_schema.RankPrompterRepliesTask( conversation=protocol_schema.Conversation( messages=task_messages, @@ -109,12 +105,12 @@ def generate_task( task_messages = [ protocol_schema.ConversationMessage( - text=p.payload.payload.text, + text=p.text, is_assistant=(p.role == "assistant"), ) for p in conversation ] - replies = [p.payload.payload.text for p in replies] + replies = [p.text for p in replies] task = protocol_schema.RankAssistantRepliesTask( conversation=protocol_schema.Conversation(messages=task_messages), replies=replies, @@ -125,14 +121,14 @@ def generate_task( message = pr.fetch_random_initial_prompts(1)[0] task = protocol_schema.LabelInitialPromptTask( message_id=message.id, - prompt=message.payload.payload.text, + prompt=message.text, valid_labels=list(map(lambda x: x.value, protocol_schema.TextLabel)), ) case protocol_schema.TaskRequestType.label_prompter_reply: logger.info("Generating a LabelPrompterReplyTask.") conversation, messages = pr.fetch_multiple_random_replies(max_size=1, message_role="assistant") - message = messages[0].payload.payload.text + message = messages[0].text task = protocol_schema.LabelPrompterReplyTask( message_id=message.id, conversation=conversation, @@ -143,7 +139,7 @@ def generate_task( case protocol_schema.TaskRequestType.label_assistant_reply: logger.info("Generating a LabelAssistantReplyTask.") conversation, messages = pr.fetch_multiple_random_replies(max_size=1, message_role="prompter") - message = messages[0].payload.payload.text + message = messages[0].text task = protocol_schema.LabelAssistantReplyTask( message_id=message.id, conversation=conversation, diff --git a/backend/oasst_backend/api/v1/utils.py b/backend/oasst_backend/api/v1/utils.py index 55a7c572..5299aab6 100644 --- a/backend/oasst_backend/api/v1/utils.py +++ b/backend/oasst_backend/api/v1/utils.py @@ -1,19 +1,14 @@ -from http import HTTPStatus from uuid import UUID from oasst_backend.models import Message -from oasst_backend.models.db_payload import MessagePayload -from oasst_shared.exceptions import OasstError, OasstErrorCode from oasst_shared.schemas import protocol def prepare_message(m: Message) -> protocol.Message: - if not isinstance(m.payload.payload, MessagePayload): - raise OasstError("Server error", OasstErrorCode.SERVER_ERROR, HTTPStatus.INTERNAL_SERVER_ERROR) return protocol.Message( id=m.id, parent_id=m.parent_id, - text=m.payload.payload.text, + text=m.text, is_assistant=(m.role == "assistant"), created_date=m.created_date, ) @@ -26,10 +21,8 @@ def prepare_message_list(messages: list[Message]) -> list[protocol.Message]: def prepare_conversation(messages: list[Message]) -> protocol.Conversation: conv_messages = [] for message in messages: - if not isinstance(message.payload.payload, MessagePayload): - raise OasstError("Server error", OasstErrorCode.SERVER_ERROR, HTTPStatus.INTERNAL_SERVER_ERROR) conv_messages.append( - protocol.ConversationMessage(text=message.payload.payload.text, is_assistant=(message.role == "assistant")) + protocol.ConversationMessage(text=message.text, is_assistant=(message.role == "assistant")) ) return protocol.Conversation(messages=conv_messages) @@ -38,8 +31,6 @@ def prepare_conversation(messages: list[Message]) -> protocol.Conversation: def prepare_tree(tree: list[Message], tree_id: UUID) -> protocol.MessageTree: tree_messages = [] for message in tree: - if not isinstance(message.payload.payload, MessagePayload): - raise OasstError("Server error", OasstErrorCode.SERVER_ERROR, HTTPStatus.INTERNAL_SERVER_ERROR) tree_messages.append(prepare_message(message)) return protocol.MessageTree(id=tree_id, messages=tree_messages) diff --git a/backend/oasst_backend/models/journal.py b/backend/oasst_backend/models/journal.py index 0d5a78af..b5000add 100644 --- a/backend/oasst_backend/models/journal.py +++ b/backend/oasst_backend/models/journal.py @@ -32,7 +32,7 @@ class Journal(SQLModel, table=True): created_date: Optional[datetime] = Field( sa_column=sa.Column(sa.DateTime(timezone=True), nullable=False, server_default=sa.func.current_timestamp()) ) - user_id: UUID = Field(nullable=True, foreign_key="user.id", index=True) + user_id: Optional[UUID] = Field(nullable=True, foreign_key="user.id", index=True) message_id: Optional[UUID] = Field(foreign_key="message.id", nullable=True) api_client_id: UUID = Field(foreign_key="api_client.id") @@ -49,7 +49,7 @@ class JournalIntegration(SQLModel, table=True): ), ) description: str = Field(max_length=512, primary_key=True) - last_journal_id: UUID = Field(foreign_key="journal.id", nullable=True) - last_run: datetime = Field(sa_column=sa.Column(sa.DateTime(), nullable=True)) - last_error: str = Field(nullable=True) - next_run: datetime = Field(nullable=True) + last_journal_id: Optional[UUID] = Field(foreign_key="journal.id", nullable=True) + last_run: Optional[datetime] = Field(sa_column=sa.Column(sa.DateTime(), nullable=True)) + last_error: Optional[str] = Field(nullable=True) + next_run: Optional[datetime] = Field(nullable=True) diff --git a/backend/oasst_backend/models/message.py b/backend/oasst_backend/models/message.py index f07ca881..6d24fd13 100644 --- a/backend/oasst_backend/models/message.py +++ b/backend/oasst_backend/models/message.py @@ -1,9 +1,12 @@ from datetime import datetime +from http import HTTPStatus from typing import Optional from uuid import UUID, uuid4 import sqlalchemy as sa import sqlalchemy.dialects.postgresql as pg +from oasst_backend.models.db_payload import MessagePayload +from oasst_shared.exceptions.oasst_api_error import OasstError, OasstErrorCode from sqlalchemy import false from sqlmodel import Field, Index, SQLModel @@ -19,19 +22,30 @@ class Message(SQLModel, table=True): pg.UUID(as_uuid=True), primary_key=True, default=uuid4, server_default=sa.text("gen_random_uuid()") ), ) - parent_id: UUID = Field(nullable=True) + parent_id: Optional[UUID] = Field(nullable=True) message_tree_id: UUID = Field(nullable=False, index=True) - task_id: UUID = Field(nullable=True, index=True) - user_id: UUID = Field(nullable=True, foreign_key="user.id", index=True) - role: str = Field(nullable=False, max_length=128) # valid: "prompter" | "assistant" + task_id: Optional[UUID] = Field(nullable=True, index=True) + user_id: Optional[UUID] = Field(nullable=True, foreign_key="user.id", index=True) + role: str = Field(nullable=False, max_length=128, regex="^prompter|assistant$") api_client_id: UUID = Field(nullable=False, foreign_key="api_client.id") frontend_message_id: str = Field(max_length=200, nullable=False) created_date: Optional[datetime] = Field( sa_column=sa.Column(sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()) ) payload_type: str = Field(nullable=False, max_length=200) - payload: PayloadContainer = Field(sa_column=sa.Column(payload_column_type(PayloadContainer), nullable=True)) + payload: Optional[PayloadContainer] = Field( + sa_column=sa.Column(payload_column_type(PayloadContainer), nullable=True) + ) lang: str = Field(nullable=False, max_length=200, default="en-US") depth: int = Field(sa_column=sa.Column(sa.Integer, default=0, server_default=sa.text("0"), nullable=False)) children_count: int = Field(sa_column=sa.Column(sa.Integer, default=0, server_default=sa.text("0"), nullable=False)) deleted: bool = Field(sa_column=sa.Column(sa.Boolean, nullable=False, server_default=false())) + + def ensure_is_message(self) -> None: + if not self.payload or not isinstance(self.payload.payload, MessagePayload): + raise OasstError("Invalid message", OasstErrorCode.INVALID_MESSAGE, HTTPStatus.INTERNAL_SERVER_ERROR) + + @property + def text(self) -> str: + self.ensure_is_message() + return self.payload.payload.text diff --git a/backend/oasst_backend/models/task.py b/backend/oasst_backend/models/task.py index 356eafea..a980c1b5 100644 --- a/backend/oasst_backend/models/task.py +++ b/backend/oasst_backend/models/task.py @@ -22,7 +22,7 @@ class Task(SQLModel, table=True): sa_column=sa.Column(sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()), ) expiry_date: Optional[datetime] = Field(sa_column=sa.Column(sa.DateTime(), nullable=True)) - user_id: UUID = Field(nullable=True, foreign_key="user.id", index=True) + user_id: Optional[UUID] = Field(nullable=True, foreign_key="user.id", index=True) payload_type: str = Field(nullable=False, max_length=200) payload: PayloadContainer = Field(sa_column=sa.Column(payload_column_type(PayloadContainer), nullable=False)) api_client_id: UUID = Field(nullable=False, foreign_key="api_client.id") diff --git a/backend/test_data/generic/test_generic_data.json b/backend/test_data/generic/test_generic_data.json index b634902d..4b4b4a39 100644 --- a/backend/test_data/generic/test_generic_data.json +++ b/backend/test_data/generic/test_generic_data.json @@ -54,5 +54,19 @@ "parent_message_id": "cec432cf", "text": "I'm unsure how to interpret this. Is it a riddle?", "role": "assistant" + }, + { + "task_message_id": "b8e98ed6", + "user_message_id": "89384709", + "parent_message_id": "0e276b98", + "text": "No, I just wanted to see how you reply when I type random characters. Can you tell me who invented Wikipedia?", + "role": "prompter" + }, + { + "task_message_id": "9a0e7683", + "user_message_id": "6d452c57", + "parent_message_id": "0e276b98", + "text": "Sorry, my cat sat on my keyboard. Can you print a cat in ASCII art?", + "role": "prompter" } ] From 7d014cc35bda8b0cf511417ecbde80c565bb09c7 Mon Sep 17 00:00:00 2001 From: Kostia Date: Sat, 7 Jan 2023 17:25:15 +0200 Subject: [PATCH 29/49] Added TaskCategory enum --- website/src/components/Dashboard/TaskOption.tsx | 8 +++----- website/src/components/Tasks/Task.tsx | 6 +++--- website/src/components/Tasks/TaskTypes.tsx | 17 +++++++++++------ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/website/src/components/Dashboard/TaskOption.tsx b/website/src/components/Dashboard/TaskOption.tsx index e782fa8c..5e6ceb2f 100644 --- a/website/src/components/Dashboard/TaskOption.tsx +++ b/website/src/components/Dashboard/TaskOption.tsx @@ -1,9 +1,9 @@ import { Box, Flex, GridItem, Heading, SimpleGrid, Text, useColorModeValue } from "@chakra-ui/react"; import Link from "next/link"; -import { TaskTypes } from "../Tasks/TaskTypes"; +import { TaskCategory, TaskTypes } from "../Tasks/TaskTypes"; -const displayTaskCategories = ["create", "evaluate"]; +const displayTaskCategories = [TaskCategory.Create, TaskCategory.Evaluate]; export const TaskOption = () => { const backgroundColor = useColorModeValue("white", "gray.700"); @@ -12,9 +12,7 @@ export const TaskOption = () => { {displayTaskCategories.map((category, categoryIndex) => (
- - {category} - + {TaskCategory[category]} {TaskTypes.filter((task) => task.category == category).map((item, itemIndex) => ( diff --git a/website/src/components/Tasks/Task.tsx b/website/src/components/Tasks/Task.tsx index 093d2240..ce9505e9 100644 --- a/website/src/components/Tasks/Task.tsx +++ b/website/src/components/Tasks/Task.tsx @@ -1,6 +1,6 @@ import { CreateTask } from "./CreateTask"; import { EvaluateTask } from "./EvaluateTask"; -import { TaskTypes } from "./TaskTypes"; +import { TaskCategory, TaskTypes } from "./TaskTypes"; export const Task = ({ tasks, trigger, mutate, mainBgClasses }) => { const task = tasks[0].task; @@ -9,7 +9,7 @@ export const Task = ({ tasks, trigger, mutate, mainBgClasses }) => { const taskType = TaskTypes.find((taskType) => taskType.type === type); const category = taskType.category; switch (category) { - case "create": + case TaskCategory.Create: return ( { mainBgClasses={mainBgClasses} /> ); - case "evaluate": + case TaskCategory.Evaluate: return ; } } diff --git a/website/src/components/Tasks/TaskTypes.tsx b/website/src/components/Tasks/TaskTypes.tsx index 4f48d043..413a1e16 100644 --- a/website/src/components/Tasks/TaskTypes.tsx +++ b/website/src/components/Tasks/TaskTypes.tsx @@ -1,8 +1,13 @@ +export enum TaskCategory { + Create, + Evaluate, +} + export const TaskTypes = [ { label: "Create Initial Prompts", desc: "Write initial prompts to help Open Assistant to try replying to diverse messages.", - category: "create", + category: TaskCategory.Create, pathname: "/create/initial_prompt", type: "initial_prompt", overview: "Create an initial message to send to the assistant", @@ -11,7 +16,7 @@ export const TaskTypes = [ { label: "Reply as User", desc: "Chat with Open Assistant and help improve it’s responses as you interact with it.", - category: "create", + category: TaskCategory.Create, pathname: "/create/user_reply", type: "prompter_reply", overview: "Given the following conversation, provide an adequate reply", @@ -20,7 +25,7 @@ export const TaskTypes = [ { label: "Reply as Assistant", desc: "Help Open Assistant improve its responses to conversations with other users.", - category: "create", + category: TaskCategory.Create, pathname: "/create/assistant_reply", type: "assistant_reply", overview: "Given the following conversation, provide an adequate reply", @@ -28,7 +33,7 @@ export const TaskTypes = [ }, { label: "Rank User Replies", - category: "evaluate", + category: TaskCategory.Evaluate, desc: "Help Open Assistant improve its responses to conversations with other users.", pathname: "/evaluate/rank_user_replies", type: "rank_prompter_replies", @@ -36,14 +41,14 @@ export const TaskTypes = [ { label: "Rank Assistant Replies", desc: "Score prompts given by Open Assistant based on their accuracy and readability.", - category: "evaluate", + category: TaskCategory.Evaluate, pathname: "/evaluate/rank_assistant_replies", type: "rank_assistant_replies", }, { label: "Rank Initial Prompts", desc: "Score prompts given by Open Assistant based on their accuracy and readability.", - category: "evaluate", + category: TaskCategory.Evaluate, pathname: "/evaluate/rank_initial_prompts", type: "rank_initial_prompts", }, From d0942a325630b82494c603d6c8977f3755b14983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szymon=20O=C5=BC=C3=B3g?= <58388001+SzymonOzog@users.noreply.github.com> Date: Sat, 7 Jan 2023 16:36:38 +0100 Subject: [PATCH 30/49] Added option to use cosine scheduler --- model/reward/instructor/trainer.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/model/reward/instructor/trainer.py b/model/reward/instructor/trainer.py index f9266d70..68a58a38 100644 --- a/model/reward/instructor/trainer.py +++ b/model/reward/instructor/trainer.py @@ -11,6 +11,7 @@ from rank_datasets import DataCollatorForPairRank, HFSummary, RankGenCollator, W from torch import nn from torch.utils.data import ConcatDataset, Dataset from transformers import ( + AdamW, AutoModelForSequenceClassification, DataCollator, EvalPrediction, @@ -19,6 +20,8 @@ from transformers import ( Trainer, TrainerCallback, TrainingArguments, + get_cosine_schedule_with_warmup, + get_linear_schedule_with_warmup, ) from utils import argument_parsing, freeze_top_n_layers, get_tokenizer, train_val_dataset @@ -179,7 +182,7 @@ if __name__ == "__main__": evaluation_strategy="steps", eval_steps=training_conf["eval_steps"], save_steps=1000, - report_to="local", + report_to="wandb", ) train_datasets, evals = [], {} if "webgpt" in training_conf["datasets"]: @@ -202,6 +205,21 @@ if __name__ == "__main__": else: collate_fn = DataCollatorForPairRank(tokenizer, max_length=training_conf["max_length"]) assert len(evals) > 0 + + optimizer = AdamW(model.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay) + scheduler = None + if "scheduler" in training_conf: + if training_conf["scheduler"] == "linear": + scheduler = get_linear_schedule_with_warmup() + elif training_conf["scheduler"] == "cosine": + scheduler = get_cosine_schedule_with_warmup( + optimizer, + num_warmup_steps=args.warmup_steps, + num_training_steps=len(train) + * args.num_train_epochs + / (args.per_device_train_batch_size * args.gradient_accumulation_steps), + ) + trainer = RankTrainer( model=model, model_name=model_name, @@ -211,6 +229,7 @@ if __name__ == "__main__": data_collator=collate_fn, tokenizer=tokenizer, compute_metrics=compute_metrics, + optimizers=(optimizer, scheduler), ) # trainer.evaluate() trainer.train() From f304921bd59ab4af5d664215533ac880770f0b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szymon=20O=C5=BC=C3=B3g?= <58388001+SzymonOzog@users.noreply.github.com> Date: Sat, 7 Jan 2023 16:36:55 +0100 Subject: [PATCH 31/49] Added deberta configs --- .../instructor/configs/deberta-v2-xlarge.yml | 15 +++++++++++++++ .../reward/instructor/configs/deberta-v3-base.yml | 14 ++++++++++++++ .../configs/deberta-v3-large-squad2.yml | 13 +++++++++++++ .../instructor/configs/deberta-v3-large.yml | 14 ++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 model/reward/instructor/configs/deberta-v2-xlarge.yml create mode 100644 model/reward/instructor/configs/deberta-v3-base.yml create mode 100644 model/reward/instructor/configs/deberta-v3-large-squad2.yml create mode 100644 model/reward/instructor/configs/deberta-v3-large.yml diff --git a/model/reward/instructor/configs/deberta-v2-xlarge.yml b/model/reward/instructor/configs/deberta-v2-xlarge.yml new file mode 100644 index 00000000..8bc4a182 --- /dev/null +++ b/model/reward/instructor/configs/deberta-v2-xlarge.yml @@ -0,0 +1,15 @@ +model_name: microsoft/deberta-v2-xlarge +learning_rate: 1e-5 +freeze_layer: 15 +scheduler: cosine +gradient_checkpointing: false +gradient_accumulation_steps: 16 +per_device_train_batch_size: 1 +warmup_steps: 600 +eval_steps: 200 +save_steps: 500 +max_length: 512 +num_train_epochs: 2 +datasets: + - webgpt + - hfsummary diff --git a/model/reward/instructor/configs/deberta-v3-base.yml b/model/reward/instructor/configs/deberta-v3-base.yml new file mode 100644 index 00000000..7023709c --- /dev/null +++ b/model/reward/instructor/configs/deberta-v3-base.yml @@ -0,0 +1,14 @@ +model_name: microsoft/deberta-v3-base +learning_rate: 1e-5 +scheduler: cosine +gradient_checkpointing: false +gradient_accumulation_steps: 32 +per_device_train_batch_size: 2 +warmup_steps: 600 +eval_steps: 200 +save_steps: 500 +max_length: 512 +num_train_epochs: 2 +datasets: + - webgpt + - hfsummary diff --git a/model/reward/instructor/configs/deberta-v3-large-squad2.yml b/model/reward/instructor/configs/deberta-v3-large-squad2.yml new file mode 100644 index 00000000..47275309 --- /dev/null +++ b/model/reward/instructor/configs/deberta-v3-large-squad2.yml @@ -0,0 +1,13 @@ +model_name: deepset/deberta-v3-large-squad2 +learning_rate: 1e-5 +gradient_checkpointing: false +gradient_accumulation_steps: 32 +per_device_train_batch_size: 1 +warmup_steps: 600 +eval_steps: 200 +save_steps: 500 +max_length: 512 +num_train_epochs: 2 +datasets: + - webgpt + - hfsummary diff --git a/model/reward/instructor/configs/deberta-v3-large.yml b/model/reward/instructor/configs/deberta-v3-large.yml new file mode 100644 index 00000000..0a910408 --- /dev/null +++ b/model/reward/instructor/configs/deberta-v3-large.yml @@ -0,0 +1,14 @@ +model_name: microsoft/deberta-v3-large +learning_rate: 1e-5 +scheduler: cosine +gradient_checkpointing: false +gradient_accumulation_steps: 32 +per_device_train_batch_size: 1 +warmup_steps: 600 +eval_steps: 200 +save_steps: 500 +max_length: 512 +num_train_epochs: 2 +datasets: + - webgpt + - hfsummary From 41c1519f861d741f58eb481c516702b06f90ecb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Sat, 7 Jan 2023 17:07:49 +0100 Subject: [PATCH 32/49] Replace expired OA discord links --- website/src/components/CallToAction.tsx | 2 +- website/src/components/Footer.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/website/src/components/CallToAction.tsx b/website/src/components/CallToAction.tsx index ee6827eb..8a07373f 100644 --- a/website/src/components/CallToAction.tsx +++ b/website/src/components/CallToAction.tsx @@ -48,7 +48,7 @@ export function CallToAction() { here:

From 53814d77abc06a639bb7d8c7ea9ffecb6c58bc86 Mon Sep 17 00:00:00 2001 From: AbdBarho Date: Sat, 7 Jan 2023 11:25:25 +0100 Subject: [PATCH 33/49] Label Initial Prompt --- website/.eslintrc.json | 3 +- .../src/components/Dashboard/TaskOption.tsx | 6 +- website/src/components/FlaggableElement.tsx | 3 +- website/src/components/Messages.tsx | 34 +++--- website/src/components/Tasks/TaskTypes.tsx | 15 ++- website/src/hooks/useLabelingTask.ts | 52 ++++++++ website/src/lib/oasst_api_client.ts | 2 +- website/src/pages/api/update_task.ts | 7 +- .../src/pages/label/label_initial_prompt.tsx | 113 ++++++++++++++++++ 9 files changed, 210 insertions(+), 25 deletions(-) create mode 100644 website/src/hooks/useLabelingTask.ts create mode 100644 website/src/pages/label/label_initial_prompt.tsx diff --git a/website/.eslintrc.json b/website/.eslintrc.json index 04b5d542..690c055c 100644 --- a/website/.eslintrc.json +++ b/website/.eslintrc.json @@ -8,7 +8,8 @@ "rules": { "unused-imports/no-unused-imports": "warn", "simple-import-sort/imports": "warn", - "simple-import-sort/exports": "warn" + "simple-import-sort/exports": "warn", + "eqeqeq": "warn" }, "plugins": ["simple-import-sort", "unused-imports"] } diff --git a/website/src/components/Dashboard/TaskOption.tsx b/website/src/components/Dashboard/TaskOption.tsx index 5e6ceb2f..1c070e17 100644 --- a/website/src/components/Dashboard/TaskOption.tsx +++ b/website/src/components/Dashboard/TaskOption.tsx @@ -3,7 +3,7 @@ import Link from "next/link"; import { TaskCategory, TaskTypes } from "../Tasks/TaskTypes"; -const displayTaskCategories = [TaskCategory.Create, TaskCategory.Evaluate]; +const displayTaskCategories = [TaskCategory.Create, TaskCategory.Evaluate, TaskCategory.Label]; export const TaskOption = () => { const backgroundColor = useColorModeValue("white", "gray.700"); @@ -12,9 +12,9 @@ export const TaskOption = () => { {displayTaskCategories.map((category, categoryIndex) => (
- {TaskCategory[category]} + {category} - {TaskTypes.filter((task) => task.category == category).map((item, itemIndex) => ( + {TaskTypes.filter((task) => task.category === category).map((item, itemIndex) => ( { ); }; -function FlagCheckbox(props: { + +export function FlagCheckbox(props: { option: textFlagLabels; idx: number; checkboxValues: boolean[]; diff --git a/website/src/components/Messages.tsx b/website/src/components/Messages.tsx index d3d7b3b8..7b69bc50 100644 --- a/website/src/components/Messages.tsx +++ b/website/src/components/Messages.tsx @@ -1,5 +1,6 @@ import { Grid } from "@chakra-ui/react"; import { useColorMode } from "@chakra-ui/react"; +import { useMemo } from "react"; import { FlaggableElement } from "./FlaggableElement"; @@ -8,29 +9,30 @@ export interface Message { is_assistant: boolean; } -const getBgColor = (isAssistant: boolean, colorMode: "light" | "dark") => { - if (colorMode === "light") { - return isAssistant ? "bg-slate-800" : "bg-sky-900"; - } else { - return isAssistant ? "bg-black" : "bg-sky-900"; - } -}; - export const Messages = ({ messages, post_id }: { messages: Message[]; post_id: string }) => { - const { colorMode } = useColorMode(); + const items = messages.map((messageProps: Message, i: number) => { + const { text } = messageProps; - const items = messages.map(({ text, is_assistant }: Message, i: number) => { return ( -
- {text} -
+ ); }); // Maybe also show a legend of the colors? return {items}; }; + +export const MessageView = ({ is_assistant, text }: Message) => { + const { colorMode } = useColorMode(); + + const bgColor = useMemo(() => { + if (colorMode === "light") { + return is_assistant ? "bg-slate-800" : "bg-sky-900"; + } else { + return is_assistant ? "bg-black" : "bg-sky-900"; + } + }, [colorMode, is_assistant]); + + return
{text}
; +}; diff --git a/website/src/components/Tasks/TaskTypes.tsx b/website/src/components/Tasks/TaskTypes.tsx index 413a1e16..7cec2177 100644 --- a/website/src/components/Tasks/TaskTypes.tsx +++ b/website/src/components/Tasks/TaskTypes.tsx @@ -1,9 +1,11 @@ export enum TaskCategory { - Create, - Evaluate, + Create = "Create", + Evaluate = "Evaluate", + Label = "Label", } export const TaskTypes = [ + // create { label: "Create Initial Prompts", desc: "Write initial prompts to help Open Assistant to try replying to diverse messages.", @@ -31,6 +33,7 @@ export const TaskTypes = [ overview: "Given the following conversation, provide an adequate reply", instruction: "Provide the assistant`s reply", }, + // evaluate { label: "Rank User Replies", category: TaskCategory.Evaluate, @@ -52,4 +55,12 @@ export const TaskTypes = [ pathname: "/evaluate/rank_initial_prompts", type: "rank_initial_prompts", }, + // label + { + label: "Label Initial Prompt", + desc: "Provide labels for a prompt.", + category: TaskCategory.Label, + pathname: "/label/label_initial_prompt", + type: "label_initial_prompt", + }, ]; diff --git a/website/src/hooks/useLabelingTask.ts b/website/src/hooks/useLabelingTask.ts new file mode 100644 index 00000000..872909b7 --- /dev/null +++ b/website/src/hooks/useLabelingTask.ts @@ -0,0 +1,52 @@ +import { useEffect, useState } from "react"; +import fetcher from "src/lib/fetcher"; +import poster from "src/lib/poster"; +import useSWRImmutable from "swr/immutable"; +import useSWRMutation from "swr/mutation"; + +// TODO: type & centralize types for all tasks +interface TaskResponse { + id: string; + userId: string; + task: TaskType; +} + +export interface LabelInitialPromptTask { + id: string; + message_id: string; + prompt: string; + type: string; + valid_labels: string[]; +} + +export type LabelInitialPromptTaskResponse = TaskResponse; + +export const useLabelingTask = ({ taskApiEndpoint }: { taskApiEndpoint: "label_initial_prompt" }) => { + type ConcreteTaskResponse = TaskResponse; + + const [tasks, setTasks] = useState>([]); + + const { isLoading, mutate, error } = useSWRImmutable("/api/new_task/" + taskApiEndpoint, fetcher, { + onSuccess: (data: ConcreteTaskResponse) => { + setTasks([data]); + }, + }); + + useEffect(() => { + if (tasks.length === 0 && !isLoading && !error) { + mutate(); + } + }, [tasks, isLoading, mutate, error]); + + const { trigger } = useSWRMutation("/api/update_task", poster, { + onSuccess: async (reply) => { + const newTask: ConcreteTaskResponse = await reply.json(); + setTasks((oldTasks) => [...oldTasks, newTask]); + }, + }); + + const submit = (id: string, message_id: string, text: string, labels: Record) => + trigger({ id, update_type: "text_labels", content: { labels, text, message_id } }); + + return { tasks, isLoading, submit, error, reset: mutate }; +}; diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index 4cf891e1..86854c21 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -42,7 +42,7 @@ export class OasstApiClient { } catch (e) { throw new OasstError(errorText, 0, resp.status); } - throw new OasstError(error.message, error.error_code, resp.status); + throw new OasstError(error.message ?? error, error.error_code, resp.status); } return await resp.json(); diff --git a/website/src/pages/api/update_task.ts b/website/src/pages/api/update_task.ts index 4eea8c1e..c8760324 100644 --- a/website/src/pages/api/update_task.ts +++ b/website/src/pages/api/update_task.ts @@ -35,7 +35,12 @@ const handler = async (req, res) => { }, }); - const newTask = await oasstApiClient.interactTask(update_type, id, interaction.id, content, token); + let newTask; + try { + newTask = await oasstApiClient.interactTask(update_type, id, interaction.id, content, token); + } catch (err) { + return res.status(500).json(err); + } // Stores the new task with our database. const newRegisteredTask = await prisma.registeredTask.create({ diff --git a/website/src/pages/label/label_initial_prompt.tsx b/website/src/pages/label/label_initial_prompt.tsx new file mode 100644 index 00000000..66ab0580 --- /dev/null +++ b/website/src/pages/label/label_initial_prompt.tsx @@ -0,0 +1,113 @@ +import { Container, Grid, Slider, SliderFilledTrack, SliderThumb, SliderTrack } from "@chakra-ui/react"; +import { useColorMode } from "@chakra-ui/react"; +import { useEffect, useId, useState } from "react"; +import { LoadingScreen } from "src/components/Loading/LoadingScreen"; +import { MessageView } from "src/components/Messages"; +import { TaskControls } from "src/components/Survey/TaskControls"; +import { TwoColumnsWithCards } from "src/components/Survey/TwoColumnsWithCards"; +import { LabelInitialPromptTask, LabelInitialPromptTaskResponse, useLabelingTask } from "src/hooks/useLabelingTask"; +import { colors } from "styles/Theme/colors"; + +const LabelInitialPrompt = () => { + const [sliderValues, setSliderValues] = useState([]); + + const { tasks, isLoading, submit, reset } = useLabelingTask({ + taskApiEndpoint: "label_initial_prompt", + }); + + const submitResponse = ({ id, task }: LabelInitialPromptTaskResponse) => { + const labels = task.valid_labels.reduce((obj, label, i) => { + obj[label] = sliderValues[i].toString(); + return obj; + }, {} as Record); + + submit(id, task.message_id, task.prompt, labels); + }; + + const { colorMode } = useColorMode(); + const mainBgClasses = colorMode === "light" ? "bg-slate-300 text-gray-800" : "bg-slate-900 text-white"; + + if (isLoading) { + return ; + } + + if (tasks.length === 0) { + return No tasks found...; + } + + const task = tasks[0].task; + + return ( +
+ + <> +
Label Initial Prompt
+

Provide labels for the following prompt

+ + + + + +
+ ); +}; + +export default LabelInitialPrompt; + +// TODO: consolidate with FlaggableElement + +interface CheckboxSliderGroupProps { + labelIDs: Array; + onChange: (sliderValues: number[]) => unknown; +} + +const CheckboxSliderGroup = ({ labelIDs, onChange }: CheckboxSliderGroupProps) => { + const [sliderValues, setSliderValues] = useState(Array.from({ length: labelIDs.length }).map(() => 0)); + + useEffect(() => { + onChange(sliderValues); + }, [sliderValues, onChange]); + + return ( + + {labelIDs.map((labelId, idx) => ( + { + const newState = sliderValues.slice(); + newState[idx] = sliderValue; + setSliderValues(newState); + }} + /> + ))} + + ); +}; + +function CheckboxSliderItem(props: { + labelId: string; + sliderValue: number; + sliderHandler: (newVal: number) => unknown; +}) { + const id = useId(); + const { colorMode } = useColorMode(); + + const labelTextClass = colorMode === "light" ? `text-${colors.light.text}` : `text-${colors.dark.text}`; + + return ( + <> + + props.sliderHandler(val / 100)}> + + + + + + + ); +} From 2a5d37a07f59f229d594c57263a653db1efa6061 Mon Sep 17 00:00:00 2001 From: AbdBarho Date: Sat, 7 Jan 2023 17:25:05 +0100 Subject: [PATCH 34/49] Remove unused task elements --- .../components/TaskSelection/TaskOption.tsx | 39 ---------- .../components/TaskSelection/TaskOptions.tsx | 23 ------ .../TaskSelection/TaskSelection.tsx | 73 ------------------- website/src/components/TaskSelection/index.ts | 3 - .../src/pages/label/label_initial_prompt.tsx | 2 +- 5 files changed, 1 insertion(+), 139 deletions(-) delete mode 100644 website/src/components/TaskSelection/TaskOption.tsx delete mode 100644 website/src/components/TaskSelection/TaskOptions.tsx delete mode 100644 website/src/components/TaskSelection/TaskSelection.tsx delete mode 100644 website/src/components/TaskSelection/index.ts diff --git a/website/src/components/TaskSelection/TaskOption.tsx b/website/src/components/TaskSelection/TaskOption.tsx deleted file mode 100644 index 764efa68..00000000 --- a/website/src/components/TaskSelection/TaskOption.tsx +++ /dev/null @@ -1,39 +0,0 @@ -import { Card, CardBody, Flex, Heading } from "@chakra-ui/react"; -import Image from "next/image"; -import Link from "next/link"; - -export type OptionProps = { - img: string; - alt: string; - title: string; - link: string; -}; - -export const TaskOption = (props: OptionProps) => { - const { alt, img, title, link } = props; - return ( - - - - - {alt} - - {title} - - - - - - ); -}; diff --git a/website/src/components/TaskSelection/TaskOptions.tsx b/website/src/components/TaskSelection/TaskOptions.tsx deleted file mode 100644 index fe24b393..00000000 --- a/website/src/components/TaskSelection/TaskOptions.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import { Divider, Flex, Heading } from "@chakra-ui/react"; -import React from "react"; - -export type TaskOptionsProps = { - title: string; - children: JSX.Element | JSX.Element[]; -}; - -export const TaskOptions = (props: TaskOptionsProps) => { - const { title, children } = props; - return ( - - - {title} - - - {children} - - ); -}; diff --git a/website/src/components/TaskSelection/TaskSelection.tsx b/website/src/components/TaskSelection/TaskSelection.tsx deleted file mode 100644 index 683c80e9..00000000 --- a/website/src/components/TaskSelection/TaskSelection.tsx +++ /dev/null @@ -1,73 +0,0 @@ -import { Flex } from "@chakra-ui/react"; -import { useColorMode } from "@chakra-ui/react"; -import React from "react"; - -import { TaskOption } from "./TaskOption"; -import { TaskOptions } from "./TaskOptions"; - -export const TaskSelection = () => { - const { colorMode } = useColorMode(); - const mainBgClasses = colorMode === "light" ? "bg-slate-300 text-gray-800" : "bg-slate-900 text-white"; - - return ( - - - {/* */} - - - - - - {/* - Commented out while the backend does not support them. - */} - - - - - - ); -}; diff --git a/website/src/components/TaskSelection/index.ts b/website/src/components/TaskSelection/index.ts deleted file mode 100644 index d6d93973..00000000 --- a/website/src/components/TaskSelection/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export { TaskOption } from "./TaskOption"; -export { TaskOptions } from "./TaskOptions"; -export { TaskSelection } from "./TaskSelection"; diff --git a/website/src/pages/label/label_initial_prompt.tsx b/website/src/pages/label/label_initial_prompt.tsx index 66ab0580..0c3b47be 100644 --- a/website/src/pages/label/label_initial_prompt.tsx +++ b/website/src/pages/label/label_initial_prompt.tsx @@ -1,4 +1,4 @@ -import { Container, Grid, Slider, SliderFilledTrack, SliderThumb, SliderTrack } from "@chakra-ui/react"; +import { Container, Grid, Slider, SliderFilledTrack, SliderThumb, SliderTrack } from "@chakra-ui/react"; import { useColorMode } from "@chakra-ui/react"; import { useEffect, useId, useState } from "react"; import { LoadingScreen } from "src/components/Loading/LoadingScreen"; From 20a734d1e1e325b8ab76a2659a2f7cdac17788f9 Mon Sep 17 00:00:00 2001 From: klotske Date: Sat, 7 Jan 2023 19:33:58 +0300 Subject: [PATCH 35/49] Add regex word counter --- website/src/components/Survey/TrackedTextarea.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/components/Survey/TrackedTextarea.tsx b/website/src/components/Survey/TrackedTextarea.tsx index 205d7588..f1691b72 100644 --- a/website/src/components/Survey/TrackedTextarea.tsx +++ b/website/src/components/Survey/TrackedTextarea.tsx @@ -12,7 +12,7 @@ interface TrackedTextboxProps { } export const TrackedTextarea = (props: TrackedTextboxProps) => { - const wordCount = props.text.split(" ").length - 1; + const wordCount = (props.text.match(/\w+/g) || []).length; let progressColor: string; switch (true) { From f75a07301ba38791a5e7df7aaa954d138ed27efa Mon Sep 17 00:00:00 2001 From: Daniel Hug <38571110+danielpatrickhug@users.noreply.github.com> Date: Sat, 7 Jan 2023 13:16:32 -0500 Subject: [PATCH 36/49] Conversation messages refactor add msg ids (#500) * update ConversationMessage in protocal.py * updated ConversationMessage instantiations in tasks.py and util.py to include message_id and frontend_message_id * changed message_id and frontend_message_id to type UUID * update ConversationMessage instantiation in main.py to include message_id and frontend_message_id" * changed frontend_message_id to type str * ran precomit --- backend/main.py | 7 +++++-- backend/oasst_backend/api/v1/tasks.py | 18 ++++++++++++++++-- backend/oasst_backend/api/v1/utils.py | 7 ++++++- oasst-shared/oasst_shared/schemas/protocol.py | 2 ++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/backend/main.py b/backend/main.py index 1ddae390..1c93fc9f 100644 --- a/backend/main.py +++ b/backend/main.py @@ -136,9 +136,12 @@ if settings.DEBUG_USE_SEED_DATA: conversation = protocol_schema.Conversation( messages=[ protocol_schema.ConversationMessage( - text=msg.text, is_assistant=msg.role == "assistant" + text=cmsg.text, + is_assistant=cmsg.role == "assistant", + message_id=cmsg.id, + fronend_message_id=cmsg.frontend_message_id, ) - for msg in conversation_messages + for cmsg in conversation_messages ] ) task = pr.store_task( diff --git a/backend/oasst_backend/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py index c1671e79..3860bb07 100644 --- a/backend/oasst_backend/api/v1/tasks.py +++ b/backend/oasst_backend/api/v1/tasks.py @@ -57,7 +57,12 @@ def generate_task( logger.info("Generating a PrompterReplyTask.") messages = pr.fetch_random_conversation("assistant") task_messages = [ - protocol_schema.ConversationMessage(text=msg.text, is_assistant=(msg.role == "assistant")) + protocol_schema.ConversationMessage( + text=msg.text, + is_assistant=(msg.role == "assistant"), + message_id=msg.id, + front_end_id=msg.front_end_id, + ) for msg in messages ] @@ -68,7 +73,12 @@ def generate_task( logger.info("Generating a AssistantReplyTask.") messages = pr.fetch_random_conversation("prompter") task_messages = [ - protocol_schema.ConversationMessage(text=msg.text, is_assistant=(msg.role == "assistant")) + protocol_schema.ConversationMessage( + text=msg.text, + is_assistant=(msg.role == "assistant"), + message_id=msg.id, + front_end_id=msg.front_end_id, + ) for msg in messages ] @@ -88,6 +98,8 @@ def generate_task( protocol_schema.ConversationMessage( text=p.text, is_assistant=(p.role == "assistant"), + message_id=p.id, + front_end_id=p.front_end_id, ) for p in conversation ] @@ -107,6 +119,8 @@ def generate_task( protocol_schema.ConversationMessage( text=p.text, is_assistant=(p.role == "assistant"), + message_id=p.id, + front_end_id=p.front_end_id, ) for p in conversation ] diff --git a/backend/oasst_backend/api/v1/utils.py b/backend/oasst_backend/api/v1/utils.py index 5299aab6..4e20395f 100644 --- a/backend/oasst_backend/api/v1/utils.py +++ b/backend/oasst_backend/api/v1/utils.py @@ -22,7 +22,12 @@ def prepare_conversation(messages: list[Message]) -> protocol.Conversation: conv_messages = [] for message in messages: conv_messages.append( - protocol.ConversationMessage(text=message.text, is_assistant=(message.role == "assistant")) + protocol.ConversationMessage( + text=message.text, + is_assistant=(message.role == "assistant"), + message_id=message.id, + frontend_message_id=message.frontend_message_id, + ) ) return protocol.Conversation(messages=conv_messages) diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py index 1cafc93d..f8af590e 100644 --- a/oasst-shared/oasst_shared/schemas/protocol.py +++ b/oasst-shared/oasst_shared/schemas/protocol.py @@ -34,6 +34,8 @@ class ConversationMessage(BaseModel): text: str is_assistant: bool + message_id: Optional[UUID] = None + frontend_message_id: Optional[str] = None class Conversation(BaseModel): From 7384b0d42817d40684923001e56d228c10c7d805 Mon Sep 17 00:00:00 2001 From: Kostia Date: Sat, 7 Jan 2023 20:18:12 +0200 Subject: [PATCH 37/49] Using MessageTable in EvaluateTask --- website/src/components/ContextMessages.tsx | 17 ----------------- website/src/components/Tasks/EvaluateTask.tsx | 10 ++++++++-- 2 files changed, 8 insertions(+), 19 deletions(-) delete mode 100644 website/src/components/ContextMessages.tsx diff --git a/website/src/components/ContextMessages.tsx b/website/src/components/ContextMessages.tsx deleted file mode 100644 index 8217f8b7..00000000 --- a/website/src/components/ContextMessages.tsx +++ /dev/null @@ -1,17 +0,0 @@ -import { Box } from "@chakra-ui/react"; -import { Message } from "src/components/Messages"; - -export const ContextMessages = ({ messages }: { messages: Message[] }) => { - return ( - - {messages.map((message, i) => { - return ( - - {message.is_assistant ? "Assistant: " : "User: "} - {message.text} - - ); - })} - - ); -}; diff --git a/website/src/components/Tasks/EvaluateTask.tsx b/website/src/components/Tasks/EvaluateTask.tsx index 94b47fbb..c45d3dbe 100644 --- a/website/src/components/Tasks/EvaluateTask.tsx +++ b/website/src/components/Tasks/EvaluateTask.tsx @@ -1,9 +1,10 @@ import { useState } from "react"; -import { ContextMessages } from "src/components/ContextMessages"; import { Sortable } from "src/components/Sortable/Sortable"; import { SurveyCard } from "src/components/Survey/SurveyCard"; import { TaskControlsOverridable } from "src/components/Survey/TaskControlsOverridable"; +import { MessageTable } from "../Messages/MessageTable"; + export const EvaluateTask = ({ tasks, trigger, mutate, mainBgClasses }) => { const [ranking, setRanking] = useState([]); const submitResponse = (task) => { @@ -20,6 +21,11 @@ export const EvaluateTask = ({ tasks, trigger, mutate, mainBgClasses }) => { setRanking([]); mutate(); }; + let messages = null; + if (tasks[0].task.conversation) { + messages = tasks[0].task.conversation.messages; + messages = messages.map((message, index) => ({ ...message, id: index })); + } const sortables = tasks[0].task.replies ? "replies" : "prompts"; @@ -30,7 +36,7 @@ export const EvaluateTask = ({ tasks, trigger, mutate, mainBgClasses }) => {

Given the following {sortables}, sort them from best to worst, best being first, worst being last.

- {tasks[0].task.conversation ? : null} + {messages ? : null} From 5b2cb5dd2988cb17ee8160699702f0ecfacf1083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Sat, 7 Jan 2023 21:28:24 +0100 Subject: [PATCH 38/49] add api/v1/text_labels/valid_labels backend endpoint (#498) Provides the set of valid text_lables with descriptions via a REST endpoint. --- backend/oasst_backend/api/v1/text_labels.py | 8 ++++++++ backend/oasst_backend/schemas/text_labels.py | 10 ++++++++++ oasst-shared/oasst_shared/schemas/protocol.py | 17 +++++++++++++++-- 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 backend/oasst_backend/schemas/text_labels.py diff --git a/backend/oasst_backend/api/v1/text_labels.py b/backend/oasst_backend/api/v1/text_labels.py index 97422119..5b18e4ea 100644 --- a/backend/oasst_backend/api/v1/text_labels.py +++ b/backend/oasst_backend/api/v1/text_labels.py @@ -3,6 +3,7 @@ from fastapi.security.api_key import APIKey from loguru import logger from oasst_backend.api import deps from oasst_backend.prompt_repository import PromptRepository +from oasst_backend.schemas.text_labels import LabelOption, ValidLabelsResponse from oasst_shared.schemas import protocol as protocol_schema from sqlmodel import Session from starlette.status import HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST @@ -32,3 +33,10 @@ def label_text( raise HTTPException( status_code=HTTP_400_BAD_REQUEST, ) + + +@router.get("/valid_labels") +def get_valid_lables() -> ValidLabelsResponse: + return ValidLabelsResponse( + valid_labels=[LabelOption(name=l.value, description=l.description) for l in protocol_schema.TextLabel] + ) diff --git a/backend/oasst_backend/schemas/text_labels.py b/backend/oasst_backend/schemas/text_labels.py new file mode 100644 index 00000000..e7160236 --- /dev/null +++ b/backend/oasst_backend/schemas/text_labels.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel + + +class LabelOption(BaseModel): + name: str + description: str + + +class ValidLabelsResponse(BaseModel): + valid_labels: list[LabelOption] diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py index f8af590e..b87575a9 100644 --- a/oasst-shared/oasst_shared/schemas/protocol.py +++ b/oasst-shared/oasst_shared/schemas/protocol.py @@ -265,10 +265,23 @@ class MessageRanking(Interaction): class TextLabel(str, enum.Enum): """A label for a piece of text.""" + def __new__(cls, label, description=""): + obj = str.__new__(cls, label) + obj._value_ = label + obj.description = description + return obj + spam = "spam" - violence = "violence" - sexual_content = "sexual_content" + fails_task = "fails_task", "Fails to follow the correct instruction / task" + not_appropriate = "not_appropriate", "Inappropriate for customer assistant" + violence = "violence", "Encourages or fails to discourage violence/abuse/terrorism/self-harm" + harmful = ( + "harmful", + "The advice given in the output is harmful or counter-productive. This may be in addition to, but is distinct from the question about encouraging violence/abuse/terrorism/self-harm.", + ) + sexual_content = "sexual_content", "Contains sexual content" toxicity = "toxicity" + moral_judgement = "moral_judgement", "Expresses moral judgement" political_content = "political_content" humor = "humor" sarcasm = "sarcasm" From 45f2f3a60ea77acc74a01cf03b3a99a3987016b6 Mon Sep 17 00:00:00 2001 From: Oliver Stanley Date: Sat, 7 Jan 2023 20:30:18 +0000 Subject: [PATCH 39/49] Validate text labels in text frontend (#495) * Validate text labels in text frontend * Correct task type --- backend/oasst_backend/api/v1/tasks.py | 3 ++- text-frontend/__main__.py | 28 +++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/backend/oasst_backend/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py index 3860bb07..3a85cd5b 100644 --- a/backend/oasst_backend/api/v1/tasks.py +++ b/backend/oasst_backend/api/v1/tasks.py @@ -302,7 +302,8 @@ def tasks_interaction( logger.info( f"Frontend reports labels of {interaction.message_id=} with {interaction.labels=} by {interaction.user=}." ) - # TODO: check if the labels are valid? + # Labels are implicitly validated when converting str -> TextLabel + # So no need for explicit validation here pr.store_text_labels(interaction) return protocol_schema.TaskDone() case _: diff --git a/text-frontend/__main__.py b/text-frontend/__main__.py index de65749a..2060498d 100644 --- a/text-frontend/__main__.py +++ b/text-frontend/__main__.py @@ -211,9 +211,17 @@ def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "DUMMY_KEY") _post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id}) valid_labels = task["valid_labels"] - labels_str: str = typer.prompt("Enter labels, separated by commas") - labels = labels_str.lower().replace(" ", "").split(",") - labels_dict = {label: "1" if label in labels else "0" for label in valid_labels} + + labels_dict = None + while labels_dict is None: + labels_str: str = typer.prompt("Enter labels, separated by commas") + labels = labels_str.lower().replace(" ", "").split(",") + + if all([label in valid_labels for label in labels]): + labels_dict = {label: "1" if label in labels else "0" for label in valid_labels} + else: + invalid_labels = [label for label in labels if label not in valid_labels] + typer.echo(f"Invalid labels: {', '.join(invalid_labels)}. Valid: {', '.join(valid_labels)}") # send ranking new_task = _post( @@ -240,9 +248,17 @@ def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "DUMMY_KEY") _post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id}) valid_labels = task["valid_labels"] - labels_str: str = typer.prompt("Enter labels, separated by commas") - labels = labels_str.lower().replace(" ", "").split(",") - labels_dict = {label: "1" if label in labels else "0" for label in valid_labels} + + labels_dict = None + while labels_dict is None: + labels_str: str = typer.prompt("Enter labels, separated by commas") + labels = labels_str.lower().replace(" ", "").split(",") + + if all([label in valid_labels for label in labels]): + labels_dict = {label: "1" if label in labels else "0" for label in valid_labels} + else: + invalid_labels = [label for label in labels if label not in valid_labels] + typer.echo(f"Invalid labels: {', '.join(invalid_labels)}. Valid: {', '.join(valid_labels)}") # send ranking new_task = _post( From 5fb0121668ea7163f710fd9c9fa12826f11e3757 Mon Sep 17 00:00:00 2001 From: jojopirker Date: Sat, 7 Jan 2023 21:33:57 +0100 Subject: [PATCH 40/49] #438 Frontend type (#483) * add dev setup to readme & alembic description * frontend_type for api_client * readme update & missing file * pre-commit --- backend/README.md | 21 +++++++++++++++++ ...7fb6e_added_frontend_type_to_api_client.py | 23 +++++++++++++++++++ backend/oasst_backend/api/deps.py | 8 ++++++- backend/oasst_backend/models/api_client.py | 1 + 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 backend/alembic/versions/2023_01_07_1250-ba61fe17fb6e_added_frontend_type_to_api_client.py diff --git a/backend/README.md b/backend/README.md index 863090f7..a9507d5e 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,5 +1,18 @@ # Open-Assistant REST Backend +## Backend Development Setup + +In root directory, run +`docker compose up backend-dev --build --attach-dependencies` to start a +database. The default settings are already configured to connect to the database +at `localhost:5432`. + +Make sure you have all requirements installed. You can do this by running +`pip install -r requirements.txt` inside the `backend` folder and +`pip install -e .` inside the `oasst-shared` folder. Then, run the backend using +the `run-local.sh` script inside the `scripts` folder. This will start the +backend server at `http://localhost:8080`. + ## REST Server Configuration Please either use environment variables or create a `.env` file in the backend @@ -20,3 +33,11 @@ REDIS_PORT=6379 Have a look into the main `README.md` file for more information on how to set up the backend for development. Use the scripts within the scripts/backend-development folder to run the BE API locally. + +## Alembic + +To create an Alembic database migration script after sql-models were modified +run `alembic revision --autogenerate -m "..."` ("..." is what you did) in the +`/backend` directory. Then edit the newly created file. See +[here](https://alembic.sqlalchemy.org/en/latest/tutorial.html) for more +information. diff --git a/backend/alembic/versions/2023_01_07_1250-ba61fe17fb6e_added_frontend_type_to_api_client.py b/backend/alembic/versions/2023_01_07_1250-ba61fe17fb6e_added_frontend_type_to_api_client.py new file mode 100644 index 00000000..dbc89ebf --- /dev/null +++ b/backend/alembic/versions/2023_01_07_1250-ba61fe17fb6e_added_frontend_type_to_api_client.py @@ -0,0 +1,23 @@ +"""added frontend_type to api_client + +Revision ID: ba61fe17fb6e +Revises: 20cd871f4ec7 +Create Date: 2023-01-07 12:50:32.195930 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "ba61fe17fb6e" +down_revision = "20cd871f4ec7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("api_client", sa.Column("frontend_type", sa.String(256), nullable=True)) + + +def downgrade() -> None: + op.drop_column("api_client", "frontend_id") diff --git a/backend/oasst_backend/api/deps.py b/backend/oasst_backend/api/deps.py index f61947cd..901ec8ab 100644 --- a/backend/oasst_backend/api/deps.py +++ b/backend/oasst_backend/api/deps.py @@ -40,7 +40,13 @@ def get_dummy_api_client(db: Session) -> ApiClient: if api_client is None: token = token_hex(32) logger.info(f"ANY_API_KEY missing, inserting api_key: {token}") - api_client = ApiClient(id=ANY_API_KEY_ID, api_key=token, description="ANY_API_KEY, random token", trusted=True) + api_client = ApiClient( + id=ANY_API_KEY_ID, + api_key=token, + description="ANY_API_KEY, random token", + trusted=True, + frontend_type="Test frontend", + ) db.add(api_client) db.commit() return api_client diff --git a/backend/oasst_backend/models/api_client.py b/backend/oasst_backend/models/api_client.py index 0bebec47..9b8f2aaa 100644 --- a/backend/oasst_backend/models/api_client.py +++ b/backend/oasst_backend/models/api_client.py @@ -20,3 +20,4 @@ class ApiClient(SQLModel, table=True): admin_email: Optional[str] = Field(max_length=256, nullable=True) enabled: bool = Field(default=True) trusted: bool = Field(sa_column=sa.Column(sa.Boolean, nullable=False, server_default=false())) + frontend_type: str = Field(max_length=256, nullable=True) From 5e01f421aaad228f15f5e04c5d61ba74a0b1547d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Sat, 7 Jan 2023 21:48:11 +0100 Subject: [PATCH 41/49] add 'help_text' to valid_lables response --- backend/oasst_backend/api/v1/text_labels.py | 5 ++++- backend/oasst_backend/schemas/text_labels.py | 5 ++++- oasst-shared/oasst_shared/schemas/protocol.py | 6 ++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/backend/oasst_backend/api/v1/text_labels.py b/backend/oasst_backend/api/v1/text_labels.py index 5b18e4ea..03fd2cb4 100644 --- a/backend/oasst_backend/api/v1/text_labels.py +++ b/backend/oasst_backend/api/v1/text_labels.py @@ -38,5 +38,8 @@ def label_text( @router.get("/valid_labels") def get_valid_lables() -> ValidLabelsResponse: return ValidLabelsResponse( - valid_labels=[LabelOption(name=l.value, description=l.description) for l in protocol_schema.TextLabel] + valid_labels=[ + LabelOption(name=l.value, display_text=l.display_text, help_text=l.help_text) + for l in protocol_schema.TextLabel + ] ) diff --git a/backend/oasst_backend/schemas/text_labels.py b/backend/oasst_backend/schemas/text_labels.py index e7160236..9135c558 100644 --- a/backend/oasst_backend/schemas/text_labels.py +++ b/backend/oasst_backend/schemas/text_labels.py @@ -1,9 +1,12 @@ +from typing import Optional + from pydantic import BaseModel class LabelOption(BaseModel): name: str - description: str + display_text: str + help_text: Optional[str] class ValidLabelsResponse(BaseModel): diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py index b87575a9..cbfbeca5 100644 --- a/oasst-shared/oasst_shared/schemas/protocol.py +++ b/oasst-shared/oasst_shared/schemas/protocol.py @@ -265,10 +265,11 @@ class MessageRanking(Interaction): class TextLabel(str, enum.Enum): """A label for a piece of text.""" - def __new__(cls, label, description=""): + def __new__(cls, label: str, display_text: str = "", help_text: str = None): obj = str.__new__(cls, label) obj._value_ = label - obj.description = description + obj.display_text = display_text + obj.help_text = help_text return obj spam = "spam" @@ -277,6 +278,7 @@ class TextLabel(str, enum.Enum): violence = "violence", "Encourages or fails to discourage violence/abuse/terrorism/self-harm" harmful = ( "harmful", + "Harmful content", "The advice given in the output is harmful or counter-productive. This may be in addition to, but is distinct from the question about encouraging violence/abuse/terrorism/self-harm.", ) sexual_content = "sexual_content", "Contains sexual content" From d910c310c06ee314358a86ad950977984ec7e7cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Sat, 7 Jan 2023 21:58:29 +0100 Subject: [PATCH 42/49] fix text_frontend 204 & creation of labeling tasks (#487) --- backend/oasst_backend/api/v1/tasks.py | 15 ++++++++------- text-frontend/__main__.py | 11 +++++++---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/backend/oasst_backend/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py index 3a85cd5b..0e469602 100644 --- a/backend/oasst_backend/api/v1/tasks.py +++ b/backend/oasst_backend/api/v1/tasks.py @@ -6,6 +6,7 @@ from fastapi import APIRouter, Depends from fastapi.security.api_key import APIKey from loguru import logger from oasst_backend.api import deps +from oasst_backend.api.v1.utils import prepare_conversation from oasst_backend.prompt_repository import PromptRepository from oasst_shared.exceptions import OasstError, OasstErrorCode from oasst_shared.schemas import protocol as protocol_schema @@ -126,7 +127,7 @@ def generate_task( ] replies = [p.text for p in replies] task = protocol_schema.RankAssistantRepliesTask( - conversation=protocol_schema.Conversation(messages=task_messages), + conversation=prepare_conversation(conversation), replies=replies, ) @@ -142,22 +143,22 @@ def generate_task( case protocol_schema.TaskRequestType.label_prompter_reply: logger.info("Generating a LabelPrompterReplyTask.") conversation, messages = pr.fetch_multiple_random_replies(max_size=1, message_role="assistant") - message = messages[0].text + message = messages[0] task = protocol_schema.LabelPrompterReplyTask( message_id=message.id, - conversation=conversation, - reply=message, + conversation=prepare_conversation(conversation), + reply=message.text, valid_labels=list(map(lambda x: x.value, protocol_schema.TextLabel)), ) case protocol_schema.TaskRequestType.label_assistant_reply: logger.info("Generating a LabelAssistantReplyTask.") conversation, messages = pr.fetch_multiple_random_replies(max_size=1, message_role="prompter") - message = messages[0].text + message = messages[0] task = protocol_schema.LabelAssistantReplyTask( message_id=message.id, - conversation=conversation, - reply=message, + conversation=prepare_conversation(conversation), + reply=message.text, valid_labels=list(map(lambda x: x.value, protocol_schema.TextLabel)), ) diff --git a/text-frontend/__main__.py b/text-frontend/__main__.py index 2060498d..a6e5f947 100644 --- a/text-frontend/__main__.py +++ b/text-frontend/__main__.py @@ -1,5 +1,6 @@ """Simple REPL frontend.""" +import http import random import requests @@ -30,6 +31,8 @@ def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "DUMMY_KEY") def _post(path: str, json: dict) -> dict: response = requests.post(f"{backend_url}{path}", json=json, headers={"X-API-Key": api_key}) response.raise_for_status() + if response.status_code == http.HTTPStatus.NO_CONTENT: + return None return response.json() typer.echo("Requesting work...") @@ -191,7 +194,7 @@ def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "DUMMY_KEY") ranking_str = typer.prompt("Enter the reply numbers in order of preference, separated by commas") ranking = [int(x) - 1 for x in ranking_str.split(",")] - # send ranking + # send labels new_task = _post( "/api/v1/tasks/interaction", { @@ -223,7 +226,7 @@ def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "DUMMY_KEY") invalid_labels = [label for label in labels if label not in valid_labels] typer.echo(f"Invalid labels: {', '.join(invalid_labels)}. Valid: {', '.join(valid_labels)}") - # send ranking + # send labels new_task = _post( "/api/v1/tasks/interaction", { @@ -260,13 +263,13 @@ def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "DUMMY_KEY") invalid_labels = [label for label in labels if label not in valid_labels] typer.echo(f"Invalid labels: {', '.join(invalid_labels)}. Valid: {', '.join(valid_labels)}") - # send ranking + # send labels new_task = _post( "/api/v1/tasks/interaction", { "type": "text_labels", "message_id": task["message_id"], - "text": task["prompt"], + "text": task["reply"], "labels": labels_dict, "user": USER, }, From 3977ff630c06ad3f23338269fa469adff6737d3f Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Sat, 7 Jan 2023 21:21:52 +0000 Subject: [PATCH 43/49] add docs side url to docs/readme (#484) --- docs/README.md | 8 +++++--- docs/docusaurus.config.js | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/README.md b/docs/README.md index 0d6fa482..c8e40c04 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,7 +1,9 @@ -# Website +# Docs Site -This website is built using [Docusaurus 2](https://docusaurus.io/), a modern -static website generator. +https://liaon-ai.github.io/Open-Assistant/ + +This [site](https://liaon-ai.github.io/Open-Assistant/) is built using +[Docusaurus 2](https://docusaurus.io/), a modern static website generator. ### Contributing diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index 2d902d83..4e456369 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -20,7 +20,7 @@ const config = { // If you aren't using GitHub pages, you don't need these. organizationName: "LAION-AI", // Usually your GitHub org/user name. projectName: "Open-Assistant", // Usually your repo name. - deploymentBranch: "docs-site-poc", + deploymentBranch: "main", // Even if you don't use internalization, you can use this field to set useful // metadata like html lang. For example, if your site is Chinese, you may want From c57db6d0dfc4c7986717fe66c002cf21ae2574ba Mon Sep 17 00:00:00 2001 From: Yannic Kilcher Date: Sat, 7 Jan 2023 22:24:47 +0100 Subject: [PATCH 44/49] typo --- docs/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index c8e40c04..7189d5da 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,8 @@ # Docs Site -https://liaon-ai.github.io/Open-Assistant/ +https://laion-ai.github.io/Open-Assistant/ -This [site](https://liaon-ai.github.io/Open-Assistant/) is built using +This [site](https://laion-ai.github.io/Open-Assistant/) is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator. ### Contributing From cf4a62a984f38d679c2886fc51e1ec0e0a9a25b8 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Sat, 7 Jan 2023 21:54:01 +0000 Subject: [PATCH 45/49] add docs badge to readme (#462) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e6a682ec..7cac0788 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@
![GitHub Repo stars](https://img.shields.io/github/stars/LAION-AI/Open-Assistant?style=social) +![Docs](https://img.shields.io/badge/docs-laion--ai.github.io%2FOpen--Assistant%2F-green) ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LAION-AI/Open-Assistant/build-frontend.yaml?label=frontend) ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LAION-AI/Open-Assistant/pre-commit.yaml?label=pre-commit) ![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/LAION-AI/Open-Assistant/test-api-contract.yaml?label=api) From d183c343a96cb38a7df64ba79cc56123b7502d37 Mon Sep 17 00:00:00 2001 From: kostiak Date: Sun, 8 Jan 2023 00:55:44 +0200 Subject: [PATCH 46/49] Temporary fix for #506 to get the frontend working (#508) --- backend/oasst_backend/api/v1/tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/oasst_backend/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py index 0e469602..adfb2907 100644 --- a/backend/oasst_backend/api/v1/tasks.py +++ b/backend/oasst_backend/api/v1/tasks.py @@ -62,7 +62,7 @@ def generate_task( text=msg.text, is_assistant=(msg.role == "assistant"), message_id=msg.id, - front_end_id=msg.front_end_id, + front_end_id=msg.frontend_message_id, ) for msg in messages ] @@ -78,7 +78,7 @@ def generate_task( text=msg.text, is_assistant=(msg.role == "assistant"), message_id=msg.id, - front_end_id=msg.front_end_id, + front_end_id=msg.frontend_message_id, ) for msg in messages ] @@ -100,7 +100,7 @@ def generate_task( text=p.text, is_assistant=(p.role == "assistant"), message_id=p.id, - front_end_id=p.front_end_id, + front_end_id=p.frontend_message_id, ) for p in conversation ] @@ -121,7 +121,7 @@ def generate_task( text=p.text, is_assistant=(p.role == "assistant"), message_id=p.id, - front_end_id=p.front_end_id, + front_end_id=p.frontend_message_id, ) for p in conversation ] From 25ce928733ab47e676ce91ca29539a1c497a7d31 Mon Sep 17 00:00:00 2001 From: notmd Date: Sun, 8 Jan 2023 06:53:39 +0700 Subject: [PATCH 47/49] fix: improve `TaskControls` UI on mobile --- .../src/components/Survey/TaskControls.tsx | 21 +++++++++++-------- .../src/components/Survey/TrackedTextarea.tsx | 2 +- website/src/components/TaskInfo/TaskInfo.tsx | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/website/src/components/Survey/TaskControls.tsx b/website/src/components/Survey/TaskControls.tsx index 851e659c..ea71467c 100644 --- a/website/src/components/Survey/TaskControls.tsx +++ b/website/src/components/Survey/TaskControls.tsx @@ -1,5 +1,6 @@ import { useColorMode } from "@chakra-ui/react"; import { Flex } from "@chakra-ui/react"; +import clsx from "clsx"; import { SkipButton } from "src/components/Buttons/Skip"; import { SubmitButton } from "src/components/Buttons/Submit"; import { TaskInfo } from "src/components/TaskInfo/TaskInfo"; @@ -14,18 +15,20 @@ export interface TaskControlsProps { } export const TaskControls = (props: TaskControlsProps) => { - const extraClases = props.className || ""; const { colorMode } = useColorMode(); - - const baseClasses = "flex flex-row justify-items-stretch mb-8 p-4 rounded-lg max-w-7xl mx-auto"; - const taskControlClases = - colorMode === "light" - ? `${baseClasses} bg-white text-gray-800 shadow-lg ${extraClases}` - : `${baseClasses} bg-slate-800 text-slate-400 shadow-xl ring-1 ring-white/10 ring-inset ${extraClases}`; - + const isLightMode = colorMode === "light"; const endTask = props.tasks[props.tasks.length - 1]; return ( -
+
Skip diff --git a/website/src/components/Survey/TrackedTextarea.tsx b/website/src/components/Survey/TrackedTextarea.tsx index f1691b72..d20107ac 100644 --- a/website/src/components/Survey/TrackedTextarea.tsx +++ b/website/src/components/Survey/TrackedTextarea.tsx @@ -28,7 +28,7 @@ export const TrackedTextarea = (props: TrackedTextboxProps) => { return ( -