From 1e2948743dd42bb78beca0a0fc33fbdab086fb63 Mon Sep 17 00:00:00 2001 From: notmd Date: Wed, 1 Feb 2023 17:22:44 +0700 Subject: [PATCH 01/23] wip --- website/next.config.js | 5 +- website/package-lock.json | 53 ------------------- website/package.json | 3 -- website/src/components/Explain.tsx | 9 +--- .../components/Survey/LabelLikertGroup.tsx | 4 +- 5 files changed, 8 insertions(+), 66 deletions(-) diff --git a/website/next.config.js b/website/next.config.js index a84ce736..31794e5a 100644 --- a/website/next.config.js +++ b/website/next.config.js @@ -1,6 +1,6 @@ -/** @type {import('next').NextConfig} */ const { i18n } = require("./next-i18next.config"); +/** @type {import('next').NextConfig} */ const nextConfig = { output: "standalone", reactStrictMode: true, @@ -19,6 +19,9 @@ const nextConfig = { // scrollRestoration: true, }, i18n, + eslint: { + ignoreDuringBuilds: true, + }, }; module.exports = nextConfig; diff --git a/website/package-lock.json b/website/package-lock.json index 0c92c0db..4cba1b5f 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -15,8 +15,6 @@ "@dnd-kit/utilities": "^3.2.1", "@emotion/react": "^11.10.5", "@emotion/styled": "^11.10.5", - "@headlessui/react": "^1.7.7", - "@heroicons/react": "^2.0.13", "@next-auth/prisma-adapter": "^1.0.5", "@next/font": "^13.1.0", "@prisma/client": "^4.7.1", @@ -32,7 +30,6 @@ "eslint-plugin-simple-import-sort": "^8.0.0", "focus-visible": "^5.2.0", "framer-motion": "^6.5.1", - "install": "^0.13.0", "lucide-react": "^0.105.0", "next": "13.0.6", "next-auth": "^4.18.6", @@ -3679,29 +3676,6 @@ "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", "dev": true }, - "node_modules/@headlessui/react": { - "version": "1.7.7", - "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-1.7.7.tgz", - "integrity": "sha512-BqDOd/tB9u2tA0T3Z0fn18ktw+KbVwMnkxxsGPIH2hzssrQhKB5n/6StZOyvLYP/FsYtvuXfi9I0YowKPv2c1w==", - "dependencies": { - "client-only": "^0.0.1" - }, - "engines": { - "node": ">=10" - }, - "peerDependencies": { - "react": "^16 || ^17 || ^18", - "react-dom": "^16 || ^17 || ^18" - } - }, - "node_modules/@heroicons/react": { - "version": "2.0.13", - "resolved": "https://registry.npmjs.org/@heroicons/react/-/react-2.0.13.tgz", - "integrity": "sha512-iSN5XwmagrnirWlYEWNPdCDj9aRYVD/lnK3JlsC9/+fqGF80k8C7rl+1HCvBX0dBoagKqOFBs6fMhJJ1hOg1EQ==", - "peerDependencies": { - "react": ">= 16" - } - }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", @@ -22038,14 +22012,6 @@ "node": ">=8" } }, - "node_modules/install": { - "version": "0.13.0", - "resolved": "https://registry.npmjs.org/install/-/install-0.13.0.tgz", - "integrity": "sha512-zDml/jzr2PKU9I8J/xyZBQn8rPCAY//UOYNmR01XwNwyfhEWObo2SWfSl1+0tm1u6PhxLwDnfsT/6jB7OUxqFA==", - "engines": { - "node": ">= 0.10" - } - }, "node_modules/internal-slot": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.4.tgz", @@ -40872,20 +40838,6 @@ "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", "dev": true }, - "@headlessui/react": { - "version": "1.7.7", - "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-1.7.7.tgz", - "integrity": "sha512-BqDOd/tB9u2tA0T3Z0fn18ktw+KbVwMnkxxsGPIH2hzssrQhKB5n/6StZOyvLYP/FsYtvuXfi9I0YowKPv2c1w==", - "requires": { - "client-only": "^0.0.1" - } - }, - "@heroicons/react": { - "version": "2.0.13", - "resolved": "https://registry.npmjs.org/@heroicons/react/-/react-2.0.13.tgz", - "integrity": "sha512-iSN5XwmagrnirWlYEWNPdCDj9aRYVD/lnK3JlsC9/+fqGF80k8C7rl+1HCvBX0dBoagKqOFBs6fMhJJ1hOg1EQ==", - "requires": {} - }, "@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", @@ -55060,11 +55012,6 @@ } } }, - "install": { - "version": "0.13.0", - "resolved": "https://registry.npmjs.org/install/-/install-0.13.0.tgz", - "integrity": "sha512-zDml/jzr2PKU9I8J/xyZBQn8rPCAY//UOYNmR01XwNwyfhEWObo2SWfSl1+0tm1u6PhxLwDnfsT/6jB7OUxqFA==" - }, "internal-slot": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.4.tgz", diff --git a/website/package.json b/website/package.json index 25fee307..c488417b 100644 --- a/website/package.json +++ b/website/package.json @@ -32,8 +32,6 @@ "@dnd-kit/utilities": "^3.2.1", "@emotion/react": "^11.10.5", "@emotion/styled": "^11.10.5", - "@headlessui/react": "^1.7.7", - "@heroicons/react": "^2.0.13", "@next-auth/prisma-adapter": "^1.0.5", "@next/font": "^13.1.0", "@prisma/client": "^4.7.1", @@ -49,7 +47,6 @@ "eslint-plugin-simple-import-sort": "^8.0.0", "focus-visible": "^5.2.0", "framer-motion": "^6.5.1", - "install": "^0.13.0", "lucide-react": "^0.105.0", "next": "13.0.6", "next-auth": "^4.18.6", diff --git a/website/src/components/Explain.tsx b/website/src/components/Explain.tsx index b571757f..fe6d23b7 100644 --- a/website/src/components/Explain.tsx +++ b/website/src/components/Explain.tsx @@ -8,7 +8,7 @@ import { PopoverTrigger, Text, } from "@chakra-ui/react"; -import { InformationCircleIcon } from "@heroicons/react/20/solid"; +import { Info } from "lucide-react"; interface ExplainProps { explanation: string[]; @@ -18,12 +18,7 @@ export const Explain = ({ explanation }: ExplainProps) => { return ( - } - > + }> diff --git a/website/src/components/Survey/LabelLikertGroup.tsx b/website/src/components/Survey/LabelLikertGroup.tsx index f7f14e7d..734c6c10 100644 --- a/website/src/components/Survey/LabelLikertGroup.tsx +++ b/website/src/components/Survey/LabelLikertGroup.tsx @@ -211,7 +211,7 @@ export const LabelLikertGroup = ({ labelIDs, onChange, isEditable = true }: Labe }} alignItems="center" > - + {textA} {descriptionA.length > 0 ? : null} @@ -229,7 +229,7 @@ export const LabelLikertGroup = ({ labelIDs, onChange, isEditable = true }: Labe /> - + {textB} {descriptionB.length > 0 ? : null} From 79a39771aa5e7ed2770c768084e5d4ce34b5caf5 Mon Sep 17 00:00:00 2001 From: notmd Date: Wed, 1 Feb 2023 17:55:29 +0700 Subject: [PATCH 02/23] remove root styles folder --- .../components/Messages/MessageTableEntry.tsx | 4 +- website/src/components/SideMenu.tsx | 10 +-- website/src/components/SideMenuLayout.tsx | 4 +- website/src/pages/messages/[id]/index.tsx | 6 +- website/src/styles/Theme/colors.ts | 2 + website/styles/Home.module.css | 41 ------------ website/styles/Theme/Components/Container.tsx | 25 -------- website/styles/Theme/colors.tsx | 18 ------ website/styles/Theme/index.tsx | 64 ------------------- 9 files changed, 12 insertions(+), 162 deletions(-) delete mode 100644 website/styles/Home.module.css delete mode 100644 website/styles/Theme/Components/Container.tsx delete mode 100644 website/styles/Theme/colors.tsx delete mode 100644 website/styles/Theme/index.tsx diff --git a/website/src/components/Messages/MessageTableEntry.tsx b/website/src/components/Messages/MessageTableEntry.tsx index e65885e2..89b0b204 100644 --- a/website/src/components/Messages/MessageTableEntry.tsx +++ b/website/src/components/Messages/MessageTableEntry.tsx @@ -23,8 +23,8 @@ import { LabelMessagePopup } from "src/components/Messages/LabelPopup"; import { getEmojiIcon, MessageEmojiButton } from "src/components/Messages/MessageEmojiButton"; import { ReportPopup } from "src/components/Messages/ReportPopup"; import { post } from "src/lib/api"; +import { colors } from "src/styles/Theme/colors"; import { Message, MessageEmojis } from "src/types/Conversation"; -import { colors } from "styles/Theme/colors"; import useSWRMutation from "swr/mutation"; interface MessageTableEntryProps { @@ -66,7 +66,7 @@ export function MessageTableEntry({ message, enabled, highlight }: MessageTableE ), [borderColor, inlineAvatar, message.is_assistant] ); - const highlightColor = useColorModeValue(colors.light.highlight, colors.dark.highlight); + const highlightColor = useColorModeValue(colors.light.active, colors.dark.active); const { trigger: sendEmojiChange } = useSWRMutation(`/api/messages/${message.id}/emoji`, post, { onSuccess: setEmojis, diff --git a/website/src/components/SideMenu.tsx b/website/src/components/SideMenu.tsx index 10e83ce4..25dabf30 100644 --- a/website/src/components/SideMenu.tsx +++ b/website/src/components/SideMenu.tsx @@ -1,8 +1,7 @@ -import { Box, Button, Text, Tooltip, useColorMode } from "@chakra-ui/react"; +import { Button, Card, Text, Tooltip, useColorMode } from "@chakra-ui/react"; import { LucideIcon, Sun } from "lucide-react"; import Link from "next/link"; import { useRouter } from "next/router"; -import { colors } from "styles/Theme/colors"; export interface MenuButtonOption { label: string; @@ -21,11 +20,8 @@ export function SideMenu(props: SideMenuProps) { return (
-
); } diff --git a/website/src/components/SideMenuLayout.tsx b/website/src/components/SideMenuLayout.tsx index 97fda2db..2c5a5054 100644 --- a/website/src/components/SideMenuLayout.tsx +++ b/website/src/components/SideMenuLayout.tsx @@ -1,6 +1,6 @@ import { Box, useColorMode } from "@chakra-ui/react"; import { MenuButtonOption, SideMenu } from "src/components/SideMenu"; -import { colors } from "styles/Theme/colors"; +import { colors } from "src/styles/Theme/colors"; interface SideMenuLayoutProps { menuButtonOptions: MenuButtonOption[]; @@ -11,7 +11,7 @@ export const SideMenuLayout = (props: SideMenuLayoutProps) => { const { colorMode } = useColorMode(); return ( - + diff --git a/website/src/pages/messages/[id]/index.tsx b/website/src/pages/messages/[id]/index.tsx index 158d28e8..2c58c367 100644 --- a/website/src/pages/messages/[id]/index.tsx +++ b/website/src/pages/messages/[id]/index.tsx @@ -1,4 +1,4 @@ -import { Box, Text, useColorModeValue } from "@chakra-ui/react"; +import { Box, Card, Text, useColorModeValue } from "@chakra-ui/react"; import Head from "next/head"; import { useTranslation } from "next-i18next"; import { serverSideTranslations } from "next-i18next/serverSideTranslations"; @@ -36,9 +36,9 @@ const MessageDetail = ({ id }: { id: string }) => { {t("parent")} - + - + )} diff --git a/website/src/styles/Theme/colors.ts b/website/src/styles/Theme/colors.ts index 9b8c447d..6ce1f161 100644 --- a/website/src/styles/Theme/colors.ts +++ b/website/src/styles/Theme/colors.ts @@ -2,10 +2,12 @@ export const colors = { light: { bg: "rgb(250,250,250)", text: "black", + active: "blue.400", }, dark: { bg: "gray.900", text: "white", + active: "blue.500", }, "dark-blue-btn": { 200: "rgb(29,78,216)", diff --git a/website/styles/Home.module.css b/website/styles/Home.module.css deleted file mode 100644 index 1dc2ec1c..00000000 --- a/website/styles/Home.module.css +++ /dev/null @@ -1,41 +0,0 @@ -.App { - text-align: center; -} - -.App-logo { - height: 40vmin; - pointer-events: none; -} - -@media (prefers-reduced-motion: no-preference) { - .App-logo { - animation: App-logo-spin infinite 20s linear; - } -} - -.AppHeader { - background: linear-gradient(217deg, rgba(255, 0, 0, 0.8), rgba(255, 0, 0, 0) 70.71%), - linear-gradient(127deg, rgba(0, 255, 0, 0.8), rgba(0, 255, 0, 0) 70.71%), - linear-gradient(336deg, rgba(0, 0, 255, 0.8), rgba(0, 0, 255, 0) 70.71%); - background: black; - min-height: 100vh; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - font-size: calc(10px + 2vmin); - color: white; -} - -.AppLink { - color: #61dafb; -} - -@keyframes App-logo-spin { - from { - transform: rotate(0deg); - } - to { - transform: rotate(360deg); - } -} diff --git a/website/styles/Theme/Components/Container.tsx b/website/styles/Theme/Components/Container.tsx deleted file mode 100644 index ec7eeed9..00000000 --- a/website/styles/Theme/Components/Container.tsx +++ /dev/null @@ -1,25 +0,0 @@ -import { - color, - defineStyle, - defineStyleConfig, - // transition, -} from "@chakra-ui/styled-system"; -import { colors } from "../colors"; - -const baseStyle = defineStyle(({ colorMode }) => ({ - minWidth: "100%", - bg: colorMode === "light" ? colors.light.bg : colors.dark.bg, - // transition: "background-color 300ms cubic-bezier(0.4, 0, 1, 1)", - color: colorMode === "light" ? colors.light.text : colors.dark.text, -})); - -const variants = { - "no-padding": { - padding: 0, - }, -}; - -export const containerTheme = defineStyleConfig({ - baseStyle, - variants, -}); diff --git a/website/styles/Theme/colors.tsx b/website/styles/Theme/colors.tsx deleted file mode 100644 index 26609185..00000000 --- a/website/styles/Theme/colors.tsx +++ /dev/null @@ -1,18 +0,0 @@ -export const colors = { - light: { - bg: "gray.100", - btn: "gray.50", - div: "white", - text: "black", - highlight: "blue.400", - active: "blue.400", - }, - dark: { - bg: "gray.900", - btn: "gray.600", - div: "gray.700", - text: "gray.200", - highlight: "blue.500", - active: "blue.500", - }, -}; diff --git a/website/styles/Theme/index.tsx b/website/styles/Theme/index.tsx deleted file mode 100644 index 7288b7ec..00000000 --- a/website/styles/Theme/index.tsx +++ /dev/null @@ -1,64 +0,0 @@ -import { type ThemeConfig, extendTheme, usePrefersReducedMotion } from "@chakra-ui/react"; -import { containerTheme } from "./Components/Container"; -import { StyleFunctionProps, Styles } from "@chakra-ui/theme-tools"; - -const config: ThemeConfig = { - initialColorMode: "system", - useSystemColorMode: false, - disableTransitionOnChange: true, -}; - -const components = { - Container: containerTheme, - Box: (props: StyleFunctionProps) => ({ - backgroundColor: props.colorMode === "light" ? "white" : "gray.800", - }), - Button: { - baseStyle: { - fontWeight: "normal", - }, - sizes: { - lg: { - fontSize: "md", - paddingY: "7", - }, - }, - variants: { - solid: (props: StyleFunctionProps) => ({ - bg: props.colorMode === "light" ? "gray.100" : "gray.600", - _hover: { - bg: props.colorMode === "light" ? "gray.200" : "#3D4A60", - }, - _active: { - bg: props.colorMode === "light" ? "gray.300" : "#374254", - }, - borderRadius: "lg", - }), - // gradient: (props: StyleFunctionProps) => ({ - // bg: `linear-gradient(${white}, ${bgColor}) padding-box, - // linear-gradient(135deg, ${lgFrom}, ${lgTo}) border-box`, - // }), - }, - }, -}; - -const breakpoints = { - sm: "640px", - md: "768px", - lg: "1024px", - xl: "1280px", - "2xl": "1536px", -}; - -const styles = { - global: (props) => ({ - main: { - fontFamily: "Inter", - }, - header: { - fontFamily: "Inter", - }, - }), -}; - -export const theme = extendTheme({ config, styles, components, breakpoints }); From fdd57eda85c4a3d96219578c56069d2ea2fcca6e Mon Sep 17 00:00:00 2001 From: notmd Date: Wed, 1 Feb 2023 19:45:43 +0700 Subject: [PATCH 03/23] setup bundle analyzer --- website/next.config.js | 7 +- website/package-lock.json | 336 ++++++++++++++++++++++++++++++++++++++ website/package.json | 2 + 3 files changed, 344 insertions(+), 1 deletion(-) diff --git a/website/next.config.js b/website/next.config.js index a84ce736..cd8805c9 100644 --- a/website/next.config.js +++ b/website/next.config.js @@ -21,4 +21,9 @@ const nextConfig = { i18n, }; -module.exports = nextConfig; +const withBundleAnalyzer = require("@next/bundle-analyzer")({ + enabled: process.env.ANALYZE === "true", + openAnalyzer: true, +}); + +module.exports = withBundleAnalyzer(nextConfig); diff --git a/website/package-lock.json b/website/package-lock.json index 41167055..b6d7343c 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -19,6 +19,7 @@ "@heroicons/react": "^2.0.13", "@marsidev/react-turnstile": "^0.0.7", "@next-auth/prisma-adapter": "^1.0.5", + "@next/bundle-analyzer": "^13.1.6", "@next/font": "^13.1.0", "@prisma/client": "^4.7.1", "@tailwindcss/forms": "^0.5.3", @@ -5774,6 +5775,14 @@ "next-auth": "^4" } }, + "node_modules/@next/bundle-analyzer": { + "version": "13.1.6", + "resolved": "https://registry.npmjs.org/@next/bundle-analyzer/-/bundle-analyzer-13.1.6.tgz", + "integrity": "sha512-rJS9CtLoGT58mL+v2ISKANosFFWP/0YKYByHQ3vTaZrbQP8b1rYRxd2QVMJmnSXaFkiP9URt1XJ6OdGyVq5b6g==", + "dependencies": { + "webpack-bundle-analyzer": "4.7.0" + } + }, "node_modules/@next/env": { "version": "13.0.6", "resolved": "https://registry.npmjs.org/@next/env/-/env-13.0.6.tgz", @@ -6187,6 +6196,11 @@ "node": ">= 8" } }, + "node_modules/@polka/url": { + "version": "1.0.0-next.21", + "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.21.tgz", + "integrity": "sha512-a5Sab1C4/icpTZVzZc5Ghpz88yQtGOyNqYXcZgOssB2uuAr+wF/MvN6bgtW32q7HHrvBki+BsZ0OuNv6EV3K9g==" + }, "node_modules/@popperjs/core": { "version": "2.11.6", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.6.tgz", @@ -18259,6 +18273,11 @@ "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==", "dev": true }, + "node_modules/duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==" + }, "node_modules/duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", @@ -21089,6 +21108,20 @@ "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } }, + "node_modules/gzip-size": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", + "integrity": "sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==", + "dependencies": { + "duplexer": "^0.1.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/handlebars": { "version": "4.7.7", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.7.tgz", @@ -27754,6 +27787,14 @@ "rimraf": "bin.js" } }, + "node_modules/mrmime": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-1.0.1.tgz", + "integrity": "sha512-hzzEagAgDyoU1Q6yg5uI+AorQgdvMCur3FcKf7NhMKWsaYg+RnbTyHRa/9IlLF9rf455MOCtcqqrQQ83pPP7Uw==", + "engines": { + "node": ">=10" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -31447,6 +31488,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/opener": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", + "integrity": "sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==", + "bin": { + "opener": "bin/opener-bin.js" + } + }, "node_modules/openid-client": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.3.1.tgz", @@ -34817,6 +34866,19 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, + "node_modules/sirv": { + "version": "1.0.19", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-1.0.19.tgz", + "integrity": "sha512-JuLThK3TnZG1TAKDwNIqNq6QA2afLOCcm+iE8D1Kj3GA40pSPsxQjjJl0J8X3tsR7T+CP1GavpzLwYkgVLWrZQ==", + "dependencies": { + "@polka/url": "^1.0.0-next.20", + "mrmime": "^1.0.0", + "totalist": "^1.0.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -36352,6 +36414,14 @@ "node": ">=0.6" } }, + "node_modules/totalist": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/totalist/-/totalist-1.1.0.tgz", + "integrity": "sha512-gduQwd1rOdDMGxFG1gEvhV88Oirdo2p+KjoYFU7k2g+i7n6AFFbDQ5kMPUsW0pNbfQsB/cwXvT1i4Bue0s9g5g==", + "engines": { + "node": ">=6" + } + }, "node_modules/tough-cookie": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", @@ -37783,6 +37853,139 @@ } } }, + "node_modules/webpack-bundle-analyzer": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/webpack-bundle-analyzer/-/webpack-bundle-analyzer-4.7.0.tgz", + "integrity": "sha512-j9b8ynpJS4K+zfO5GGwsAcQX4ZHpWV+yRiHDiL+bE0XHJ8NiPYLTNVQdlFYWxtpg9lfAQNlwJg16J9AJtFSXRg==", + "dependencies": { + "acorn": "^8.0.4", + "acorn-walk": "^8.0.0", + "chalk": "^4.1.0", + "commander": "^7.2.0", + "gzip-size": "^6.0.0", + "lodash": "^4.17.20", + "opener": "^1.5.2", + "sirv": "^1.0.7", + "ws": "^7.3.1" + }, + "bin": { + "webpack-bundle-analyzer": "lib/bin/analyzer.js" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/webpack-bundle-analyzer/node_modules/acorn": { + "version": "8.8.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", + "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/webpack-bundle-analyzer/node_modules/acorn-walk": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/webpack-bundle-analyzer/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/webpack-bundle-analyzer/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/webpack-bundle-analyzer/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/webpack-bundle-analyzer/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/webpack-bundle-analyzer/node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "engines": { + "node": ">= 10" + } + }, + "node_modules/webpack-bundle-analyzer/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/webpack-bundle-analyzer/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/webpack-bundle-analyzer/node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/webpack-dev-middleware": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-4.3.0.tgz", @@ -42529,6 +42732,14 @@ "integrity": "sha512-VqMS11IxPXrPGXw6Oul6jcyS/n8GLOWzRMrPr3EMdtD6eOalM6zz05j08PcNiis8QzkfuYnCv49OvufTuaEwYQ==", "requires": {} }, + "@next/bundle-analyzer": { + "version": "13.1.6", + "resolved": "https://registry.npmjs.org/@next/bundle-analyzer/-/bundle-analyzer-13.1.6.tgz", + "integrity": "sha512-rJS9CtLoGT58mL+v2ISKANosFFWP/0YKYByHQ3vTaZrbQP8b1rYRxd2QVMJmnSXaFkiP9URt1XJ6OdGyVq5b6g==", + "requires": { + "webpack-bundle-analyzer": "4.7.0" + } + }, "@next/env": { "version": "13.0.6", "resolved": "https://registry.npmjs.org/@next/env/-/env-13.0.6.tgz", @@ -42758,6 +42969,11 @@ } } }, + "@polka/url": { + "version": "1.0.0-next.21", + "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.21.tgz", + "integrity": "sha512-a5Sab1C4/icpTZVzZc5Ghpz88yQtGOyNqYXcZgOssB2uuAr+wF/MvN6bgtW32q7HHrvBki+BsZ0OuNv6EV3K9g==" + }, "@popperjs/core": { "version": "2.11.6", "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.6.tgz", @@ -52149,6 +52365,11 @@ "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==", "dev": true }, + "duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==" + }, "duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", @@ -54367,6 +54588,14 @@ "integrity": "sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw==", "dev": true }, + "gzip-size": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", + "integrity": "sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==", + "requires": { + "duplexer": "^0.1.2" + } + }, "handlebars": { "version": "4.7.7", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.7.tgz", @@ -59420,6 +59649,11 @@ } } }, + "mrmime": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-1.0.1.tgz", + "integrity": "sha512-hzzEagAgDyoU1Q6yg5uI+AorQgdvMCur3FcKf7NhMKWsaYg+RnbTyHRa/9IlLF9rf455MOCtcqqrQQ83pPP7Uw==" + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -61934,6 +62168,11 @@ "is-wsl": "^2.2.0" } }, + "opener": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", + "integrity": "sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==" + }, "openid-client": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.3.1.tgz", @@ -64472,6 +64711,16 @@ } } }, + "sirv": { + "version": "1.0.19", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-1.0.19.tgz", + "integrity": "sha512-JuLThK3TnZG1TAKDwNIqNq6QA2afLOCcm+iE8D1Kj3GA40pSPsxQjjJl0J8X3tsR7T+CP1GavpzLwYkgVLWrZQ==", + "requires": { + "@polka/url": "^1.0.0-next.20", + "mrmime": "^1.0.0", + "totalist": "^1.0.0" + } + }, "sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -65697,6 +65946,11 @@ "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", "dev": true }, + "totalist": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/totalist/-/totalist-1.1.0.tgz", + "integrity": "sha512-gduQwd1rOdDMGxFG1gEvhV88Oirdo2p+KjoYFU7k2g+i7n6AFFbDQ5kMPUsW0pNbfQsB/cwXvT1i4Bue0s9g5g==" + }, "tough-cookie": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", @@ -66809,6 +67063,88 @@ } } }, + "webpack-bundle-analyzer": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/webpack-bundle-analyzer/-/webpack-bundle-analyzer-4.7.0.tgz", + "integrity": "sha512-j9b8ynpJS4K+zfO5GGwsAcQX4ZHpWV+yRiHDiL+bE0XHJ8NiPYLTNVQdlFYWxtpg9lfAQNlwJg16J9AJtFSXRg==", + "requires": { + "acorn": "^8.0.4", + "acorn-walk": "^8.0.0", + "chalk": "^4.1.0", + "commander": "^7.2.0", + "gzip-size": "^6.0.0", + "lodash": "^4.17.20", + "opener": "^1.5.2", + "sirv": "^1.0.7", + "ws": "^7.3.1" + }, + "dependencies": { + "acorn": { + "version": "8.8.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.2.tgz", + "integrity": "sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==" + }, + "acorn-walk": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==" + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "requires": { + "has-flag": "^4.0.0" + } + }, + "ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "requires": {} + } + } + }, "webpack-dev-middleware": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-4.3.0.tgz", diff --git a/website/package.json b/website/package.json index 8c495a2c..4390c8fe 100644 --- a/website/package.json +++ b/website/package.json @@ -6,6 +6,7 @@ "scripts": { "dev": "next dev", "build": "next build", + "build:analyze": "ANALYZE=true next build", "start": "next start", "lint": "next lint", "typecheck": "tsc --noEmit", @@ -36,6 +37,7 @@ "@heroicons/react": "^2.0.13", "@marsidev/react-turnstile": "^0.0.7", "@next-auth/prisma-adapter": "^1.0.5", + "@next/bundle-analyzer": "^13.1.6", "@next/font": "^13.1.0", "@prisma/client": "^4.7.1", "@tailwindcss/forms": "^0.5.3", From d145f6677ae8378b752bd15614f7b52edaccd4c9 Mon Sep 17 00:00:00 2001 From: sryu1 <95025816+sryu1@users.noreply.github.com> Date: Wed, 1 Feb 2023 23:00:23 +1000 Subject: [PATCH 04/23] Minor updates to prompting guide (#1050) --- docs/docs/guides/README.md | 2 +- docs/docs/guides/prompting.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/guides/README.md b/docs/docs/guides/README.md index 5276baa5..2ecfec29 100644 --- a/docs/docs/guides/README.md +++ b/docs/docs/guides/README.md @@ -1,3 +1,3 @@ # Guides -Useful guides. +Useful guides to using [Open-Assistant](https://open-assistant.io/). diff --git a/docs/docs/guides/prompting.md b/docs/docs/guides/prompting.md index 86cc0d27..032c680f 100644 --- a/docs/docs/guides/prompting.md +++ b/docs/docs/guides/prompting.md @@ -91,7 +91,7 @@ following query: **Agent**: "I apologize, I'm not sure what you're trying to say. Could you please rephrase it for me?" -**User**: "You dumb motherf\*&ker AI sh\*t?!" +**User**: "You dumb motherf\*&ker AI sh\*t!" **Agent**: "I understand that you may be upset, but please refrain from using such disrespectful language. I'm here to help you. Is there something specific From ff94e3a6e04e243bc12ca5863deb7a4f8a29cb40 Mon Sep 17 00:00:00 2001 From: notmd Date: Wed, 1 Feb 2023 20:30:04 +0700 Subject: [PATCH 05/23] support window --- website/package-lock.json | 28 ++++++++++++++++++++++++++++ website/package.json | 3 ++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/website/package-lock.json b/website/package-lock.json index b6d7343c..ba6002e6 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -74,6 +74,7 @@ "@types/react": "18.0.26", "@typescript-eslint/eslint-plugin": "^5.47.1", "babel-loader": "^8.3.0", + "cross-env": "^7.0.3", "cypress": "^12.2.0", "cypress-image-diff-js": "^1.23.0", "eslint-plugin-storybook": "^0.6.8", @@ -17174,6 +17175,24 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "devOptional": true }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -51522,6 +51541,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "devOptional": true }, + "cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.1" + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", diff --git a/website/package.json b/website/package.json index 4390c8fe..0ec21cc2 100644 --- a/website/package.json +++ b/website/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "next dev", "build": "next build", - "build:analyze": "ANALYZE=true next build", + "build:analyze": "cross-env ANALYZE=true next build", "start": "next start", "lint": "next lint", "typecheck": "tsc --noEmit", @@ -92,6 +92,7 @@ "@types/react": "18.0.26", "@typescript-eslint/eslint-plugin": "^5.47.1", "babel-loader": "^8.3.0", + "cross-env": "^7.0.3", "cypress": "^12.2.0", "cypress-image-diff-js": "^1.23.0", "eslint-plugin-storybook": "^0.6.8", From 91d30455957aadbabf3fc131fc9b460db20fbb01 Mon Sep 17 00:00:00 2001 From: Alan Jean Date: Wed, 1 Feb 2023 23:00:00 +0400 Subject: [PATCH 06/23] fix(website): fix display of task instruction Fixes the misuse of the i18n lib which caused the task instruction not to appear above the response text-area when an 'instruction' translation key exists for the current task. Fixes #1058 --- website/src/components/Tasks/CreateTask.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/src/components/Tasks/CreateTask.tsx b/website/src/components/Tasks/CreateTask.tsx index 04f840e4..8d406fd7 100644 --- a/website/src/components/Tasks/CreateTask.tsx +++ b/website/src/components/Tasks/CreateTask.tsx @@ -49,9 +49,9 @@ export const CreateTask = ({ <> - {!!i18n.exists(`task.${taskType.id}.instruction`) && ( + {!!i18n.exists(`tasks:${taskType.id}.instruction`) && ( - {t(getTypeSafei18nKey(`${taskType.id}.instruction`))} + {t(getTypeSafei18nKey(`tasks:${taskType.id}.instruction`))} )} Date: Wed, 1 Feb 2023 23:49:21 +0100 Subject: [PATCH 07/23] Add infty prompting + prompt-lottery (#1055) * first bits of prompt-lottery * add tree prompt lottery * minor text changes --- ...8ff8_add_won_prompt_lottery_date_to_mts.py | 27 +++++ backend/oasst_backend/config.py | 4 +- .../models/message_tree_state.py | 7 ++ backend/oasst_backend/tree_manager.py | 105 +++++++++++++++--- 4 files changed, 126 insertions(+), 17 deletions(-) create mode 100644 backend/alembic/versions/2023_02_01_1010-f60958968ff8_add_won_prompt_lottery_date_to_mts.py diff --git a/backend/alembic/versions/2023_02_01_1010-f60958968ff8_add_won_prompt_lottery_date_to_mts.py b/backend/alembic/versions/2023_02_01_1010-f60958968ff8_add_won_prompt_lottery_date_to_mts.py new file mode 100644 index 00000000..f82f28fd --- /dev/null +++ b/backend/alembic/versions/2023_02_01_1010-f60958968ff8_add_won_prompt_lottery_date_to_mts.py @@ -0,0 +1,27 @@ +"""add won_prompt_lottery_date to mts + +Revision ID: f60958968ff8 +Revises: 7b8f0011e0b0 +Create Date: 2023-02-01 10:10:38.301707 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f60958968ff8" +down_revision = "7b8f0011e0b0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("message_tree_state", sa.Column("won_prompt_lottery_date", sa.DateTime(timezone=True), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("message_tree_state", "won_prompt_lottery_date") + # ### end Alembic commands ### diff --git a/backend/oasst_backend/config.py b/backend/oasst_backend/config.py index 449543d5..17935021 100644 --- a/backend/oasst_backend/config.py +++ b/backend/oasst_backend/config.py @@ -13,7 +13,7 @@ class TreeManagerConfiguration(BaseModel): No new initial prompt tasks are handed out to users if this number is reached.""" - max_tree_depth: int = 6 + max_tree_depth: int = 3 """Maximum depth of message tree.""" max_children_count: int = 3 @@ -22,7 +22,7 @@ class TreeManagerConfiguration(BaseModel): num_prompter_replies: int = 1 """Number of prompter replies to collect per assistant reply.""" - goal_tree_size: int = 15 + goal_tree_size: int = 12 """Total number of messages to gather per tree.""" num_reviews_initial_prompt: int = 3 diff --git a/backend/oasst_backend/models/message_tree_state.py b/backend/oasst_backend/models/message_tree_state.py index a286d483..199f475b 100644 --- a/backend/oasst_backend/models/message_tree_state.py +++ b/backend/oasst_backend/models/message_tree_state.py @@ -1,4 +1,6 @@ +from datetime import datetime from enum import Enum +from typing import Optional from uuid import UUID import sqlalchemy as sa @@ -46,6 +48,9 @@ class State(str, Enum): BACKLOG_RANKING = "backlog_ranking" """Imported tree ready to be activated and ranked by users (currently inactive).""" + PROMPT_LOTTERY_WAITING = "prompt_lottery_waiting" + """Initial prompt has passed spam check, waiting to be drawn to grow.""" + VALID_STATES = ( State.INITIAL_PROMPT_REVIEW, @@ -63,6 +68,7 @@ TERMINAL_STATES = ( State.SCORING_FAILED, State.HALTED_BY_MODERATOR, State.BACKLOG_RANKING, + State.PROMPT_LOTTERY_WAITING, ) @@ -78,3 +84,4 @@ class MessageTreeState(SQLModel, table=True): state: str = Field(nullable=False, max_length=128, index=True) active: bool = Field(nullable=False, index=True) origin: str = Field(sa_column=sa.Column(sa.String(1024), nullable=True)) + won_prompt_lottery_date: Optional[datetime] = Field(sa_column=sa.Column(sa.DateTime(timezone=True), nullable=True)) diff --git a/backend/oasst_backend/tree_manager.py b/backend/oasst_backend/tree_manager.py index b1d50d35..709fc622 100644 --- a/backend/oasst_backend/tree_manager.py +++ b/backend/oasst_backend/tree_manager.py @@ -21,6 +21,7 @@ from oasst_backend.utils.hugging_face import HfClassificationModel, HfEmbeddingM from oasst_backend.utils.ranking import ranked_pairs from oasst_shared.exceptions.oasst_api_error import OasstError, OasstErrorCode from oasst_shared.schemas import protocol as protocol_schema +from oasst_shared.utils import utcnow from sqlmodel import Session, func, not_, or_, text, update @@ -153,7 +154,7 @@ class TreeManager: def _determine_task_availability_internal( self, - num_active_trees: int, + num_missing_prompts: int, extendible_parents: list[ExtendibleParentRow], prompts_need_review: list[Message], replies_need_review: list[Message], @@ -161,8 +162,7 @@ class TreeManager: ) -> dict[protocol_schema.TaskRequestType, int]: task_count_by_type: dict[protocol_schema.TaskRequestType, int] = {t: 0 for t in protocol_schema.TaskRequestType} - num_missing_prompts = max(0, self.cfg.max_active_trees - num_active_trees) - task_count_by_type[protocol_schema.TaskRequestType.initial_prompt] = num_missing_prompts + task_count_by_type[protocol_schema.TaskRequestType.initial_prompt] = max(1, num_missing_prompts) task_count_by_type[protocol_schema.TaskRequestType.prompter_reply] = len( list(filter(lambda x: x.parent_role == "assistant", extendible_parents)) @@ -194,6 +194,72 @@ class TreeManager: return task_count_by_type + def _prompt_lottery(self, lang: str) -> int: + MAX_RETRIES = 5 + retry = 0 + while True: + num_active_trees = self.query_num_active_trees(lang=lang, exclude_ranking=True) + num_missing_prompts = self.cfg.max_active_trees - num_active_trees + if num_missing_prompts <= 0: + return 0 + + # select among distinct users + authors_qry = ( + self.db.query(Message.user_id) + .select_from(MessageTreeState) + .join(Message, MessageTreeState.message_tree_id == Message.id) + .filter( + MessageTreeState.state == message_tree_state.State.PROMPT_LOTTERY_WAITING, + Message.lang == lang, + not_(Message.deleted), + Message.review_result, + ) + .distinct(Message.user_id) + ) + + author_ids = authors_qry.all() + if len(author_ids) == 0: + logger.info( + f"No prompts for prompt lottery available ({num_missing_prompts} trees missing for {lang=})." + ) + return num_missing_prompts + + # first select an authour + prompt_author_id: UUID = random.choice(author_ids)["user_id"] + logger.info(f"Selected random prompt author {prompt_author_id} among {len(author_ids)} candidates.") + + # select random prompt of author + qry = ( + self.db.query(MessageTreeState, Message) + .select_from(MessageTreeState) + .join(Message, MessageTreeState.message_tree_id == Message.id) + .filter( + MessageTreeState.state == message_tree_state.State.PROMPT_LOTTERY_WAITING, + Message.user_id == prompt_author_id, + Message.lang == lang, + not_(Message.deleted), + Message.review_result, + ) + .limit(100) + ) + + prompt_candidates = qry.all() + if len(prompt_candidates) == 0: + retry += 1 # not sure if this can happen with repeatable read isolation level, just in case we retry + if retry < MAX_RETRIES: + continue + else: + logger.warning("Max retries in prompt lottery reached.") + return num_missing_prompts + + winner_prompt = random.choice(prompt_candidates) + message: Message = winner_prompt.Message + logger.info(f"Prompt lottery winner: {message.id=}") + + mts: MessageTreeState = winner_prompt.MessageTreeState + self._enter_state(mts, message_tree_state.State.GROWING) + self.db.flush() + def determine_task_availability(self, lang: str) -> dict[protocol_schema.TaskRequestType, int]: self.pr.ensure_user_is_enabled() @@ -201,14 +267,14 @@ class TreeManager: lang = "en" logger.warning("Task availability request without lang tag received, assuming lang='en'.") - num_active_trees = self.query_num_active_trees(lang=lang, exclude_ranking=True) + num_missing_prompts = self._prompt_lottery(lang=lang) extendible_parents, _ = self.query_extendible_parents(lang=lang) prompts_need_review = self.query_prompts_need_review(lang=lang) replies_need_review = self.query_replies_need_review(lang=lang) incomplete_rankings = self.query_incomplete_rankings(lang=lang) return self._determine_task_availability_internal( - num_active_trees=num_active_trees, + num_missing_prompts=num_missing_prompts, extendible_parents=extendible_parents, prompts_need_review=prompts_need_review, replies_need_review=replies_need_review, @@ -238,7 +304,8 @@ class TreeManager: lang = "en" logger.warning("Task request without lang tag received, assuming 'en'.") - num_active_trees = self.query_num_active_trees(lang=lang, exclude_ranking=True) + num_missing_prompts = self._prompt_lottery(lang=lang) + prompts_need_review = self.query_prompts_need_review(lang=lang) replies_need_review = self.query_replies_need_review(lang=lang) extendible_parents, active_tree_sizes = self.query_extendible_parents(lang=lang) @@ -256,7 +323,7 @@ class TreeManager: num_ranking_tasks=len(incomplete_rankings), num_replies_need_review=len(replies_need_review), num_prompts_need_review=len(prompts_need_review), - num_missing_prompts=max(0, self.cfg.max_active_trees - num_active_trees), + num_missing_prompts=num_missing_prompts, num_missing_replies=num_missing_replies, ) @@ -268,7 +335,7 @@ class TreeManager: ) else: task_count_by_type = self._determine_task_availability_internal( - num_active_trees=num_active_trees, + num_missing_prompts=num_missing_prompts, extendible_parents=extendible_parents, prompts_need_review=prompts_need_review, replies_need_review=replies_need_review, @@ -611,7 +678,7 @@ class TreeManager: ) else: self.enter_low_grade_state(msg.message_tree_id) - self.check_condition_for_growing_state(msg.message_tree_id) + self.check_condition_for_prompt_lottery(msg.message_tree_id) elif msg.review_count >= self.cfg.num_reviews_reply: if not msg.review_result and acceptance_score > self.cfg.acceptance_threshold_reply: msg.review_result = True @@ -649,6 +716,8 @@ class TreeManager: if len(incomplete_rankings) < self.cfg.min_active_rankings_per_lang: self.activate_backlog_tree(lang=root_msg.lang) else: + if mts.state == message_tree_state.State.GROWING and mts.won_prompt_lottery_date is None: + mts.won_prompt_lottery_date = utcnow() logger.info(f"Tree entered '{mts.state}' state ({mts.message_tree_id=})") def enter_low_grade_state(self, message_tree_id: UUID) -> None: @@ -656,8 +725,8 @@ class TreeManager: mts = self.pr.fetch_tree_state(message_tree_id) self._enter_state(mts, message_tree_state.State.ABORTED_LOW_GRADE) - def check_condition_for_growing_state(self, message_tree_id: UUID) -> bool: - logger.debug(f"check_condition_for_growing_state({message_tree_id=})") + def check_condition_for_prompt_lottery(self, message_tree_id: UUID) -> bool: + logger.debug(f"check_condition_for_prompt_lottery({message_tree_id=})") mts = self.pr.fetch_tree_state(message_tree_id) if not mts.active or mts.state != message_tree_state.State.INITIAL_PROMPT_REVIEW: @@ -670,7 +739,7 @@ class TreeManager: logger.debug(f"False {initial_prompt.review_result=}") return False - self._enter_state(mts, message_tree_state.State.GROWING) + self._enter_state(mts, message_tree_state.State.PROMPT_LOTTERY_WAITING) return True def check_condition_for_ranking_state(self, message_tree_id: UUID) -> bool: @@ -1097,7 +1166,7 @@ LEFT JOIN message_reaction mr ON mr.task_id = t.id AND mr.payload_type = 'Rankin f"Checking state of {len(prompt_review_trees)} active message trees in 'initial_prompt_review' state." ) for t in prompt_review_trees: - self.check_condition_for_growing_state(t.message_tree_id) + self.check_condition_for_prompt_lottery(t.message_tree_id) growing_trees: list[MessageTreeState] = ( self.db.query(MessageTreeState) @@ -1288,11 +1357,17 @@ DELETE FROM message WHERE message_tree_id = :message_tree_id; logger.debug(f"purge_message_tree({message_tree_id=}) {r.rowcount} rows.") def _reactivate_tree(self, mts: MessageTreeState): - self._enter_state(mts, message_tree_state.State.INITIAL_PROMPT_REVIEW) + if mts.state == message_tree_state.State.PROMPT_LOTTERY_WAITING: + return + tree_id = mts.message_tree_id - if self.check_condition_for_growing_state(tree_id): + if mts.won_prompt_lottery_date is not None: + self._enter_state(mts, message_tree_state.State.GROWING) if self.check_condition_for_ranking_state(tree_id): self.check_condition_for_scoring_state(tree_id) + else: + self._enter_state(mts, message_tree_state.State.INITIAL_PROMPT_REVIEW) + self.check_condition_for_prompt_lottery(tree_id) @managed_tx_method(CommitMode.FLUSH) def purge_user_messages( From bbf038677cd9fe247db15d6758dd687ce9db1abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Wed, 1 Feb 2023 23:53:21 +0100 Subject: [PATCH 08/23] Add terms of service acceptance date to user table (#1046) * add tos_acceptance_date column to user * send 451 UNAVAILABLE_FOR_LEGAL_REASONS status * add create user REST endpoint * adapt text-frontend to ToS requirements * set DEBUG_IGNORE_TOS_ACCEPTANCE default to True (temporary change) * update down revision to f60958968ff8 --- ...f323d12_add_tos_acceptance_date_to_user.py | 34 +++++++++++++++++++ backend/main.py | 1 + .../oasst_backend/api/v1/frontend_users.py | 31 +++++++++++++++++ backend/oasst_backend/api/v1/users.py | 3 +- backend/oasst_backend/config.py | 3 ++ backend/oasst_backend/models/user.py | 4 +++ backend/oasst_backend/prompt_repository.py | 16 ++++++--- backend/oasst_backend/user_repository.py | 14 ++++++-- .../exceptions/oasst_api_error.py | 1 + oasst-shared/oasst_shared/schemas/protocol.py | 8 +++++ text-frontend/__main__.py | 10 ++++++ text-frontend/auto_main.py | 10 ++++++ 12 files changed, 127 insertions(+), 8 deletions(-) create mode 100644 backend/alembic/versions/2023_02_01_0022-55361f323d12_add_tos_acceptance_date_to_user.py diff --git a/backend/alembic/versions/2023_02_01_0022-55361f323d12_add_tos_acceptance_date_to_user.py b/backend/alembic/versions/2023_02_01_0022-55361f323d12_add_tos_acceptance_date_to_user.py new file mode 100644 index 00000000..bca17b4f --- /dev/null +++ b/backend/alembic/versions/2023_02_01_0022-55361f323d12_add_tos_acceptance_date_to_user.py @@ -0,0 +1,34 @@ +"""add tos_acceptance_date to user + +Revision ID: 55361f323d12 +Revises: 7b8f0011e0b0 +Create Date: 2023-02-01 00:22:08.280251 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "55361f323d12" +down_revision = "f60958968ff8" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("user", sa.Column("tos_acceptance_date", sa.DateTime(timezone=True), nullable=True)) + op.drop_column("user_stats", "streak_days") + op.drop_column("user_stats", "streak_last_day_date") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "user_stats", sa.Column("streak_last_day_date", postgresql.TIMESTAMP(), autoincrement=False, nullable=True) + ) + op.add_column("user_stats", sa.Column("streak_days", sa.INTEGER(), autoincrement=False, nullable=True)) + op.drop_column("user", "tos_acceptance_date") + # ### end Alembic commands ### diff --git a/backend/main.py b/backend/main.py index 07d0b45b..8e30b78e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -147,6 +147,7 @@ if settings.DEBUG_USE_SEED_DATA: ur = UserRepository(db=session, api_client=api_client) tr = TaskRepository(db=session, api_client=api_client, client_user=dummy_user, user_repository=ur) + ur.update_user(tr.user_id, enabled=True, show_on_leaderboard=False, tos_acceptance=True) pr = PromptRepository( db=session, api_client=api_client, client_user=dummy_user, user_repository=ur, task_repository=tr ) diff --git a/backend/oasst_backend/api/v1/frontend_users.py b/backend/oasst_backend/api/v1/frontend_users.py index a4ca6380..114a3a9c 100644 --- a/backend/oasst_backend/api/v1/frontend_users.py +++ b/backend/oasst_backend/api/v1/frontend_users.py @@ -59,6 +59,37 @@ def query_frontend_user( return user.to_protocol_frontend_user() +@router.post("/", response_model=protocol.FrontEndUser) +def create_frontend_user( + *, + create_user: protocol.CreateFrontendUserRequest, + api_client: ApiClient = Depends(deps.get_api_client), + db: Session = Depends(deps.get_db), +): + ur = UserRepository(db, api_client) + user = ur.lookup_client_user(create_user, create_missing=True) + + def changed(a, b) -> bool: + return a is not None and a != b + + # only call update_user if something changed + if ( + changed(create_user.enabled, user.enabled) + or changed(create_user.show_on_leaderboard, user.show_on_leaderboard) + or changed(create_user.notes, user.notes) + or (create_user.tos_acceptance and user.tos_acceptance_date is None) + ): + user = ur.update_user( + user.id, + enabled=create_user.enabled, + show_on_leaderboard=create_user.show_on_leaderboard, + tos_acceptance=create_user.tos_acceptance, + notes=create_user.notes, + ) + + return user.to_protocol_frontend_user() + + @router.get("/{auth_method}/{username}/messages", response_model=list[protocol.Message]) def query_frontend_user_messages( auth_method: str, diff --git a/backend/oasst_backend/api/v1/users.py b/backend/oasst_backend/api/v1/users.py index 2ced40c1..b3604c3f 100644 --- a/backend/oasst_backend/api/v1/users.py +++ b/backend/oasst_backend/api/v1/users.py @@ -191,6 +191,7 @@ def update_user( enabled: Optional[bool] = None, notes: Optional[str] = None, show_on_leaderboard: Optional[bool] = None, + tos_acceptance: Optional[bool] = None, db: Session = Depends(deps.get_db), api_client: ApiClient = Depends(deps.get_trusted_api_client), ): @@ -198,7 +199,7 @@ def update_user( Update a user by global user ID. Only trusted clients can update users. """ ur = UserRepository(db, api_client) - ur.update_user(user_id, enabled, notes, show_on_leaderboard) + ur.update_user(user_id, enabled, notes, show_on_leaderboard, tos_acceptance) @router.delete("/{user_id}", status_code=HTTP_204_NO_CONTENT) diff --git a/backend/oasst_backend/config.py b/backend/oasst_backend/config.py index 17935021..f40f5637 100644 --- a/backend/oasst_backend/config.py +++ b/backend/oasst_backend/config.py @@ -158,6 +158,9 @@ class Settings(BaseSettings): DEBUG_SKIP_EMBEDDING_COMPUTATION: bool = False DEBUG_SKIP_TOXICITY_CALCULATION: bool = False DEBUG_DATABASE_ECHO: bool = False + DEBUG_IGNORE_TOS_ACCEPTANCE: bool = ( # ignore whether users accepted the ToS + True # TODO: set False after ToS acceptance UI was added to web-frontend + ) DUPLICATE_MESSAGE_FILTER_WINDOW_MINUTES: int = 120 diff --git a/backend/oasst_backend/models/user.py b/backend/oasst_backend/models/user.py index 3d3bd6a9..6c73089c 100644 --- a/backend/oasst_backend/models/user.py +++ b/backend/oasst_backend/models/user.py @@ -41,6 +41,9 @@ class User(SQLModel, table=True): sa_column=sa.Column(sa.DateTime(timezone=True), nullable=True, server_default=sa.func.current_timestamp()) ) + # terms of service acceptance date + tos_acceptance_date: Optional[datetime] = Field(sa_column=sa.Column(sa.DateTime(timezone=True), nullable=True)) + def to_protocol_frontend_user(self): return protocol.FrontEndUser( user_id=self.id, @@ -55,4 +58,5 @@ class User(SQLModel, table=True): streak_days=self.streak_days, streak_last_day_date=self.streak_last_day_date, last_activity_date=self.last_activity_date, + tos_acceptance_date=self.tos_acceptance_date, ) diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py index e889e73b..5d46aa2d 100644 --- a/backend/oasst_backend/prompt_repository.py +++ b/backend/oasst_backend/prompt_repository.py @@ -35,7 +35,6 @@ from oasst_shared.utils import unaware_to_utc, utcnow from sqlalchemy.orm import Query from sqlalchemy.orm.attributes import flag_modified from sqlmodel import JSON, Session, and_, func, literal_column, not_, or_, text, update -from starlette.status import HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND class PromptRepository: @@ -77,7 +76,14 @@ class PromptRepository: raise OasstError("User required", OasstErrorCode.USER_NOT_SPECIFIED) if self.user.deleted or not self.user.enabled: - raise OasstError("User account disabled", OasstErrorCode.USER_DISABLED) + raise OasstError("User account disabled", OasstErrorCode.USER_DISABLED, HTTPStatus.SERVICE_UNAVAILABLE) + + if self.user.tos_acceptance_date is None and not settings.DEBUG_IGNORE_TOS_ACCEPTANCE: + raise OasstError( + "User has not accepted terms of service.", + OasstErrorCode.USER_HAS_NOT_ACCEPTED_TOS, + HTTPStatus.UNAVAILABLE_FOR_LEGAL_REASONS, + ) def fetch_message_by_frontend_message_id(self, frontend_message_id: str, fail_if_missing: bool = True) -> Message: validate_frontend_message_id(frontend_message_id) @@ -90,7 +96,7 @@ class PromptRepository: raise OasstError( f"Message with frontend_message_id {frontend_message_id} not found.", OasstErrorCode.MESSAGE_NOT_FOUND, - HTTP_404_NOT_FOUND, + HTTPStatus.NOT_FOUND, ) return message @@ -675,7 +681,7 @@ class PromptRepository: message = self.db.query(Message).filter(Message.id == message_id).one_or_none() if fail_if_missing and not message: - raise OasstError("Message not found", OasstErrorCode.MESSAGE_NOT_FOUND, HTTP_404_NOT_FOUND) + raise OasstError("Message not found", OasstErrorCode.MESSAGE_NOT_FOUND, HTTPStatus.NOT_FOUND) return message def fetch_non_task_text_labels(self, message_id: UUID, user_id: UUID) -> Optional[TextLabels]: @@ -874,7 +880,7 @@ class PromptRepository: if api_client_id != self.api_client.id: # Unprivileged api client asks for foreign messages - raise OasstError("Forbidden", OasstErrorCode.API_CLIENT_NOT_AUTHORIZED, HTTP_403_FORBIDDEN) + raise OasstError("Forbidden", OasstErrorCode.API_CLIENT_NOT_AUTHORIZED, HTTPStatus.FORBIDDEN) qry = self.db.query(Message) if user_id: diff --git a/backend/oasst_backend/user_repository.py b/backend/oasst_backend/user_repository.py index 984964b6..ba6d1a10 100644 --- a/backend/oasst_backend/user_repository.py +++ b/backend/oasst_backend/user_repository.py @@ -73,7 +73,8 @@ class UserRepository: enabled: Optional[bool] = None, notes: Optional[str] = None, show_on_leaderboard: Optional[bool] = None, - ) -> None: + tos_acceptance: Optional[bool] = None, + ) -> User: """ Update a user by global user ID to disable or set admin notes. Only trusted clients may update users. @@ -94,8 +95,11 @@ class UserRepository: user.notes = notes if show_on_leaderboard is not None: user.show_on_leaderboard = show_on_leaderboard + if tos_acceptance: + user.tos_acceptance_date = utcnow() self.db.add(user) + return user @managed_tx_method(CommitMode.COMMIT) def mark_user_deleted(self, id: UUID) -> None: @@ -143,8 +147,10 @@ class UserRepository: display_name=display_name, api_client_id=self.api_client.id, auth_method=auth_method, - show_on_leaderboard=(auth_method != "system"), # don't show system users, e.g. import user ) + if auth_method == "system": + user.show_on_leaderboard = False # don't show system users, e.g. import user + user.tos_acceptance_date = utcnow() self.db.add(user) elif display_name and display_name != user.display_name: # we found the user but the display name changed @@ -156,6 +162,10 @@ class UserRepository: def lookup_client_user(self, client_user: protocol_schema.User, create_missing: bool = True) -> User | None: if not client_user: return None + + if not (client_user.auth_method and client_user.id): + raise OasstError("Auth method or username missing.", OasstErrorCode.AUTH_AND_USERNAME_REQUIRED) + num_retries = settings.DATABASE_MAX_TX_RETRY_COUNT for i in range(num_retries): try: diff --git a/oasst-shared/oasst_shared/exceptions/oasst_api_error.py b/oasst-shared/oasst_shared/exceptions/oasst_api_error.py index 2c3650a6..a8682a32 100644 --- a/oasst-shared/oasst_shared/exceptions/oasst_api_error.py +++ b/oasst-shared/oasst_shared/exceptions/oasst_api_error.py @@ -80,6 +80,7 @@ class OasstErrorCode(IntEnum): USER_NOT_SPECIFIED = 4000 USER_DISABLED = 4001 USER_NOT_FOUND = 4002 + USER_HAS_NOT_ACCEPTED_TOS = 4003 EMOJI_OP_UNSUPPORTED = 5000 diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py index 8929251c..1139286c 100644 --- a/oasst-shared/oasst_shared/schemas/protocol.py +++ b/oasst-shared/oasst_shared/schemas/protocol.py @@ -39,6 +39,7 @@ class FrontEndUser(User): streak_days: Optional[int] = None streak_last_day_date: Optional[datetime] = None last_activity_date: Optional[datetime] = None + tos_acceptance_date: Optional[datetime] = None class PageResult(BaseModel): @@ -499,3 +500,10 @@ class MessageEmojiRequest(BaseModel): user: User op: EmojiOp = EmojiOp.togggle emoji: EmojiCode + + +class CreateFrontendUserRequest(User): + show_on_leaderboard: bool = True + enabled: bool = True + tos_acceptance: Optional[bool] = None + notes: Optional[str] = None diff --git a/text-frontend/__main__.py b/text-frontend/__main__.py index 18c1f124..b3f4d925 100644 --- a/text-frontend/__main__.py +++ b/text-frontend/__main__.py @@ -28,6 +28,16 @@ def _render_message(message: dict) -> str: def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "1234"): """Simple REPL frontend.""" + # make sure dummy user has accepted the terms of service + create_user_request = dict(USER) + create_user_request["tos_acceptance"] = True + response = requests.post( + f"{backend_url}/api/v1/frontend_users/", json=create_user_request, headers={"X-API-Key": api_key} + ) + response.raise_for_status() + user = response.json() + typer.echo(f"user: {user}") + def _post(path: str, json: dict) -> dict: response = requests.post(f"{backend_url}{path}", json=json, headers={"X-API-Key": api_key}) response.raise_for_status() diff --git a/text-frontend/auto_main.py b/text-frontend/auto_main.py index 485ee1cb..2775d98c 100644 --- a/text-frontend/auto_main.py +++ b/text-frontend/auto_main.py @@ -29,6 +29,16 @@ def _render_message(message: dict) -> str: def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "1234"): """automates tasks""" + # make sure dummy user has accepted the terms of service + create_user_request = dict(USER) + create_user_request["tos_acceptance"] = True + response = requests.post( + f"{backend_url}/api/v1/frontend_users/", json=create_user_request, headers={"X-API-Key": api_key} + ) + response.raise_for_status() + user = response.json() + typer.echo(f"user: {user}") + def _post(path: str, json: dict) -> dict: response = requests.post(f"{backend_url}{path}", json=json, headers={"X-API-Key": api_key}) response.raise_for_status() From 779f522385b90be60212ba57fa1c58bacb0fe101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Thu, 2 Feb 2023 00:07:08 +0100 Subject: [PATCH 09/23] Skip tasks permanently based on special skip emoji codes (#1064) * add skip emojis * extend tree_manager queries to exclude tasks based on skip-emojis * remove unrelated columns * change skip_reason max_length to 512 * update down_revision to 55361f323d12 --- ...9e3f2_add_skip_bool_skip_reason_to_task.py | 30 ++++++++++ backend/oasst_backend/api/v1/tasks.py | 2 +- backend/oasst_backend/models/task.py | 2 + backend/oasst_backend/prompt_repository.py | 41 +++++++++++++- backend/oasst_backend/task_repository.py | 15 ----- backend/oasst_backend/tree_manager.py | 55 ++++++++++++++++--- oasst-shared/oasst_shared/schemas/protocol.py | 5 ++ 7 files changed, 126 insertions(+), 24 deletions(-) create mode 100644 backend/alembic/versions/2023_02_01_2146-9e7ec4a9e3f2_add_skip_bool_skip_reason_to_task.py diff --git a/backend/alembic/versions/2023_02_01_2146-9e7ec4a9e3f2_add_skip_bool_skip_reason_to_task.py b/backend/alembic/versions/2023_02_01_2146-9e7ec4a9e3f2_add_skip_bool_skip_reason_to_task.py new file mode 100644 index 00000000..92c36adc --- /dev/null +++ b/backend/alembic/versions/2023_02_01_2146-9e7ec4a9e3f2_add_skip_bool_skip_reason_to_task.py @@ -0,0 +1,30 @@ +"""add skip bool & skip_reason to task + +Revision ID: 9e7ec4a9e3f2 +Revises: 7b8f0011e0b0 +Create Date: 2023-02-01 21:46:49.971052 + +""" +import sqlalchemy as sa +import sqlmodel +from alembic import op + +# revision identifiers, used by Alembic. +revision = "9e7ec4a9e3f2" +down_revision = "55361f323d12" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("task", sa.Column("skipped", sa.Boolean(), server_default=sa.text("false"), nullable=False)) + op.add_column("task", sa.Column("skip_reason", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("task", "skip_reason") + op.drop_column("task", "skipped") + # ### end Alembic commands ### diff --git a/backend/oasst_backend/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py index 7db42662..c817ba67 100644 --- a/backend/oasst_backend/api/v1/tasks.py +++ b/backend/oasst_backend/api/v1/tasks.py @@ -131,7 +131,7 @@ def tasks_acknowledge_failure( logger.info(f"Frontend reports failure to implement task {task_id=}, {nack_request=}.") api_client = deps.api_auth(api_key, db) pr = PromptRepository(db, api_client, frontend_user=frontend_user) - pr.task_repository.acknowledge_task_failure(task_id) + pr.skip_task(task_id=task_id, reason=nack_request.reason) except (KeyError, RuntimeError): logger.exception("Failed to not acknowledge task.") raise OasstError("Failed to not acknowledge task.", OasstErrorCode.TASK_NACK_FAILED) diff --git a/backend/oasst_backend/models/task.py b/backend/oasst_backend/models/task.py index 7f91b157..ad41c78a 100644 --- a/backend/oasst_backend/models/task.py +++ b/backend/oasst_backend/models/task.py @@ -31,6 +31,8 @@ class Task(SQLModel, table=True): api_client_id: UUID = Field(nullable=False, foreign_key="api_client.id") ack: Optional[bool] = None done: bool = Field(sa_column=sa.Column(sa.Boolean, nullable=False, server_default=false())) + skipped: bool = Field(sa_column=sa.Column(sa.Boolean, nullable=False, server_default=false())) + skip_reason: str = Field(nullable=True, max_length=512) frontend_message_id: Optional[str] = None message_tree_id: Optional[UUID] = None parent_message_id: Optional[UUID] = None diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py index 5d46aa2d..b805a4d7 100644 --- a/backend/oasst_backend/prompt_repository.py +++ b/backend/oasst_backend/prompt_repository.py @@ -36,6 +36,21 @@ from sqlalchemy.orm import Query from sqlalchemy.orm.attributes import flag_modified from sqlmodel import JSON, Session, and_, func, literal_column, not_, or_, text, update +_task_type_and_reaction = ( + ( + (db_payload.PrompterReplyPayload, db_payload.AssistantReplyPayload), + protocol_schema.EmojiCode.skip_reply, + ), + ( + (db_payload.LabelInitialPromptPayload, db_payload.LabelConversationReplyPayload), + protocol_schema.EmojiCode.skip_labeling, + ), + ( + (db_payload.RankInitialPromptsPayload, db_payload.RankConversationRepliesPayload), + protocol_schema.EmojiCode.skip_ranking, + ), +) + class PromptRepository: def __init__( @@ -1001,7 +1016,31 @@ WHERE message.id = cc.id; message_trees=result.get(None, 0), ) - def handle_message_emoji(self, message_id: UUID, op: protocol_schema.EmojiOp, emoji: protocol_schema) -> Message: + @managed_tx_method() + def skip_task(self, task_id: UUID, reason: str): + self.ensure_user_is_enabled() + + task = self.task_repository.fetch_task_by_id(task_id) + self._validate_task(task) + + if not task.collective: + task.skipped = True + task.skip_reason = reason + self.db.add(task) + + def handle_cancel_emoji(task_payload: db_payload.TaskPayload) -> Message | None: + for types, emoji in _task_type_and_reaction: + for t in types: + if isinstance(task_payload, t): + return self.handle_message_emoji(task.parent_message_id, protocol_schema.EmojiOp.add, emoji) + return None + + task_payload: db_payload.TaskPayload = task.payload.payload + handle_cancel_emoji(task_payload) + + def handle_message_emoji( + self, message_id: UUID, op: protocol_schema.EmojiOp, emoji: protocol_schema.EmojiCode + ) -> Message: self.ensure_user_is_enabled() message = self.fetch_message(message_id) diff --git a/backend/oasst_backend/task_repository.py b/backend/oasst_backend/task_repository.py index db4ba576..b721caf5 100644 --- a/backend/oasst_backend/task_repository.py +++ b/backend/oasst_backend/task_repository.py @@ -167,21 +167,6 @@ class TaskRepository: task.done = True self.db.add(task) - @managed_tx_method(CommitMode.COMMIT) - def acknowledge_task_failure(self, task_id): - # find task - task: Task = self.db.query(Task).filter(Task.id == task_id, Task.api_client_id == self.api_client.id).first() - if task is None: - raise OasstError(f"Task for {task_id=} not found", OasstErrorCode.TASK_NOT_FOUND, HTTP_404_NOT_FOUND) - if task.expired: - raise OasstError("Task already expired.", OasstErrorCode.TASK_EXPIRED) - if task.done or task.ack is not None: - raise OasstError("Task already updated.", OasstErrorCode.TASK_ALREADY_UPDATED) - - task.ack = False - # ToDo: check race-condition, transaction - self.db.add(task) - @managed_tx_method(CommitMode.COMMIT) def insert_task( self, diff --git a/backend/oasst_backend/tree_manager.py b/backend/oasst_backend/tree_manager.py index 709fc622..672c9081 100644 --- a/backend/oasst_backend/tree_manager.py +++ b/backend/oasst_backend/tree_manager.py @@ -13,7 +13,16 @@ from fastapi.encoders import jsonable_encoder from loguru import logger from oasst_backend.api.v1.utils import prepare_conversation, prepare_conversation_message_list from oasst_backend.config import TreeManagerConfiguration, settings -from oasst_backend.models import Message, MessageReaction, MessageTreeState, Task, TextLabels, User, message_tree_state +from oasst_backend.models import ( + Message, + MessageEmoji, + MessageReaction, + MessageTreeState, + Task, + TextLabels, + User, + message_tree_state, +) from oasst_backend.prompt_repository import PromptRepository from oasst_backend.utils import tree_export from oasst_backend.utils.database_utils import CommitMode, async_managed_tx_method, managed_tx_method @@ -22,7 +31,7 @@ from oasst_backend.utils.ranking import ranked_pairs from oasst_shared.exceptions.oasst_api_error import OasstError, OasstErrorCode from oasst_shared.schemas import protocol as protocol_schema from oasst_shared.utils import utcnow -from sqlmodel import Session, func, not_, or_, text, update +from sqlmodel import Session, and_, func, not_, or_, text, update class TaskType(Enum): @@ -889,6 +898,14 @@ class TreeManager: self.db.query(Message) .select_from(MessageTreeState) .join(Message, MessageTreeState.message_tree_id == Message.message_tree_id) + .outerjoin( + MessageEmoji, + and_( + Message.id == MessageEmoji.message_id, + MessageEmoji.user_id == self.pr.user_id, + MessageEmoji.emoji == protocol_schema.EmojiCode.skip_labeling, + ), + ) .filter( MessageTreeState.active, MessageTreeState.state == state, @@ -896,6 +913,7 @@ class TreeManager: not_(Message.deleted), Message.review_count < required_reviews, Message.lang == lang, + MessageEmoji.message_id.is_(None), ) ) @@ -948,12 +966,18 @@ SELECT m.parent_id, m.role, COUNT(m.id) children_count, MIN(m.ranking_count) chi mts.message_tree_id FROM message_tree_state mts INNER JOIN message m ON mts.message_tree_id = m.message_tree_id + LEFT JOIN message_emoji me on + (m.id = me.message_id + AND :skip_user_id IS NOT NULL + AND me.user_id = :skip_user_id + AND me.emoji = :skip_ranking) WHERE mts.active -- only consider active trees AND mts.state = :ranking_state -- message tree must be in ranking state AND m.review_result -- must be reviewed AND m.lang = :lang -- matches lang AND NOT m.deleted -- not deleted AND m.parent_id IS NOT NULL -- ignore initial prompts + AND me.message_id IS NULL -- no skip ranking emoji for user GROUP BY m.parent_id, m.role, mts.message_tree_id HAVING COUNT(m.id) > 1 and MIN(m.ranking_count) < :num_required_rankings """ @@ -979,6 +1003,8 @@ HAVING(COUNT(mr.message_id) FILTER (WHERE mr.user_id = :user_id) = 0) "ranking_state": message_tree_state.State.RANKING, "lang": lang, "user_id": user_id, + "skip_user_id": self.pr.user_id, + "skip_ranking": protocol_schema.EmojiCode.skip_ranking, }, ) return [IncompleteRankingsRow.from_orm(x) for x in r.all()] @@ -988,6 +1014,11 @@ HAVING(COUNT(mr.message_id) FILTER (WHERE mr.user_id = :user_id) = 0) SELECT m.id as parent_id, m.role as parent_role, m.depth, m.message_tree_id, COUNT(c.id) active_children_count FROM message_tree_state mts INNER JOIN message m ON mts.message_tree_id = m.message_tree_id -- all elements of message tree + LEFT JOIN message_emoji me ON + (m.id = me.message_id + AND :skip_user_id IS NOT NULL + AND me.user_id = :skip_user_id + AND me.emoji = :skip_reply) LEFT JOIN message c ON m.id = c.parent_id -- child nodes WHERE mts.active -- only consider active trees AND mts.state = :growing_state -- message tree must be growing @@ -995,6 +1026,7 @@ WHERE mts.active -- only consider active trees AND m.depth < mts.max_depth -- ignore leaf nodes as parents AND m.review_result -- parent node must have positive review AND m.lang = :lang -- parent matches lang + AND me.message_id IS NULL -- no skip reply emoji for user AND NOT coalesce(c.deleted, FALSE) -- don't count deleted children AND (c.review_result OR coalesce(c.review_count, 0) < :num_reviews_reply) -- don't count children with negative review but count elements under review GROUP BY m.id, m.role, m.depth, m.message_tree_id, mts.max_children_count @@ -1015,6 +1047,8 @@ HAVING COUNT(c.id) < mts.max_children_count -- below maximum number of children "num_prompter_replies": self.cfg.num_prompter_replies, "lang": lang, "user_id": user_id, + "skip_user_id": self.pr.user_id, + "skip_reply": protocol_schema.EmojiCode.skip_reply, }, ) @@ -1053,6 +1087,8 @@ HAVING COUNT(m.id) < mts.goal_tree_size "num_prompter_replies": self.cfg.num_prompter_replies, "lang": lang, "user_id": user_id, + "skip_user_id": self.pr.user_id, + "skip_reply": protocol_schema.EmojiCode.skip_reply, }, ) return [ActiveTreeSizeRow.from_orm(x) for x in r.all()] @@ -1525,7 +1561,8 @@ if __name__ == "__main__": with Session(engine) as db: api_client = api_auth(settings.OFFICIAL_WEB_API_KEY, db=db) # api_client = create_api_client(session=db, description="test", frontend_type="bot") - dummy_user = protocol_schema.User(id="__dummy_user__", display_name="Dummy User", auth_method="local") + # dummy_user = protocol_schema.User(id="__dummy_user__", display_name="Dummy User", auth_method="local") + dummy_user = protocol_schema.User(id="1234", display_name="bulb", auth_method="local") pr = PromptRepository(db=db, api_client=api_client, client_user=dummy_user) cfg = TreeManagerConfiguration() @@ -1540,14 +1577,18 @@ if __name__ == "__main__": # print("query_incomplete_rankings", tm.query_incomplete_rankings()) # print("query_replies_need_review", tm.query_replies_need_review()) # print("query_incomplete_reply_reviews", tm.query_replies_need_review()) - # print("query_incomplete_initial_prompt_reviews", tm.query_prompts_need_review()) + xs = tm.query_prompts_need_review(lang="en") + print("xs", len(xs)) + for x in xs: + print(x.id, x.emojis) + # print("query_incomplete_initial_prompt_reviews", tm.query_prompts_need_review(lang="en")) # print("query_extendible_trees", tm.query_extendible_trees()) # print("query_extendible_parents", tm.query_extendible_parents()) # print("next_task:", tm.next_task()) - print( - ".query_tree_ranking_results", tm.query_tree_ranking_results(UUID("21f9d585-d22c-44ab-a696-baa3d83b5f1b")) - ) + # print( + # ".query_tree_ranking_results", tm.query_tree_ranking_results(UUID("21f9d585-d22c-44ab-a696-baa3d83b5f1b")) + # ) # print(tm.export_trees_to_file(message_tree_ids=["7e75fb38-e664-4e2b-817c-b9a0b01b0074"], file="lol.jsonl")) diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py index 1139286c..6bd95dc5 100644 --- a/oasst-shared/oasst_shared/schemas/protocol.py +++ b/oasst-shared/oasst_shared/schemas/protocol.py @@ -489,6 +489,11 @@ class EmojiCode(str, enum.Enum): poop = "poop" # 💩 skull = "skull" # 💀 + # skip task system uses special emoji codes + skip_reply = "skip_reply" + skip_ranking = "skip_ranking" + skip_labeling = "skip_labeling" + class EmojiOp(str, enum.Enum): togggle = "toggle" From d33fb9ce88b89ef72b2882fa0cd55372ed1caa42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Thu, 2 Feb 2023 00:15:49 +0100 Subject: [PATCH 10/23] fix alembic downgrade script of 7b8f0011e0b0 --- ...1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/alembic/versions/2023_01_29_1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py b/backend/alembic/versions/2023_01_29_1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py index 9cf3a233..9117ad52 100644 --- a/backend/alembic/versions/2023_01_29_1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py +++ b/backend/alembic/versions/2023_01_29_1207-7b8f0011e0b0_move_user_streak_from_user_stats_to_.py @@ -36,6 +36,7 @@ def upgrade() -> None: def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_column("user_stats", "streak_days") - op.drop_column("user_stats", "streak_last_day_date") + op.drop_column("user", "streak_days") + op.drop_column("user", "streak_last_day_date") + op.drop_column("user", "last_activity_date") # ### end Alembic commands ### From f4b534eb1751778d0be03d4b6ea9e34448d705b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Thu, 2 Feb 2023 00:27:27 +0100 Subject: [PATCH 11/23] no task ack check for skip --- backend/oasst_backend/prompt_repository.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py index b805a4d7..dacd5f9b 100644 --- a/backend/oasst_backend/prompt_repository.py +++ b/backend/oasst_backend/prompt_repository.py @@ -160,7 +160,12 @@ class PromptRepository: return message def _validate_task( - self, task: Task, *, task_id: Optional[UUID] = None, frontend_message_id: Optional[str] = None + self, + task: Task, + *, + task_id: Optional[UUID] = None, + frontend_message_id: Optional[str] = None, + check_ack: bool = True, ) -> Task: if task is None: if task_id: @@ -171,7 +176,7 @@ class PromptRepository: if task.expired: raise OasstError("Task already expired.", OasstErrorCode.TASK_EXPIRED) - if not task.ack: + if check_ack and not task.ack: raise OasstError("Task is not acknowledged.", OasstErrorCode.TASK_NOT_ACK) if task.done: raise OasstError("Task already done.", OasstErrorCode.TASK_ALREADY_DONE) @@ -1021,7 +1026,7 @@ WHERE message.id = cc.id; self.ensure_user_is_enabled() task = self.task_repository.fetch_task_by_id(task_id) - self._validate_task(task) + self._validate_task(task, check_ack=False) if not task.collective: task.skipped = True From 5f7013fb81f15b7f5e869a63bb7b822c10d272e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Thu, 2 Feb 2023 00:51:20 +0100 Subject: [PATCH 12/23] add '_' prefix to special emoji codes, fix skip rankings --- backend/oasst_backend/tree_manager.py | 2 +- oasst-shared/oasst_shared/schemas/protocol.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/oasst_backend/tree_manager.py b/backend/oasst_backend/tree_manager.py index 672c9081..e09f22d1 100644 --- a/backend/oasst_backend/tree_manager.py +++ b/backend/oasst_backend/tree_manager.py @@ -967,7 +967,7 @@ SELECT m.parent_id, m.role, COUNT(m.id) children_count, MIN(m.ranking_count) chi FROM message_tree_state mts INNER JOIN message m ON mts.message_tree_id = m.message_tree_id LEFT JOIN message_emoji me on - (m.id = me.message_id + (m.parent_id = me.message_id AND :skip_user_id IS NOT NULL AND me.user_id = :skip_user_id AND me.emoji = :skip_ranking) diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py index 6bd95dc5..3570d3fd 100644 --- a/oasst-shared/oasst_shared/schemas/protocol.py +++ b/oasst-shared/oasst_shared/schemas/protocol.py @@ -490,9 +490,9 @@ class EmojiCode(str, enum.Enum): skull = "skull" # 💀 # skip task system uses special emoji codes - skip_reply = "skip_reply" - skip_ranking = "skip_ranking" - skip_labeling = "skip_labeling" + skip_reply = "_skip_reply" + skip_ranking = "_skip_ranking" + skip_labeling = "_skip_labeling" class EmojiOp(str, enum.Enum): From aa39019e1c8e4675ae115b94c58ef7c580740c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Thu, 2 Feb 2023 00:23:33 +0000 Subject: [PATCH 13/23] hide system emojis starting with '_' --- .../components/Messages/MessageTableEntry.tsx | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/website/src/components/Messages/MessageTableEntry.tsx b/website/src/components/Messages/MessageTableEntry.tsx index 89b0b204..74f523bb 100644 --- a/website/src/components/Messages/MessageTableEntry.tsx +++ b/website/src/components/Messages/MessageTableEntry.tsx @@ -97,14 +97,16 @@ export function MessageTableEntry({ message, enabled, highlight }: MessageTableE style={{ float: "right", position: "relative", right: "-0.3em", bottom: "-0em", marginLeft: "1em" }} onClick={(e) => e.stopPropagation()} > - {Object.entries(emojiState.emojis).map(([emoji, count]) => ( - react(emoji, !emojiState.user_emojis.includes(emoji))} - /> - ))} + {Object.entries(emojiState.emojis) + .filter(([k, _]) => !k.startsWith("_")) + .map(([emoji, count]) => ( + react(emoji, !emojiState.user_emojis.includes(emoji))} + /> + ))} Date: Thu, 2 Feb 2023 09:52:39 +0300 Subject: [PATCH 14/23] Add new datasets contribution guide (#1061) * Add readme * change link in CONTRIBUTING --- CONTRIBUTING.md | 2 +- openassistant/datasets/README.md | 108 +++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 openassistant/datasets/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 39773b41..c0d77f69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -140,4 +140,4 @@ automatically deploy the built release to the dev machine. ### Contribute a Dataset See -[here](https://github.com/LAION-AI/Open-Assistant/blob/main/docs/docs/data/datasets.md) +[here](https://github.com/LAION-AI/Open-Assistant/blob/main/openassistant/datasets/README.md) diff --git a/openassistant/datasets/README.md b/openassistant/datasets/README.md new file mode 100644 index 00000000..4d523f1f --- /dev/null +++ b/openassistant/datasets/README.md @@ -0,0 +1,108 @@ +# **Datasets** + +This folder contains datasets loading scripts that are used to train +OpenAssistant. The current list of datasets can be found +[here](https://docs.google.com/spreadsheets/d/1NYYa6vHiRnk5kwnyYaCT0cBO62--Tm3w4ihdBtp4ISk). + +## **Adding a New Dataset** + +To add a new dataset to OpenAssistant, follow these steps: + +1. **Create an issue**: Create a new + [issue](https://github.com/LAION-AI/Open-Assistant/issues/new) and describe + your proposal for the new dataset. + +2. **Create a dataset on HuggingFace**: Create a dataset on + [HuggingFace](https://huggingface.co). See + [below](#creating-a-dataset-on-huggingface) for more details. + +3. **Make a pull request**: Add a new dataset loading script to this folder and + link the issue in the pull request description. For more information, see + [below](#making-a-pull-request). + +## **Creating a Dataset on HuggingFace** + +To create a new dataset on HuggingFace, follow these steps: + +#### 1. Convert your dataset file(s) to the Parquet format using the [pandas](https://pandas.pydata.org/) library: + +```python +import pandas as pd + +# Create a pandas dataframe from your dataset file(s) +df = pd.read_json(...) # or any other way + +# Save the file in the Parquet format +df.to_parquet("dataset.parquet", row_group_size=100, engine="pyarrow") +``` + +#### 2. Install HuggingFace CLI + +```bash +pip install huggingface-cli +``` + +#### 3. Log in to HuggingFace + +Use your [access token](https://huggingface.co/docs/hub/security-tokens) to +login: + +- Via terminal + +```bash +huggingface-cli login +``` + +- in Jupyter notebook + +```python +from huggingface_hub import notebook_login +notebook_login() +``` + +#### 4. Push the Parquet file to HuggingFace using the following code: + +```python +from datasets import Dataset +ds = Dataset.from_parquet("dataset.parquet") +ds.push_to_hub("your_huggingface_name/dataset_name") +``` + +#### 5. Update the `README.md` file + +Update the `README.md` file of your dataset by visiting this link: +https://huggingface.co/datasets/your_huggingface_name/dataset_name/edit/main/README.md +(paste your HuggingFace name and dataset) + +## **Making a Pull Request** + +#### 1. Fork this repository + +#### 2. Create a new branch in your fork + +#### 3. Add your dataset to the repository + +- Create a folder with the name of your dataset. +- Add a loading script that loads your dataset from HuggingFace, for example: + + ```python + from datasets import load_dataset + + if __name__ == "__main__": + ds = load_dataset("your_huggingface_name/dataset_name") + print(ds["train"]) + ``` + +- Optionally, add any other files that describe your dataset and its creation, + such as a README, notebooks, scrapers, etc. + +#### 4. Stage your changes and run the pre-commit hook + +```bash +pre-commit run +``` + +#### 5. Submit a pull request + +- Submit a pull request and include a link to the issue it resolves in the + description, for example: `Resolves #123` From 9fa22be66ee3eff0126b8f5de9ce0f5c04955794 Mon Sep 17 00:00:00 2001 From: notmd Date: Thu, 2 Feb 2023 16:07:45 +0700 Subject: [PATCH 15/23] fix side menu regression --- website/src/components/SideMenu.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/website/src/components/SideMenu.tsx b/website/src/components/SideMenu.tsx index 25dabf30..ee7dba4d 100644 --- a/website/src/components/SideMenu.tsx +++ b/website/src/components/SideMenu.tsx @@ -21,8 +21,9 @@ export function SideMenu(props: SideMenuProps) { return (