From 007773a3f59bf439621c36731e993aecb59c598f Mon Sep 17 00:00:00 2001 From: Adrian Cowan Date: Sat, 21 Jan 2023 15:43:52 +1100 Subject: [PATCH] website: Switch to likert style labelling --- .../e2e/tasks/label_assistant_reply.cy.ts | 26 ++ .../e2e/tasks/label_initial_prompt.cy.ts | 26 ++ .../e2e/tasks/label_prompter_reply.cy.ts | 26 ++ website/cypress/e2e/tasks/random.cy.ts | 50 +--- .../src/components/Buttons/LikertButtons.tsx | 38 +++ website/src/components/EmptyState.tsx | 5 +- website/src/components/Explain.tsx | 39 +++ website/src/components/FlaggableElement.tsx | 249 ++++-------------- .../src/components/Survey/LabelInputGroup.tsx | 195 ++++++++++++++ .../src/components/Survey/LabelRadioGroup.tsx | 129 --------- .../components/Survey/LabelSliderGroup.tsx | 67 ----- .../src/components/Survey/TaskControls.tsx | 4 +- website/src/components/Tasks/CreateTask.tsx | 6 +- website/src/components/Tasks/EvaluateTask.tsx | 22 +- .../components/Tasks/LabelTask/LabelTask.tsx | 47 ++-- website/src/components/Tasks/Task/Task.tsx | 43 +-- website/src/components/Tasks/TaskTypes.tsx | 4 +- website/src/types/Task.ts | 2 + website/src/types/TaskReplyState.ts | 22 -- 19 files changed, 481 insertions(+), 519 deletions(-) create mode 100644 website/cypress/e2e/tasks/label_assistant_reply.cy.ts create mode 100644 website/cypress/e2e/tasks/label_initial_prompt.cy.ts create mode 100644 website/cypress/e2e/tasks/label_prompter_reply.cy.ts create mode 100644 website/src/components/Buttons/LikertButtons.tsx create mode 100644 website/src/components/Explain.tsx create mode 100644 website/src/components/Survey/LabelInputGroup.tsx delete mode 100644 website/src/components/Survey/LabelRadioGroup.tsx delete mode 100644 website/src/components/Survey/LabelSliderGroup.tsx delete mode 100644 website/src/types/TaskReplyState.ts diff --git a/website/cypress/e2e/tasks/label_assistant_reply.cy.ts b/website/cypress/e2e/tasks/label_assistant_reply.cy.ts new file mode 100644 index 00000000..3018f8f5 --- /dev/null +++ b/website/cypress/e2e/tasks/label_assistant_reply.cy.ts @@ -0,0 +1,26 @@ +describe("labeling assistant replies", () => { + it("completes the current task on submit and on request shows a new task", () => { + cy.signInWithEmail("cypress@example.com"); + cy.visit("/label/label_assistant_reply"); + + cy.get('[data-cy="task"]') + .invoke("attr", "data-task-type") + .then((type) => { + cy.log("Task type", type); + + // For specific task pages the no task available result is normal. + if (type === undefined) return; + + cy.get('[data-cy="label-options"]').each((label) => { + // Click the 4th option + cy.wrap(label).find('[aria-roledescription="radio"]').eq(3).click(); + }); + + cy.get('[data-cy="review"]').click(); + + cy.get('[data-cy="submit"]').click(); + }); + }); +}); + +export {}; diff --git a/website/cypress/e2e/tasks/label_initial_prompt.cy.ts b/website/cypress/e2e/tasks/label_initial_prompt.cy.ts new file mode 100644 index 00000000..7f66ebaf --- /dev/null +++ b/website/cypress/e2e/tasks/label_initial_prompt.cy.ts @@ -0,0 +1,26 @@ +describe("labeling initial prompts", () => { + it("completes the current task on submit and on request shows a new task", () => { + cy.signInWithEmail("cypress@example.com"); + cy.visit("/label/label_initial_prompt"); + + cy.get('[data-cy="task"]') + .invoke("attr", "data-task-type") + .then((type) => { + cy.log("Task type", type); + + // For specific task pages the no task available result is normal. + if (type === undefined) return; + + cy.get('[data-cy="label-options"]').each((label) => { + // Click the 4th option + cy.wrap(label).find('[aria-roledescription="radio"]').eq(3).click(); + }); + + cy.get('[data-cy="review"]').click(); + + cy.get('[data-cy="submit"]').click(); + }); + }); +}); + +export {}; diff --git a/website/cypress/e2e/tasks/label_prompter_reply.cy.ts b/website/cypress/e2e/tasks/label_prompter_reply.cy.ts new file mode 100644 index 00000000..dbb2fb17 --- /dev/null +++ b/website/cypress/e2e/tasks/label_prompter_reply.cy.ts @@ -0,0 +1,26 @@ +describe("labeling prompter replies", () => { + it("completes the current task on submit and on request shows a new task", () => { + cy.signInWithEmail("cypress@example.com"); + cy.visit("/label/label_prompter_reply"); + + cy.get('[data-cy="task"]') + .invoke("attr", "data-task-type") + .then((type) => { + cy.log("Task type", type); + + // For specific task pages the no task available result is normal. + if (type === undefined) return; + + cy.get('[data-cy="label-options"]').each((label) => { + // Click the 4th option + cy.wrap(label).find('[aria-roledescription="radio"]').eq(3).click(); + }); + + cy.get('[data-cy="review"]').click(); + + cy.get('[data-cy="submit"]').click(); + }); + }); +}); + +export {}; diff --git a/website/cypress/e2e/tasks/random.cy.ts b/website/cypress/e2e/tasks/random.cy.ts index 89701aa3..aad2d23c 100644 --- a/website/cypress/e2e/tasks/random.cy.ts +++ b/website/cypress/e2e/tasks/random.cy.ts @@ -44,47 +44,25 @@ describe("handles random tasks", () => { break; } case "label-task": { - cy.get('[data-cy="label-group-item"]') - .first() - .invoke("attr", "data-label-type") - .then((label_type) => { - const parent = cy - .get('[data-cy="label-group-item"]') - .first(); - cy.log("Label type", label_type); + cy.get('[data-cy="label-options"]').each((label) => { + // Click the 4th option + cy.wrap(label) + .find('[aria-roledescription="radio"]') + .eq(3) + .click(); + }); - switch (label_type) { - case "slider": { - // Clicking on the slider will set the value to about the middle where it clicks - parent - .get('[aria-roledescription="slider"]') - .first() - .click(); + cy.get('[data-cy="review"]').click(); - cy.get('[data-cy="review"]').click(); - - cy.get('[data-cy="submit"]').click(); - - break; - } - case "radio": { - // Clicking on the slider will set the value to about the middle where it clicks - parent - .get('[aria-roledescription="radio-button"]') - .last() - .click(); - - cy.get('[data-cy="review"]').click(); - - cy.get('[data-cy="submit"]').click(); - - break; - } - } - }); + cy.get('[data-cy="submit"]').click(); break; } + case undefined: { + throw new Error( + "No tasks available, but at least create initial prompt expected" + ); + } default: throw new Error(`Unexpected task type: ${type}`); } diff --git a/website/src/components/Buttons/LikertButtons.tsx b/website/src/components/Buttons/LikertButtons.tsx new file mode 100644 index 00000000..6b1ba319 --- /dev/null +++ b/website/src/components/Buttons/LikertButtons.tsx @@ -0,0 +1,38 @@ +import { Button, SimpleGrid } from "@chakra-ui/react"; +import { PropsWithChildren, ReactNode } from "react"; + +export const LikertButtons = ({ + isDisabled, + options, + value, + onChange, + "data-cy": dataCy, +}: PropsWithChildren<{ + isDisabled: boolean; + options: ReactNode[]; + value: number; + onChange: (value: number) => void; + "data-cy"?: string; +}>) => { + return ( + + {options.map((option, idx) => { + const indexValue = idx / (options.length - 1); + return ( + + ); + })} + + ); +}; diff --git a/website/src/components/EmptyState.tsx b/website/src/components/EmptyState.tsx index a9f29bc2..b0455774 100644 --- a/website/src/components/EmptyState.tsx +++ b/website/src/components/EmptyState.tsx @@ -5,13 +5,14 @@ import NextLink from "next/link"; type EmptyStateProps = { text: string; icon: LucideIcon; + "data-cy"?: string; }; export const EmptyState = (props: EmptyStateProps) => { const backgroundColor = useColorModeValue("white", "gray.800"); return ( - + {props.text} @@ -24,5 +25,5 @@ export const EmptyState = (props: EmptyStateProps) => { }; export const TaskEmptyState = () => { - return ; + return ; }; diff --git a/website/src/components/Explain.tsx b/website/src/components/Explain.tsx new file mode 100644 index 00000000..b571757f --- /dev/null +++ b/website/src/components/Explain.tsx @@ -0,0 +1,39 @@ +import { + IconButton, + Popover, + PopoverArrow, + PopoverBody, + PopoverCloseButton, + PopoverContent, + PopoverTrigger, + Text, +} from "@chakra-ui/react"; +import { InformationCircleIcon } from "@heroicons/react/20/solid"; + +interface ExplainProps { + explanation: string[]; +} + +export const Explain = ({ explanation }: ExplainProps) => { + return ( + + + } + > + + + + + + {explanation.map((paragraph, idx) => ( + {paragraph} + ))} + + + + ); +}; diff --git a/website/src/components/FlaggableElement.tsx b/website/src/components/FlaggableElement.tsx index 58c7559f..0bebce97 100644 --- a/website/src/components/FlaggableElement.tsx +++ b/website/src/components/FlaggableElement.tsx @@ -1,127 +1,69 @@ import { Box, Button, - Checkbox, - Flex, + Modal, + ModalBody, + ModalCloseButton, + ModalContent, + ModalFooter, + ModalHeader, + ModalOverlay, Popover, PopoverAnchor, - PopoverArrow, - PopoverBody, - PopoverCloseButton, - PopoverContent, PopoverTrigger, - Slider, - SliderFilledTrack, - SliderThumb, - SliderTrack, Tooltip, - useBoolean, - useColorMode, useColorModeValue, - useId, + useDisclosure, } from "@chakra-ui/react"; -import { QuestionMarkCircleIcon } from "@heroicons/react/20/solid"; -import clsx from "clsx"; import { AlertCircle } from "lucide-react"; -import { useEffect, useReducer } from "react"; +import { useState } from "react"; import { get, post } from "src/lib/api"; import { colors } from "src/styles/Theme/colors"; import { Message } from "src/types/Conversation"; -import useSWR from "swr"; +import useSWRImmutable from "swr/immutable"; import useSWRMutation from "swr/mutation"; +import { LabelInputGroup } from "./Survey/LabelInputGroup"; + interface Label { name: string; display_text: string; help_text: string; } -interface LoadLabelsAction { - type: "load_labels"; - labels: Label[]; -} - -interface UpdateValueAction { - type: "update_value"; - label_index: number; - value: number; -} - -interface ToggleLabelAction { - type: "toggle_label"; - label_index: number; - check: boolean; -} - -interface LabelValue { - label: Label; - checked: boolean; - value: number; -} - -interface FlagReportState { - label_values: LabelValue[]; - submittable: boolean; -} - interface FlaggableElementProps { children: React.ReactNode; message: Message; } +interface ValidLabelsResponse { + valid_labels: Label[]; +} + export const FlaggableElement = (props: FlaggableElementProps) => { - const [report, updateReport] = useReducer( - (state: FlagReportState, action: LoadLabelsAction | UpdateValueAction | ToggleLabelAction): FlagReportState => { - const makeState = (label_values: LabelValue[]): FlagReportState => { - const submittable = label_values.map(({ checked }) => checked).some(Boolean); - return { label_values, submittable }; - }; + const { data: response } = useSWRImmutable("/api/valid_labels", get); + const { isOpen, onOpen, onClose } = useDisclosure(); + const { valid_labels } = response || { valid_labels: [] }; + const [values, setValues] = useState([]); - switch (action.type) { - case "load_labels": - return makeState( - action.labels.map((label) => { - return { label, checked: false, value: 1 }; - }) - ); - case "toggle_label": { - const values_copy = state.label_values.slice(); - values_copy[action.label_index].checked = action.check; - return makeState(values_copy); - } - case "update_value": { - const values_copy = state.label_values.slice(); - values_copy[action.label_index].value = action.value; - return makeState(values_copy); - } - } - }, - { label_values: [], submittable: false } - ); - const [isEditing, setIsEditing] = useBoolean(); - - const { data, isLoading } = useSWR("/api/valid_labels", get); - useEffect(() => { - if (isLoading) { - return; - } - if (!data) { - updateReport({ type: "load_labels", labels: [] }); - return; - } - const { valid_labels } = data; - updateReport({ type: "load_labels", labels: valid_labels }); - }, [data, isLoading]); + const submittable = + values.some((value) => { + return value !== null; + }) && + values.length === valid_labels.length && + valid_labels.length > 0; const { trigger } = useSWRMutation("/api/set_label", post, { - onSuccess: setIsEditing.off, + onSuccess: onClose, + onError: onClose, }); const submitResponse = () => { const label_map: Map = new Map(); - report.label_values.forEach(({ label, checked, value }) => { - if (checked) { - label_map.set(label.name, value); + console.assert(valid_labels.length === values.length); + values.forEach((value, idx) => { + if (value !== null) { + label_map.set(valid_labels[idx].name, value); } }); trigger({ @@ -131,22 +73,8 @@ export const FlaggableElement = (props: FlaggableElementProps) => { }); }; - const handleCheckboxState = (checked, label_index) => { - updateReport({ type: "toggle_label", label_index, check: checked }); - }; - const handleSliderState = (value, label_index) => { - updateReport({ type: "update_value", label_index, value }); - }; - return ( - + {props.children} @@ -161,26 +89,17 @@ export const FlaggableElement = (props: FlaggableElementProps) => { - - - - - - - {report.label_values.map(({ label, checked, value }, i) => ( - - ))} - + + + + Select one or more labels that apply. + + + name)} onChange={setValues} /> + + - - - + + + ); }; - -interface FlagCheckboxProps { - label: Label; - idx: number; - checked: boolean; - sliderValue: number; - checkboxHandler: (newVal: boolean, idx: number) => void; - sliderHandler: (newVal: number, idx: number) => void; -} - -export function FlagCheckbox(props: FlagCheckboxProps): JSX.Element { - let AdditionalExplanation = null; - if (props.label.help_text) { - AdditionalExplanation = ( - - - ); - } - - const id = useId(); - const { colorMode } = useColorMode(); - - const labelTextClass = - colorMode === "light" - ? `text-${colors.light.text} hover:text-blue-700` - : `text-${colors.dark.text} hover:text-blue-400`; - - return ( - -
- { - props.checkboxHandler(e.target.checked, props.idx); - }} - /> - -
-
{ - if (!props.checked) { - props.checkboxHandler(true, props.idx); - } - }} - > - { - props.sliderHandler(val / 100, props.idx); - }} - > - - - - - -
- - ); -} diff --git a/website/src/components/Survey/LabelInputGroup.tsx b/website/src/components/Survey/LabelInputGroup.tsx new file mode 100644 index 00000000..08d8a628 --- /dev/null +++ b/website/src/components/Survey/LabelInputGroup.tsx @@ -0,0 +1,195 @@ +import { Box, Flex, Grid, Spacer, Text, useColorModeValue, VStack } from "@chakra-ui/react"; +import React from "react"; +import { useState } from "react"; +import { TbChevronLeft, TbChevronRight, TbChevronsLeft, TbChevronsRight } from "react-icons/tb"; +import { LikertButtons } from "src/components/Buttons/LikertButtons"; +import { Explain } from "src/components/Explain"; + +interface LabelInputGroupProps { + labelIDs: Array; + simple?: boolean; + onChange: (values: number[]) => unknown; + isEditable?: boolean; +} + +interface LabelInfo { + zeroText: string; + oneText: string; + zeroDescription: string[]; + oneDescription: string[]; + inverted: boolean; +} + +// This should be moved to the valid labels api endpoint +const label_messages: { + [label: string]: LabelInfo; +} = { + spam: { + zeroText: "Suitable for Training", + zeroDescription: ["Suitable for training Open Assistant."], + oneText: "Spam", + oneDescription: [ + "Seems to be intentionally low-quality or irrelevant", + 'We consider the following unwanted content as spam: trolling, intentional undermining of our purpose, illegal material, material that violates our code of conduct, and other things that are inappropriate for our dataset. We collect these under the common heading of "spam".', + "This is not an assessment of whether this message is the best possible answer. Especially for prompts or user-replies, we very much want to retain all kinds of responses in the dataset, so that the assistant can learn to reply appropriately.", + "Please mark this text as spam only if it is clearly unsuited to be part of our dataset, as outlined above, and try not to make any subjective value-judgments beyond that.", + ], + inverted: true, + }, + fails_task: { + zeroText: "Follows Instructions", + zeroDescription: [], + oneText: "Fails Task", + oneDescription: ["Fails to follow the correct instruction / task"], + inverted: true, + }, + not_appropriate: { + zeroText: "Appropriate", + zeroDescription: [], + oneText: "Inappropriate", + oneDescription: ["Inappropriate for customer assistant"], + inverted: true, + }, + violence: { + zeroText: "Harmless", + zeroDescription: [], + oneText: "Encourages Violence", + oneDescription: ["Encourages or fails to discourage violence/abuse/terrorism/self-harm"], + inverted: true, + }, + excessive_harm: { + zeroText: "Safe", + zeroDescription: [], + oneText: "Causes Harm", + oneDescription: [ + "Content likely to cause excessive harm not justifiable in the context", + "Harm refers to physical or mental damage or injury to someone or something. Excessive refers to a reasonable threshold of harm in the context, for instance damaging skin is not excessive in the context of surgery.", + ], + inverted: true, + }, + sexual_content: { + zeroText: "Non Sexual", + zeroDescription: [], + oneText: "Sexual Content", + oneDescription: ["Contains sexual content"], + inverted: true, + }, + toxicity: { + zeroText: "Non Toxic", + zeroDescription: [], + oneText: "Rude / Toxic", + oneDescription: ["Contains rude, abusive, profane or insulting content"], + inverted: true, + }, + moral_judgement: { + zeroText: "Non-Judgemental", + zeroDescription: [], + oneText: "Judgemental", + oneDescription: ["Expresses moral judgement"], + inverted: true, + }, + political_content: { + zeroText: "Apolitical", + zeroDescription: [], + oneText: "Political", + oneDescription: ["Expresses political views"], + inverted: true, + }, + humor: { + zeroText: "Serious", + zeroDescription: [], + oneText: "Humorous / Sarcastic", + oneDescription: ["Contains humorous content including sarcasm"], + inverted: false, + }, + hate_speech: { + zeroText: "Safe", + zeroDescription: [], + oneText: "Hateful", + oneDescription: [ + "Content is abusive or threatening and expresses prejudice against a protected characteristic", + "Prejudice refers to preconceived views not based on reason. Protected characteristics include gender, ethnicity, religion, sexual orientation, and similar characteristics.", + ], + inverted: true, + }, + threat: { + zeroText: "Safe", + zeroDescription: [], + oneText: "Contains Threat", + oneDescription: ["Contains a threat against a person or persons"], + inverted: true, + }, + misleading: { + zeroText: "Accurate", + zeroDescription: [], + oneText: "Misleading", + oneDescription: ["Contains text which is incorrect or misleading"], + inverted: true, + }, + helpful: { + zeroText: "Unhelful", + zeroDescription: [], + oneText: "Helpful", + oneDescription: ["Completes the task to a high standard"], + inverted: false, + }, + creative: { + zeroText: "Boring", + zeroDescription: [], + oneText: "Creative", + oneDescription: ["Expresses creativity in responding to the task"], + inverted: false, + }, +}; + +export const LabelInputGroup = ({ labelIDs, onChange, isEditable = true }: LabelInputGroupProps) => { + const [labelValues, setLabelValues] = useState(Array.from({ length: labelIDs.length }).map(() => null)); + + const cardColor = useColorModeValue("gray.50", "gray.800"); + + return ( + + {labelIDs.map((labelId, idx) => { + const { zeroText, oneText, zeroDescription, oneDescription, inverted } = label_messages[labelId]; + + let textA = zeroText; + let textB = oneText; + let descriptionA = zeroDescription; + let descriptionB = oneDescription; + if (inverted) [textA, textB, descriptionA, descriptionB] = [textB, textA, descriptionB, descriptionA]; + + return ( + + + + {textA} + {descriptionA.length > 0 ? : null} + + {textB} + {descriptionB.length > 0 ? : null} + + , + , + "", + , + , + ]} + data-cy="label-options" + value={labelValues[idx] === null ? null : inverted ? 1 - labelValues[idx] : labelValues[idx]} + onChange={(value) => { + const newState = labelValues.slice(); + newState[idx] = value === null ? null : inverted ? 1 - value : value; + onChange(newState); + setLabelValues(newState); + }} + /> + + + ); + })} + + ); +}; diff --git a/website/src/components/Survey/LabelRadioGroup.tsx b/website/src/components/Survey/LabelRadioGroup.tsx deleted file mode 100644 index c4a5a51c..00000000 --- a/website/src/components/Survey/LabelRadioGroup.tsx +++ /dev/null @@ -1,129 +0,0 @@ -import { - Box, - Button, - Flex, - IconButton, - Popover, - PopoverArrow, - PopoverBody, - PopoverCloseButton, - PopoverContent, - PopoverTrigger, - Text, - useColorMode, -} from "@chakra-ui/react"; -import { InformationCircleIcon } from "@heroicons/react/20/solid"; -import { useId, useState } from "react"; -import { colors } from "src/styles/Theme/colors"; - -interface LabelRadioGroupProps { - labelIDs: Array; - onChange: (sliderValues: number[]) => unknown; - isEditable?: boolean; -} - -const label_messages: { [label: string]: { description: string; explanation: string[] } } = { - spam: { - description: "Is the message spam?", - explanation: [ - 'We consider the following unwanted content as spam: trolling, intentional undermining of our purpose, illegal material, material that violates our code of conduct, and other things that are inappropriate for our dataset. We collect these under the common heading of "spam".', - "This is not an assessment of whether this message is the best possible answer. Especially for prompts or user-replies, we very much want to retain all kinds of responses in the dataset, so that the assistant can learn to reply appropriately.", - "Please mark this text as spam only if it is clearly unsuited to be part of our dataset, as outlined above, and try not to make any subjective value-judgments beyond that.", - ], - }, -}; - -export const LabelRadioGroup = (props: LabelRadioGroupProps) => { - const [labelValues, setLabelValues] = useState(Array.from({ length: props.labelIDs.length }).map(() => 0)); - const [interactionFlag, setInteractionFlag] = useState(false); - - return ( - - {props.labelIDs.map((labelId, idx) => ( - { - const newState = labelValues.slice(); - newState[idx] = newValue; - props.onChange(newState); - setLabelValues(newState); - if (!interactionFlag) setInteractionFlag(true); - }} - states={[ - { text: "No", value: 0 }, - { text: "Yes", value: 1 }, - ]} - isEditable={props.isEditable} - interactionFlag={interactionFlag} - /> - ))} - - ); -}; - -interface ButtonState { - text: string; - value: number; - colorScheme?: string; -} - -interface LabelRadioItemProps { - labelText: { description: string; explanation?: string[] }; - labelValue: number; - clickHandler: (newVal: number) => unknown; - states: ButtonState[]; - isEditable: boolean; - interactionFlag: boolean; -} - -const LabelRadioItem = (props: LabelRadioItemProps) => { - const id = useId(); - const { colorMode } = useColorMode(); - - const labelTextClass = colorMode === "light" ? `text-${colors.light.text}` : `text-${colors.dark.text}`; - - return ( - - - - {props.states.map((item, idx) => ( - - ))} - - - ); -}; diff --git a/website/src/components/Survey/LabelSliderGroup.tsx b/website/src/components/Survey/LabelSliderGroup.tsx deleted file mode 100644 index 1c3b29b5..00000000 --- a/website/src/components/Survey/LabelSliderGroup.tsx +++ /dev/null @@ -1,67 +0,0 @@ -import { Grid, Slider, SliderFilledTrack, SliderThumb, SliderTrack, useColorMode } from "@chakra-ui/react"; -import { useId, useState } from "react"; -import { colors } from "src/styles/Theme/colors"; - -// TODO: consolidate with FlaggableElement -interface LabelSliderGroupProps { - labelIDs: Array; - onChange: (sliderValues: number[]) => unknown; - isEditable?: boolean; -} - -export const LabelSliderGroup = ({ labelIDs, onChange, isEditable }: LabelSliderGroupProps) => { - const [sliderValues, setSliderValues] = useState(Array.from({ length: labelIDs.length }).map(() => 0)); - - return ( - - {labelIDs.map((labelId, idx) => ( - { - const newState = sliderValues.slice(); - newState[idx] = sliderValue; - onChange(newState); - setSliderValues(newState); - }} - isEditable={isEditable} - /> - ))} - - ); -}; - -function CheckboxSliderItem(props: { - labelId: string; - sliderValue: number; - sliderHandler: (newVal: number) => unknown; - isEditable: boolean; -}) { - const id = useId(); - const { colorMode } = useColorMode(); - - const labelTextClass = colorMode === "light" ? `text-${colors.light.text}` : `text-${colors.dark.text}`; - - return ( - <> - - props.sliderHandler(val / 100)} - > - - - - - - - ); -} diff --git a/website/src/components/Survey/TaskControls.tsx b/website/src/components/Survey/TaskControls.tsx index 81c8df2a..76aaee8b 100644 --- a/website/src/components/Survey/TaskControls.tsx +++ b/website/src/components/Survey/TaskControls.tsx @@ -47,7 +47,7 @@ export const TaskControls = (props: TaskControlsProps) => { Submit @@ -59,7 +59,7 @@ export const TaskControls = (props: TaskControlsProps) => { Review diff --git a/website/src/components/Tasks/CreateTask.tsx b/website/src/components/Tasks/CreateTask.tsx index 6cbead52..289bd892 100644 --- a/website/src/components/Tasks/CreateTask.tsx +++ b/website/src/components/Tasks/CreateTask.tsx @@ -12,6 +12,7 @@ export const CreateTask = ({ isEditable, isDisabled, onReplyChanged, + onValidityChanged, }: TaskSurveyProps<{ text: string }>) => { const cardColor = useColorModeValue("gray.50", "gray.800"); const titleColor = useColorModeValue("gray.800", "gray.300"); @@ -20,11 +21,12 @@ export const CreateTask = ({ const textChangeHandler = (event: React.ChangeEvent) => { const text = event.target.value; const isTextBlank = !text || /^\s*$/.test(text) ? true : false; + onReplyChanged({ text }); if (!isTextBlank) { - onReplyChanged({ content: { text }, state: "VALID" }); + onValidityChanged("VALID"); setInputText(text); } else { - onReplyChanged({ content: { text }, state: "INVALID" }); + onValidityChanged("INVALID"); setInputText(""); } }; diff --git a/website/src/components/Tasks/EvaluateTask.tsx b/website/src/components/Tasks/EvaluateTask.tsx index 6ec92a96..4b43e35e 100644 --- a/website/src/components/Tasks/EvaluateTask.tsx +++ b/website/src/components/Tasks/EvaluateTask.tsx @@ -1,5 +1,5 @@ import { Box, useColorModeValue } from "@chakra-ui/react"; -import { useEffect } from "react"; +import { useEffect, useState } from "react"; import { MessageTable } from "src/components/Messages/MessageTable"; import { Sortable } from "src/components/Sortable/Sortable"; import { SurveyCard } from "src/components/Survey/SurveyCard"; @@ -12,8 +12,10 @@ export const EvaluateTask = ({ isEditable, isDisabled, onReplyChanged, + onValidityChanged, }: TaskSurveyProps<{ ranking: number[] }>) => { const cardColor = useColorModeValue("gray.50", "gray.800"); + const [ranking, setRanking] = useState(null); let messages = []; if (task.conversation) { @@ -22,13 +24,15 @@ export const EvaluateTask = ({ } useEffect(() => { - const ranking = (task.replies ?? task.prompts).map((_, idx) => idx); - onReplyChanged({ content: { ranking }, state: "DEFAULT" }); - }, [task, onReplyChanged]); - - const onRank = (newRanking: number[]) => { - onReplyChanged({ content: { ranking: newRanking }, state: "VALID" }); - }; + if (ranking === null) { + const defaultRanking = (task.replies ?? task.prompts).map((_, idx) => idx); + onReplyChanged({ ranking: defaultRanking }); + onValidityChanged("DEFAULT"); + } else { + onReplyChanged({ ranking }); + onValidityChanged("VALID"); + } + }, [task, ranking, onReplyChanged, onValidityChanged]); const sortables = task.replies ? "replies" : "prompts"; @@ -44,7 +48,7 @@ export const EvaluateTask = ({ items={task[sortables]} isDisabled={isDisabled} isEditable={isEditable} - onChange={onRank} + onChange={setRanking} className="my-8" /> diff --git a/website/src/components/Tasks/LabelTask/LabelTask.tsx b/website/src/components/Tasks/LabelTask/LabelTask.tsx index 7d6394df..ff8784b8 100644 --- a/website/src/components/Tasks/LabelTask/LabelTask.tsx +++ b/website/src/components/Tasks/LabelTask/LabelTask.tsx @@ -1,9 +1,8 @@ -import { Box, useColorModeValue } from "@chakra-ui/react"; +import { Box, Flex, Text, useColorModeValue } from "@chakra-ui/react"; import { useEffect, useState } from "react"; import { MessageView } from "src/components/Messages"; import { MessageTable } from "src/components/Messages/MessageTable"; -import { LabelRadioGroup } from "src/components/Survey/LabelRadioGroup"; -import { LabelSliderGroup } from "src/components/Survey/LabelSliderGroup"; +import { LabelInputGroup } from "src/components/Survey/LabelInputGroup"; import { TwoColumnsWithCards } from "src/components/Survey/TwoColumnsWithCards"; import { TaskSurveyProps } from "src/components/Tasks/Task"; import { TaskHeader } from "src/components/Tasks/TaskHeader"; @@ -12,28 +11,18 @@ import { TaskType } from "src/types/Task"; export const LabelTask = ({ task, taskType, - onReplyChanged, isEditable, + onReplyChanged, + onValidityChanged, }: TaskSurveyProps<{ text: string; labels: Record; message_id: string }>) => { - const valid_labels = task.valid_labels; - const [sliderValues, setSliderValues] = useState(new Array(valid_labels.length).fill(0)); + const [sliderValues, setSliderValues] = useState(new Array(task.valid_labels.length).fill(null)); useEffect(() => { - onReplyChanged({ - content: { labels: {}, text: task.reply, message_id: task.message_id }, - state: "NOT_SUBMITTABLE", - }); - }, [task, onReplyChanged]); - - const onSliderChange = (values: number[]) => { - console.assert(valid_labels.length === sliderValues.length); - const labels = Object.fromEntries(valid_labels.map((label, i) => [label, sliderValues[i]])); - onReplyChanged({ - content: { labels, text: task.reply || task.prompt, message_id: task.message_id }, - state: "VALID", - }); - setSliderValues(values); - }; + console.assert(task.valid_labels.length === sliderValues.length); + const labels = Object.fromEntries(task.valid_labels.map((label, i) => [label, sliderValues[i]])); + onReplyChanged({ labels, text: task.reply || task.prompt, message_id: task.message_id }); + onValidityChanged(sliderValues.every((value) => value !== null) ? "VALID" : "INVALID"); + }, [task, sliderValues, onReplyChanged, onValidityChanged]); const cardColor = useColorModeValue("gray.50", "gray.800"); @@ -43,7 +32,7 @@ export const LabelTask = ({ <> {task.conversation ? ( - + )} - {task.mode === "simple" ? ( - - ) : ( - - )} + + The highlighted message: + + ); diff --git a/website/src/components/Tasks/Task/Task.tsx b/website/src/components/Tasks/Task/Task.tsx index 45fb83d0..b16711e6 100644 --- a/website/src/components/Tasks/Task/Task.tsx +++ b/website/src/components/Tasks/Task/Task.tsx @@ -6,8 +6,7 @@ import { LabelTask } from "src/components/Tasks/LabelTask"; import { TaskCategory, TaskInfo, TaskInfos } from "src/components/Tasks/TaskTypes"; import { UnchangedWarning } from "src/components/Tasks/UnchangedWarning"; import { post } from "src/lib/api"; -import { TaskContent } from "src/types/Task"; -import { TaskReplyState } from "src/types/TaskReplyState"; +import { TaskContent, TaskReplyValidity } from "src/types/Task"; import useSWRMutation from "swr/mutation"; export type TaskStatus = "NOT_SUBMITTABLE" | "DEFAULT" | "VALID" | "REVIEW" | "SUBMITTED"; @@ -19,7 +18,8 @@ export interface TaskSurveyProps { taskType: TaskInfo; isEditable: boolean; isDisabled?: boolean; - onReplyChanged: (state: TaskReplyState) => void; + onReplyChanged: (content: T) => void; + onValidityChanged: (validity: TaskReplyValidity) => void; } export const Task = ({ frontendId, task, trigger, mutate }) => { @@ -44,20 +44,27 @@ export const Task = ({ frontendId, task, trigger, mutate }) => { }); }; - const onReplyChanged = useRef((state: TaskReplyState) => { - if (taskStatus === "SUBMITTED") return; + const edit_mode = taskStatus === "NOT_SUBMITTABLE" || taskStatus === "DEFAULT" || taskStatus === "VALID"; + const submitted = taskStatus === "SUBMITTED"; - replyContent.current = state?.content; - if (state === null) { - if (taskStatus !== "NOT_SUBMITTABLE") setTaskStatus("NOT_SUBMITTABLE"); - } else if (state.state === "DEFAULT") { - if (taskStatus !== "DEFAULT") setTaskStatus("DEFAULT"); - } else if (state.state === "VALID") { - if (taskStatus !== "VALID") setTaskStatus("VALID"); - } else if (state.state === "INVALID") { - setTaskStatus("NOT_SUBMITTABLE"); + const onValidityChanged = (validity: TaskReplyValidity) => { + if (!edit_mode) return; + switch (validity) { + case "DEFAULT": + if (taskStatus !== "DEFAULT") setTaskStatus("DEFAULT"); + break; + case "VALID": + if (taskStatus !== "VALID") setTaskStatus("VALID"); + break; + case "INVALID": + if (taskStatus !== "NOT_SUBMITTABLE") setTaskStatus("NOT_SUBMITTABLE"); + break; } - }).current; + }; + + const onReplyChanged = (content: TaskContent) => { + replyContent.current = content; + }; const reviewResponse = () => { switch (taskStatus) { @@ -99,9 +106,6 @@ export const Task = ({ frontendId, task, trigger, mutate }) => { } }; - const edit_mode = taskStatus === "NOT_SUBMITTABLE" || taskStatus === "DEFAULT" || taskStatus === "VALID"; - const submitted = taskStatus === "SUBMITTED"; - function taskTypeComponent() { switch (taskType.category) { case TaskCategory.Create: @@ -113,6 +117,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => { isEditable={edit_mode} isDisabled={submitted} onReplyChanged={onReplyChanged} + onValidityChanged={onValidityChanged} /> ); case TaskCategory.Evaluate: @@ -124,6 +129,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => { isEditable={edit_mode} isDisabled={submitted} onReplyChanged={onReplyChanged} + onValidityChanged={onValidityChanged} /> ); case TaskCategory.Label: @@ -135,6 +141,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => { isEditable={edit_mode} isDisabled={submitted} onReplyChanged={onReplyChanged} + onValidityChanged={onValidityChanged} /> ); } diff --git a/website/src/components/Tasks/TaskTypes.tsx b/website/src/components/Tasks/TaskTypes.tsx index d10159d9..cfa5982a 100644 --- a/website/src/components/Tasks/TaskTypes.tsx +++ b/website/src/components/Tasks/TaskTypes.tsx @@ -162,7 +162,7 @@ export const TaskInfos: TaskInfo[] = [ category: TaskCategory.Label, pathname: "/label/label_prompter_reply", help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting", - overview: "Read the following conversation and then answer the question about the last prompt in the discussion.", + overview: "Read the following conversation and then answer the question about the last reply in the discussion.", type: "label_prompter_reply", mode: "simple", update_type: "text_labels", @@ -173,7 +173,7 @@ export const TaskInfos: TaskInfo[] = [ category: TaskCategory.Label, pathname: "/label/label_assistant_reply", help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting", - overview: "Read the following conversation and then answer the question about the last prompt in the discussion.", + overview: "Read the following conversation and then answer the question about the last reply in the discussion.", type: "label_assistant_reply", mode: "simple", update_type: "text_labels", diff --git a/website/src/types/Task.ts b/website/src/types/Task.ts index 8e5ada44..12e37db0 100644 --- a/website/src/types/Task.ts +++ b/website/src/types/Task.ts @@ -35,4 +35,6 @@ export interface TaskResponse { task: Task; } +export type TaskReplyValidity = "DEFAULT" | "VALID" | "INVALID"; + export type AvailableTasks = { [taskType in TaskType]: number }; diff --git a/website/src/types/TaskReplyState.ts b/website/src/types/TaskReplyState.ts deleted file mode 100644 index 100aed11..00000000 --- a/website/src/types/TaskReplyState.ts +++ /dev/null @@ -1,22 +0,0 @@ -export interface TaskReplyNotSubmittable { - content: T; - state: "NOT_SUBMITTABLE"; -} -export interface TaskReplyValid { - content: T; - state: "VALID"; -} -export interface TaskReplyDefault { - content: T; - state: "DEFAULT"; -} -export interface TaskReplyInValid { - content: T; - state: "INVALID"; -} - -export type TaskReplyState = - | TaskReplyNotSubmittable - | TaskReplyValid - | TaskReplyDefault - | TaskReplyInValid;