website: Support new widget types for labelling (#966)

* website: Support new widget types for labelling

Adds proper support for yes/no spam style questions as well as a simple interface for flag style labels.

Also cleaned up the Task component to fix some rerender issues.

* website: Fix some UI text, adjust yes/no button alignment

* website: Remove left over console.log

Co-authored-by: notmd <33456881+notmd@users.noreply.github.com>

---------

Co-authored-by: notmd <33456881+notmd@users.noreply.github.com>

website/cypress/e2e/tasks/label_assistant_reply.cy.ts

@@ -11,6 +11,10 @@ describe("labeling assistant replies", () => {
         // For specific task pages the no task available result is normal.
         if (type === undefined) return;
 
+        cy.get('[data-cy="label-question"]').each((label) => {
+          // Click the no button, this generally approves the spam check
+          cy.wrap(label).find('[data-cy="no"]').click();
+        });
         cy.get('[data-cy="label-options"]').each((label) => {
           // Click the 4th option
           cy.wrap(label).find('[data-cy="radio-option"]').eq(3).click();

website/cypress/e2e/tasks/label_initial_prompt.cy.ts

@@ -11,6 +11,10 @@ describe("labeling initial prompts", () => {
         // For specific task pages the no task available result is normal.
         if (type === undefined) return;
 
+        cy.get('[data-cy="label-question"]').each((label) => {
+          // Click the no button, this generally approves the spam check
+          cy.wrap(label).find('[data-cy="no"]').click();
+        });
         cy.get('[data-cy="label-options"]').each((label) => {
           // Click the 4th option
           cy.wrap(label).find('[data-cy="radio-option"]').eq(3).click();

website/cypress/e2e/tasks/label_prompter_reply.cy.ts

@@ -11,6 +11,10 @@ describe("labeling prompter replies", () => {
         // For specific task pages the no task available result is normal.
         if (type === undefined) return;
 
+        cy.get('[data-cy="label-question"]').each((label) => {
+          // Click the no button, this generally approves the spam check
+          cy.wrap(label).find('[data-cy="no"]').click();
+        });
         cy.get('[data-cy="label-options"]').each((label) => {
           // Click the 4th option
           cy.wrap(label).find('[data-cy="radio-option"]').eq(3).click();

website/cypress/e2e/tasks/random.cy.ts

@@ -44,6 +44,10 @@ describe("handles random tasks", () => {
                 break;
               }
               case "label-task": {
+                cy.get('[data-cy="label-question"]').each((label) => {
+                  // Click the no button, this generally approves the spam check
+                  cy.wrap(label).find('[data-cy="no"]').click();
+                });
                 cy.get('[data-cy="label-options"]').each((label) => {
                   // Click the 4th option
                   cy.wrap(label).find('[data-cy="radio-option"]').eq(3).click();
@@ -55,15 +59,6 @@ describe("handles random tasks", () => {
 
                 break;
               }
-              case "spam-task": {
-                cy.get('[data-cy="not-spam-button"]').click();
-
-                cy.get('[data-cy="review"]').click();
-
-                cy.get('[data-cy="submit"]').click();
-
-                break;
-              }
               case undefined: {
                 throw new Error("No tasks available, but at least create initial prompt expected");
               }

website/public/locales/en/common.json

@@ -16,5 +16,7 @@
   "sign_in": "Sign In",
   "sign_out": "Sign Out",
   "terms_of_service": "Terms of Service",
-  "title": "Open Assistant"
+  "title": "Open Assistant",
+  "yes": "Yes",
+  "no": "No"
 }

website/public/locales/en/labelling.json

@@ -0,0 +1,16 @@
+{
+  "label_highlighted_yes_no_instruction": "Answer the following question(s) about the highlighted message:",
+  "label_highlighted_flag_instruction": "Select any that apply to the highlighted message:",
+  "label_highlighted_likert_instruction": "Rate the highlighted message:",
+  "label_message_yes_no_instruction": "Answer the following question(s) about the message:",
+  "label_message_flag_instruction": "Select any that apply to the message:",
+  "label_message_likert_instruction": "Rate the message:",
+  "spam.question": "Is the message spam?",
+  "fails_task.question": "Does the reply fail the propmpters task?",
+  "not_appropriate": "Not Appropriate",
+  "pii": "Contains PII",
+  "hate_speech": "Hate Speech",
+  "sexual_content": "Sexual Content",
+  "moral_judgement": "Judges Morality",
+  "political_content": "Politcal"
+}

website/src/components/FlaggableElement.tsx

@@ -1,116 +0,0 @@
-import {
-  Box,
-  Button,
-  Modal,
-  ModalBody,
-  ModalCloseButton,
-  ModalContent,
-  ModalFooter,
-  ModalHeader,
-  ModalOverlay,
-  Popover,
-  PopoverAnchor,
-  PopoverTrigger,
-  Tooltip,
-  useColorModeValue,
-  useDisclosure,
-} from "@chakra-ui/react";
-import { AlertCircle } from "lucide-react";
-import { useState } from "react";
-import { get, post } from "src/lib/api";
-import { colors } from "src/styles/Theme/colors";
-import { Message } from "src/types/Conversation";
-import useSWRImmutable from "swr/immutable";
-import useSWRMutation from "swr/mutation";
-
-import { LabelInputGroup } from "./Survey/LabelInputGroup";
-
-interface Label {
-  name: string;
-  display_text: string;
-  help_text: string;
-}
-
-interface FlaggableElementProps {
-  children: React.ReactNode;
-  message: Message;
-}
-
-interface ValidLabelsResponse {
-  valid_labels: Label[];
-}
-
-export const FlaggableElement = (props: FlaggableElementProps) => {
-  const { data: response } = useSWRImmutable<ValidLabelsResponse>("/api/valid_labels", get);
-  const { isOpen, onOpen, onClose } = useDisclosure();
-  const { valid_labels } = response || { valid_labels: [] };
-  const [values, setValues] = useState<number[]>([]);
-
-  const submittable =
-    values.some((value) => {
-      return value !== null;
-    }) &&
-    values.length === valid_labels.length &&
-    valid_labels.length > 0;
-
-  const { trigger } = useSWRMutation("/api/set_label", post, {
-    onSuccess: onClose,
-    onError: onClose,
-  });
-
-  const submitResponse = () => {
-    const label_map: Map<string, number> = new Map();
-    console.assert(valid_labels.length === values.length);
-    values.forEach((value, idx) => {
-      if (value !== null) {
-        label_map.set(valid_labels[idx].name, value);
-      }
-    });
-    trigger({
-      message_id: props.message.id,
-      label_map: Object.fromEntries(label_map),
-      text: props.message.text,
-    });
-  };
-
-  return (
-    <Popover isOpen={isOpen} onOpen={onOpen} onClose={onClose} closeOnBlur={false} isLazy lazyBehavior="keepMounted">
-      <Box display="flex" alignItems="center" flexDirection={["column", "row"]} gap="2">
-        <PopoverAnchor>{props.children}</PopoverAnchor>
-
-        <Tooltip label="Report" bg="red.500" aria-label="A tooltip">
-          <Box>
-            <PopoverTrigger>
-              <Box as="button" display="flex" alignItems="center" justifyContent="center" borderRadius="full" p="1">
-                <AlertCircle size="20" className="text-red-400" aria-hidden="true" />
-              </Box>
-            </PopoverTrigger>
-          </Box>
-        </Tooltip>
-      </Box>
-
-      <Modal isOpen={isOpen} onClose={onClose}>
-        <ModalOverlay />
-        <ModalContent>
-          <ModalHeader>Select one or more labels that apply.</ModalHeader>
-          <ModalCloseButton />
-          <ModalBody>
-            <LabelInputGroup labelIDs={valid_labels.map(({ name }) => name)} onChange={setValues} />
-          </ModalBody>
-          <ModalFooter>
-            <Button
-              isDisabled={!submittable}
-              onClick={submitResponse}
-              className={`bg-indigo-600 text-${useColorModeValue(
-                colors.light.text,
-                colors.dark.text
-              )} hover:bg-indigo-700`}
-            >
-              Report
-            </Button>
-          </ModalFooter>
-        </ModalContent>
-      </Modal>
-    </Popover>
-  );
-};

website/src/components/Messages.tsx

@@ -1,25 +1,7 @@
-import { Box, forwardRef, Grid, useColorMode } from "@chakra-ui/react";
+import { Box, forwardRef, useColorMode } from "@chakra-ui/react";
 import { useMemo } from "react";
 import { Message } from "src/types/Conversation";
 
-import { FlaggableElement } from "./FlaggableElement";
-
-interface MessagesProps {
-  messages: Message[];
-}
-
-export const Messages = ({ messages }: MessagesProps) => {
-  const items = messages.map((messageProps: Message, i: number) => {
-    return (
-      <FlaggableElement message={messageProps} key={i + messageProps.id}>
-        <MessageView {...messageProps} />
-      </FlaggableElement>
-    );
-  });
-  // Maybe also show a legend of the colors?
-  return <Grid gap={2}>{items}</Grid>;
-};
-
 export const MessageView = forwardRef<Partial<Message>, "div">((message: Partial<Message>, ref) => {
   const { colorMode } = useColorMode();
 

website/src/components/Messages/LabelFlagGroup.tsx

@@ -0,0 +1,32 @@
+import { Button, Flex } from "@chakra-ui/react";
+import { useTranslation } from "next-i18next";
+import { getTypeSafei18nKey } from "src/lib/i18n";
+
+interface LabelFlagGroupProps {
+  values: number[];
+  labelNames: string[];
+  isEditable?: boolean;
+  onChange: (values: number[]) => void;
+}
+
+export const LabelFlagGroup = ({ values, labelNames, isEditable = true, onChange }: LabelFlagGroupProps) => {
+  const { t } = useTranslation("labelling");
+  return (
+    <Flex wrap="wrap" gap="4">
+      {labelNames.map((name, idx) => (
+        <Button
+          key={name}
+          onClick={() => {
+            const newValues = values.slice();
+            newValues[idx] = newValues[idx] ? 0 : 1;
+            onChange(newValues);
+          }}
+          isDisabled={!isEditable}
+          colorScheme={values[idx] === 1 ? "blue" : undefined}
+        >
+          {t(getTypeSafei18nKey(name))}
+        </Button>
+      ))}
+    </Flex>
+  );
+};

website/src/components/Messages/LabelInputGroup.tsx

@@ -0,0 +1,84 @@
+import { Text, VStack } from "@chakra-ui/react";
+import { Label } from "src/types/Tasks";
+
+import { LabelLikertGroup } from "../Survey/LabelLikertGroup";
+import { LabelFlagGroup } from "./LabelFlagGroup";
+import { LabelYesNoGroup } from "./LabelYesNoGroup";
+
+export interface LabelInputInstructions {
+  yesNoInstruction: string;
+  flagInstruction: string;
+  likertInstruction: string;
+}
+
+interface LabelInputGroupProps {
+  values: number[];
+  labels: Label[];
+  requiredLabels?: string[];
+  isEditable?: boolean;
+  instructions: LabelInputInstructions;
+  onChange: (values: number[]) => void;
+}
+
+export const LabelInputGroup = ({
+  labels,
+  values,
+  requiredLabels,
+  isEditable,
+  instructions,
+  onChange,
+}: LabelInputGroupProps) => {
+  const yesNoIndexes = labels.map((label, idx) => (label.widget === "yes_no" ? idx : null)).filter((v) => v !== null);
+  const flagIndexes = labels.map((label, idx) => (label.widget === "flag" ? idx : null)).filter((v) => v !== null);
+  const likertIndexes = labels.map((label, idx) => (label.widget === "likert" ? idx : null)).filter((v) => v !== null);
+
+  return (
+    <VStack alignItems="stretch" spacing={6}>
+      {yesNoIndexes.length > 0 && (
+        <VStack alignItems="stretch" spacing={2}>
+          <Text>{instructions.yesNoInstruction}</Text>
+          <LabelYesNoGroup
+            values={yesNoIndexes.map((idx) => values[idx])}
+            labelNames={yesNoIndexes.map((idx) => labels[idx].name)}
+            isEditable={isEditable}
+            requiredLabels={requiredLabels}
+            onChange={(yesNoValues) => {
+              const newValues = values.slice();
+              yesNoIndexes.forEach((idx, yesNoIndex) => (newValues[idx] = yesNoValues[yesNoIndex]));
+              onChange(newValues);
+            }}
+          />
+        </VStack>
+      )}
+      {flagIndexes.length > 0 && (
+        <VStack alignItems="stretch" spacing={2}>
+          <Text>{instructions.flagInstruction}</Text>
+          <LabelFlagGroup
+            values={flagIndexes.map((idx) => values[idx])}
+            labelNames={flagIndexes.map((idx) => labels[idx].name)}
+            isEditable={isEditable}
+            onChange={(flagValues) => {
+              const newValues = values.slice();
+              flagIndexes.forEach((idx, flagIndex) => (newValues[idx] = flagValues[flagIndex]));
+              onChange(newValues);
+            }}
+          />
+        </VStack>
+      )}
+      {likertIndexes.length > 0 && (
+        <VStack alignItems="stretch" spacing={2}>
+          <Text>{instructions.likertInstruction}</Text>
+          <LabelLikertGroup
+            labelIDs={likertIndexes.map((idx) => labels[idx].name)}
+            isEditable={isEditable}
+            onChange={(likertValues) => {
+              const newValues = values.slice();
+              likertIndexes.forEach((idx, likertIndex) => (newValues[idx] = likertValues[likertIndex]));
+              onChange(newValues);
+            }}
+          />
+        </VStack>
+      )}
+    </VStack>
+  );
+};

website/src/components/Messages/LabelPopup.tsx

@@ -9,9 +9,10 @@ import {
   ModalOverlay,
 } from "@chakra-ui/react";
 import { useTranslation } from "next-i18next";
-import { useState } from "react";
+import { useEffect, useState } from "react";
-import { LabelInputGroup } from "src/components/Survey/LabelInputGroup";
+import { LabelInputGroup } from "src/components/Messages/LabelInputGroup";
 import { get, post } from "src/lib/api";
+import { Label } from "src/types/Tasks";
 import useSWRImmutable from "swr/immutable";
 import useSWRMutation from "swr/mutation";
 
@@ -21,21 +22,19 @@ interface LabelMessagePopupProps {
   onClose: () => void;
 }
 
-interface Label {
-  name: string;
-  display_text: string;
-  help_text: string;
-}
-
 interface ValidLabelsResponse {
   valid_labels: Label[];
 }
 
 export const LabelMessagePopup = ({ messageId, show, onClose }: LabelMessagePopupProps) => {
-  const { t } = useTranslation("message");
+  const { t } = useTranslation();
   const { data: response } = useSWRImmutable<ValidLabelsResponse>("/api/valid_labels", get);
   const valid_labels = response?.valid_labels ?? [];
-  const [values, setValues] = useState<number[]>(null);
+  const [values, setValues] = useState<number[]>(new Array(valid_labels.length).fill(null));
+
+  useEffect(() => {
+    setValues(new Array(valid_labels.length).fill(null));
+  }, [messageId, valid_labels.length]);
 
   const { trigger: setLabels } = useSWRMutation("/api/set_label", post);
 
@@ -60,14 +59,23 @@ export const LabelMessagePopup = ({ messageId, show, onClose }: LabelMessagePopu
     <Modal isOpen={show} onClose={onClose}>
       <ModalOverlay />
       <ModalContent>
-        <ModalHeader>{t("label_title")}</ModalHeader>
+        <ModalHeader>{t("message:label_title")}</ModalHeader>
         <ModalCloseButton />
         <ModalBody>
-          <LabelInputGroup labelIDs={valid_labels.map(({ name }) => name)} onChange={setValues} />
+          <LabelInputGroup
+            labels={valid_labels}
+            values={values}
+            instructions={{
+              yesNoInstruction: t("labelling:label_message_yes_no_instruction"),
+              flagInstruction: t("labelling:label_message_flag_instruction"),
+              likertInstruction: t("labelling:label_message_likert_instruction"),
+            }}
+            onChange={setValues}
+          />
         </ModalBody>
         <ModalFooter>
           <Button colorScheme="blue" mr={3} onClick={submit}>
-            {t("submit_labels")}
+            {t("message:submit_labels")}
           </Button>
         </ModalFooter>
       </ModalContent>

website/src/components/Messages/LabelYesNoGroup.tsx

@@ -0,0 +1,89 @@
+import { Button, HStack, Text, Tooltip } from "@chakra-ui/react";
+import { useTranslation } from "next-i18next";
+import { getTypeSafei18nKey } from "src/lib/i18n";
+
+interface LabelYesNoGroupProps {
+  values: number[];
+  labelNames: string[];
+  requiredLabels?: string[];
+  isEditable?: boolean;
+  onChange: (values: number[]) => void;
+}
+
+export const LabelYesNoGroup = ({
+  values,
+  labelNames,
+  requiredLabels = [],
+  isEditable = true,
+  onChange,
+}: LabelYesNoGroupProps) => {
+  const { t } = useTranslation("labelling");
+  return (
+    <>
+      {labelNames.map((name, idx) => {
+        return (
+          <YesNoQuestion
+            key={name}
+            question={t(getTypeSafei18nKey(`${name}.question`))}
+            value={values[idx] === null ? null : values[idx] > 0.1 ? true : false}
+            onChange={(value) => {
+              const newValues = values.slice();
+              newValues[idx] = value;
+              onChange(newValues);
+            }}
+            isEditable={isEditable}
+            isRequired={requiredLabels.includes(name)}
+          />
+        );
+      })}
+    </>
+  );
+};
+
+const YesNoQuestion = ({
+  isEditable,
+  question,
+  value,
+  isRequired,
+  onChange,
+}: {
+  isEditable: boolean;
+  question: string;
+  value: boolean;
+  isRequired?: boolean;
+  onChange: (boolean) => void;
+}) => {
+  const { t } = useTranslation();
+  return (
+    <div data-cy="label-question" style={{ maxWidth: "30em" }}>
+      <Text display="inline">
+        {question}
+        {isRequired ? <RequiredMark /> : undefined}
+      </Text>
+      <HStack style={{ float: "right" }}>
+        <Button
+          data-cy="yes"
+          isDisabled={!isEditable}
+          colorScheme={value === true ? "blue" : undefined}
+          onClick={() => onChange(isRequired ? true : value === null ? true : null)}
+        >
+          {t("yes")}
+        </Button>
+        <Button
+          data-cy="no"
+          isDisabled={!isEditable}
+          colorScheme={value === false ? "blue" : undefined}
+          onClick={() => onChange(isRequired ? false : value === null ? false : null)}
+        >
+          {t("no")}
+        </Button>
+      </HStack>
+    </div>
+  );
+};
+
+const RequiredMark = () => (
+  <Tooltip label="Required">
+    <span style={{ color: "red" }}>*</span>
+  </Tooltip>
+);

website/src/components/Messages/MessageTableEntry.tsx

@@ -36,7 +36,7 @@ export function MessageTableEntry({ message, enabled, highlight }: MessageTableE
   const router = useRouter();
   const [emojiState, setEmojis] = useState<MessageEmojis>({ emojis: {}, user_emojis: [] });
   useEffect(() => {
-    setEmojis({ emojis: message.emojis, user_emojis: message.user_emojis });
+    setEmojis({ emojis: message.emojis || {}, user_emojis: message.user_emojis || [] });
   }, [message.emojis, message.user_emojis]);
 
   const goToMessage = useCallback(() => router.push(`/messages/${message.id}`), [router, message.id]);

website/src/components/Survey/LabelLikertGroup.tsx

@@ -135,7 +135,7 @@ const getLabelInfo = (label: string): LabelInfo => {
         oneDescription: ["Contains text which is incorrect or misleading"],
         inverted: true,
       };
-    case "helpful":
+    case "helpfulness":
       return {
         zeroText: "Unhelful",
         zeroDescription: [],
@@ -186,7 +186,7 @@ const getLabelInfo = (label: string): LabelInfo => {
   }
 };
 
-export const LabelInputGroup = ({ labelIDs, onChange, isEditable = true }: LabelInputGroupProps) => {
+export const LabelLikertGroup = ({ labelIDs, onChange, isEditable = true }: LabelInputGroupProps) => {
   const [labelValues, setLabelValues] = useState<number[]>(Array.from({ length: labelIDs.length }).map(() => null));
 
   const cardColor = useColorModeValue("gray.50", "gray.800");

website/src/components/Survey/TaskControls.tsx

@@ -4,12 +4,10 @@ import { SkipButton } from "src/components/Buttons/Skip";
 import { SubmitButton } from "src/components/Buttons/Submit";
 import { TaskInfo } from "src/components/TaskInfo/TaskInfo";
 import { TaskStatus } from "src/components/Tasks/Task";
+import { BaseTask } from "src/types/Task";
 
 export interface TaskControlsProps {
-  // we need a task type
+  task: BaseTask;
-  // eslint-disable-next-line  @typescript-eslint/no-explicit-any
-  task: any;
-  className?: string;
   taskStatus: TaskStatus;
   onEdit: () => void;
   onReview: () => void;
@@ -17,7 +15,7 @@ export interface TaskControlsProps {
   onSkip: (reason: string) => void;
 }
 
-export const TaskControls = (props: TaskControlsProps) => {
+export const TaskControls = ({ task, taskStatus, onEdit, onReview, onSubmit, onSkip }: TaskControlsProps) => {
   const backgroundColor = useColorModeValue("white", "gray.800");
 
   return (
@@ -31,38 +29,32 @@ export const TaskControls = (props: TaskControlsProps) => {
       shadow="base"
       gap="4"
     >
-      <TaskInfo id={props.task.id} output="Submit your answer" />
+      <TaskInfo id={task.id} output="Submit your answer" />
       <Flex width={["full", "fit-content"]} justify="center" ml="auto" gap={2}>
-        {props.taskStatus === "REVIEW" || props.taskStatus === "SUBMITTED" ? (
+        {taskStatus.mode === "EDIT" ? (
           <>
-            <Tooltip label="Edit">
+            <SkipButton onSkip={onSkip} />
-              <IconButton
-                size="lg"
-                data-cy="edit"
-                aria-label="edit"
-                onClick={props.onEdit}
-                icon={<Edit2 size="1em" />}
-              />
-            </Tooltip>
             <SubmitButton
-              colorScheme="green"
+              colorScheme="blue"
-              data-cy="submit"
+              data-cy="review"
-              isDisabled={props.taskStatus === "SUBMITTED"}
+              isDisabled={taskStatus.replyValidity === "INVALID"}
-              onClick={props.onSubmit}
+              onClick={onReview}
             >
-              Submit
+              Review
             </SubmitButton>
           </>
         ) : (
           <>
-            <SkipButton onSkip={props.onSkip} />
+            <Tooltip label="Edit">
+              <IconButton size="lg" data-cy="edit" aria-label="edit" onClick={onEdit} icon={<Edit2 size="1em" />} />
+            </Tooltip>
             <SubmitButton
-              colorScheme="blue"
+              colorScheme="green"
-              data-cy="review"
+              data-cy="submit"
-              isDisabled={props.taskStatus === "NOT_SUBMITTABLE"}
+              isDisabled={taskStatus.mode === "SUBMITTED"}
-              onClick={props.onReview}
+              onClick={onSubmit}
             >
-              Review
+              Submit
             </SubmitButton>
           </>
         )}

website/src/components/Tasks/CreateTask.tsx

@@ -7,6 +7,8 @@ import { TwoColumnsWithCards } from "src/components/Survey/TwoColumnsWithCards";
 import { TaskSurveyProps } from "src/components/Tasks/Task";
 import { TaskHeader } from "src/components/Tasks/TaskHeader";
 import { getTypeSafei18nKey } from "src/lib/i18n";
+import { TaskType } from "src/types/Task";
+import { CreateAssistantReplyTask, CreateInitialPromptTask, CreatePrompterReplyTask } from "src/types/Tasks";
 
 export const CreateTask = ({
   task,
@@ -15,7 +17,7 @@ export const CreateTask = ({
   isDisabled,
   onReplyChanged,
   onValidityChanged,
-}: TaskSurveyProps<{ text: string }>) => {
+}: TaskSurveyProps<CreateInitialPromptTask | CreateAssistantReplyTask | CreatePrompterReplyTask, { text: string }>) => {
   const { t, i18n } = useTranslation(["tasks", "common"]);
   const cardColor = useColorModeValue("gray.50", "gray.800");
   const titleColor = useColorModeValue("gray.800", "gray.300");
@@ -39,7 +41,7 @@ export const CreateTask = ({
       <TwoColumnsWithCards>
         <>
           <TaskHeader taskType={taskType} />
-          {!!task.conversation && (
+          {task.type !== TaskType.initial_prompt && (
             <Box mt="4" borderRadius="lg" bg={cardColor} className="p-3 sm:p-6">
               <MessageTable messages={task.conversation.messages} highlightLastMessage />
             </Box>

website/src/components/Tasks/EvaluateTask.tsx

@@ -5,6 +5,8 @@ import { Sortable } from "src/components/Sortable/Sortable";
 import { SurveyCard } from "src/components/Survey/SurveyCard";
 import { TaskSurveyProps } from "src/components/Tasks/Task";
 import { TaskHeader } from "src/components/Tasks/TaskHeader";
+import { TaskType } from "src/types/Task";
+import { RankAssistantRepliesTask, RankInitialPromptsTask, RankPrompterRepliesTask } from "src/types/Tasks";
 
 export const EvaluateTask = ({
   task,
@@ -13,19 +15,25 @@ export const EvaluateTask = ({
   isDisabled,
   onReplyChanged,
   onValidityChanged,
-}: TaskSurveyProps<{ ranking: number[] }>) => {
+}: TaskSurveyProps<
+  RankInitialPromptsTask | RankAssistantRepliesTask | RankPrompterRepliesTask,
+  { ranking: number[] }
+>) => {
   const cardColor = useColorModeValue("gray.50", "gray.800");
   const [ranking, setRanking] = useState<number[]>(null);
 
   let messages = [];
-  if (task.conversation) {
+  if (task.type !== TaskType.rank_initial_prompts) {
     messages = task.conversation.messages;
   }
 
   useEffect(() => {
     if (ranking === null) {
-      const defaultRanking = (task.replies ?? task.prompts).map((_, idx) => idx);
+      if (task.type === TaskType.rank_initial_prompts) {
-      onReplyChanged({ ranking: defaultRanking });
+        onReplyChanged({ ranking: task.prompts.map((_, idx) => idx) });
+      } else {
+        onReplyChanged({ ranking: task.replies.map((_, idx) => idx) });
+      }
       onValidityChanged("DEFAULT");
     } else {
       onReplyChanged({ ranking });
@@ -33,7 +41,7 @@ export const EvaluateTask = ({
     }
   }, [task, ranking, onReplyChanged, onValidityChanged]);
 
-  const sortables = task.replies ? "replies" : "prompts";
+  const sortables = task.type === TaskType.rank_initial_prompts ? "prompts" : "replies";
 
   return (
     <div data-cy="task" data-task-type="evaluate-task">

website/src/components/Tasks/LabelTask/LabelTask.tsx

@@ -1,11 +1,18 @@
-import { Box, Button, Flex, HStack, Text, useColorModeValue } from "@chakra-ui/react";
+import { Box, useBoolean, useColorModeValue } from "@chakra-ui/react";
+import { useTranslation } from "next-i18next";
 import { useEffect, useState } from "react";
 import { MessageView } from "src/components/Messages";
+import { LabelInputGroup } from "src/components/Messages/LabelInputGroup";
 import { MessageTable } from "src/components/Messages/MessageTable";
-import { LabelInputGroup } from "src/components/Survey/LabelInputGroup";
 import { TwoColumnsWithCards } from "src/components/Survey/TwoColumnsWithCards";
 import { TaskSurveyProps } from "src/components/Tasks/Task";
 import { TaskHeader } from "src/components/Tasks/TaskHeader";
+import { TaskType } from "src/types/Task";
+import { LabelTaskType } from "src/types/Tasks";
+
+const isRequired = (labelName: string, requiredLabels?: string[]) => {
+  return requiredLabels ? requiredLabels.includes(labelName) : false;
+};
 
 export const LabelTask = ({
   task,
@@ -13,15 +20,33 @@ export const LabelTask = ({
   isEditable,
   onReplyChanged,
   onValidityChanged,
-}: TaskSurveyProps<{ text: string; labels: Record<string, number>; message_id: string }>) => {
+}: TaskSurveyProps<LabelTaskType, { text: string; labels: Record<string, number>; message_id: string }>) => {
-  const [sliderValues, setSliderValues] = useState<number[]>(new Array(task.valid_labels.length).fill(null));
+  const { t } = useTranslation("labelling");
+  const [values, setValues] = useState<number[]>(new Array(task.labels.length).fill(null));
+  const [userInputMade, setUserInputMade] = useBoolean(false);
 
+  // Initial setup to run when the task changes
   useEffect(() => {
-    console.assert(task.valid_labels.length === sliderValues.length);
+    setValues(new Array(task.labels.length).fill(null));
-    const labels = Object.fromEntries(task.valid_labels.map((label, i) => [label, sliderValues[i]]));
+    onValidityChanged(task.labels.some(({ name }) => isRequired(name, task.mandatory_labels)) ? "INVALID" : "DEFAULT");
-    onReplyChanged({ labels, text: task.reply || task.prompt, message_id: task.message_id });
+    setUserInputMade.off();
-    onValidityChanged(sliderValues.every((value) => value !== null) ? "VALID" : "INVALID");
+  }, [task, setUserInputMade, onValidityChanged]);
-  }, [task, sliderValues, onReplyChanged, onValidityChanged]);
+
+  // Update the reply and validity when the values change
+  useEffect(() => {
+    onReplyChanged({
+      text: "unused?",
+      labels: Object.fromEntries(task.labels.map(({ name }, idx) => [name, values[idx] || 0])),
+      message_id: task.message_id,
+    });
+    onValidityChanged(
+      task.labels.some(({ name }, idx) => values[idx] === null && isRequired(name, task.mandatory_labels))
+        ? "INVALID"
+        : userInputMade
+        ? "VALID"
+        : "DEFAULT"
+    );
+  }, [task, values, onReplyChanged, userInputMade, onValidityChanged]);
 
   const cardColor = useColorModeValue("gray.50", "gray.800");
   const isSpamTask = task.mode === "simple" && task.valid_labels.length === 1 && task.valid_labels[0] === "spam";
@@ -31,12 +56,9 @@ export const LabelTask = ({
       <TwoColumnsWithCards>
         <>
           <TaskHeader taskType={taskType} />
-          {task.conversation ? (
+          {task.type !== TaskType.label_initial_prompt ? (
             <Box mt="4" p={[4, 6]} borderRadius="lg" bg={cardColor}>
-              <MessageTable
+              <MessageTable messages={task.conversation.messages} highlightLastMessage />
-                messages={[...(task.conversation?.messages ?? []), task.reply_message]}
-                highlightLastMessage
-              />
             </Box>
           ) : (
             <Box mt="4">
@@ -44,51 +66,22 @@ export const LabelTask = ({
             </Box>
           )}
         </>
-        {isSpamTask ? (
+        <LabelInputGroup
-          <SpamTaskInput
+          labels={task.labels}
-            value={sliderValues[0]}
+          values={values}
-            onChange={(value) => setSliderValues([value])}
+          requiredLabels={task.mandatory_labels}
-            isEditable={isEditable}
+          isEditable={isEditable}
-          />
+          instructions={{
-        ) : (
+            yesNoInstruction: t("label_highlighted_yes_no_instruction"),
-          <Flex direction="column" alignItems="stretch">
+            flagInstruction: t("label_highlighted_flag_instruction"),
-            <Text>The highlighted message:</Text>
+            likertInstruction: t("label_highlighted_likert_instruction"),
-            <LabelInputGroup labelIDs={task.valid_labels} isEditable={isEditable} onChange={setSliderValues} />
+          }}
-          </Flex>
+          onChange={(values) => {
-        )}
+            setValues(values);
+            setUserInputMade.on();
+          }}
+        />
       </TwoColumnsWithCards>
     </div>
   );
 };
-
-const SpamTaskInput = ({
-  isEditable,
-  value,
-  onChange,
-}: {
-  isEditable: boolean;
-  value: number;
-  onChange: (number) => void;
-}) => {
-  return (
-    <HStack>
-      <Text>Is the highlighted message spam?</Text>
-      <Button
-        data-cy="spam-button"
-        isDisabled={!isEditable}
-        colorScheme={value === 1 ? "blue" : undefined}
-        onClick={() => onChange(1)}
-      >
-        Yes
-      </Button>
-      <Button
-        data-cy="not-spam-button"
-        isDisabled={!isEditable}
-        colorScheme={value === 0 ? "blue" : undefined}
-        onClick={() => onChange(0)}
-      >
-        No
-      </Button>
-    </HStack>
-  );
-};

website/src/components/Tasks/Task/Task.tsx

@@ -1,5 +1,6 @@
 import { useTranslation } from "next-i18next";
-import { useRef, useState } from "react";
+import { useCallback, useEffect, useReducer } from "react";
+import { useMemo, useRef } from "react";
 import { TaskControls } from "src/components/Survey/TaskControls";
 import { CreateTask } from "src/components/Tasks/CreateTask";
 import { EvaluateTask } from "src/components/Tasks/EvaluateTask";
@@ -8,15 +9,52 @@ import { TaskCategory, TaskInfo, TaskInfos } from "src/components/Tasks/TaskType
 import { UnchangedWarning } from "src/components/Tasks/UnchangedWarning";
 import { post } from "src/lib/api";
 import { getTypeSafei18nKey } from "src/lib/i18n";
-import { TaskContent, TaskReplyValidity } from "src/types/Task";
+import { BaseTask, TaskContent, TaskReplyValidity } from "src/types/Task";
 import useSWRMutation from "swr/mutation";
 
-export type TaskStatus = "NOT_SUBMITTABLE" | "DEFAULT" | "VALID" | "REVIEW" | "SUBMITTED";
+interface EditMode {
+  mode: "EDIT";
+  replyValidity: TaskReplyValidity;
+}
+interface ReviewMode {
+  mode: "REVIEW";
+}
+interface DefaultWarnMode {
+  mode: "DEFAULT_WARN";
+}
+interface SubmittedMode {
+  mode: "SUBMITTED";
+}
 
-export interface TaskSurveyProps<T> {
+export type TaskStatus = EditMode | DefaultWarnMode | ReviewMode | SubmittedMode;
-  // we need a task type
+
-  // eslint-disable-next-line  @typescript-eslint/no-explicit-any
+interface NewTask {
-  task: any;
+  action: "NEW_TASK";
+}
+
+interface Review {
+  action: "REVIEW";
+}
+
+interface SetSubmitted {
+  action: "SET_SUBMITTED";
+}
+
+interface ReturnToEdit {
+  action: "RETURN_EDIT";
+}
+
+interface AcceptDefault {
+  action: "ACCEPT_DEFAULT";
+}
+
+interface UpdateValidity {
+  action: "UPDATE_VALIDITY";
+  replyValidity: TaskReplyValidity;
+}
+
+export interface TaskSurveyProps<TaskType extends BaseTask, T> {
+  task: TaskType;
   taskType: TaskInfo;
   isEditable: boolean;
   isDisabled?: boolean;
@@ -26,13 +64,63 @@ export interface TaskSurveyProps<T> {
 
 export const Task = ({ frontendId, task, trigger, mutate }) => {
   const { t } = useTranslation("tasks");
-  const [taskStatus, setTaskStatus] = useState<TaskStatus>("NOT_SUBMITTABLE");
+  const [taskStatus, taskEvent] = useReducer(
+    (
+      status: TaskStatus,
+      event: NewTask | UpdateValidity | AcceptDefault | Review | ReturnToEdit | SetSubmitted
+    ): TaskStatus => {
+      switch (event.action) {
+        case "NEW_TASK":
+          return { mode: "EDIT", replyValidity: "INVALID" };
+        case "UPDATE_VALIDITY":
+          return status.mode === "EDIT" ? { mode: "EDIT", replyValidity: event.replyValidity } : status;
+        case "ACCEPT_DEFAULT":
+          return status.mode === "DEFAULT_WARN" ? { mode: "REVIEW" } : status;
+        case "REVIEW": {
+          if (status.mode === "EDIT") {
+            switch (status.replyValidity) {
+              case "DEFAULT":
+                return { mode: "DEFAULT_WARN" };
+              case "VALID":
+                return { mode: "REVIEW" };
+            }
+          }
+          return status;
+        }
+        case "RETURN_EDIT": {
+          switch (status.mode) {
+            case "REVIEW":
+              return { mode: "EDIT", replyValidity: "VALID" };
+            case "DEFAULT_WARN":
+              return { mode: "EDIT", replyValidity: "DEFAULT" };
+            default:
+              return status;
+          }
+        }
+        case "SET_SUBMITTED": {
+          return status.mode === "REVIEW" ? { mode: "SUBMITTED" } : status;
+        }
+      }
+    },
+    { mode: "EDIT", replyValidity: "INVALID" }
+  );
+
   const replyContent = useRef<TaskContent>(null);
-  const [showUnchangedWarning, setShowUnchangedWarning] = useState(false);
+  const updateValidity = useCallback(
+    (replyValidity: TaskReplyValidity) => taskEvent({ action: "UPDATE_VALIDITY", replyValidity }),
+    [taskEvent]
+  );
+
+  useEffect(() => {
+    taskEvent({ action: "NEW_TASK" });
+  }, [task.id, updateValidity]);
 
   const rootEl = useRef<HTMLDivElement>(null);
 
-  const taskType = TaskInfos.find((taskType) => taskType.type === task.type && taskType.mode === task.mode);
+  const taskType = useMemo(
+    () => TaskInfos.find((taskType) => taskType.type === task.type && taskType.mode === task.mode),
+    [task.type, task.mode]
+  );
 
   const { trigger: sendRejection } = useSWRMutation("/api/reject_task", post, {
     onSuccess: async () => {
@@ -47,79 +135,36 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
     });
   };
 
-  const edit_mode = taskStatus === "NOT_SUBMITTABLE" || taskStatus === "DEFAULT" || taskStatus === "VALID";
+  const onReplyChanged = useCallback(
-  const submitted = taskStatus === "SUBMITTED";
+    (content: TaskContent) => {
-
+      replyContent.current = content;
-  const onValidityChanged = (validity: TaskReplyValidity) => {
+    },
-    if (!edit_mode) return;
+    [replyContent]
-    switch (validity) {
+  );
-      case "DEFAULT":
-        if (taskStatus !== "DEFAULT") setTaskStatus("DEFAULT");
-        break;
-      case "VALID":
-        if (taskStatus !== "VALID") setTaskStatus("VALID");
-        break;
-      case "INVALID":
-        if (taskStatus !== "NOT_SUBMITTABLE") setTaskStatus("NOT_SUBMITTABLE");
-        break;
-    }
-  };
-
-  const onReplyChanged = (content: TaskContent) => {
-    replyContent.current = content;
-  };
-
-  const reviewResponse = () => {
-    switch (taskStatus) {
-      case "DEFAULT":
-        setShowUnchangedWarning(true);
-        break;
-      case "VALID":
-        setTaskStatus("REVIEW");
-        break;
-      default:
-        return;
-    }
-  };
-
-  const editResponse = () => {
-    switch (taskStatus) {
-      case "REVIEW":
-        setTaskStatus("VALID");
-        break;
-      default:
-        return;
-    }
-  };
 
   const submitResponse = () => {
-    switch (taskStatus) {
+    if (taskStatus.mode === "REVIEW") {
-      case "REVIEW": {
+      trigger({
-        trigger({
+        id: frontendId,
-          id: frontendId,
+        update_type: taskType.update_type,
-          update_type: taskType.update_type,
+        content: replyContent.current,
-          content: replyContent.current,
+      });
-        });
+      taskEvent({ action: "SET_SUBMITTED" });
-        setTaskStatus("SUBMITTED");
+      scrollToTop(rootEl.current);
-        scrollToTop(rootEl.current);
-        break;
-      }
-      default:
-        return;
     }
   };
 
-  function taskTypeComponent() {
+  const taskTypeComponent = useMemo(() => {
     switch (taskType.category) {
       case TaskCategory.Create:
         return (
           <CreateTask
             task={task}
             taskType={taskType}
-            isEditable={edit_mode}
+            isEditable={taskStatus.mode === "EDIT"}
-            isDisabled={submitted}
+            isDisabled={taskStatus.mode === "SUBMITTED"}
             onReplyChanged={onReplyChanged}
-            onValidityChanged={onValidityChanged}
+            onValidityChanged={updateValidity}
           />
         );
       case TaskCategory.Evaluate:
@@ -127,10 +172,10 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
           <EvaluateTask
             task={task}
             taskType={taskType}
-            isEditable={edit_mode}
+            isEditable={taskStatus.mode === "EDIT"}
-            isDisabled={submitted}
+            isDisabled={taskStatus.mode === "SUBMITTED"}
             onReplyChanged={onReplyChanged}
-            onValidityChanged={onValidityChanged}
+            onValidityChanged={updateValidity}
           />
         );
       case TaskCategory.Label:
@@ -138,37 +183,34 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
           <LabelTask
             task={task}
             taskType={taskType}
-            isEditable={edit_mode}
+            isEditable={taskStatus.mode === "EDIT"}
-            isDisabled={submitted}
+            isDisabled={taskStatus.mode === "SUBMITTED"}
             onReplyChanged={onReplyChanged}
-            onValidityChanged={onValidityChanged}
+            onValidityChanged={updateValidity}
           />
         );
     }
-  }
+  }, [task, taskType, taskStatus.mode, onReplyChanged, updateValidity]);
 
   return (
     <div ref={rootEl}>
-      {taskTypeComponent()}
+      {taskTypeComponent}
       <TaskControls
         task={task}
         taskStatus={taskStatus}
-        onEdit={editResponse}
+        onEdit={() => taskEvent({ action: "RETURN_EDIT" })}
-        onReview={reviewResponse}
+        onReview={() => taskEvent({ action: "REVIEW" })}
         onSubmit={submitResponse}
         onSkip={rejectTask}
       />
       <UnchangedWarning
-        show={showUnchangedWarning}
+        show={taskStatus.mode === "DEFAULT_WARN"}
         title={t(getTypeSafei18nKey(`${taskType.id}.unchanged_title`)) || t("default.unchanged_title")}
         message={t(getTypeSafei18nKey(`${taskType.id}.unchanged_message`)) || t("default.unchanged_message")}
         continueButtonText={"Continue anyway"}
-        onClose={() => setShowUnchangedWarning(false)}
+        onClose={() => taskEvent({ action: "RETURN_EDIT" })}
         onContinueAnyway={() => {
-          if (taskStatus === "DEFAULT") {
+          taskEvent({ action: "ACCEPT_DEFAULT" });
-            setTaskStatus("REVIEW");
-            setShowUnchangedWarning(false);
-          }
         }}
       />
     </div>

website/src/types/Tasks.ts

@@ -33,31 +33,39 @@ export interface RankPrompterRepliesTask extends BaseTask {
   replies: string[];
 }
 
-export interface LabelAssistantReplyTask extends BaseTask {
+export interface Label {
+  display_text: string;
+  help_text: string;
+  name: string;
+  widget: "flag" | "yes_no" | "likert";
+}
+
+export interface BaseLabelTask extends BaseTask {
+  message_id: string;
+  labels: Label[];
+  valid_labels: string[];
+  disposition: "spam" | "quality";
+  mode: "simple" | "full";
+  mandatory_labels?: string[];
+}
+
+export interface LabelAssistantReplyTask extends BaseLabelTask {
   type: TaskType.label_assistant_reply;
-  message_id: string;
   conversation: Conversation;
   reply_message: Message;
   reply: string;
-  valid_labels: string[];
-  mode: "simple" | "full";
-  mandatory_labels?: string[];
 }
 
-export interface LabelPrompterReplyTask extends BaseTask {
+export interface LabelPrompterReplyTask extends BaseLabelTask {
   type: TaskType.label_prompter_reply;
-  message_id: string;
   conversation: Conversation;
   reply_message: Message;
   reply: string;
-  valid_labels: string[];
-  mode: "simple" | "full";
-  mandatory_labels?: string[];
 }
 
-export interface LabelInitialPromptTask extends BaseTask {
+export interface LabelInitialPromptTask extends BaseLabelTask {
   type: TaskType.label_initial_prompt;
-  message_id: string;
-  valid_labels: string[];
   prompt: string;
 }
+
+export type LabelTaskType = LabelAssistantReplyTask | LabelPrompterReplyTask | LabelInitialPromptTask;

website/types/i18next.d.ts

@@ -3,6 +3,7 @@ import type dashboard from "public/locales/en/dashboard.json";
 import type index from "public/locales/en/index.json";
 import type leaderboard from "public/locales/en/leaderboard.json";
 import type message from "public/locales/en/message.json";
+import type labelling from "public/locales/en/labelling.json";
 import type tasks from "public/locales/en/tasks.json";
 
 declare module "i18next" {
@@ -14,6 +15,7 @@ declare module "i18next" {
       leaderboard: typeof leaderboard;
       tasks: typeof tasks;
       message: typeof message;
+      labelling: typeof labelling;
     };
   }
 }