Merge pull request #970 from othrayte/improve-e2e-test-reliability

website: Improve reliability of e2e tests

website/cypress/support/commands.ts

@@ -46,6 +46,7 @@ Cypress.Commands.add("signInUsingEmailedLink", (emailAddress) => {
     // Find and use login link
     const loginLink = emails.pop().html.match(/href="[^"]+(\/api\/auth\/callback\/[^"]+?)"/)[1];
     cy.visit(loginLink);
+    cy.url().should("include", "/dashboard");
   });
 });
 

website/src/components/Loading/LoadingScreen.jsx

@@ -1,4 +1,4 @@
-import { Box, Center, Progress, Text, useColorModeValue } from "@chakra-ui/react";
+import { Box, Center, Progress, Text } from "@chakra-ui/react";
 
 export const LoadingScreen = ({ text = "Loading..." } = {}) => {
   return (

website/src/components/Survey/TaskControls.tsx

@@ -1,4 +1,4 @@
-import { Box, Flex, IconButton, Tooltip, useColorModeValue } from "@chakra-ui/react";
+import { Box, Flex, IconButton, Progress, Tooltip, useColorModeValue } from "@chakra-ui/react";
 import { Edit2 } from "lucide-react";
 import { SkipButton } from "src/components/Buttons/Skip";
 import { SubmitButton } from "src/components/Buttons/Submit";
@@ -9,55 +9,58 @@ import { BaseTask } from "src/types/Task";
 export interface TaskControlsProps {
   task: BaseTask;
   taskStatus: TaskStatus;
+  isLoading: boolean;
   onEdit: () => void;
   onReview: () => void;
   onSubmit: () => void;
   onSkip: (reason: string) => void;
 }
 
-export const TaskControls = ({ task, taskStatus, onEdit, onReview, onSubmit, onSkip }: TaskControlsProps) => {
+export const TaskControls = ({
+  task,
+  taskStatus,
+  isLoading,
+  onEdit,
+  onReview,
+  onSubmit,
+  onSkip,
+}: TaskControlsProps) => {
   const backgroundColor = useColorModeValue("white", "gray.800");
 
   return (
-    <Box
-      width="full"
-      bg={backgroundColor}
-      borderRadius="xl"
-      p="6"
-      display="flex"
-      flexDirection={["column", "row"]}
-      shadow="base"
-      gap="4"
-    >
-      <TaskInfo id={task.id} output="Submit your answer" />
-      <Flex width={["full", "fit-content"]} justify="center" ml="auto" gap={2}>
-        {taskStatus.mode === "EDIT" ? (
-          <>
-            <SkipButton onSkip={onSkip} />
-            <SubmitButton
-              colorScheme="blue"
-              data-cy="review"
-              isDisabled={taskStatus.replyValidity === "INVALID"}
-              onClick={onReview}
-            >
-              Review
-            </SubmitButton>
-          </>
-        ) : (
-          <>
-            <Tooltip label="Edit">
-              <IconButton size="lg" data-cy="edit" aria-label="edit" onClick={onEdit} icon={<Edit2 size="1em" />} />
-            </Tooltip>
-            <SubmitButton
-              colorScheme="green"
-              data-cy="submit"
-              isDisabled={taskStatus.mode === "SUBMITTED"}
-              onClick={onSubmit}
-            >
-              Submit
-            </SubmitButton>
-          </>
-        )}
+    <Box width="full" bg={backgroundColor} borderRadius="xl" shadow="base">
+      {isLoading && <Progress size="sm" isIndeterminate />}
+      <Flex p="6" gap="4" direction={["column", "row"]}>
+        <TaskInfo id={task.id} output="Submit your answer" />
+        <Flex width={["full", "fit-content"]} justify="center" ml="auto" gap={2}>
+          {taskStatus.mode === "EDIT" ? (
+            <>
+              <SkipButton onSkip={onSkip} />
+              <SubmitButton
+                colorScheme="blue"
+                data-cy="review"
+                isDisabled={taskStatus.replyValidity === "INVALID"}
+                onClick={onReview}
+              >
+                Review
+              </SubmitButton>
+            </>
+          ) : (
+            <>
+              <Tooltip label="Edit">
+                <IconButton size="lg" data-cy="edit" aria-label="edit" onClick={onEdit} icon={<Edit2 size="1em" />} />
+              </Tooltip>
+              <SubmitButton
+                colorScheme="green"
+                data-cy="submit"
+                isDisabled={taskStatus.mode === "SUBMITTED"}
+                onClick={onSubmit}
+              >
+                Submit
+              </SubmitButton>
+            </>
+          )}
+        </Flex>
       </Flex>
     </Box>
   );

website/src/components/TaskPage/TaskPage.tsx

@@ -4,9 +4,10 @@ import { TaskEmptyState } from "src/components/EmptyState";
 import { LoadingScreen } from "src/components/Loading/LoadingScreen";
 import { Task } from "src/components/Tasks/Task";
 import { TaskInfos } from "src/components/Tasks/TaskTypes";
-import { ERROR_CODES, taskApiHooks } from "src/lib/constants";
+import { taskApiHooks } from "src/lib/constants";
 import { getTypeSafei18nKey } from "src/lib/i18n";
 import { TaskType } from "src/types/Task";
+import { KnownTaskType } from "src/types/Tasks";
 
 type TaskPageProps = {
   type: TaskType;
@@ -15,26 +16,38 @@ type TaskPageProps = {
 export const TaskPage = ({ type }: TaskPageProps) => {
   const { t } = useTranslation(["tasks", "common"]);
   const taskApiHook = taskApiHooks[type];
-  const { tasks, isLoading, reset, trigger, error } = taskApiHook(type);
+  const { response, isLoading, completeTask, skipTask } = taskApiHook(type);
   const taskInfo = TaskInfos.find((taskType) => taskType.type === type);
 
-  if (isLoading) {
-    return <LoadingScreen text={t("common:loading")} />;
+  let body;
+  switch (response.taskAvailability) {
+    case "AWAITING_INITIAL":
+      body = <LoadingScreen text={t("common:loading")} />;
+      break;
+    case "NONE_AVAILABLE":
+      body = <TaskEmptyState />;
+      break;
+    case "AVAILABLE":
+      body = (
+        <Task
+          key={response.task.id}
+          frontendId={response.id}
+          task={response.task as KnownTaskType}
+          isLoading={isLoading}
+          completeTask={completeTask}
+          skipTask={skipTask}
+        />
+      );
+      break;
   }
 
-  if (tasks.length === 0 || error?.errorCode === ERROR_CODES.TASK_REQUESTED_TYPE_NOT_AVAILABLE) {
-    return <TaskEmptyState />;
-  }
-
-  const task = tasks[0];
-
   return (
     <>
       <Head>
         <title>{t(getTypeSafei18nKey(`${taskInfo.id}.label`))}</title>
         <meta name="description" content={t(getTypeSafei18nKey(`${taskInfo.id}.desc`))} />
       </Head>
-      <Task key={task.task.id} frontendId={task.id} task={task.task} trigger={trigger} mutate={reset} />
+      {body}
     </>
   );
 };

website/src/components/Tasks/CreateTask.tsx

@@ -8,7 +8,7 @@ import { TaskSurveyProps } from "src/components/Tasks/Task";
 import { TaskHeader } from "src/components/Tasks/TaskHeader";
 import { getTypeSafei18nKey } from "src/lib/i18n";
 import { TaskType } from "src/types/Task";
-import { CreateAssistantReplyTask, CreateInitialPromptTask, CreatePrompterReplyTask } from "src/types/Tasks";
+import { CreateTaskType } from "src/types/Tasks";
 
 export const CreateTask = ({
   task,
@@ -17,7 +17,7 @@ export const CreateTask = ({
   isDisabled,
   onReplyChanged,
   onValidityChanged,
-}: TaskSurveyProps<CreateInitialPromptTask | CreateAssistantReplyTask | CreatePrompterReplyTask, { text: string }>) => {
+}: TaskSurveyProps<CreateTaskType, { text: string }>) => {
   const { t, i18n } = useTranslation(["tasks", "common"]);
   const cardColor = useColorModeValue("gray.50", "gray.800");
   const titleColor = useColorModeValue("gray.800", "gray.300");

website/src/components/Tasks/EvaluateTask.tsx

@@ -6,7 +6,7 @@ import { SurveyCard } from "src/components/Survey/SurveyCard";
 import { TaskSurveyProps } from "src/components/Tasks/Task";
 import { TaskHeader } from "src/components/Tasks/TaskHeader";
 import { TaskType } from "src/types/Task";
-import { RankAssistantRepliesTask, RankInitialPromptsTask, RankPrompterRepliesTask } from "src/types/Tasks";
+import { RankTaskType } from "src/types/Tasks";
 
 export const EvaluateTask = ({
   task,
@@ -15,10 +15,7 @@ export const EvaluateTask = ({
   isDisabled,
   onReplyChanged,
   onValidityChanged,
-}: TaskSurveyProps<
-  RankInitialPromptsTask | RankAssistantRepliesTask | RankPrompterRepliesTask,
-  { ranking: number[] }
->) => {
+}: TaskSurveyProps<RankTaskType, { ranking: number[] }>) => {
   const cardColor = useColorModeValue("gray.50", "gray.800");
   const [ranking, setRanking] = useState<number[]>(null);
 

website/src/components/Tasks/Task/Task.stories.tsx

@@ -7,8 +7,10 @@ export default {
   component: Task,
 };
 
-const Template = ({ frontendId, task, trigger, mutate }) => {
-  return <Task frontendId={frontendId} task={task} trigger={trigger} mutate={mutate} />;
+const Template = ({ frontendId, task, isLoading, completeTask, skipTask }) => {
+  return (
+    <Task frontendId={frontendId} task={task} isLoading={isLoading} completeTask={completeTask} skipTask={skipTask} />
+  );
 };
 
 export const Default = Template.bind({});
@@ -23,10 +25,11 @@ Default.args = {
     type: "label_prompter_reply",
     valid_labels: ["spam", "fails_task"],
   },
-  trigger: (id, update_type, content) => {
+  isLoading: false,
+  completeTask: (id, update_type, content) => {
     console.log(content);
   },
-  mutate: () => {
-    console.log("mutate");
+  skipTask: () => {
+    console.log("skip");
   },
 };

website/src/components/Tasks/Task/Task.tsx

@@ -10,6 +10,7 @@ import { UnchangedWarning } from "src/components/Tasks/UnchangedWarning";
 import { post } from "src/lib/api";
 import { getTypeSafei18nKey } from "src/lib/i18n";
 import { BaseTask, TaskContent, TaskReplyValidity } from "src/types/Task";
+import { CreateTaskType, KnownTaskType, LabelTaskType, RankTaskType } from "src/types/Tasks";
 import useSWRMutation from "swr/mutation";
 
 interface EditMode {
@@ -62,7 +63,15 @@ export interface TaskSurveyProps<TaskType extends BaseTask, T> {
   onValidityChanged: (validity: TaskReplyValidity) => void;
 }
 
-export const Task = ({ frontendId, task, trigger, mutate }) => {
+interface TaskProps {
+  frontendId: string;
+  task: KnownTaskType;
+  isLoading: boolean;
+  completeTask: (TaskContent) => void;
+  skipTask: () => void;
+}
+
+export const Task = ({ frontendId, task, isLoading, completeTask, skipTask }: TaskProps) => {
   const { t } = useTranslation("tasks");
   const [taskStatus, taskEvent] = useReducer(
     (
@@ -117,14 +126,13 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
 
   const rootEl = useRef<HTMLDivElement>(null);
 
-  const taskType = useMemo(
-    () => TaskInfos.find((taskType) => taskType.type === task.type && taskType.mode === task.mode),
-    [task.type, task.mode]
-  );
+  const taskType = useMemo(() => {
+    return TaskInfos.find((taskType) => taskType.type === task.type);
+  }, [task.type]);
 
   const { trigger: sendRejection } = useSWRMutation("/api/reject_task", post, {
     onSuccess: async () => {
-      mutate();
+      skipTask();
     },
   });
 
@@ -144,7 +152,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
 
   const submitResponse = () => {
     if (taskStatus.mode === "REVIEW") {
-      trigger({
+      completeTask({
         id: frontendId,
         update_type: taskType.update_type,
         content: replyContent.current,
@@ -159,7 +167,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
       case TaskCategory.Create:
         return (
           <CreateTask
-            task={task}
+            task={task as CreateTaskType}
             taskType={taskType}
             isEditable={taskStatus.mode === "EDIT"}
             isDisabled={taskStatus.mode === "SUBMITTED"}
@@ -170,7 +178,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
       case TaskCategory.Evaluate:
         return (
           <EvaluateTask
-            task={task}
+            task={task as RankTaskType}
             taskType={taskType}
             isEditable={taskStatus.mode === "EDIT"}
             isDisabled={taskStatus.mode === "SUBMITTED"}
@@ -181,7 +189,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
       case TaskCategory.Label:
         return (
           <LabelTask
-            task={task}
+            task={task as LabelTaskType}
             taskType={taskType}
             isEditable={taskStatus.mode === "EDIT"}
             isDisabled={taskStatus.mode === "SUBMITTED"}
@@ -198,6 +206,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
       <TaskControls
         task={task}
         taskStatus={taskStatus}
+        isLoading={isLoading}
         onEdit={() => taskEvent({ action: "RETURN_EDIT" })}
         onReview={() => taskEvent({ action: "REVIEW" })}
         onSubmit={submitResponse}

website/src/hooks/tasks/useGenericTaskAPI.tsx

@@ -1,26 +1,43 @@
 import { useState } from "react";
 import { get, post } from "src/lib/api";
-import { BaseTask, TaskResponse, TaskType as TaskTypeEnum } from "src/types/Task";
+import { TaskApiHook } from "src/types/Hooks";
+import { BaseTask, TaskAvailableResponse, TaskResponse, TaskType as TaskTypeEnum } from "src/types/Task";
 import useSWRImmutable from "swr/immutable";
 import useSWRMutation from "swr/mutation";
 
-export const useGenericTaskAPI = <TaskType extends BaseTask>(taskType: TaskTypeEnum) => {
-  type ConcreteTaskResponse = TaskResponse<TaskType>;
+export const useGenericTaskAPI = <TaskType extends BaseTask>(taskType: TaskTypeEnum): TaskApiHook<TaskType> => {
+  const [response, setReponse] = useState<TaskResponse<TaskType>>({ taskAvailability: "AWAITING_INITIAL" });
+  const [isLoading, setIsLoading] = useState(true);
 
-  const [tasks, setTasks] = useState<ConcreteTaskResponse[]>([]);
+  const { mutate: requestNewTask } = useSWRImmutable<TaskAvailableResponse<TaskType>>(
+    "/api/new_task/" + taskType,
+    get,
+    {
+      onSuccess: (response) => {
+        setIsLoading(false);
+        setReponse({ taskAvailability: "AVAILABLE", ...response });
+      },
+      onError: () => {
+        // We could check for code 503 here for truely unavailable, but we need to do something with other errors anyway.
+        setIsLoading(false);
+        setReponse({ taskAvailability: "NONE_AVAILABLE" });
+      },
+      revalidateOnMount: true,
+      dedupingInterval: 500,
+    }
+  );
 
-  const { isLoading, mutate, error } = useSWRImmutable<ConcreteTaskResponse>("/api/new_task/" + taskType, get, {
-    onSuccess: (data) => setTasks([data]),
-    revalidateOnMount: true,
-    dedupingInterval: 500,
-  });
-
-  const { trigger } = useSWRMutation("/api/update_task", post, {
-    onSuccess: async (newTask: ConcreteTaskResponse) => {
-      setTasks((oldTasks) => [...oldTasks, newTask]);
-      mutate();
+  const { trigger: completeTask } = useSWRMutation<TaskAvailableResponse<TaskType>>("/api/update_task", post, {
+    onSuccess: () => {
+      setIsLoading(true);
+      requestNewTask();
+    },
+    onError: () => {
+      // We could check for code 503 here for truely unavailable, but we need to do something with other errors anyway.
+      setIsLoading(false);
+      setReponse({ taskAvailability: "NONE_AVAILABLE" });
     },
   });
 
-  return { tasks, isLoading, trigger, error, reset: mutate };
+  return { response, isLoading, completeTask, skipTask: requestNewTask };
 };

website/src/pages/api/new_task/[task_type].ts

@@ -1,4 +1,6 @@
 import { withoutRole } from "src/lib/auth";
+import { ERROR_CODES } from "src/lib/constants";
+import { OasstError } from "src/lib/oasst_api_client";
 import { createApiClientFromUser } from "src/lib/oasst_client_factory";
 import prisma from "src/lib/prismadb";
 import { getBackendUserCore, getUserLanguage } from "src/lib/users";
@@ -22,8 +24,12 @@ const handler = withoutRole("banned", async (req, res, token) => {
   try {
     task = await oasstApiClient.fetchTask(task_type as string, user, userLanguage);
   } catch (err) {
-    console.error(err);
-    res.status(500).json(err);
+    if (err instanceof OasstError && err.errorCode === ERROR_CODES.TASK_REQUESTED_TYPE_NOT_AVAILABLE) {
+      res.status(503).json({});
+    } else {
+      console.error(err);
+      res.status(500).json(err);
+    }
     return;
   }
 

website/src/types/Hooks.ts

@@ -1,26 +1,16 @@
-import { MutatorCallback, MutatorOptions } from "swr";
+import { BaseTask, TaskContent, TaskResponse, TaskType } from "src/types/Task";
 
-import { BaseTask, TaskResponse, TaskType } from "./Task";
+interface TaskInteraction {
+  id: string;
+  update_type: string;
+  content: TaskContent;
+}
 
-type ConcreteTaskResponse = TaskResponse<BaseTask>;
-type TaskError = { errorCode: number; message: string };
-
-type Trigger = (
-  extraArgument?: unknown,
-  options?: MutatorOptions<ConcreteTaskResponse>
-) => Promise<ConcreteTaskResponse>;
-
-type Reset = (
-  data?: ConcreteTaskResponse | Promise<ConcreteTaskResponse> | MutatorCallback<ConcreteTaskResponse>,
-  opts?: boolean | MutatorOptions<ConcreteTaskResponse>
-) => Promise<ConcreteTaskResponse>;
-
-type TaskAPIHook = {
-  tasks: TaskResponse<BaseTask>[];
+export type TaskApiHook<Task extends BaseTask> = {
+  response: TaskResponse<Task>;
   isLoading: boolean;
-  error: TaskError;
-  trigger: Trigger;
-  reset: Reset;
+  completeTask: (interaction: TaskInteraction) => void;
+  skipTask: () => void;
 };
 
-export type TaskApiHooks = Record<TaskType, (args: TaskType) => TaskAPIHook>;
+export type TaskApiHooks = Record<TaskType, (args: TaskType) => TaskApiHook<BaseTask>>;

website/src/types/Task.ts

@@ -29,12 +29,26 @@ export interface BaseTask {
   type: TaskType;
 }
 
-export interface TaskResponse<Task extends BaseTask> {
+export interface TaskAvailableResponse<Task extends BaseTask> {
   id: string;
   userId: string;
   task: Task;
 }
 
+interface TaskAvailable<Task extends BaseTask> extends TaskAvailableResponse<Task> {
+  taskAvailability: "AVAILABLE";
+}
+
+interface AwaitingInitialTask {
+  taskAvailability: "AWAITING_INITIAL";
+}
+
+interface NoTaskAvailable {
+  taskAvailability: "NONE_AVAILABLE";
+}
+
+export type TaskResponse<Task extends BaseTask> = TaskAvailable<Task> | AwaitingInitialTask | NoTaskAvailable;
+
 export type TaskReplyValidity = "DEFAULT" | "VALID" | "INVALID";
 
 export type AvailableTasks = { [taskType in TaskType]: number };

website/src/types/Tasks.ts

@@ -16,6 +16,8 @@ export interface CreatePrompterReplyTask extends BaseTask {
   conversation: Conversation;
 }
 
+export type CreateTaskType = CreateInitialPromptTask | CreateAssistantReplyTask | CreatePrompterReplyTask;
+
 export interface RankInitialPromptsTask extends BaseTask {
   type: TaskType.rank_initial_prompts;
   prompts: string[];
@@ -33,6 +35,8 @@ export interface RankPrompterRepliesTask extends BaseTask {
   replies: string[];
 }
 
+export type RankTaskType = RankInitialPromptsTask | RankAssistantRepliesTask | RankPrompterRepliesTask;
+
 export interface Label {
   display_text: string;
   help_text: string;
@@ -68,4 +72,6 @@ export interface LabelInitialPromptTask extends BaseLabelTask {
   prompt: string;
 }
 
-export type LabelTaskType = LabelAssistantReplyTask | LabelPrompterReplyTask | LabelInitialPromptTask;
+export type LabelTaskType = LabelInitialPromptTask | LabelAssistantReplyTask | LabelPrompterReplyTask;
+
+export type KnownTaskType = CreateTaskType | RankTaskType | LabelTaskType;