Merging with main

This commit is contained in:
Keith Stevens
2023-01-11 08:47:16 +09:00
41 changed files with 457 additions and 515 deletions
+14 -12
View File
@@ -4,21 +4,22 @@ import clsx from "clsx";
import { SkipButton } from "src/components/Buttons/Skip";
import { SubmitButton } from "src/components/Buttons/Submit";
import { TaskInfo } from "src/components/TaskInfo/TaskInfo";
import { TaskStatus } from "src/components/Tasks/Task";
export interface TaskControlsProps {
// we need a task type
// eslint-disable-next-line @typescript-eslint/no-explicit-any
tasks: any[];
task: any;
className?: string;
onSubmitResponse: (task: { id: string }) => void;
onSkipTask: (task: { id: string }, reason: string) => void;
taskStatus: TaskStatus;
onSubmit: () => void;
onSkip: (reason: string) => void;
onNextTask: () => void;
}
export const TaskControls = (props: TaskControlsProps) => {
const { colorMode } = useColorMode();
const isLightMode = colorMode === "light";
const endTask = props.tasks[props.tasks.length - 1];
return (
<section
className={clsx(
@@ -30,15 +31,16 @@ export const TaskControls = (props: TaskControlsProps) => {
}
)}
>
<TaskInfo id={props.tasks[0].id} output="Submit your answer" />
<TaskInfo id={props.task.id} output="Submit your answer" />
<Flex justify="center" ml="auto" gap={2}>
<SkipButton
onSkip={(reason: string) => {
props.onSkipTask(props.tasks[0], reason);
}}
/>
{endTask.task.type !== "task_done" ? (
<SubmitButton colorScheme="blue" data-cy="submit" onClick={() => props.onSubmitResponse(props.tasks[0])}>
<SkipButton onSkip={props.onSkip} disabled={props.taskStatus === "SUBMITTED"} />
{props.taskStatus !== "SUBMITTED" ? (
<SubmitButton
colorScheme="blue"
data-cy="submit"
disabled={props.taskStatus === "NOT_SUBMITTABLE"}
onClick={props.onSubmit}
>
Submit
</SubmitButton>
) : (
@@ -1,60 +0,0 @@
import {
Button,
Flex,
Modal,
ModalBody,
ModalCloseButton,
ModalContent,
ModalFooter,
ModalHeader,
ModalOverlay,
useDisclosure,
} from "@chakra-ui/react";
import { TaskControls, TaskControlsProps } from "src/components/Survey/TaskControls";
interface TaskControlsOverridableProps extends TaskControlsProps {
isValid: boolean;
prepareForSubmit: () => void;
}
export const TaskControlsOverridable = (props: TaskControlsOverridableProps) => {
const { isValid, onSubmitResponse, ...rest } = props;
const { isOpen: isModalOpen, onOpen: onOpenModal, onClose: onModalClose } = useDisclosure();
const unchangedResponsePrompt = () => {
onOpenModal();
// Ideally this happens when the user clicks submit, but we can't
// reliably wait for it to be executed before submitting the response
// without significant refactoring.
// As a result, modal will only display once even if the user doesn't proceed
props.prepareForSubmit();
};
const onSubmitResponseOverride = () => {
onSubmitResponse(props.tasks[0]);
onModalClose();
};
return (
<>
<Modal isOpen={isModalOpen} onClose={onModalClose} isCentered>
<ModalOverlay />
<ModalContent>
<ModalCloseButton />
<ModalHeader>Order Unchanged</ModalHeader>
<ModalBody>You have not changed the order of the prompts. Are you sure you would like to submit?</ModalBody>
<ModalFooter>
<Flex justify="center" ml="auto" gap={2}>
<Button variant={"ghost"} onClick={onModalClose}>
Cancel
</Button>
<Button onClick={onSubmitResponseOverride}>Submit anyway</Button>
</Flex>
</ModalFooter>
</ModalContent>
</Modal>
<TaskControls onSubmitResponse={isValid ? props.onSubmitResponse : unchangedResponsePrompt} {...rest} />
</>
);
};
+21 -54
View File
@@ -1,67 +1,34 @@
import { useState } from "react";
import { Messages } from "src/components/Messages";
import { TaskControls } from "src/components/Survey/TaskControls";
import { TrackedTextarea } from "src/components/Survey/TrackedTextarea";
import { TwoColumnsWithCards } from "src/components/Survey/TwoColumnsWithCards";
import { TaskInfo } from "src/components/Tasks/TaskTypes";
import { TaskSurveyProps } from "src/components/Tasks/Task";
export interface CreateTaskProps {
// we need a task type
// eslint-disable-next-line @typescript-eslint/no-explicit-any
tasks: any[];
taskType: TaskInfo;
trigger: (update: { id: string; update_type: string; content: { text: string } }) => void;
onSkipTask: (task: { id: string }, reason: string) => void;
onNextTask: () => void;
mainBgClasses: string;
}
export const CreateTask = ({ tasks, taskType, trigger, onSkipTask, onNextTask, mainBgClasses }: CreateTaskProps) => {
const task = tasks[0].task;
export const CreateTask = ({ task, taskType, onReplyChanged }: TaskSurveyProps<{ text: string }>) => {
const [inputText, setInputText] = useState("");
const submitResponse = (task: { id: string }) => {
const text = inputText.trim();
trigger({
id: task.id,
update_type: "text_reply_to_message",
content: {
text,
},
});
};
const textChangeHandler = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
setInputText(event.target.value);
const text = event.target.value;
onReplyChanged({ content: { text }, state: "VALID" });
setInputText(text);
};
return (
<div className={`p-12 ${mainBgClasses}`}>
<TwoColumnsWithCards>
<>
<h5 className="text-lg font-semibold">{taskType.label}</h5>
<p className="text-lg py-1">{taskType.overview}</p>
{task.conversation ? <Messages messages={task.conversation.messages} post_id={task.id} /> : null}
</>
<>
<h5 className="text-lg font-semibold">{taskType.instruction}</h5>
<TrackedTextarea
text={inputText}
onTextChange={textChangeHandler}
thresholds={{ low: 20, medium: 40, goal: 50 }}
textareaProps={{ placeholder: "Reply..." }}
/>
</>
</TwoColumnsWithCards>
<TaskControls
tasks={tasks}
onSubmitResponse={submitResponse}
onSkipTask={(task, reason) => {
setInputText("");
onSkipTask(task, reason);
}}
onNextTask={onNextTask}
/>
</div>
<TwoColumnsWithCards>
<>
<h5 className="text-lg font-semibold">{taskType.label}</h5>
<p className="text-lg py-1">{taskType.overview}</p>
{task.conversation ? <Messages messages={task.conversation.messages} post_id={task.id} /> : null}
</>
<>
<h5 className="text-lg font-semibold">{taskType.instruction}</h5>
<TrackedTextarea
text={inputText}
onTextChange={textChangeHandler}
thresholds={{ low: 20, medium: 40, goal: 50 }}
textareaProps={{ placeholder: "Reply..." }}
/>
</>
</TwoColumnsWithCards>
);
};
+29 -52
View File
@@ -1,62 +1,39 @@
import { useState } from "react";
import { useEffect } from "react";
import { MessageTable } from "src/components/Messages/MessageTable";
import { Sortable } from "src/components/Sortable/Sortable";
import { SurveyCard } from "src/components/Survey/SurveyCard";
import { TaskControlsOverridable } from "src/components/Survey/TaskControlsOverridable";
import { TaskSurveyProps } from "src/components/Tasks/Task";
import { MessageTable } from "../Messages/MessageTable";
export interface EvaluateTaskProps {
// we need a task type
// eslint-disable-next-line @typescript-eslint/no-explicit-any
tasks: any[];
trigger: (update: { id: string; update_type: string; content: { ranking: number[] } }) => void;
onSkipTask: (task: { id: string }, reason: string) => void;
onNextTask: () => void;
mainBgClasses: string;
}
export const EvaluateTask = ({ tasks, trigger, onSkipTask, onNextTask, mainBgClasses }: EvaluateTaskProps) => {
const [ranking, setRanking] = useState<number[]>([]);
const submitResponse = (task) => {
trigger({
id: task.id,
update_type: "message_ranking",
content: {
ranking,
},
});
};
let messages = null;
if (tasks[0].task.conversation) {
messages = tasks[0].task.conversation.messages;
export const EvaluateTask = ({ task, onReplyChanged }: TaskSurveyProps<{ ranking: number[] }>) => {
let messages = [];
if (task.conversation) {
messages = task.conversation.messages;
messages = messages.map((message, index) => ({ ...message, id: index }));
}
const sortables = tasks[0].task.replies ? "replies" : "prompts";
useEffect(() => {
const conversationMsgs = task.conversation ? task.conversation.messages : [];
const defaultRanking = conversationMsgs.map((message, index) => index);
onReplyChanged({
content: { ranking: defaultRanking },
state: "DEFAULT",
});
}, [task.conversation, onReplyChanged]);
const onRank = (newRanking: number[]) => {
onReplyChanged({ content: { ranking: newRanking }, state: "VALID" });
};
const sortables = task.replies ? "replies" : "prompts";
return (
<div className={`p-12 ${mainBgClasses}`}>
<SurveyCard className="max-w-7xl mx-auto h-fit mb-24">
<h5 className="text-lg font-semibold mb-4">Instructions</h5>
<p className="text-lg py-1">
Given the following {sortables}, sort them from best to worst, best being first, worst being last.
</p>
{messages ? <MessageTable messages={messages} /> : null}
<Sortable items={tasks[0].task[sortables]} onChange={setRanking} className="my-8" />
</SurveyCard>
<TaskControlsOverridable
tasks={tasks}
isValid={ranking.length === tasks[0].task[sortables].length}
prepareForSubmit={() => setRanking(tasks[0].task[sortables].map((_, idx) => idx))}
onSubmitResponse={submitResponse}
onSkipTask={(task, reason) => {
setRanking([]);
onSkipTask(task, reason);
}}
onNextTask={onNextTask}
/>
</div>
<SurveyCard className="max-w-7xl mx-auto h-fit mb-24">
<h5 className="text-lg font-semibold mb-4">Instructions</h5>
<p className="text-lg py-1">
Given the following {sortables}, sort them from best to worst, best being first, worst being last.
</p>
<MessageTable messages={messages} />
<Sortable items={task[sortables]} onChange={onRank} className="my-8" />
</SurveyCard>
);
};
+45 -36
View File
@@ -1,44 +1,56 @@
import { Grid, Slider, SliderFilledTrack, SliderThumb, SliderTrack } from "@chakra-ui/react";
import { useColorMode } from "@chakra-ui/react";
import { ReactNode, useEffect, useId, useMemo, useState } from "react";
import { useEffect, useId, useState } from "react";
import { MessageView } from "src/components/Messages";
import { MessageTable } from "src/components/Messages/MessageTable";
import { TwoColumnsWithCards } from "src/components/Survey/TwoColumnsWithCards";
import { TaskSurveyProps } from "src/components/Tasks/Task";
import { TaskType } from "src/types/Task";
import { colors } from "styles/Theme/colors";
export const LabelTask = ({
title,
desc,
messages,
inputs,
controls,
}: {
title: string;
desc: string;
messages: ReactNode;
inputs: ReactNode;
controls: ReactNode;
}) => {
const { colorMode } = useColorMode();
const mainBgClasses = colorMode === "light" ? "bg-slate-300 text-gray-800" : "bg-slate-900 text-white";
task,
taskType,
onReplyChanged,
}: TaskSurveyProps<{ text: string; labels: { [k: string]: number }; message_id: string }>) => {
const [sliderValues, setSliderValues] = useState<number[]>([]);
const card = useMemo(
() => (
<>
<h5 className="text-lg font-semibold">{title}</h5>
<p className="text-lg py-1">{desc}</p>
{messages}
</>
),
[title, desc, messages]
);
const valid_labels = task.valid_labels;
useEffect(() => {
onReplyChanged({ content: { labels: {}, text: task.reply, message_id: task.message_id }, state: "DEFAULT" });
}, [task.reply, task.message_id, onReplyChanged]);
const onSliderChange = (values: number[]) => {
console.assert(valid_labels.length === sliderValues.length);
const labels = Object.fromEntries(valid_labels.map((label, i) => [label, sliderValues[i]]));
onReplyChanged({ content: { labels, text: task.reply, message_id: task.message_id }, state: "VALID" });
setSliderValues(values);
};
return (
<div className={`p-12 ${mainBgClasses}`}>
<TwoColumnsWithCards>
{card}
{inputs}
</TwoColumnsWithCards>
{controls}
</div>
<TwoColumnsWithCards>
<>
<h5 className="text-lg font-semibold">{taskType.label}</h5>
<p className="text-lg py-1">{taskType.overview}</p>
{task.conversation ? (
<MessageTable
messages={[
...(task.conversation ? task.conversation.messages : []),
{
text: task.reply,
is_assistant: task.type === TaskType.label_assistant_reply,
message_id: task.message_id,
},
]}
/>
) : (
<MessageView text={task.prompt} is_assistant={false} message_id={task.message_id} />
)}
</>
<LabelSliderGroup labelIDs={task.valid_labels} onChange={onSliderChange} />
</TwoColumnsWithCards>
);
};
@@ -51,10 +63,6 @@ interface LabelSliderGroupProps {
export const LabelSliderGroup = ({ labelIDs, onChange }: LabelSliderGroupProps) => {
const [sliderValues, setSliderValues] = useState<number[]>(Array.from({ length: labelIDs.length }).map(() => 0));
useEffect(() => {
onChange(sliderValues);
}, [sliderValues, onChange]);
return (
<Grid templateColumns="auto 1fr" rowGap={1} columnGap={3}>
{labelIDs.map((labelId, idx) => (
@@ -65,6 +73,7 @@ export const LabelSliderGroup = ({ labelIDs, onChange }: LabelSliderGroupProps)
sliderHandler={(sliderValue) => {
const newState = sliderValues.slice();
newState[idx] = sliderValue;
onChange(sliderValues);
setSliderValues(newState);
}}
/>
+97 -28
View File
@@ -1,11 +1,35 @@
import { useColorMode } from "@chakra-ui/react";
import { useRef, useState } from "react";
import { TaskControls } from "src/components/Survey/TaskControls";
import { CreateTask } from "src/components/Tasks/CreateTask";
import { EvaluateTask } from "src/components/Tasks/EvaluateTask";
import { TaskCategory, TaskTypes } from "src/components/Tasks/TaskTypes";
import { LabelTask } from "src/components/Tasks/LabelTask";
import { TaskCategory, TaskInfo, TaskTypes } from "src/components/Tasks/TaskTypes";
import { UnchangedWarning } from "src/components/Tasks/UnchangedWarning";
import poster from "src/lib/poster";
import { TaskContent } from "src/types/Task";
import { TaskReplyState } from "src/types/TaskReplyState";
import useSWRMutation from "swr/mutation";
export const Task = ({ tasks, trigger, mutate, mainBgClasses }) => {
const task = tasks[0].task;
export type TaskStatus = "NOT_SUBMITTABLE" | "DEFAULT" | "SUBMITABLE" | "SUBMITTED";
export interface TaskSurveyProps<T> {
// we need a task type
// eslint-disable-next-line @typescript-eslint/no-explicit-any
task: any;
taskType: TaskInfo;
onReplyChanged: (state: TaskReplyState<T>) => void;
}
export const Task = ({ task, trigger, mutate }) => {
const [taskStatus, setTaskStatus] = useState<TaskStatus>("NOT_SUBMITTABLE");
const replyContent = useRef<TaskContent>(null);
const [showUnchangedWarning, setShowUnchangedWarning] = useState(false);
const taskType = TaskTypes.find((taskType) => taskType.type === task.type);
const { colorMode } = useColorMode();
const mainBgClasses = colorMode === "light" ? "bg-slate-300 text-gray-800" : "bg-slate-900 text-white";
const { trigger: sendRejection } = useSWRMutation("/api/reject_task", poster, {
onSuccess: async () => {
@@ -13,40 +37,85 @@ export const Task = ({ tasks, trigger, mutate, mainBgClasses }) => {
},
});
const rejectTask = (task: { id: string }, reason: string) => {
const rejectTask = (reason: string) => {
sendRejection({
id: task.id,
reason,
});
};
function taskTypeComponent(type) {
const taskType = TaskTypes.find((taskType) => taskType.type === type);
const category = taskType.category;
switch (category) {
const onReplyChanged = useRef((state: TaskReplyState<TaskContent>) => {
if (taskStatus === "SUBMITTED") return;
replyContent.current = state?.content;
if (state === null) {
if (taskStatus !== "NOT_SUBMITTABLE") setTaskStatus("NOT_SUBMITTABLE");
} else if (state.state === "DEFAULT") {
if (taskStatus !== "DEFAULT") setTaskStatus("DEFAULT");
} else if (state.state === "VALID") {
if (taskStatus !== "SUBMITABLE") setTaskStatus("SUBMITABLE");
}
}).current;
const submitResponse = () => {
switch (taskStatus) {
case "NOT_SUBMITTABLE":
return;
case "DEFAULT":
setShowUnchangedWarning(true);
break;
case "SUBMITABLE": {
trigger({
id: task.id,
update_type: taskType.update_type,
content: replyContent.current,
});
setTaskStatus("SUBMITTED");
break;
}
case "SUBMITTED":
return;
}
};
function taskTypeComponent() {
switch (taskType.category) {
case TaskCategory.Create:
return (
<CreateTask
tasks={tasks}
trigger={trigger}
onSkipTask={rejectTask}
onNextTask={mutate}
taskType={taskType}
mainBgClasses={mainBgClasses}
/>
);
return <CreateTask key={task.id} task={task} taskType={taskType} onReplyChanged={onReplyChanged} />;
case TaskCategory.Evaluate:
return (
<EvaluateTask
tasks={tasks}
trigger={trigger}
onSkipTask={rejectTask}
onNextTask={mutate}
mainBgClasses={mainBgClasses}
/>
);
return <EvaluateTask key={task.id} task={task} taskType={taskType} onReplyChanged={onReplyChanged} />;
case TaskCategory.Label:
return <LabelTask key={task.id} task={task} taskType={taskType} onReplyChanged={onReplyChanged} />;
}
}
return taskTypeComponent(task.type);
return (
<div className={`p-12 ${mainBgClasses}`}>
{taskTypeComponent()}
<TaskControls
task={task}
taskStatus={taskStatus}
onSubmit={submitResponse}
onSkip={rejectTask}
onNextTask={mutate}
/>
<UnchangedWarning
show={showUnchangedWarning}
title={taskType.unchanged_title || "No changes"}
message={taskType.unchanged_message || "Are you sure you would like to submit?"}
onClose={() => setShowUnchangedWarning(false)}
onSubmitAnyway={() => {
if (taskStatus === "DEFAULT") {
trigger({
id: task.id,
update_type: taskType.update_type,
content: replyContent.current,
});
setTaskStatus("SUBMITTED");
setShowUnchangedWarning(false);
}
}}
/>
</div>
);
};
@@ -12,6 +12,9 @@ export interface TaskInfo {
type: string;
overview?: string;
instruction?: string;
update_type: string;
unchanged_title?: string;
unchanged_message?: string;
}
export const TaskTypes: TaskInfo[] = [
@@ -24,6 +27,7 @@ export const TaskTypes: TaskInfo[] = [
type: "initial_prompt",
overview: "Create an initial message to send to the assistant",
instruction: "Provide the initial prompt",
update_type: "text_reply_to_message",
},
{
label: "Reply as User",
@@ -33,6 +37,7 @@ export const TaskTypes: TaskInfo[] = [
type: "prompter_reply",
overview: "Given the following conversation, provide an adequate reply",
instruction: "Provide the user`s reply",
update_type: "text_reply_to_message",
},
{
label: "Reply as Assistant",
@@ -42,6 +47,7 @@ export const TaskTypes: TaskInfo[] = [
type: "assistant_reply",
overview: "Given the following conversation, provide an adequate reply",
instruction: "Provide the assistant`s reply",
update_type: "text_reply_to_message",
},
// evaluate
{
@@ -50,6 +56,9 @@ export const TaskTypes: TaskInfo[] = [
desc: "Help Open Assistant improve its responses to conversations with other users.",
pathname: "/evaluate/rank_user_replies",
type: "rank_prompter_replies",
update_type: "message_ranking",
unchanged_title: "Order Unchanged",
unchanged_message: "You have not changed the order of the prompts. Are you sure you would like to submit?",
},
{
label: "Rank Assistant Replies",
@@ -57,6 +66,9 @@ export const TaskTypes: TaskInfo[] = [
category: TaskCategory.Evaluate,
pathname: "/evaluate/rank_assistant_replies",
type: "rank_assistant_replies",
update_type: "message_ranking",
unchanged_title: "Order Unchanged",
unchanged_message: "You have not changed the order of the prompts. Are you sure you would like to submit?",
},
{
label: "Rank Initial Prompts",
@@ -64,6 +76,9 @@ export const TaskTypes: TaskInfo[] = [
category: TaskCategory.Evaluate,
pathname: "/evaluate/rank_initial_prompts",
type: "rank_initial_prompts",
update_type: "message_ranking",
unchanged_title: "Order Unchanged",
unchanged_message: "You have not changed the order of the prompts. Are you sure you would like to submit?",
},
// label
{
@@ -71,20 +86,26 @@ export const TaskTypes: TaskInfo[] = [
desc: "Provide labels for a prompt.",
category: TaskCategory.Label,
pathname: "/label/label_initial_prompt",
overview: "Provide labels for the following prompt",
type: "label_initial_prompt",
update_type: "text_labels",
},
{
label: "Label Prompter Reply",
desc: "Provide labels for a prompt.",
category: TaskCategory.Label,
pathname: "/label/label_prompter_reply",
overview: "Given the following discussion, provide labels for the final promp",
type: "label_prompter_reply",
update_type: "text_labels",
},
{
label: "Label Assistant Reply",
desc: "Provide labels for a prompt.",
category: TaskCategory.Label,
pathname: "/label/label_assistant_reply",
overview: "Given the following discussion, provide labels for the final prompt.",
type: "label_assistant_reply",
update_type: "text_labels",
},
];
@@ -0,0 +1,42 @@
import {
Button,
Flex,
Modal,
ModalBody,
ModalCloseButton,
ModalContent,
ModalFooter,
ModalHeader,
ModalOverlay,
} from "@chakra-ui/react";
interface UnchangedWarningProps {
show: boolean;
title: string;
message: string;
onClose: () => void;
onSubmitAnyway: () => void;
}
export const UnchangedWarning = (props: UnchangedWarningProps) => {
return (
<>
<Modal isOpen={props.show} onClose={props.onClose} isCentered>
<ModalOverlay />
<ModalContent>
<ModalCloseButton />
<ModalHeader>{props.title}</ModalHeader>
<ModalBody>{props.message}</ModalBody>
<ModalFooter>
<Flex justify="center" ml="auto" gap={2}>
<Button variant={"ghost"} onClick={props.onClose}>
Cancel
</Button>
<Button onClick={props.onSubmitAnyway}>Submit anyway</Button>
</Flex>
</ModalFooter>
</ModalContent>
</Modal>
</>
);
};