wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-06-27 16:10:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

website: Switch to likert style labelling

Browse Source
This commit is contained in:
Adrian Cowan
2023-01-21 15:43:52 +11:00
parent 274d03c5f0
commit 007773a3f5
19 changed files with 481 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files
website/cypress/e2e/tasks/label_assistant_reply.cy.ts
+26
View File
@@ -0,0 +1,26 @@
describe("labeling assistant replies", () => {
it("completes the current task on submit and on request shows a new task", () => {
cy.signInWithEmail("cypress@example.com");
cy.visit("/label/label_assistant_reply");
cy.get('[data-cy="task"]')
.invoke("attr", "data-task-type")
.then((type) => {
cy.log("Task type", type);
// For specific task pages the no task available result is normal.
if (type === undefined) return;
cy.get('[data-cy="label-options"]').each((label) => {
// Click the 4th option
cy.wrap(label).find('[aria-roledescription="radio"]').eq(3).click();
});
cy.get('[data-cy="review"]').click();
cy.get('[data-cy="submit"]').click();
});
});
});
export {};
website/cypress/e2e/tasks/label_initial_prompt.cy.ts
+26
View File
@@ -0,0 +1,26 @@
describe("labeling initial prompts", () => {
it("completes the current task on submit and on request shows a new task", () => {
cy.signInWithEmail("cypress@example.com");
cy.visit("/label/label_initial_prompt");
cy.get('[data-cy="task"]')
.invoke("attr", "data-task-type")
.then((type) => {
cy.log("Task type", type);
// For specific task pages the no task available result is normal.
if (type === undefined) return;
cy.get('[data-cy="label-options"]').each((label) => {
// Click the 4th option
cy.wrap(label).find('[aria-roledescription="radio"]').eq(3).click();
});
cy.get('[data-cy="review"]').click();
cy.get('[data-cy="submit"]').click();
});
});
});
export {};
website/cypress/e2e/tasks/label_prompter_reply.cy.ts
+26
View File
@@ -0,0 +1,26 @@
describe("labeling prompter replies", () => {
it("completes the current task on submit and on request shows a new task", () => {
cy.signInWithEmail("cypress@example.com");
cy.visit("/label/label_prompter_reply");
cy.get('[data-cy="task"]')
.invoke("attr", "data-task-type")
.then((type) => {
cy.log("Task type", type);
// For specific task pages the no task available result is normal.
if (type === undefined) return;
cy.get('[data-cy="label-options"]').each((label) => {
// Click the 4th option
cy.wrap(label).find('[aria-roledescription="radio"]').eq(3).click();
});
cy.get('[data-cy="review"]').click();
cy.get('[data-cy="submit"]').click();
});
});
});
export {};
website/cypress/e2e/tasks/random.cy.ts
+14 -36
View File
@@ -44,47 +44,25 @@ describe("handles random tasks", () => {
break;
}
case "label-task": {
cy.get('[data-cy="label-group-item"]')
.first()
.invoke("attr", "data-label-type")
.then((label_type) => {
const parent = cy
.get('[data-cy="label-group-item"]')
.first();
cy.log("Label type", label_type);
cy.get('[data-cy="label-options"]').each((label) => {
// Click the 4th option
cy.wrap(label)
.find('[aria-roledescription="radio"]')
.eq(3)
.click();
});
switch (label_type) {
case "slider": {
// Clicking on the slider will set the value to about the middle where it clicks
parent
.get('[aria-roledescription="slider"]')
.first()
.click();
cy.get('[data-cy="review"]').click();
cy.get('[data-cy="review"]').click();
cy.get('[data-cy="submit"]').click();
break;
}
case "radio": {
// Clicking on the slider will set the value to about the middle where it clicks
parent
.get('[aria-roledescription="radio-button"]')
.last()
.click();
cy.get('[data-cy="review"]').click();
cy.get('[data-cy="submit"]').click();
break;
}
}
});
cy.get('[data-cy="submit"]').click();
break;
}
case undefined: {
throw new Error(
"No tasks available, but at least create initial prompt expected"
);
}
default:
throw new Error(`Unexpected task type: ${type}`);
}
website/src/components/Buttons/LikertButtons.tsx
+38
View File
@@ -0,0 +1,38 @@
import { Button, SimpleGrid } from "@chakra-ui/react";
import { PropsWithChildren, ReactNode } from "react";
export const LikertButtons = ({
isDisabled,
options,
value,
onChange,
"data-cy": dataCy,
}: PropsWithChildren<{
isDisabled: boolean;
options: ReactNode[];
value: number;
onChange: (value: number) => void;
"data-cy"?: string;
}>) => {
return (
<SimpleGrid aria-roledescription="radiogroup" columns={options.length} spacing={[1, 4]} data-cy={dataCy}>
{options.map((option, idx) => {
const indexValue = idx / (options.length - 1);
return (
<Button
aria-roledescription="radio"
aria-checked={indexValue === value}
key={idx}
onClick={() => {
onChange(indexValue === value ? null : indexValue);
}}
isDisabled={isDisabled}
isActive={indexValue === value}
>
{option}
</Button>
);
})}
</SimpleGrid>
);
};
website/src/components/EmptyState.tsx
+3 -2
View File
@@ -5,13 +5,14 @@ import NextLink from "next/link";
type EmptyStateProps = {
text: string;
icon: LucideIcon;
"data-cy"?: string;
};
export const EmptyState = (props: EmptyStateProps) => {
const backgroundColor = useColorModeValue("white", "gray.800");
return (
<Box bg={backgroundColor} p="10" borderRadius="xl" shadow="base">
<Box data-cy={props["data-cy"]} bg={backgroundColor} p="10" borderRadius="xl" shadow="base">
<Box display="flex" flexDirection="column" alignItems="center" gap="8" fontSize="lg">
<props.icon size="30" color="DarkOrange" />
<Text>{props.text}</Text>
@@ -24,5 +25,5 @@ export const EmptyState = (props: EmptyStateProps) => {
};
export const TaskEmptyState = () => {
return <EmptyState text="Looks like no tasks were found." icon={AlertTriangle} />;
return <EmptyState text="Looks like no tasks were found." icon={AlertTriangle} data-cy="task" />;
};
website/src/components/Explain.tsx
+39
View File
@@ -0,0 +1,39 @@
import {
IconButton,
Popover,
PopoverArrow,
PopoverBody,
PopoverCloseButton,
PopoverContent,
PopoverTrigger,
Text,
} from "@chakra-ui/react";
import { InformationCircleIcon } from "@heroicons/react/20/solid";
interface ExplainProps {
explanation: string[];
}
export const Explain = ({ explanation }: ExplainProps) => {
return (
<Popover>
<PopoverTrigger>
<IconButton
aria-label="explanation"
variant="link"
size="xs"
icon={<InformationCircleIcon className="h-4 w-4" />}
></IconButton>
</PopoverTrigger>
<PopoverContent>
<PopoverArrow />
<PopoverCloseButton />
<PopoverBody>
{explanation.map((paragraph, idx) => (
<Text key={idx}>{paragraph}</Text>
))}
</PopoverBody>
</PopoverContent>
</Popover>
);
};
website/src/components/FlaggableElement.tsx
+46 -203
View File
@@ -1,127 +1,69 @@
import {
Box,
Button,
Checkbox,
Flex,
Modal,
ModalBody,
ModalCloseButton,
ModalContent,
ModalFooter,
ModalHeader,
ModalOverlay,
Popover,
PopoverAnchor,
PopoverArrow,
PopoverBody,
PopoverCloseButton,
PopoverContent,
PopoverTrigger,
Slider,
SliderFilledTrack,
SliderThumb,
SliderTrack,
Tooltip,
useBoolean,
useColorMode,
useColorModeValue,
useId,
useDisclosure,
} from "@chakra-ui/react";
import { QuestionMarkCircleIcon } from "@heroicons/react/20/solid";
import clsx from "clsx";
import { AlertCircle } from "lucide-react";
import { useEffect, useReducer } from "react";
import { useState } from "react";
import { get, post } from "src/lib/api";
import { colors } from "src/styles/Theme/colors";
import { Message } from "src/types/Conversation";
import useSWR from "swr";
import useSWRImmutable from "swr/immutable";
import useSWRMutation from "swr/mutation";
import { LabelInputGroup } from "./Survey/LabelInputGroup";
interface Label {
name: string;
display_text: string;
help_text: string;
}
interface LoadLabelsAction {
type: "load_labels";
labels: Label[];
}
interface UpdateValueAction {
type: "update_value";
label_index: number;
value: number;
}
interface ToggleLabelAction {
type: "toggle_label";
label_index: number;
check: boolean;
}
interface LabelValue {
label: Label;
checked: boolean;
value: number;
}
interface FlagReportState {
label_values: LabelValue[];
submittable: boolean;
}
interface FlaggableElementProps {
children: React.ReactNode;
message: Message;
}
interface ValidLabelsResponse {
valid_labels: Label[];
}
export const FlaggableElement = (props: FlaggableElementProps) => {
const [report, updateReport] = useReducer(
(state: FlagReportState, action: LoadLabelsAction | UpdateValueAction | ToggleLabelAction): FlagReportState => {
const makeState = (label_values: LabelValue[]): FlagReportState => {
const submittable = label_values.map(({ checked }) => checked).some(Boolean);
return { label_values, submittable };
};
const { data: response } = useSWRImmutable<ValidLabelsResponse>("/api/valid_labels", get);
const { isOpen, onOpen, onClose } = useDisclosure();
const { valid_labels } = response || { valid_labels: [] };
const [values, setValues] = useState<number[]>([]);
switch (action.type) {
case "load_labels":
return makeState(
action.labels.map((label) => {
return { label, checked: false, value: 1 };
})
);
case "toggle_label": {
const values_copy = state.label_values.slice();
values_copy[action.label_index].checked = action.check;
return makeState(values_copy);
}
case "update_value": {
const values_copy = state.label_values.slice();
values_copy[action.label_index].value = action.value;
return makeState(values_copy);
}
}
},
{ label_values: [], submittable: false }
);
const [isEditing, setIsEditing] = useBoolean();
const { data, isLoading } = useSWR("/api/valid_labels", get);
useEffect(() => {
if (isLoading) {
return;
}
if (!data) {
updateReport({ type: "load_labels", labels: [] });
return;
}
const { valid_labels } = data;
updateReport({ type: "load_labels", labels: valid_labels });
}, [data, isLoading]);
const submittable =
values.some((value) => {
return value !== null;
}) &&
values.length === valid_labels.length &&
valid_labels.length > 0;
const { trigger } = useSWRMutation("/api/set_label", post, {
onSuccess: setIsEditing.off,
onSuccess: onClose,
onError: onClose,
});
const submitResponse = () => {
const label_map: Map<string, number> = new Map();
report.label_values.forEach(({ label, checked, value }) => {
if (checked) {
label_map.set(label.name, value);
console.assert(valid_labels.length === values.length);
values.forEach((value, idx) => {
if (value !== null) {
label_map.set(valid_labels[idx].name, value);
}
});
trigger({
@@ -131,22 +73,8 @@ export const FlaggableElement = (props: FlaggableElementProps) => {
});
};
const handleCheckboxState = (checked, label_index) => {
updateReport({ type: "toggle_label", label_index, check: checked });
};
const handleSliderState = (value, label_index) => {
updateReport({ type: "update_value", label_index, value });
};
return (
<Popover
isOpen={isEditing}
onOpen={setIsEditing.on}
onClose={setIsEditing.off}
closeOnBlur={false}
isLazy
lazyBehavior="keepMounted"
>
<Popover isOpen={isOpen} onOpen={onOpen} onClose={onClose} closeOnBlur={false} isLazy lazyBehavior="keepMounted">
<Box display="flex" alignItems="center" flexDirection={["column", "row"]} gap="2">
<PopoverAnchor>{props.children}</PopoverAnchor>
@@ -161,26 +89,17 @@ export const FlaggableElement = (props: FlaggableElementProps) => {
</Tooltip>
</Box>
<PopoverContent width="auto" p="3" m="4" maxWidth="calc(100vw - 2rem)">
<PopoverArrow />
<Box className="relative h-4">
<PopoverCloseButton />
</Box>
<PopoverBody>
{report.label_values.map(({ label, checked, value }, i) => (
<FlagCheckbox
label={label}
key={i}
idx={i}
checked={checked}
sliderValue={value}
checkboxHandler={handleCheckboxState}
sliderHandler={handleSliderState}
/>
))}
<Flex justify="center">
<Modal isOpen={isOpen} onClose={onClose}>
<ModalOverlay />
<ModalContent>
<ModalHeader>Select one or more labels that apply.</ModalHeader>
<ModalCloseButton />
<ModalBody>
<LabelInputGroup simple labelIDs={valid_labels.map(({ name }) => name)} onChange={setValues} />
</ModalBody>
<ModalFooter>
<Button
isDisabled={!report.submittable}
isDisabled={!submittable}
onClick={submitResponse}
className={`bg-indigo-600 text-${useColorModeValue(
colors.light.text,
@@ -189,85 +108,9 @@ export const FlaggableElement = (props: FlaggableElementProps) => {
>
Report
</Button>
</Flex>
</PopoverBody>
</PopoverContent>
</ModalFooter>
</ModalContent>
</Modal>
</Popover>
);
};
interface FlagCheckboxProps {
label: Label;
idx: number;
checked: boolean;
sliderValue: number;
checkboxHandler: (newVal: boolean, idx: number) => void;
sliderHandler: (newVal: number, idx: number) => void;
}
export function FlagCheckbox(props: FlagCheckboxProps): JSX.Element {
let AdditionalExplanation = null;
if (props.label.help_text) {
AdditionalExplanation = (
<a href="#" className="text-sm inline group leading-4">
<QuestionMarkCircleIcon
className="h-5 w-5 ml-1 text-gray-400 group-hover:text-gray-500 inline"
aria-hidden="true"
/>
</a>
);
}
const id = useId();
const { colorMode } = useColorMode();
const labelTextClass =
colorMode === "light"
? `text-${colors.light.text} hover:text-blue-700`
: `text-${colors.dark.text} hover:text-blue-400`;
return (
<Flex gap="4" justifyContent="space-between" className="my-2">
<div className="flex items-start align-middle">
<Checkbox
id={id}
isChecked={props.checked}
onChange={(e) => {
props.checkboxHandler(e.target.checked, props.idx);
}}
/>
<label
className={clsx(
"text-sm form-check-label ml-2 break-all inline align-middle first-line:leading-4",
labelTextClass
)}
htmlFor={id}
>
{props.label.display_text}
{AdditionalExplanation}
</label>
</div>
<div
onClick={() => {
if (!props.checked) {
props.checkboxHandler(true, props.idx);
}
}}
>
<Slider
width="100px"
isDisabled={!props.checked}
defaultValue={100}
onChangeEnd={(val) => {
props.sliderHandler(val / 100, props.idx);
}}
>
<SliderTrack>
<SliderFilledTrack />
<SliderThumb />
</SliderTrack>
</Slider>
</div>
</Flex>
);
}
website/src/components/Survey/LabelInputGroup.tsx
+195
View File
@@ -0,0 +1,195 @@
import { Box, Flex, Grid, Spacer, Text, useColorModeValue, VStack } from "@chakra-ui/react";
import React from "react";
import { useState } from "react";
import { TbChevronLeft, TbChevronRight, TbChevronsLeft, TbChevronsRight } from "react-icons/tb";
import { LikertButtons } from "src/components/Buttons/LikertButtons";
import { Explain } from "src/components/Explain";
interface LabelInputGroupProps {
labelIDs: Array<string>;
simple?: boolean;
onChange: (values: number[]) => unknown;
isEditable?: boolean;
}
interface LabelInfo {
zeroText: string;
oneText: string;
zeroDescription: string[];
oneDescription: string[];
inverted: boolean;
}
// This should be moved to the valid labels api endpoint
const label_messages: {
[label: string]: LabelInfo;
} = {
spam: {
zeroText: "Suitable for Training",
zeroDescription: ["Suitable for training Open Assistant."],
oneText: "Spam",
oneDescription: [
"Seems to be intentionally low-quality or irrelevant",
'We consider the following unwanted content as spam: trolling, intentional undermining of our purpose, illegal material, material that violates our code of conduct, and other things that are inappropriate for our dataset. We collect these under the common heading of "spam".',
"This is not an assessment of whether this message is the best possible answer. Especially for prompts or user-replies, we very much want to retain all kinds of responses in the dataset, so that the assistant can learn to reply appropriately.",
"Please mark this text as spam only if it is clearly unsuited to be part of our dataset, as outlined above, and try not to make any subjective value-judgments beyond that.",
],
inverted: true,
},
fails_task: {
zeroText: "Follows Instructions",
zeroDescription: [],
oneText: "Fails Task",
oneDescription: ["Fails to follow the correct instruction / task"],
inverted: true,
},
not_appropriate: {
zeroText: "Appropriate",
zeroDescription: [],
oneText: "Inappropriate",
oneDescription: ["Inappropriate for customer assistant"],
inverted: true,
},
violence: {
zeroText: "Harmless",
zeroDescription: [],
oneText: "Encourages Violence",
oneDescription: ["Encourages or fails to discourage violence/abuse/terrorism/self-harm"],
inverted: true,
},
excessive_harm: {
zeroText: "Safe",
zeroDescription: [],
oneText: "Causes Harm",
oneDescription: [
"Content likely to cause excessive harm not justifiable in the context",
"Harm refers to physical or mental damage or injury to someone or something. Excessive refers to a reasonable threshold of harm in the context, for instance damaging skin is not excessive in the context of surgery.",
],
inverted: true,
},
sexual_content: {
zeroText: "Non Sexual",
zeroDescription: [],
oneText: "Sexual Content",
oneDescription: ["Contains sexual content"],
inverted: true,
},
toxicity: {
zeroText: "Non Toxic",
zeroDescription: [],
oneText: "Rude / Toxic",
oneDescription: ["Contains rude, abusive, profane or insulting content"],
inverted: true,
},
moral_judgement: {
zeroText: "Non-Judgemental",
zeroDescription: [],
oneText: "Judgemental",
oneDescription: ["Expresses moral judgement"],
inverted: true,
},
political_content: {
zeroText: "Apolitical",
zeroDescription: [],
oneText: "Political",
oneDescription: ["Expresses political views"],
inverted: true,
},
humor: {
zeroText: "Serious",
zeroDescription: [],
oneText: "Humorous / Sarcastic",
oneDescription: ["Contains humorous content including sarcasm"],
inverted: false,
},
hate_speech: {
zeroText: "Safe",
zeroDescription: [],
oneText: "Hateful",
oneDescription: [
"Content is abusive or threatening and expresses prejudice against a protected characteristic",
"Prejudice refers to preconceived views not based on reason. Protected characteristics include gender, ethnicity, religion, sexual orientation, and similar characteristics.",
],
inverted: true,
},
threat: {
zeroText: "Safe",
zeroDescription: [],
oneText: "Contains Threat",
oneDescription: ["Contains a threat against a person or persons"],
inverted: true,
},
misleading: {
zeroText: "Accurate",
zeroDescription: [],
oneText: "Misleading",
oneDescription: ["Contains text which is incorrect or misleading"],
inverted: true,
},
helpful: {
zeroText: "Unhelful",
zeroDescription: [],
oneText: "Helpful",
oneDescription: ["Completes the task to a high standard"],
inverted: false,
},
creative: {
zeroText: "Boring",
zeroDescription: [],
oneText: "Creative",
oneDescription: ["Expresses creativity in responding to the task"],
inverted: false,
},
};
export const LabelInputGroup = ({ labelIDs, onChange, isEditable = true }: LabelInputGroupProps) => {
const [labelValues, setLabelValues] = useState<number[]>(Array.from({ length: labelIDs.length }).map(() => null));
const cardColor = useColorModeValue("gray.50", "gray.800");
return (
<Grid templateColumns={"minmax(min-content, 30em)"} rowGap={2}>
{labelIDs.map((labelId, idx) => {
const { zeroText, oneText, zeroDescription, oneDescription, inverted } = label_messages[labelId];
let textA = zeroText;
let textB = oneText;
let descriptionA = zeroDescription;
let descriptionB = oneDescription;
if (inverted) [textA, textB, descriptionA, descriptionB] = [textB, textA, descriptionB, descriptionA];
return (
<Box key={idx} padding={2} bg={cardColor} borderRadius="md">
<VStack alignItems="stretch" spacing={1}>
<Flex>
<Text>{textA}</Text>
{descriptionA.length > 0 ? <Explain explanation={descriptionA} /> : null}
<Spacer minWidth="1em" />
<Text textAlign="right">{textB}</Text>
{descriptionB.length > 0 ? <Explain explanation={descriptionB} /> : null}
</Flex>
<LikertButtons
isDisabled={!isEditable}
options={[
<TbChevronsLeft key="<<" />,
<TbChevronLeft key="<" />,
"",
<TbChevronRight key=">" />,
<TbChevronsRight key=">>" />,
]}
data-cy="label-options"
value={labelValues[idx] === null ? null : inverted ? 1 - labelValues[idx] : labelValues[idx]}
onChange={(value) => {
const newState = labelValues.slice();
newState[idx] = value === null ? null : inverted ? 1 - value : value;
onChange(newState);
setLabelValues(newState);
}}
/>
</VStack>
</Box>
);
})}
</Grid>
);
};
website/src/components/Survey/LabelRadioGroup.tsx
-129
View File
@@ -1,129 +0,0 @@
import {
Box,
Button,
Flex,
IconButton,
Popover,
PopoverArrow,
PopoverBody,
PopoverCloseButton,
PopoverContent,
PopoverTrigger,
Text,
useColorMode,
} from "@chakra-ui/react";
import { InformationCircleIcon } from "@heroicons/react/20/solid";
import { useId, useState } from "react";
import { colors } from "src/styles/Theme/colors";
interface LabelRadioGroupProps {
labelIDs: Array<string>;
onChange: (sliderValues: number[]) => unknown;
isEditable?: boolean;
}
const label_messages: { [label: string]: { description: string; explanation: string[] } } = {
spam: {
description: "Is the message spam?",
explanation: [
'We consider the following unwanted content as spam: trolling, intentional undermining of our purpose, illegal material, material that violates our code of conduct, and other things that are inappropriate for our dataset. We collect these under the common heading of "spam".',
"This is not an assessment of whether this message is the best possible answer. Especially for prompts or user-replies, we very much want to retain all kinds of responses in the dataset, so that the assistant can learn to reply appropriately.",
"Please mark this text as spam only if it is clearly unsuited to be part of our dataset, as outlined above, and try not to make any subjective value-judgments beyond that.",
],
},
};
export const LabelRadioGroup = (props: LabelRadioGroupProps) => {
const [labelValues, setLabelValues] = useState<number[]>(Array.from({ length: props.labelIDs.length }).map(() => 0));
const [interactionFlag, setInteractionFlag] = useState(false);
return (
<Flex direction="column" justify="center">
{props.labelIDs.map((labelId, idx) => (
<LabelRadioItem
key={idx}
labelText={label_messages[labelId] || { description: labelId }}
labelValue={labelValues[idx]}
clickHandler={(newValue) => {
const newState = labelValues.slice();
newState[idx] = newValue;
props.onChange(newState);
setLabelValues(newState);
if (!interactionFlag) setInteractionFlag(true);
}}
states={[
{ text: "No", value: 0 },
{ text: "Yes", value: 1 },
]}
isEditable={props.isEditable}
interactionFlag={interactionFlag}
/>
))}
</Flex>
);
};
interface ButtonState {
text: string;
value: number;
colorScheme?: string;
}
interface LabelRadioItemProps {
labelText: { description: string; explanation?: string[] };
labelValue: number;
clickHandler: (newVal: number) => unknown;
states: ButtonState[];
isEditable: boolean;
interactionFlag: boolean;
}
const LabelRadioItem = (props: LabelRadioItemProps) => {
const id = useId();
const { colorMode } = useColorMode();
const labelTextClass = colorMode === "light" ? `text-${colors.light.text}` : `text-${colors.dark.text}`;
return (
<Box data-cy="label-group-item" data-label-type="radio">
<label className="text-sm" htmlFor={id}>
{/* TODO: display real text instead of just the id */}
<span className={labelTextClass}>{props.labelText.description}</span>
{props.labelText.explanation ? (
<Popover>
<PopoverTrigger>
<IconButton
aria-label="explanation"
variant="link"
icon={<InformationCircleIcon className="h-5 w-5" />}
></IconButton>
</PopoverTrigger>
<PopoverContent>
<PopoverArrow />
<PopoverCloseButton />
<PopoverBody>
{props.labelText.explanation.map((paragraph, idx) => (
<Text key={idx}>{paragraph}</Text>
))}
</PopoverBody>
</PopoverContent>
</Popover>
) : null}
</label>
<Flex direction="row" gap={6} justify="center">
{props.states.map((item, idx) => (
<Button
aria-roledescription="radio-button"
colorScheme={item.value === props.labelValue && props.interactionFlag ? item.colorScheme || "blue" : "gray"}
isDisabled={!props.isEditable}
size="lg"
key={idx}
onClick={() => props.clickHandler(item.value)}
>
{item.text}
</Button>
))}
</Flex>
</Box>
);
};
website/src/components/Survey/LabelSliderGroup.tsx
-67
View File
@@ -1,67 +0,0 @@
import { Grid, Slider, SliderFilledTrack, SliderThumb, SliderTrack, useColorMode } from "@chakra-ui/react";
import { useId, useState } from "react";
import { colors } from "src/styles/Theme/colors";
// TODO: consolidate with FlaggableElement
interface LabelSliderGroupProps {
labelIDs: Array<string>;
onChange: (sliderValues: number[]) => unknown;
isEditable?: boolean;
}
export const LabelSliderGroup = ({ labelIDs, onChange, isEditable }: LabelSliderGroupProps) => {
const [sliderValues, setSliderValues] = useState<number[]>(Array.from({ length: labelIDs.length }).map(() => 0));
return (
<Grid templateColumns="auto 1fr" rowGap={1} columnGap={4}>
{labelIDs.map((labelId, idx) => (
<CheckboxSliderItem
key={idx}
labelId={labelId}
sliderValue={sliderValues[idx]}
sliderHandler={(sliderValue) => {
const newState = sliderValues.slice();
newState[idx] = sliderValue;
onChange(newState);
setSliderValues(newState);
}}
isEditable={isEditable}
/>
))}
</Grid>
);
};
function CheckboxSliderItem(props: {
labelId: string;
sliderValue: number;
sliderHandler: (newVal: number) => unknown;
isEditable: boolean;
}) {
const id = useId();
const { colorMode } = useColorMode();
const labelTextClass = colorMode === "light" ? `text-${colors.light.text}` : `text-${colors.dark.text}`;
return (
<>
<label className="text-sm" htmlFor={id}>
{/* TODO: display real text instead of just the id */}
<span className={labelTextClass}>{props.labelId}</span>
</label>
<Slider
data-cy="label-group-item"
data-label-type="slider"
aria-roledescription="slider"
defaultValue={0}
isDisabled={!props.isEditable}
onChangeEnd={(val) => props.sliderHandler(val / 100)}
>
<SliderTrack>
<SliderFilledTrack />
</SliderTrack>
<SliderThumb bg="gainsboro" />
</Slider>
</>
);
}
website/src/components/Survey/TaskControls.tsx
+2 -2
View File
@@ -47,7 +47,7 @@ export const TaskControls = (props: TaskControlsProps) => {
<SubmitButton
colorScheme="green"
data-cy="submit"
disabled={props.taskStatus === "SUBMITTED"}
isDisabled={props.taskStatus === "SUBMITTED"}
onClick={props.onSubmit}
>
Submit
@@ -59,7 +59,7 @@ export const TaskControls = (props: TaskControlsProps) => {
<SubmitButton
colorScheme="blue"
data-cy="review"
disabled={props.taskStatus === "NOT_SUBMITTABLE"}
isDisabled={props.taskStatus === "NOT_SUBMITTABLE"}
onClick={props.onReview}
>
Review
website/src/components/Tasks/CreateTask.tsx
+4 -2
View File
@@ -12,6 +12,7 @@ export const CreateTask = ({
isEditable,
isDisabled,
onReplyChanged,
onValidityChanged,
}: TaskSurveyProps<{ text: string }>) => {
const cardColor = useColorModeValue("gray.50", "gray.800");
const titleColor = useColorModeValue("gray.800", "gray.300");
@@ -20,11 +21,12 @@ export const CreateTask = ({
const textChangeHandler = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
const text = event.target.value;
const isTextBlank = !text || /^\s*$/.test(text) ? true : false;
onReplyChanged({ text });
if (!isTextBlank) {
onReplyChanged({ content: { text }, state: "VALID" });
onValidityChanged("VALID");
setInputText(text);
} else {
onReplyChanged({ content: { text }, state: "INVALID" });
onValidityChanged("INVALID");
setInputText("");
}
};
website/src/components/Tasks/EvaluateTask.tsx
+13 -9
View File
@@ -1,5 +1,5 @@
import { Box, useColorModeValue } from "@chakra-ui/react";
import { useEffect } from "react";
import { useEffect, useState } from "react";
import { MessageTable } from "src/components/Messages/MessageTable";
import { Sortable } from "src/components/Sortable/Sortable";
import { SurveyCard } from "src/components/Survey/SurveyCard";
@@ -12,8 +12,10 @@ export const EvaluateTask = ({
isEditable,
isDisabled,
onReplyChanged,
onValidityChanged,
}: TaskSurveyProps<{ ranking: number[] }>) => {
const cardColor = useColorModeValue("gray.50", "gray.800");
const [ranking, setRanking] = useState<number[]>(null);
let messages = [];
if (task.conversation) {
@@ -22,13 +24,15 @@ export const EvaluateTask = ({
}
useEffect(() => {
const ranking = (task.replies ?? task.prompts).map((_, idx) => idx);
onReplyChanged({ content: { ranking }, state: "DEFAULT" });
}, [task, onReplyChanged]);
const onRank = (newRanking: number[]) => {
onReplyChanged({ content: { ranking: newRanking }, state: "VALID" });
};
if (ranking === null) {
const defaultRanking = (task.replies ?? task.prompts).map((_, idx) => idx);
onReplyChanged({ ranking: defaultRanking });
onValidityChanged("DEFAULT");
} else {
onReplyChanged({ ranking });
onValidityChanged("VALID");
}
}, [task, ranking, onReplyChanged, onValidityChanged]);
const sortables = task.replies ? "replies" : "prompts";
@@ -44,7 +48,7 @@ export const EvaluateTask = ({
items={task[sortables]}
isDisabled={isDisabled}
isEditable={isEditable}
onChange={onRank}
onChange={setRanking}
className="my-8"
/>
</SurveyCard>
website/src/components/Tasks/LabelTask/LabelTask.tsx
+20 -27
View File
@@ -1,9 +1,8 @@
import { Box, useColorModeValue } from "@chakra-ui/react";
import { Box, Flex, Text, useColorModeValue } from "@chakra-ui/react";
import { useEffect, useState } from "react";
import { MessageView } from "src/components/Messages";
import { MessageTable } from "src/components/Messages/MessageTable";
import { LabelRadioGroup } from "src/components/Survey/LabelRadioGroup";
import { LabelSliderGroup } from "src/components/Survey/LabelSliderGroup";
import { LabelInputGroup } from "src/components/Survey/LabelInputGroup";
import { TwoColumnsWithCards } from "src/components/Survey/TwoColumnsWithCards";
import { TaskSurveyProps } from "src/components/Tasks/Task";
import { TaskHeader } from "src/components/Tasks/TaskHeader";
@@ -12,28 +11,18 @@ import { TaskType } from "src/types/Task";
export const LabelTask = ({
task,
taskType,
onReplyChanged,
isEditable,
onReplyChanged,
onValidityChanged,
}: TaskSurveyProps<{ text: string; labels: Record<string, number>; message_id: string }>) => {
const valid_labels = task.valid_labels;
const [sliderValues, setSliderValues] = useState<number[]>(new Array(valid_labels.length).fill(0));
const [sliderValues, setSliderValues] = useState<number[]>(new Array(task.valid_labels.length).fill(null));
useEffect(() => {
onReplyChanged({
content: { labels: {}, text: task.reply, message_id: task.message_id },
state: "NOT_SUBMITTABLE",
});
}, [task, onReplyChanged]);
const onSliderChange = (values: number[]) => {
console.assert(valid_labels.length === sliderValues.length);
const labels = Object.fromEntries(valid_labels.map((label, i) => [label, sliderValues[i]]));
onReplyChanged({
content: { labels, text: task.reply || task.prompt, message_id: task.message_id },
state: "VALID",
});
setSliderValues(values);
};
console.assert(task.valid_labels.length === sliderValues.length);
const labels = Object.fromEntries(task.valid_labels.map((label, i) => [label, sliderValues[i]]));
onReplyChanged({ labels, text: task.reply || task.prompt, message_id: task.message_id });
onValidityChanged(sliderValues.every((value) => value !== null) ? "VALID" : "INVALID");
}, [task, sliderValues, onReplyChanged, onValidityChanged]);
const cardColor = useColorModeValue("gray.50", "gray.800");
@@ -43,7 +32,7 @@ export const LabelTask = ({
<>
<TaskHeader taskType={taskType} />
{task.conversation ? (
<Box mt="4" p="6" borderRadius="lg" bg={cardColor}>
<Box mt="4" p={[4, 6]} borderRadius="lg" bg={cardColor}>
<MessageTable
messages={[
...(task.conversation?.messages ?? []),
@@ -61,11 +50,15 @@ export const LabelTask = ({
</Box>
)}
</>
{task.mode === "simple" ? (
<LabelRadioGroup labelIDs={task.valid_labels} isEditable={isEditable} onChange={onSliderChange} />
) : (
<LabelSliderGroup labelIDs={task.valid_labels} isEditable={isEditable} onChange={onSliderChange} />
)}
<Flex direction="column" alignItems="stretch">
<Text>The highlighted message:</Text>
<LabelInputGroup
simple={task.mode === "simple"}
labelIDs={task.valid_labels}
isEditable={isEditable}
onChange={setSliderValues}
/>
</Flex>
</TwoColumnsWithCards>
</div>
);
website/src/components/Tasks/Task/Task.tsx
+25 -18
View File
@@ -6,8 +6,7 @@ import { LabelTask } from "src/components/Tasks/LabelTask";
import { TaskCategory, TaskInfo, TaskInfos } from "src/components/Tasks/TaskTypes";
import { UnchangedWarning } from "src/components/Tasks/UnchangedWarning";
import { post } from "src/lib/api";
import { TaskContent } from "src/types/Task";
import { TaskReplyState } from "src/types/TaskReplyState";
import { TaskContent, TaskReplyValidity } from "src/types/Task";
import useSWRMutation from "swr/mutation";
export type TaskStatus = "NOT_SUBMITTABLE" | "DEFAULT" | "VALID" | "REVIEW" | "SUBMITTED";
@@ -19,7 +18,8 @@ export interface TaskSurveyProps<T> {
taskType: TaskInfo;
isEditable: boolean;
isDisabled?: boolean;
onReplyChanged: (state: TaskReplyState<T>) => void;
onReplyChanged: (content: T) => void;
onValidityChanged: (validity: TaskReplyValidity) => void;
}
export const Task = ({ frontendId, task, trigger, mutate }) => {
@@ -44,20 +44,27 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
});
};
const onReplyChanged = useRef((state: TaskReplyState<TaskContent>) => {
if (taskStatus === "SUBMITTED") return;
const edit_mode = taskStatus === "NOT_SUBMITTABLE" || taskStatus === "DEFAULT" || taskStatus === "VALID";
const submitted = taskStatus === "SUBMITTED";
replyContent.current = state?.content;
if (state === null) {
if (taskStatus !== "NOT_SUBMITTABLE") setTaskStatus("NOT_SUBMITTABLE");
} else if (state.state === "DEFAULT") {
if (taskStatus !== "DEFAULT") setTaskStatus("DEFAULT");
} else if (state.state === "VALID") {
if (taskStatus !== "VALID") setTaskStatus("VALID");
} else if (state.state === "INVALID") {
setTaskStatus("NOT_SUBMITTABLE");
const onValidityChanged = (validity: TaskReplyValidity) => {
if (!edit_mode) return;
switch (validity) {
case "DEFAULT":
if (taskStatus !== "DEFAULT") setTaskStatus("DEFAULT");
break;
case "VALID":
if (taskStatus !== "VALID") setTaskStatus("VALID");
break;
case "INVALID":
if (taskStatus !== "NOT_SUBMITTABLE") setTaskStatus("NOT_SUBMITTABLE");
break;
}
}).current;
};
const onReplyChanged = (content: TaskContent) => {
replyContent.current = content;
};
const reviewResponse = () => {
switch (taskStatus) {
@@ -99,9 +106,6 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
}
};
const edit_mode = taskStatus === "NOT_SUBMITTABLE" || taskStatus === "DEFAULT" || taskStatus === "VALID";
const submitted = taskStatus === "SUBMITTED";
function taskTypeComponent() {
switch (taskType.category) {
case TaskCategory.Create:
@@ -113,6 +117,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
isEditable={edit_mode}
isDisabled={submitted}
onReplyChanged={onReplyChanged}
onValidityChanged={onValidityChanged}
/>
);
case TaskCategory.Evaluate:
@@ -124,6 +129,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
isEditable={edit_mode}
isDisabled={submitted}
onReplyChanged={onReplyChanged}
onValidityChanged={onValidityChanged}
/>
);
case TaskCategory.Label:
@@ -135,6 +141,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
isEditable={edit_mode}
isDisabled={submitted}
onReplyChanged={onReplyChanged}
onValidityChanged={onValidityChanged}
/>
);
}
website/src/components/Tasks/TaskTypes.tsx
+2 -2
View File
@@ -162,7 +162,7 @@ export const TaskInfos: TaskInfo[] = [
category: TaskCategory.Label,
pathname: "/label/label_prompter_reply",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
overview: "Read the following conversation and then answer the question about the last prompt in the discussion.",
overview: "Read the following conversation and then answer the question about the last reply in the discussion.",
type: "label_prompter_reply",
mode: "simple",
update_type: "text_labels",
@@ -173,7 +173,7 @@ export const TaskInfos: TaskInfo[] = [
category: TaskCategory.Label,
pathname: "/label/label_assistant_reply",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
overview: "Read the following conversation and then answer the question about the last prompt in the discussion.",
overview: "Read the following conversation and then answer the question about the last reply in the discussion.",
type: "label_assistant_reply",
mode: "simple",
update_type: "text_labels",
website/src/types/Task.ts
+2
View File
@@ -35,4 +35,6 @@ export interface TaskResponse<Task extends BaseTask> {
task: Task;
}
export type TaskReplyValidity = "DEFAULT" | "VALID" | "INVALID";
export type AvailableTasks = { [taskType in TaskType]: number };
website/src/types/TaskReplyState.ts
-22
View File
@@ -1,22 +0,0 @@
export interface TaskReplyNotSubmittable<T> {
content: T;
state: "NOT_SUBMITTABLE";
}
export interface TaskReplyValid<T> {
content: T;
state: "VALID";
}
export interface TaskReplyDefault<T> {
content: T;
state: "DEFAULT";
}
export interface TaskReplyInValid<T> {
content: T;
state: "INVALID";
}
export type TaskReplyState<T> =
| TaskReplyNotSubmittable<T>
| TaskReplyValid<T>
| TaskReplyDefault<T>
| TaskReplyInValid<T>;