From 1011fa527c97e7b0f04aea905a588279177420fa Mon Sep 17 00:00:00 2001 From: Adrian Cowan Date: Fri, 27 Jan 2023 13:25:07 +1100 Subject: [PATCH] website: Fix handling of new and unknown labels --- .../src/components/Survey/LabelInputGroup.tsx | 288 ++++++++++-------- 1 file changed, 167 insertions(+), 121 deletions(-) diff --git a/website/src/components/Survey/LabelInputGroup.tsx b/website/src/components/Survey/LabelInputGroup.tsx index 06e0888b..9a06b6ca 100644 --- a/website/src/components/Survey/LabelInputGroup.tsx +++ b/website/src/components/Survey/LabelInputGroup.tsx @@ -18,126 +18,172 @@ interface LabelInfo { inverted: boolean; } -// This should be moved to the valid labels api endpoint -const label_messages: { - [label: string]: LabelInfo; -} = { - spam: { - zeroText: "Not Spam", - zeroDescription: ["Suitable for training Open Assistant."], - oneText: "Spam", - oneDescription: [ - "Seems to be intentionally low-quality or irrelevant", - 'We consider the following unwanted content as spam: trolling, intentional undermining of our purpose, illegal material, material that violates our code of conduct, and other things that are inappropriate for our dataset. We collect these under the common heading of "spam".', - "This is not an assessment of whether this message is the best possible answer. Especially for prompts or user-replies, we very much want to retain all kinds of responses in the dataset, so that the assistant can learn to reply appropriately.", - "Please mark this text as spam only if it is clearly unsuited to be part of our dataset, as outlined above, and try not to make any subjective value-judgments beyond that.", - ], - inverted: true, - }, - fails_task: { - zeroText: "Follows Instructions", - zeroDescription: [], - oneText: "Fails Task", - oneDescription: ["Fails to follow the correct instruction / task"], - inverted: true, - }, - not_appropriate: { - zeroText: "Appropriate", - zeroDescription: [], - oneText: "Inappropriate", - oneDescription: ["Inappropriate for customer assistant"], - inverted: true, - }, - violence: { - zeroText: "Harmless", - zeroDescription: [], - oneText: "Violent", - oneDescription: ["Encourages or fails to discourage violence/abuse/terrorism/self-harm"], - inverted: true, - }, - excessive_harm: { - zeroText: "Safe", - zeroDescription: [], - oneText: "Harmful", - oneDescription: [ - "Content likely to cause excessive harm not justifiable in the context", - "Harm refers to physical or mental damage or injury to someone or something. Excessive refers to a reasonable threshold of harm in the context, for instance damaging skin is not excessive in the context of surgery.", - ], - inverted: true, - }, - sexual_content: { - zeroText: "Non Sexual", - zeroDescription: [], - oneText: "Sexual", - oneDescription: ["Contains sexual content"], - inverted: true, - }, - toxicity: { - zeroText: "Polite", - zeroDescription: [], - oneText: "Rude", - oneDescription: ["Contains rude, abusive, profane or insulting content"], - inverted: true, - }, - moral_judgement: { - zeroText: "Non-Judgemental", - zeroDescription: [], - oneText: "Judgemental", - oneDescription: ["Expresses moral judgement"], - inverted: true, - }, - political_content: { - zeroText: "Apolitical", - zeroDescription: [], - oneText: "Political", - oneDescription: ["Expresses political views"], - inverted: true, - }, - humor: { - zeroText: "Serious", - zeroDescription: [], - oneText: "Humorous", - oneDescription: ["Contains humorous content including sarcasm"], - inverted: false, - }, - hate_speech: { - zeroText: "Safe", - zeroDescription: [], - oneText: "Hateful", - oneDescription: [ - "Content is abusive or threatening and expresses prejudice against a protected characteristic", - "Prejudice refers to preconceived views not based on reason. Protected characteristics include gender, ethnicity, religion, sexual orientation, and similar characteristics.", - ], - inverted: true, - }, - threat: { - zeroText: "Safe", - zeroDescription: [], - oneText: "Threatening", - oneDescription: ["Contains a threat against a person or persons"], - inverted: true, - }, - misleading: { - zeroText: "Accurate", - zeroDescription: [], - oneText: "Misleading", - oneDescription: ["Contains text which is incorrect or misleading"], - inverted: true, - }, - helpful: { - zeroText: "Unhelful", - zeroDescription: [], - oneText: "Helpful", - oneDescription: ["Completes the task to a high standard"], - inverted: false, - }, - creative: { - zeroText: "Boring", - zeroDescription: [], - oneText: "Creative", - oneDescription: ["Expresses creativity in responding to the task"], - inverted: false, - }, +const getLabelInfo = (label: string): LabelInfo => { + switch (label) { + case "spam": + return { + zeroText: "Not Spam", + zeroDescription: ["Suitable for training Open Assistant."], + oneText: "Spam", + oneDescription: [ + "Seems to be intentionally low-quality or irrelevant", + 'We consider the following unwanted content as spam: trolling, intentional undermining of our purpose, illegal material, material that violates our code of conduct, and other things that are inappropriate for our dataset. We collect these under the common heading of "spam".', + "This is not an assessment of whether this message is the best possible answer. Especially for prompts or user-replies, we very much want to retain all kinds of responses in the dataset, so that the assistant can learn to reply appropriately.", + "Please mark this text as spam only if it is clearly unsuited to be part of our dataset, as outlined above, and try not to make any subjective value-judgments beyond that.", + ], + inverted: true, + }; + case "fails_task": + return { + zeroText: "Follows Instructions", + zeroDescription: [], + oneText: "Fails Task", + oneDescription: ["Fails to follow the correct instruction / task"], + inverted: true, + }; + case "not_appropriate": + return { + zeroText: "Appropriate", + zeroDescription: [], + oneText: "Inappropriate", + oneDescription: ["Inappropriate for customer assistant"], + inverted: true, + }; + case "violence": + return { + zeroText: "Harmless", + zeroDescription: [], + oneText: "Violent", + oneDescription: ["Encourages or fails to discourage violence/abuse/terrorism/self-harm"], + inverted: true, + }; + case "excessive_harm": + return { + zeroText: "Safe", + zeroDescription: [], + oneText: "Harmful", + oneDescription: [ + "Content likely to cause excessive harm not justifiable in the context", + "Harm refers to physical or mental damage or injury to someone or something. Excessive refers to a reasonable threshold of harm in the context, for instance damaging skin is not excessive in the context of surgery.", + ], + inverted: true, + }; + case "sexual_content": + return { + zeroText: "Non Sexual", + zeroDescription: [], + oneText: "Sexual", + oneDescription: ["Contains sexual content"], + inverted: true, + }; + case "toxicity": + return { + zeroText: "Polite", + zeroDescription: [], + oneText: "Rude", + oneDescription: ["Contains rude, abusive, profane or insulting content"], + inverted: true, + }; + case "moral_judgement": + return { + zeroText: "Non-Judgemental", + zeroDescription: [], + oneText: "Judgemental", + oneDescription: ["Expresses moral judgement"], + inverted: true, + }; + case "political_content": + return { + zeroText: "Apolitical", + zeroDescription: [], + oneText: "Political", + oneDescription: ["Expresses political views"], + inverted: true, + }; + case "humor": + return { + zeroText: "Serious", + zeroDescription: [], + oneText: "Humorous", + oneDescription: ["Contains humorous content including sarcasm"], + inverted: false, + }; + case "hate_speech": + return { + zeroText: "Safe", + zeroDescription: [], + oneText: "Hateful", + oneDescription: [ + "Content is abusive or threatening and expresses prejudice against a protected characteristic", + "Prejudice refers to preconceived views not based on reason. Protected characteristics include gender, ethnicity, religion, sexual orientation, and similar characteristics.", + ], + inverted: true, + }; + case "threat": + return { + zeroText: "Safe", + zeroDescription: [], + oneText: "Threatening", + oneDescription: ["Contains a threat against a person or persons"], + inverted: true, + }; + case "misleading": + return { + zeroText: "Accurate", + zeroDescription: [], + oneText: "Misleading", + oneDescription: ["Contains text which is incorrect or misleading"], + inverted: true, + }; + case "helpful": + return { + zeroText: "Unhelful", + zeroDescription: [], + oneText: "Helpful", + oneDescription: ["Completes the task to a high standard"], + inverted: false, + }; + case "creative": + return { + zeroText: "Boring", + zeroDescription: [], + oneText: "Creative", + oneDescription: ["Expresses creativity in responding to the task"], + inverted: false, + }; + case "pii": + return { + zeroText: "Clean", + zeroDescription: [], + oneText: "Contains PII", + oneDescription: ["Contains personally identifing information"], + inverted: false, + }; + case "quality": + return { + zeroText: "Low Quality", + zeroDescription: [], + oneText: "High Quality", + oneDescription: [], + inverted: false, + }; + case "creativity": + return { + zeroText: "Ordinary", + zeroDescription: [], + oneText: "Creative", + oneDescription: [], + inverted: false, + }; + default: + return { + zeroText: `!${label}`, + zeroDescription: [], + oneText: label, + oneDescription: [], + inverted: false, + }; + } }; export const LabelInputGroup = ({ labelIDs, onChange, isEditable = true }: LabelInputGroupProps) => { @@ -148,7 +194,7 @@ export const LabelInputGroup = ({ labelIDs, onChange, isEditable = true }: Label return ( {labelIDs.map((labelId, idx) => { - const { zeroText, oneText, zeroDescription, oneDescription, inverted } = label_messages[labelId]; + const { zeroText, oneText, zeroDescription, oneDescription, inverted } = getLabelInfo(labelId); let textA = zeroText; let textB = oneText;