wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-08-14 12:00:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

add lang_mismatch label to prompt/reply acceptance check (#984)

Browse Source 
* add lang_mismatch label to acceptance check

* make acceptance formula clearer

* make lang_mismatch flag, remove helpful from prompter labels
This commit is contained in:
Andreas Köpf
2023-01-29 17:34:37 +01:00
committed by GitHub
parent 61b2949122
commit 0601d874e8
4 changed files with 29 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/oasst_backend/config.py
+3 -2
View File
@@ -57,8 +57,8 @@ class TreeManagerConfiguration(BaseModel):
labels_initial_prompt: list[TextLabel] = [
TextLabel.spam,
TextLabel.lang_mismatch,
TextLabel.quality,
TextLabel.helpfulness,
TextLabel.creativity,
TextLabel.humor,
TextLabel.toxicity,
@@ -71,6 +71,7 @@ class TreeManagerConfiguration(BaseModel):
labels_assistant_reply: list[TextLabel] = [
TextLabel.spam,
TextLabel.lang_mismatch,
TextLabel.fails_task,
TextLabel.quality,
TextLabel.helpfulness,
@@ -86,8 +87,8 @@ class TreeManagerConfiguration(BaseModel):
labels_prompter_reply: list[TextLabel] = [
TextLabel.spam,
TextLabel.lang_mismatch,
TextLabel.quality,
TextLabel.helpfulness,
TextLabel.humor,
TextLabel.creativity,
TextLabel.toxicity,
backend/oasst_backend/tree_manager.py
+12 -2
View File
@@ -376,6 +376,8 @@ class TreeManager:
label_mode = protocol_schema.LabelTaskMode.simple
label_disposition = protocol_schema.LabelTaskDisposition.spam
valid_labels = list(self.cfg.mandatory_labels_assistant_reply)
if protocol_schema.TextLabel.lang_mismatch not in valid_labels:
valid_labels.append(protocol_schema.TextLabel.lang_mismatch)
if protocol_schema.TextLabel.quality not in valid_labels:
valid_labels.append(protocol_schema.TextLabel.quality)
@@ -400,6 +402,8 @@ class TreeManager:
label_mode = protocol_schema.LabelTaskMode.simple
label_disposition = protocol_schema.LabelTaskDisposition.spam
valid_labels = list(self.cfg.mandatory_labels_prompter_reply)
if protocol_schema.TextLabel.lang_mismatch not in valid_labels:
valid_labels.append(protocol_schema.TextLabel.lang_mismatch)
if protocol_schema.TextLabel.quality not in valid_labels:
valid_labels.append(protocol_schema.TextLabel.quality)
@@ -483,6 +487,8 @@ class TreeManager:
valid_labels = self.cfg.mandatory_labels_initial_prompt
label_mode = protocol_schema.LabelTaskMode.simple
label_disposition = protocol_schema.LabelTaskDisposition.spam
if protocol_schema.TextLabel.lang_mismatch not in valid_labels:
valid_labels.append(protocol_schema.TextLabel.lang_mismatch)
logger.info(f"Generating a LabelInitialPromptTask ({label_mode=:s}).")
task = protocol_schema.LabelInitialPromptTask(
@@ -803,8 +809,12 @@ class TreeManager:
return backlog_tree
def _calculate_acceptance(self, labels: list[TextLabels]):
# calculate acceptance based on spam label
return np.mean([1 - l.labels[protocol_schema.TextLabel.spam] for l in labels])
# calculate acceptance based on lang_mismatch & spam label
lang_mismatch = np.mean([(l.labels.get(protocol_schema.TextLabel.lang_mismatch) or 0) for l in labels])
spam = np.mean([l.labels[protocol_schema.TextLabel.spam] for l in labels])
acceptance_score = 1 - (spam + lang_mismatch)
logger.debug(f"{acceptance_score=} ({spam=}, {lang_mismatch=})")
return acceptance_score
def _query_need_review(
self, state: message_tree_state.State, required_reviews: int, root: bool, lang: str