[feature] add rank dataset for webgpt and human feedback summary

2026-09-09 11:15:08 +08:00 · 2022-12-30 17:25:50 +00:00
parent a8e7cee73c
commit ad98a28241
10 changed files with 300 additions and 0 deletions
@@ -0,0 +1,18 @@
+import re
+
+re_reference_remove = re.compile(r'\[([0-9])+\]|\[([0-9])+,([0-9])+\]')
+
+def webgpt_return_format(row):
+    if row['score_0'] >= row['score_1']:
+        # remove this to prevent information leak, since we are not using reference
+        return {
+                'question': row['question']['full_text'],
+                     'pos': re_reference_remove.sub('', row['answer_0']),
+                     'neg': re_reference_remove.sub('', row['answer_1'])
+                }
+
+    return {
+            'question': row['question']['full_text'],
+                 'pos': re_reference_remove.sub('', row['answer_1']),
+                 'neg': re_reference_remove.sub('', row['answer_0'])
+            }