[feature] add pythia and limit translation pair

2026-08-08 11:13:58 +08:00 · 2023-01-22 00:56:17 +00:00
parent 62a203fd8c
commit f5b2a34857
2 changed files with 6 additions and 0 deletions
@@ -100,6 +100,8 @@ class WMT2019(TranslationPair):
            else:  # translating in reverse direction
                source = random.choice(TRANSLATION_PROMPT[src]).format(row[tgt])
                self.pairs.append((source, row[src]))
+            if len(self.pairs) > 100000:
+                break


 class DiveMT(TranslationPair):