Filter supported languages (SEA only) of seacrowd_qa schema for MKQA S…

…EACrowd#34
fhudi · Jan 31, 2024 · 9ed7b79 · 9ed7b79
1 parent 76ca2e8
commit 9ed7b79
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/seacrowd/sea_datasets/mkqa/mkqa.py b/seacrowd/sea_datasets/mkqa/mkqa.py
@@ -230,7 +230,7 @@ def _generate_examples(self, filepath: Path) -> Tuple[int, Dict]:
 
         elif self.config.schema == "seacrowd_qa":
             for cur in datas:
-                for cur_lang in [lang] if lang else self._SOURCE_LANGUAGES:
+                for cur_lang in [lang] if lang else map(lambda k: self._LANG_3TO2.get(k, k), _LANGUAGES):
                     ret = {
                         "id": f'{cur["example_id"]}_{cur_lang}',
                         "question_id": cur["example_id"],