Regenerated wikidata-tekGen Vicuna-13B eval metrics V2

cenguix · May 12, 2023 · 08c92bd · 08c92bd
1 parent 21f8788
commit 08c92bd
Show file tree

Hide file tree

Showing 20 changed files with 4,005 additions and 8,340 deletions.
diff --git a/.idea/.gitignore b/.idea/.gitignore
diff --git a/.idea/Text2KGBench.iml b/.idea/Text2KGBench.iml
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/.idea/modules.xml b/.idea/modules.xml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_10_culture_llm_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_10_culture_llm_stats_v2.jsonl
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_1_movie_llm_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_1_movie_llm_stats_v2.jsonl
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_2_music_llm_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_2_music_llm_stats_v2.jsonl
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_3_sport_llm_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_3_sport_llm_stats_v2.jsonl
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_4_book_llm_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_4_book_llm_stats_v2.jsonl
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_5_military_llm_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_5_military_llm_stats_v2.jsonl
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_6_computer_llm_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_6_computer_llm_stats_v2.jsonl
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_7_space_llm_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_7_space_llm_stats_v2.jsonl
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_8_politics_llm_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_8_politics_llm_stats_v2.jsonl
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_9_nature_llm_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_9_nature_llm_stats_v2.jsonl
diff --git a/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_llm_avg_stats_v2.jsonl b/data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_llm_avg_stats_v2.jsonl
diff --git a/src/evaluation/run_eval.py b/src/evaluation/run_eval.py
@@ -172,9 +172,9 @@ def main():
                 if  f1 < 1  and len(filtered_system_triples) > 0 and subj_hallucination == 0 and obj_hallucination == 0:
                     print(f"sent: {sentence}\nf1: {f1}\nsys:{filtered_system_triples}\nground:{gt_triples}\n\n")
 
-                eval_metrics = {"id": sent_id, "precision": precision, "recall": recall, "f1": f1,
-                                "onto_conf": ont_conformance, "rel_halluc": rel_hallucination,
-                                "sub_halluc": subj_hallucination, "obj_halluc": obj_hallucination,
+                eval_metrics = {"id": sent_id, "precision": f"{precision:.2f}" , "recall": f"{recall:.2f}", "f1": f"{f1:.2f}",
+                                "onto_conf": f"{ont_conformance:.2f}", "rel_halluc": f"{rel_hallucination:.2f}",
+                                "sub_halluc": f"{subj_hallucination:.2f}", "obj_halluc": f"{obj_hallucination:.2f}",
                                 "llm_triples": system_triples, "filtered_llm_triples": filtered_system_triples,
                                 "gt_triples": gt_triples, "sent": sentence}
                 eval_metrics_list.append(eval_metrics)

diff --git a/src/evaluation/tekgen_vicuna_config.json b/src/evaluation/tekgen_vicuna_config.json
@@ -0,0 +1,13 @@
+{
+  "onto_list" : [
+    "1_movie",  "2_music", "3_sport", "4_book", "5_military", "6_computer", "7_space", "8_politics", "9_nature", "10_culture"
+  ],
+  "path_patterns": {
+    "sys": "../../data/wikidata_tekgen/baselines/Vicuna-13B/llm_responses/ont_$$onto$$_llm_responses.jsonl",
+    "gt":"../../data/wikidata_tekgen/ground_truth/ont_$$onto$$_ground_truth.jsonl",
+    "selected_ids": "../../data/wikidata_tekgen/manually_verified_sentences/selected_ont_$$onto$$.txt",
+    "onto": "../../data/wikidata_tekgen/ontologies/$$onto$$_ontology.json",
+    "output": "../../data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_$$onto$$_llm_stats_v2.jsonl"
+  },
+  "avg_out_file": "../../data/wikidata_tekgen/baselines/Vicuna-13B/eval_metrics/ont_llm_avg_stats_v2.jsonl"
+}
diff --git a/src/evaluation/tikgen_vicuna_config.json b/src/evaluation/tikgen_vicuna_config.json
@@ -3,7 +3,7 @@
     "1_movie",  "2_music", "3_sport", "4_book", "5_military", "6_computer", "7_space", "8_politics", "9_nature", "10_culture"
   ],
   "path_patterns": {
-    "sys": "../../data/baselines/vicuna_13B/llm_responses/ont_$$onto$$_llm_responses_v2.jsonl",
+    "sys": "../../data/baselines/vicuna_13B/llm_responses/ont_$$onto$$_llm_responses.jsonl",
     "gt":"../../data/ground_truth/ont_$$onto$$_ground_truth_v1.jsonl",
     "selected_ids": "../../data/ground_truth/selected_ids/selected_ont_$$onto$$.txt",
     "onto": "../../data/ontology/$$onto$$_ontology.json",