Add more robust processing

scripts/import_mutants.py

@@ -1,7 +1,11 @@
 import argparse
+import re
 import json
 
 BULLET = "•"
+BULLET_UNICODE = "•"
+DASH_UNICODE = "—"
+MINUS_UNICODE = "–"
 TYPE_SEPARATOR = "."
 ITEM_SEPARATOR = ","
 PL_TAG = "PL"
@@ -12,7 +16,10 @@
 SKILLS_TAG = "Skills"
 OFFENSE_TAG = "Offense"
 DEFENSE_TAG = "Defense"
+POWERS_TAG = "Defense"
 TOTALS_TAG = "Totals"
+STANDARD_TAGS = [EQUIP_TAG, ADV_TAG, SKILLS_TAG, OFFENSE_TAG, DEFENSE_TAG, TOTALS_TAG]
+FINAL_LINE_REGEX = re.compile(r"Total .* points.")
 
 
 def process_args():
@@ -27,27 +34,36 @@ def process_args():
 
 
 def process_file(filename):
+
     json_data = {}
     # After the first 2 lines, we concatenate the rest of the lines
     # because the remaning lines are not uniform
     remaning_lines = ""
-    with open(filename, "r") as f:
+    with open(filename, "r", encoding="utf-8") as f:
         for line in f:
             print(line)
-            if BULLET in line:
+            if BULLET_UNICODE in line:
                 json_data.update(parse_name_line(line))
             elif "STR " in line:
                 json_data.update(parse_attribute_line(line))
+            elif FINAL_LINE_REGEX.search(line):
+                remaning_lines += line.strip("\n")
+                json_data.update(parse_standard_tags(remaning_lines))
+                print("Saving character data for: " + json_data[NAME_TAG])
+                print(json_data)
+                remaning_lines = ""
             else:
                 remaning_lines += line.strip("\n") + " "
     # print(remaning_lines)
-    json_data.update(parse_tag(EQUIP_TAG, remaning_lines))
-    json_data.update(parse_tag(ADV_TAG, remaning_lines))
-    json_data.update(parse_tag(SKILLS_TAG, remaning_lines))
-    json_data.update(parse_tag(OFFENSE_TAG, remaning_lines))
-    json_data.update(parse_tag(DEFENSE_TAG, remaning_lines))
-    json_data.update(parse_tag(TOTALS_TAG, remaning_lines))
-    print(json_data)
+
+
+def parse_standard_tags(lines):
+    json_data = {}
+    for tag in STANDARD_TAGS:
+        if tag + ":" in lines:
+            print("Parsing tag: " + tag)
+            json_data.update(parse_tag(tag, lines))
+    return json_data
 
 
 def parse_tag(tag, remaning_lines):
@@ -65,6 +81,9 @@ def clear_spaces(tup):
 
 def parse_attribute_line(line):
     # STR 0 STA 0 AGL 0 DEX 1 FGT 1 INT 2 AWE 2 PRE 3
+    line = line.replace(DASH_UNICODE, "0")
+    line = line.replace(MINUS_UNICODE, "-")
+    print(line)
     attributes = line.split()
     attribute_dict = {}
     for i in range(0, len(attributes), 2):
@@ -78,14 +97,12 @@ def parse_attribute_line(line):
 def parse_name_line(line):
     # data = line.split("\u2022")
     name_line = {}
-    data = line.split(BULLET)
+    data = line.split(BULLET_UNICODE)
     name_pl = data[0].split(PL_TAG)
     name_line[NAME_TAG] = name_pl[0].strip()
     name_line[PL_TAG] = name_pl[1].strip()
     name_line[MR_TAG] = data[1].strip()
     print(name_line)
-    # json_data = json.dumps(returnDict, indent=2)
-    # print(json_data)
     return name_line