Remove duplicates

applications/main/nfc/resources/nfc/assets/mf_classic_dict.nfc

@@ -1,4 +1,4 @@
-# Key dictionary from https://github.com/RfidResearchGroup/proxmark3/
+# Key dictionary from https://github.com/RfidResearchGroup/proxmark3/blob/master/client/dictionaries/mfc_default_keys.dic
 #
 # Mifare Default Keys
 #   -- iceman fork version --
@@ -2507,8 +2507,6 @@ AA034F342A55
 456776908C48
 
 # BusFacil - Brazilian public transport card for some cities
-7b296f353c6b
-3fa7217ec575
 fae9b14365a9
 c567dd4a6004
 c567dd4a6005
@@ -2600,9 +2598,7 @@ b1ea40b2caa6
 # Sector 15 - see above
 # SKGT personalised subscription card
 # Sector 0, 2, 16, key A
-a0a1a2a3a4a5
 # Sector 8-14, 17-39, key A
-ffffffffffff
 # Sector 1, key A
 # blue
 f1df0ca8948b

scripts/fix_mfc_dict.py

@@ -2,33 +2,31 @@
 import pathlib
 import string
 
+KEY_LENGTH = 12
+
 file = (
     pathlib.Path(__file__).parent
     / "../applications/main/nfc/resources/nfc/assets/mf_classic_dict.nfc"
 )
-try:
-    lines = file.read_text(encoding="ascii").splitlines()
-except UnicodeDecodeError:
-    print(
-        "Fix non-ASCII characters: https://pteo.paranoiaworks.mobi/diacriticsremover/"
-    )
-    exit()
+lines = file.read_text().split("\n")
 
 total = 0
-for i, line in enumerate(lines):
-    lines[i] = line = line.strip()
+for i, line in reversed(list(enumerate(lines))):
     if line.startswith("#"):
         continue
-    lines[i] = line = line.upper()
     if not line:
         continue
-    if len(line) != 12 or any(char not in string.hexdigits for char in line):
+    key = line[:KEY_LENGTH]
+    if len(key) != KEY_LENGTH or any(char not in string.hexdigits for char in key):
         print(f"Line {i + 1} is not correct: {line}")
-    else:
-        total = total + 1
-    for j in reversed(range(i + 1, len(lines))):
-        if lines[j].upper().strip() == line:
-            del lines[j]
+    for check in lines[:i]:
+        if check.upper()[:KEY_LENGTH] == line.upper()[:KEY_LENGTH]:
+            print(f"Line {i + 1} is a duplicate: {line}")
+            del lines[i]
+            break
+    else:  # Didn't break
+        total += 1
+
 
-file.write_text("\n".join(line for line in lines if line.removeprefix("#")))
+file.write_text("\n".join(lines))
 print(f"Total keys: {total}")