changed Movie usm conversion tool

.gitignore

@@ -10,4 +10,5 @@ Test.py
 *.spec
 Run.exe
 PriconneUtils.exe
+PriconneUtilsPrivate.exe
 *.ico

config.json

@@ -1,6 +1,6 @@
 {
- "TruthVersion": 10039800,
- "hash": "5b6d3e1264489008ce6044d4a69d106a",
+ "TruthVersion": 10048500,
+ "hash": "56ff9804a36e2b0d7c4110c79afaaff3",
  "assetmanifest": "http://prd-priconne-redive.akamaized.net/dl/Resources/version/Jpn/AssetBundles/Windows/manifest/manifest_assetmanifest",
  "masterdata": "http://prd-priconne-redive.akamaized.net/dl/Resources/version/Jpn/AssetBundles/Windows/manifest/masterdata_assetmanifest",
  "movie": "http://prd-priconne-redive.akamaized.net/dl/Resources/version/Jpn/Movie/PC/High/manifest/moviemanifest",

scripts/Constants.py

@@ -1,4 +1,5 @@
-import os, re
+import os
+import re
 
 CUR_DIR = os.getcwd()
 

scripts/DBCheck.py

@@ -1,4 +1,8 @@
-import os, requests, json
+import os
+import json
+
+import requests
+
 import Constants
 
 # Function to update the db if there is one
@@ -63,7 +67,7 @@ def check_update():
 
         # Checks if the TruthVersion is the same as in the latest_ver.json
         if version == int(c["TruthVersion"]):
-            print('> Database version is up to date!')
+            print('> Database version is up to date!\n')
 
         else:
             print(f'New version available: [{version}], updating database...')

scripts/Movie.py

@@ -1,93 +1,197 @@
-import os, requests, re, multiprocessing
-import Constants
+import os
+import re
+import traceback
+import subprocess
+import multiprocessing
 from concurrent.futures import ThreadPoolExecutor
 
-name = []
-hash = []
+import requests
+import PyCriCodecs as pcc
+
+import Constants
 
-# Generates name and hash list for ThreadPoolExecutor and multiprocessing
-def generate_list(mov_name, dir_name):
+
+def generate_list(mov_name: str, dir_name: str) -> tuple[list[str], list[str]]:
+    """
+    Generates name and hash list for ThreadPoolExecutor and multiprocessing
+    """
+
     # print(f'mov_name: {mov_name} | dir_name: {dir_name} | isdir: {os.path.isdir(dir_name)}')
     if not os.path.isdir(dir_name):
         os.makedirs(dir_name)
 
     # print(mov_name)
-    with open(Constants.MOVIEMANIFEST, 'r') as m:
+    name = []
+    hash = []
+    with open(Constants.MOVIEMANIFEST, "r") as m:
         for lines in m:
-            l = lines.split(',')
-            n = l[0].split('/')[-1]
+            l = lines.split(",")
+            n = l[0].split("/")[-1]
             h = l[1]
 
             if re.fullmatch(mov_name, n):
                 name.append(os.path.join(dir_name, n))
                 hash.append(h)
 
-    # return name, hash
+    return name, hash
+
+
+def download_file(name: str, hash: str) -> None:
+    """
+    Downloads all video files that doesn't exist yet
+    """
 
-# Downloads all video files that doesn't exist yet
-def download_file(name: str, hash):
     # print(f'Filename is {name}')
     if not os.path.isfile(name) and not os.path.isfile(f'{name.split(".")[0]}.mp4'):
-        print(f'Downloading {os.path.basename(name)}...')
+        print(f'Downloading [{os.path.basename(name)}]...')
         r = requests.get(f'{Constants.MOVIE_URL}/{hash[:2]}/{hash}').content
-        with open(name, 'wb') as usm:
+        with open(name, "wb") as usm:
             usm.write(r)
 
-    else:
-        print(f'File [{os.path.basename(name)}] already exists')
-    # return
+    # else:
+    #     print(f'File [{os.path.basename(name)}] already exists')
+
 
+def convert_file(name: str) -> None:
+    """
+    Converts all of the usm files to mp4 files
+    """
 
-# Converts all of the usm files to mp4 files
-def convert_file(name: str):
     try:
-        if not os.path.isfile(f'{name.split(".")[0]}.mp4'):
-            os.system(f'UsmToolkit convert -c "{name}" -o "{os.path.dirname(name)}"')
+        name_mp4 = name.split(".")[0] + ".mp4"
+        if not os.path.isfile(name_mp4):
+            usm_obj = pcc.USM(name)
+            usm_obj.demux()
+            filenames = extract(usm_obj, os.path.dirname(name))
+            ffmpeg_combine = "ffmpeg -hide_banner -loglevel quiet -y "
+            for fname in filenames[1:]:
+                ffmpeg_combine += f'-i {fname} '
+
+            ffmpeg_combine += name_mp4
+            print(f'Converting [{os.path.basename(name)}]...')
+            subprocess.run(ffmpeg_combine, check=True)
+
+            # Unlink the usm_obj so the usm file can be removed
+            del usm_obj
+            for file in filenames:
+                os.remove(file)
 
-        # else:
-        #     print(f'File [.\\{str(name).split(".")[1]}.mp4] already exists')
+    except subprocess.CalledProcessError:
+        print("FFmpeg not found!")
+        print("Please install FFmpeg first or make sure its in the PATH variable")
+        input("Press ENTER to exit")
+        exit(1)
+
+    except NotImplementedError as nie:
+        print(nie)
+
+    except Exception:
+        print(f'An ERROR occured\n{traceback.format_exc()}')
+
+
+def extract(usm: pcc.USM, dirname: str = "") -> list[str]:
+    """
+    Slightly modified extract() function from PyCriCodecs's usm.py
+    to accomodate Priconne's USM\n
+    Returns a list of all filenames inside of the USM
+    """
+
+    # Gets a table consisting of video/audio metadata
+    table = usm.get_metadata()[0]["CRIUSF_DIR_STREAM"]
+    filenames = []
+    for obj in table:
+        fname: str = obj["filename"][1]
+
+        # Taken from PyCriCodecs' usm.py extract() method
+        # Adjust filenames and/or paths to extract them into the current directory.
+        if ":\\" in fname: # Absolute paths.
+            fname = fname.split(":\\", 1)[1]
+        elif ":/" in fname: # Absolute paths.
+            fname = fname.split(":/", 1)[1]
+        elif ":"+os.sep in fname: # Absolute paths.
+            fname = fname.split(":"+os.sep, 1)[1]
+        elif ".."+os.sep in fname: # Relative paths.
+            fname = fname.rsplit(".."+os.sep, 1)[1]
+        elif "../" in fname: # Relative paths.
+            fname = fname.rsplit("../", 1)[1]
+        elif "..\\" in fname: # Relative paths.
+            fname = fname.rsplit("..\\", 1)[1]
+        fname = ''.join(x for x in fname if x not in ':?*<>|"') # removes illegal characters.
+
+        fname = os.path.join(dirname, fname)
+        if fname not in filenames:
+            filenames.append(fname)
 
-    except Exception as e:
-        print(f'An ERROR occured\n{e}')
+    for i, (k, data) in enumerate(usm.output.items()):
+        if "SFV" in k:
+            with open(filenames[i+1], "wb") as out:
+                out.write(data)
+
+        elif "SFA" in k:
+            audio = pcc.ADX(data)
+            with open(filenames[i+1], "wb") as out:
+                out.write(audio.decode())
+
+        else:
+            raise NotImplementedError(f'Unknown type of data: {k}\nHeader: {data[:4]}')
+
+    return filenames
+
 
-    # return
+def movie() -> None:
+    """
+    Runs the Movie download and conversion logic
+    """
 
-# Main function to run
-def movie():
     if not os.path.isfile(Constants.MOVIEMANIFEST):
-        print("Please do DBCheck first before using this as the file needed to download stuff in this script is downloaded from DBCheck")
+        print(
+            "Please do DBCheck first before using this "
+            "as the file needed to download stuff in this script "
+            "is downloaded from DBCheck"
+        )
         input("Press ENTER to continue")
 
     else:
-        mov_type = input("Select type: 'cutin' 'l2d' 'summon' 'event'\n")
+        print("Select type: (write the number)")
+        print("1. cutin\n2. l2d\n3. summon\n4. event")
+        mov_type = input(">> ").lower().strip()
 
         try:
-            dir_name = Constants.MOVIE_TYPES['dir'][mov_type.strip()]
-            mov_name = Constants.MOVIE_TYPES['name'][mov_type.strip()]
-            generate_list(mov_name, dir_name)
-
-        except:
-            print("> INVALID TYPE! <\nCurrent types are only 'cutin', 'l2d', 'summon', or 'event'\n")
+            mov_dict = {1: "cutin", 2: "l2d", 3: "summon", 4: "event"}
+            mov_type = mov_dict[int(mov_type)]
+            dir_name = Constants.MOVIE_TYPES["dir"][mov_type]
+            mov_name = Constants.MOVIE_TYPES["name"][mov_type]
+            name, hash = generate_list(mov_name, dir_name)
+
+        except (KeyError, ValueError):
+            print("> INVALID TYPE! <")
+            print("Current types are only 'cutin', 'l2d', 'summon', or 'event'\n")
             input("Press ENTER to continue")
 
         else:
-            # ThreadPoolExecutor and multiprocessing makes it so the downloading and converting of a file doesn't happen 1 by 1
-            # Instead, a few files are downloaded at a time and after all of those files finished downloading, they get
-            # converted a few files at the same time as well
-            # The number of files downloaded or converted at a time will depend on your system and download speed (for download)
-            with ThreadPoolExecutor() as thread:
-                thread.map(download_file, name, hash)
-                thread.shutdown(wait=True)
-
-            with multiprocessing.Pool() as pool:
-                pool.map(convert_file, name)
-                pool.terminate()
-
-        finally:
-            input(">> Download and conversion completed!\nPress ENTER to continue")
+            if mov_type in ["cutin", "l2d", "summon"]:
+                pass
+
+            else:
+                # ThreadPoolExecutor and multiprocessing makes it so the 
+                # downloading and converting of a file doesn't happen 1 by 1
+                # Instead, a few files are downloaded at a time and after all 
+                # of those files finished downloading, they get converted a 
+                # few files at the same time as well
+                # The number of files downloaded or converted at a time will 
+                # depend on your system and download speed (for download)
+                with ThreadPoolExecutor() as thread:
+                    thread.map(download_file, name, hash)
+                    thread.shutdown(wait=True)
+
+                with multiprocessing.Pool() as pool:
+                    pool.map(convert_file, name)
+                    pool.terminate()
+
             name.clear()
             hash.clear()
+            input(">> Download and conversion completed!\nPress ENTER to continue")
 
 
-# if __name__ == '__main__':
-#     movie()
+if __name__ == "__main__":
+    movie()

scripts/Run.py

@@ -1,4 +1,5 @@
-import sys, multiprocessing
+import sys
+import multiprocessing
 from DBCheck import update_db
 from Movie import movie
 from Sound import sound

scripts/Sound.py

@@ -1,7 +1,13 @@
-import os, requests, re, shutil, multiprocessing
-import Constants
+import os
+import re
+import shutil
+import multiprocessing
 from concurrent.futures import ThreadPoolExecutor
 
+import requests
+
+import Constants
+
 name = []
 hash = []
 
@@ -82,7 +88,7 @@ def convert_file(name: str):
 
 def check_file(fn, fd):
     for file in os.listdir(fd):
-        a = re.search(fn, file)
+        a: re.Match = re.search(fn, file)
         if a: 
             return a.group()