goalc: use iso_data build_info to inform custom level build process (…

…#2959)

CMakePresets.json

@@ -58,6 +58,16 @@
         "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}"
       }
     },
+    {
+      "name": "base-linux-debug",
+      "hidden": true,
+      "inherits": "base",
+      "binaryDir": "${sourceDir}/build/Release/bin",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}"
+      }
+    },
     {
       "name": "base-macos-release",
       "hidden": true,
@@ -173,6 +183,16 @@
         "ZYDIS_BUILD_SHARED_LIB": "OFF"
       }
     },
+    {
+      "name": "Debug-linux-clang-static",
+      "displayName": "Linux Static Release (clang)",
+      "description": "Build with Clang as Release without Debug Symbols but statically linked",
+      "inherits": ["base-linux-debug", "base-clang"],
+      "cacheVariables": {
+        "STATICALLY_LINK": "true",
+        "ZYDIS_BUILD_SHARED_LIB": "OFF"
+      }
+    },
     {
       "name": "Release-linux-gcc",
       "displayName": "Linux Release (gcc)",

decompiler/CMakeLists.txt

@@ -35,6 +35,8 @@ add_library(
         Disasm/OpcodeInfo.cpp
         Disasm/Register.cpp
 
+        extractor/extractor_util.cpp
+
         Function/BasicBlocks.cpp
         Function/CfgVtx.cpp
         Function/Function.cpp
@@ -108,7 +110,7 @@ target_link_libraries(decompiler
 
 
 add_executable(extractor
-        extractor/main.cpp extractor/extractor_util.hpp )
+        extractor/main.cpp extractor/extractor_util.cpp)
 
 target_link_libraries(extractor
         decomp

decompiler/extractor/extractor_util.hpp → decompiler/extractor/extractor_util.cpp

@@ -1,4 +1,4 @@
-#pragma once
+#include "extractor_util.h"
 
 #include <optional>
 #include <regex>
@@ -15,94 +15,34 @@
 #include "third-party/json.hpp"
 #include "third-party/zstd/lib/common/xxhash.h"
 
-enum class ExtractorErrorCode {
-  SUCCESS = 0,
-  INVALID_CLI_INPUT = 3990,
-  VALIDATION_CANT_LOCATE_ELF = 4000,
-  VALIDATION_SERIAL_MISSING_FROM_DB = 4001,
-  VALIDATION_ELF_MISSING_FROM_DB = 4002,
-  VALIDATION_BAD_ISO_CONTENTS = 4010,
-  VALIDATION_INCORRECT_EXTRACTION_COUNT = 4011,
-  VALIDATION_FILE_CONTENTS_UNEXPECTED = 4012,
-  VALIDATION_BAD_EXTRACTION = 4020,
-  DECOMPILATION_GENERIC_ERROR = 4030,
-  EXTRACTION_INVALID_ISO_PATH = 4040,
-  EXTRACTION_ISO_UNEXPECTED_SIZE = 4041,
-  COMPILATION_BAD_PROJECT_PATH = 4050,
-};
-
-enum GameIsoFlags { FLAG_JAK1_BLACK_LABEL = (1 << 0) };
-
-static const std::unordered_map<std::string, GameIsoFlags> sGameIsoFlagNames = {
+const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names = {
     {"jak1-black-label", FLAG_JAK1_BLACK_LABEL}};
 
-static const std::unordered_map<int, std::string> sGameIsoTerritoryMap = {
+const std::unordered_map<int, std::string> game_iso_territory_map = {
     {GAME_TERRITORY_SCEA, "NTSC-U"},
     {GAME_TERRITORY_SCEE, "PAL"},
     {GAME_TERRITORY_SCEI, "NTSC-J"},
     {GAME_TERRITORY_SCEK, "NTSC-K"}};
 
 std::string get_territory_name(int territory) {
-  ASSERT_MSG(sGameIsoTerritoryMap.count(territory),
+  ASSERT_MSG(game_iso_territory_map.count(territory),
              fmt::format("territory {} not found in territory name map"));
-  return sGameIsoTerritoryMap.at(territory);
+  return game_iso_territory_map.at(territory);
 }
 
 // used for - decompiler_out/<jak1> and iso_data/<jak1>
-std::unordered_map<std::string, std::string> data_subfolders = {{"jak1", "jak1"}};
-
-struct ISOMetadata {
-  std::string canonical_name;
-  int region;  // territory code
-  int num_files;
-  uint64_t contents_hash;
-  std::string decomp_config_version;
-  std::string game_name;
-  std::vector<std::string> flags;
-};
-
-// This is all we need to re-fetch info from the database
-// - if this changes such that we have a collision in the future,
-//   then the database isn't adequate and everything must change
-struct BuildInfo {
-  std::string serial = "";
-  uint64_t elf_hash = 0;
-};
-
-void to_json(nlohmann::json& j, const BuildInfo& info) {
-  j = nlohmann::json{{"serial", info.serial}, {"elf_hash", info.elf_hash}};
-}
-
-void from_json(const nlohmann::json& j, BuildInfo& info) {
-  j[0].at("serial").get_to(info.serial);
-  j[0].at("elf_hash").get_to(info.elf_hash);
-}
-
-std::optional<BuildInfo> get_buildinfo_from_path(fs::path iso_data_path) {
-  if (!fs::exists(iso_data_path / "buildinfo.json")) {
-    return {};
-  }
-  auto buildinfo_path = (iso_data_path / "buildinfo.json").string();
-  try {
-    return parse_commented_json(file_util::read_text_file(buildinfo_path), buildinfo_path)
-        .get<BuildInfo>();
-  } catch (std::exception& e) {
-    lg::error("JSON parsing error on buildinfo.json - {}", e.what());
-    return {};
-  }
-}
+const std::unordered_map<std::string, std::string> data_subfolders = {{"jak1", "jak1"}};
 
-static const ISOMetadata jak1_ntsc_black_label_info = {
-    "Jak & Daxter™: The Precursor Legacy (Black Label)",
-    GAME_TERRITORY_SCEA,
-    337,
-    11363853835861842434U,
-    "ntsc_v1",
-    "jak1",
-    {"jak1-black-label"}};
+const ISOMetadata jak1_ntsc_black_label_info = {"Jak & Daxter™: The Precursor Legacy (Black Label)",
+                                                GAME_TERRITORY_SCEA,
+                                                337,
+                                                11363853835861842434U,
+                                                "ntsc_v1",
+                                                "jak1",
+                                                {"jak1-black-label"}};
 
 // { SERIAL : { ELF_HASH : ISOMetadataDatabase } }
-static const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>> isoDatabase{
+const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>> iso_database = {
     {"SCUS-97124",
      {{7280758013604870207U, jak1_ntsc_black_label_info},
       {744661860962747854,
@@ -132,12 +72,36 @@ static const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMet
         "jak1",
         {}}}}}};
 
+void to_json(nlohmann::json& j, const BuildInfo& info) {
+  j = nlohmann::json{{"serial", info.serial}, {"elf_hash", info.elf_hash}};
+}
+
+void from_json(const nlohmann::json& j, BuildInfo& info) {
+  j[0].at("serial").get_to(info.serial);
+  j[0].at("elf_hash").get_to(info.elf_hash);
+}
+
+std::optional<BuildInfo> get_buildinfo_from_path(fs::path iso_data_path) {
+  if (!fs::exists(iso_data_path / "buildinfo.json")) {
+    return {};
+  }
+  auto buildinfo_path = (iso_data_path / "buildinfo.json").string();
+  try {
+    const auto buildinfo_data = file_util::read_text_file(buildinfo_path);
+    lg::info("Found version info - {}", buildinfo_data);
+    return parse_commented_json(buildinfo_data, buildinfo_path).get<BuildInfo>();
+  } catch (std::exception& e) {
+    lg::error("JSON parsing error on buildinfo.json - {}", e.what());
+    return {};
+  }
+}
+
 std::optional<ISOMetadata> get_version_info_from_build_info(const BuildInfo& build_info) {
   if (build_info.serial.empty() || build_info.elf_hash == 0) {
     return {};
   }
-  auto dbEntry = isoDatabase.find(build_info.serial);
-  if (dbEntry == isoDatabase.end()) {
+  auto dbEntry = iso_database.find(build_info.serial);
+  if (dbEntry == iso_database.end()) {
     return {};
   }
 

decompiler/extractor/extractor_util.h

@@ -0,0 +1,90 @@
+#pragma once
+
+#include <optional>
+#include <regex>
+#include <unordered_map>
+
+#include "common/log/log.h"
+#include "common/util/Assert.h"
+#include "common/util/FileUtil.h"
+#include "common/util/json_util.h"
+#include "common/util/read_iso_file.h"
+
+#include "game/kernel/common/kboot.h"
+
+#include "third-party/json.hpp"
+#include "third-party/zstd/lib/common/xxhash.h"
+
+enum class ExtractorErrorCode {
+  SUCCESS = 0,
+  INVALID_CLI_INPUT = 3990,
+  VALIDATION_CANT_LOCATE_ELF = 4000,
+  VALIDATION_SERIAL_MISSING_FROM_DB = 4001,
+  VALIDATION_ELF_MISSING_FROM_DB = 4002,
+  VALIDATION_BAD_ISO_CONTENTS = 4010,
+  VALIDATION_INCORRECT_EXTRACTION_COUNT = 4011,
+  VALIDATION_FILE_CONTENTS_UNEXPECTED = 4012,
+  VALIDATION_BAD_EXTRACTION = 4020,
+  DECOMPILATION_GENERIC_ERROR = 4030,
+  EXTRACTION_INVALID_ISO_PATH = 4040,
+  EXTRACTION_ISO_UNEXPECTED_SIZE = 4041,
+  COMPILATION_BAD_PROJECT_PATH = 4050,
+};
+
+enum GameIsoFlags { FLAG_JAK1_BLACK_LABEL = (1 << 0) };
+
+extern const std::unordered_map<std::string, GameIsoFlags> game_iso_flag_names;
+
+extern const std::unordered_map<int, std::string> game_iso_territory_map;
+
+// used for - decompiler_out/<jak1> and iso_data/<jak1>
+extern const std::unordered_map<std::string, std::string> data_subfolders;
+
+std::string get_territory_name(int territory);
+
+struct ISOMetadata {
+  std::string canonical_name;
+  int region;  // territory code
+  int num_files;
+  uint64_t contents_hash;
+  std::string decomp_config_version;
+  std::string game_name;
+  std::vector<std::string> flags;
+};
+
+extern const ISOMetadata jak1_ntsc_black_label_info;
+
+// { SERIAL : { ELF_HASH : ISOMetadataDatabase } }
+extern const std::unordered_map<std::string, std::unordered_map<uint64_t, ISOMetadata>>
+    iso_database;
+
+// This is all we need to re-fetch info from the database
+// - if this changes such that we have a collision in the future,
+//   then the database isn't adequate and everything must change
+struct BuildInfo {
+  std::string serial = "";
+  uint64_t elf_hash = 0;
+};
+void to_json(nlohmann::json& j, const BuildInfo& info);
+void from_json(const nlohmann::json& j, BuildInfo& info);
+
+std::optional<BuildInfo> get_buildinfo_from_path(fs::path iso_data_path);
+
+std::optional<ISOMetadata> get_version_info_from_build_info(const BuildInfo& build_info);
+
+ISOMetadata get_version_info_or_default(const fs::path& iso_data_path);
+
+std::tuple<std::optional<std::string>, std::optional<uint64_t>> findElfFile(
+    const fs::path& extracted_iso_path);
+
+void log_potential_new_db_entry(ExtractorErrorCode error_code,
+                                const std::string& serial,
+                                const uint64_t elf_hash,
+                                const int files_extracted,
+                                const uint64_t contents_hash);
+
+std::tuple<bool, ExtractorErrorCode> is_iso_file(fs::path path_to_supposed_iso);
+
+std::tuple<uint64_t, int> calculate_extraction_hash(const IsoFile& iso_file);
+
+std::tuple<uint64_t, int> calculate_extraction_hash(const fs::path& extracted_iso_path);

decompiler/extractor/main.cpp

@@ -2,7 +2,7 @@
 #include <regex>
 #include <unordered_map>
 
-#include "extractor_util.hpp"
+#include "extractor_util.h"
 
 #include "common/log/log.h"
 #include "common/util/FileUtil.h"
@@ -52,8 +52,8 @@ std::tuple<std::optional<ISOMetadata>, ExtractorErrorCode> validate(
   }
 
   // Find the game in our tracking database
-  auto dbEntry = isoDatabase.find(serial.value());
-  if (dbEntry == isoDatabase.end()) {
+  auto dbEntry = iso_database.find(serial.value());
+  if (dbEntry == iso_database.end()) {
     lg::error("Serial '{}' not found in the validation database", serial.value());
     log_potential_new_db_entry(ExtractorErrorCode::VALIDATION_SERIAL_MISSING_FROM_DB,
                                serial.value(), elf_hash.value(), expected_num_files, expected_hash);
@@ -203,7 +203,7 @@ ExtractorErrorCode compile(const fs::path& iso_data_path, const std::string& dat
 
   int flags = 0;
   for (const auto& flag : version_info.flags) {
-    if (auto it = sGameIsoFlagNames.find(flag); it != sGameIsoFlagNames.end()) {
+    if (auto it = game_iso_flag_names.find(flag); it != game_iso_flag_names.end()) {
       flags |= it->second;
     }
   }
@@ -320,7 +320,7 @@ int main(int argc, char** argv) {
     lg::error("Error: input game name '{}' is not valid", game_name);
     return static_cast<int>(ExtractorErrorCode::INVALID_CLI_INPUT);
   }
-  std::string data_subfolder = data_subfolders[game_name];
+  std::string data_subfolder = data_subfolders.at(game_name);
 
   if (flag_extract) {
     // we extract to a temporary location because we don't know what we're extracting yet!
@@ -356,7 +356,7 @@ int main(int argc, char** argv) {
       } else {
         // We know the version since we just extracted it, so the user didn't need to provide this
         // explicitly
-        data_subfolder = data_subfolders[version_info->game_name];
+        data_subfolder = data_subfolders.at(version_info->game_name);
         iso_data_path = file_util::get_jak_project_dir() / "iso_data" / data_subfolder;
         if (fs::exists(iso_data_path)) {
           fs::remove_all(iso_data_path);