From 07d97bce8c01272e55f6b85bc736c615e4857073 Mon Sep 17 00:00:00 2001 From: Tyler Wilding Date: Sun, 3 Sep 2023 16:17:35 -0600 Subject: [PATCH] goalc: use iso_data `build_info` to inform custom level build process (#2959) --- CMakePresets.json | 20 +++ decompiler/CMakeLists.txt | 4 +- ...{extractor_util.hpp => extractor_util.cpp} | 116 ++++++------------ decompiler/extractor/extractor_util.h | 90 ++++++++++++++ decompiler/extractor/main.cpp | 12 +- goalc/build_level/build_level.cpp | 41 ++++--- 6 files changed, 184 insertions(+), 99 deletions(-) rename decompiler/extractor/{extractor_util.hpp => extractor_util.cpp} (79%) create mode 100644 decompiler/extractor/extractor_util.h diff --git a/CMakePresets.json b/CMakePresets.json index 010dcc1adbb..2e75ce28cbf 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -58,6 +58,16 @@ "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}" } }, + { + "name": "base-linux-debug", + "hidden": true, + "inherits": "base", + "binaryDir": "${sourceDir}/build/Release/bin", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_INSTALL_PREFIX": "${sourceDir}/build/install/${presetName}" + } + }, { "name": "base-macos-release", "hidden": true, @@ -173,6 +183,16 @@ "ZYDIS_BUILD_SHARED_LIB": "OFF" } }, + { + "name": "Debug-linux-clang-static", + "displayName": "Linux Static Release (clang)", + "description": "Build with Clang as Release without Debug Symbols but statically linked", + "inherits": ["base-linux-debug", "base-clang"], + "cacheVariables": { + "STATICALLY_LINK": "true", + "ZYDIS_BUILD_SHARED_LIB": "OFF" + } + }, { "name": "Release-linux-gcc", "displayName": "Linux Release (gcc)", diff --git a/decompiler/CMakeLists.txt b/decompiler/CMakeLists.txt index 2a6a6d2c421..31738c1b9a7 100644 --- a/decompiler/CMakeLists.txt +++ b/decompiler/CMakeLists.txt @@ -35,6 +35,8 @@ add_library( Disasm/OpcodeInfo.cpp Disasm/Register.cpp + extractor/extractor_util.cpp + Function/BasicBlocks.cpp Function/CfgVtx.cpp Function/Function.cpp @@ -108,7 +110,7 @@ target_link_libraries(decompiler add_executable(extractor - extractor/main.cpp extractor/extractor_util.hpp ) + extractor/main.cpp extractor/extractor_util.cpp) target_link_libraries(extractor decomp diff --git a/decompiler/extractor/extractor_util.hpp b/decompiler/extractor/extractor_util.cpp similarity index 79% rename from decompiler/extractor/extractor_util.hpp rename to decompiler/extractor/extractor_util.cpp index 3331601a5cf..fd565cc033c 100644 --- a/decompiler/extractor/extractor_util.hpp +++ b/decompiler/extractor/extractor_util.cpp @@ -1,4 +1,4 @@ -#pragma once +#include "extractor_util.h" #include #include @@ -15,94 +15,34 @@ #include "third-party/json.hpp" #include "third-party/zstd/lib/common/xxhash.h" -enum class ExtractorErrorCode { - SUCCESS = 0, - INVALID_CLI_INPUT = 3990, - VALIDATION_CANT_LOCATE_ELF = 4000, - VALIDATION_SERIAL_MISSING_FROM_DB = 4001, - VALIDATION_ELF_MISSING_FROM_DB = 4002, - VALIDATION_BAD_ISO_CONTENTS = 4010, - VALIDATION_INCORRECT_EXTRACTION_COUNT = 4011, - VALIDATION_FILE_CONTENTS_UNEXPECTED = 4012, - VALIDATION_BAD_EXTRACTION = 4020, - DECOMPILATION_GENERIC_ERROR = 4030, - EXTRACTION_INVALID_ISO_PATH = 4040, - EXTRACTION_ISO_UNEXPECTED_SIZE = 4041, - COMPILATION_BAD_PROJECT_PATH = 4050, -}; - -enum GameIsoFlags { FLAG_JAK1_BLACK_LABEL = (1 << 0) }; - -static const std::unordered_map sGameIsoFlagNames = { +const std::unordered_map game_iso_flag_names = { {"jak1-black-label", FLAG_JAK1_BLACK_LABEL}}; -static const std::unordered_map sGameIsoTerritoryMap = { +const std::unordered_map game_iso_territory_map = { {GAME_TERRITORY_SCEA, "NTSC-U"}, {GAME_TERRITORY_SCEE, "PAL"}, {GAME_TERRITORY_SCEI, "NTSC-J"}, {GAME_TERRITORY_SCEK, "NTSC-K"}}; std::string get_territory_name(int territory) { - ASSERT_MSG(sGameIsoTerritoryMap.count(territory), + ASSERT_MSG(game_iso_territory_map.count(territory), fmt::format("territory {} not found in territory name map")); - return sGameIsoTerritoryMap.at(territory); + return game_iso_territory_map.at(territory); } // used for - decompiler_out/ and iso_data/ -std::unordered_map data_subfolders = {{"jak1", "jak1"}}; - -struct ISOMetadata { - std::string canonical_name; - int region; // territory code - int num_files; - uint64_t contents_hash; - std::string decomp_config_version; - std::string game_name; - std::vector flags; -}; - -// This is all we need to re-fetch info from the database -// - if this changes such that we have a collision in the future, -// then the database isn't adequate and everything must change -struct BuildInfo { - std::string serial = ""; - uint64_t elf_hash = 0; -}; - -void to_json(nlohmann::json& j, const BuildInfo& info) { - j = nlohmann::json{{"serial", info.serial}, {"elf_hash", info.elf_hash}}; -} - -void from_json(const nlohmann::json& j, BuildInfo& info) { - j[0].at("serial").get_to(info.serial); - j[0].at("elf_hash").get_to(info.elf_hash); -} - -std::optional get_buildinfo_from_path(fs::path iso_data_path) { - if (!fs::exists(iso_data_path / "buildinfo.json")) { - return {}; - } - auto buildinfo_path = (iso_data_path / "buildinfo.json").string(); - try { - return parse_commented_json(file_util::read_text_file(buildinfo_path), buildinfo_path) - .get(); - } catch (std::exception& e) { - lg::error("JSON parsing error on buildinfo.json - {}", e.what()); - return {}; - } -} +const std::unordered_map data_subfolders = {{"jak1", "jak1"}}; -static const ISOMetadata jak1_ntsc_black_label_info = { - "Jak & Daxter™: The Precursor Legacy (Black Label)", - GAME_TERRITORY_SCEA, - 337, - 11363853835861842434U, - "ntsc_v1", - "jak1", - {"jak1-black-label"}}; +const ISOMetadata jak1_ntsc_black_label_info = {"Jak & Daxter™: The Precursor Legacy (Black Label)", + GAME_TERRITORY_SCEA, + 337, + 11363853835861842434U, + "ntsc_v1", + "jak1", + {"jak1-black-label"}}; // { SERIAL : { ELF_HASH : ISOMetadataDatabase } } -static const std::unordered_map> isoDatabase{ +const std::unordered_map> iso_database = { {"SCUS-97124", {{7280758013604870207U, jak1_ntsc_black_label_info}, {744661860962747854, @@ -132,12 +72,36 @@ static const std::unordered_map get_buildinfo_from_path(fs::path iso_data_path) { + if (!fs::exists(iso_data_path / "buildinfo.json")) { + return {}; + } + auto buildinfo_path = (iso_data_path / "buildinfo.json").string(); + try { + const auto buildinfo_data = file_util::read_text_file(buildinfo_path); + lg::info("Found version info - {}", buildinfo_data); + return parse_commented_json(buildinfo_data, buildinfo_path).get(); + } catch (std::exception& e) { + lg::error("JSON parsing error on buildinfo.json - {}", e.what()); + return {}; + } +} + std::optional get_version_info_from_build_info(const BuildInfo& build_info) { if (build_info.serial.empty() || build_info.elf_hash == 0) { return {}; } - auto dbEntry = isoDatabase.find(build_info.serial); - if (dbEntry == isoDatabase.end()) { + auto dbEntry = iso_database.find(build_info.serial); + if (dbEntry == iso_database.end()) { return {}; } diff --git a/decompiler/extractor/extractor_util.h b/decompiler/extractor/extractor_util.h new file mode 100644 index 00000000000..acfaf69fc05 --- /dev/null +++ b/decompiler/extractor/extractor_util.h @@ -0,0 +1,90 @@ +#pragma once + +#include +#include +#include + +#include "common/log/log.h" +#include "common/util/Assert.h" +#include "common/util/FileUtil.h" +#include "common/util/json_util.h" +#include "common/util/read_iso_file.h" + +#include "game/kernel/common/kboot.h" + +#include "third-party/json.hpp" +#include "third-party/zstd/lib/common/xxhash.h" + +enum class ExtractorErrorCode { + SUCCESS = 0, + INVALID_CLI_INPUT = 3990, + VALIDATION_CANT_LOCATE_ELF = 4000, + VALIDATION_SERIAL_MISSING_FROM_DB = 4001, + VALIDATION_ELF_MISSING_FROM_DB = 4002, + VALIDATION_BAD_ISO_CONTENTS = 4010, + VALIDATION_INCORRECT_EXTRACTION_COUNT = 4011, + VALIDATION_FILE_CONTENTS_UNEXPECTED = 4012, + VALIDATION_BAD_EXTRACTION = 4020, + DECOMPILATION_GENERIC_ERROR = 4030, + EXTRACTION_INVALID_ISO_PATH = 4040, + EXTRACTION_ISO_UNEXPECTED_SIZE = 4041, + COMPILATION_BAD_PROJECT_PATH = 4050, +}; + +enum GameIsoFlags { FLAG_JAK1_BLACK_LABEL = (1 << 0) }; + +extern const std::unordered_map game_iso_flag_names; + +extern const std::unordered_map game_iso_territory_map; + +// used for - decompiler_out/ and iso_data/ +extern const std::unordered_map data_subfolders; + +std::string get_territory_name(int territory); + +struct ISOMetadata { + std::string canonical_name; + int region; // territory code + int num_files; + uint64_t contents_hash; + std::string decomp_config_version; + std::string game_name; + std::vector flags; +}; + +extern const ISOMetadata jak1_ntsc_black_label_info; + +// { SERIAL : { ELF_HASH : ISOMetadataDatabase } } +extern const std::unordered_map> + iso_database; + +// This is all we need to re-fetch info from the database +// - if this changes such that we have a collision in the future, +// then the database isn't adequate and everything must change +struct BuildInfo { + std::string serial = ""; + uint64_t elf_hash = 0; +}; +void to_json(nlohmann::json& j, const BuildInfo& info); +void from_json(const nlohmann::json& j, BuildInfo& info); + +std::optional get_buildinfo_from_path(fs::path iso_data_path); + +std::optional get_version_info_from_build_info(const BuildInfo& build_info); + +ISOMetadata get_version_info_or_default(const fs::path& iso_data_path); + +std::tuple, std::optional> findElfFile( + const fs::path& extracted_iso_path); + +void log_potential_new_db_entry(ExtractorErrorCode error_code, + const std::string& serial, + const uint64_t elf_hash, + const int files_extracted, + const uint64_t contents_hash); + +std::tuple is_iso_file(fs::path path_to_supposed_iso); + +std::tuple calculate_extraction_hash(const IsoFile& iso_file); + +std::tuple calculate_extraction_hash(const fs::path& extracted_iso_path); diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index a1f553a0590..87eccd6e0d5 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -2,7 +2,7 @@ #include #include -#include "extractor_util.hpp" +#include "extractor_util.h" #include "common/log/log.h" #include "common/util/FileUtil.h" @@ -52,8 +52,8 @@ std::tuple, ExtractorErrorCode> validate( } // Find the game in our tracking database - auto dbEntry = isoDatabase.find(serial.value()); - if (dbEntry == isoDatabase.end()) { + auto dbEntry = iso_database.find(serial.value()); + if (dbEntry == iso_database.end()) { lg::error("Serial '{}' not found in the validation database", serial.value()); log_potential_new_db_entry(ExtractorErrorCode::VALIDATION_SERIAL_MISSING_FROM_DB, serial.value(), elf_hash.value(), expected_num_files, expected_hash); @@ -203,7 +203,7 @@ ExtractorErrorCode compile(const fs::path& iso_data_path, const std::string& dat int flags = 0; for (const auto& flag : version_info.flags) { - if (auto it = sGameIsoFlagNames.find(flag); it != sGameIsoFlagNames.end()) { + if (auto it = game_iso_flag_names.find(flag); it != game_iso_flag_names.end()) { flags |= it->second; } } @@ -320,7 +320,7 @@ int main(int argc, char** argv) { lg::error("Error: input game name '{}' is not valid", game_name); return static_cast(ExtractorErrorCode::INVALID_CLI_INPUT); } - std::string data_subfolder = data_subfolders[game_name]; + std::string data_subfolder = data_subfolders.at(game_name); if (flag_extract) { // we extract to a temporary location because we don't know what we're extracting yet! @@ -356,7 +356,7 @@ int main(int argc, char** argv) { } else { // We know the version since we just extracted it, so the user didn't need to provide this // explicitly - data_subfolder = data_subfolders[version_info->game_name]; + data_subfolder = data_subfolders.at(version_info->game_name); iso_data_path = file_util::get_jak_project_dir() / "iso_data" / data_subfolder; if (fs::exists(iso_data_path)) { fs::remove_all(iso_data_path); diff --git a/goalc/build_level/build_level.cpp b/goalc/build_level/build_level.cpp index 6b21ab8adf2..270411aec51 100644 --- a/goalc/build_level/build_level.cpp +++ b/goalc/build_level/build_level.cpp @@ -7,6 +7,7 @@ #include "common/util/json_util.h" #include "common/util/string_util.h" +#include "decompiler/extractor/extractor_util.h" #include "decompiler/level_extractor/extract_merc.h" #include "goalc/build_level/Entity.h" #include "goalc/build_level/FileInfo.h" @@ -144,37 +145,45 @@ bool run_build_level(const std::string& input_file, // Add textures and models // TODO remove hardcoded config settings if (level_json.contains("art_groups") && !level_json.at("art_groups").empty()) { - decompiler::Config config; - try { - config = decompiler::read_config_file( - file_util::get_jak_project_dir() / "decompiler/config/jak1/jak1_config.jsonc", "ntsc_v1", - R"({"decompile_code": false, "find_functions": false, "levels_extract": true, "allowed_objects": []})"); - } catch (const std::exception& e) { - lg::error("Failed to parse config: {}", e.what()); - return false; - } - - fs::path in_folder; + fs::path iso_folder = ""; lg::info("Looking for ISO path..."); + // TODO - add to file_util for (const auto& entry : fs::directory_iterator(file_util::get_jak_project_dir() / "iso_data")) { + // TODO - hard-coded to jak 1 if (entry.is_directory() && entry.path().filename().string().find("jak1") != std::string::npos) { lg::info("Found ISO path: {}", entry.path().string()); - in_folder = entry.path(); + iso_folder = entry.path(); } } - if (!fs::exists(in_folder)) { - lg::error("Could not find ISO path!"); + + if (iso_folder.empty() || !fs::exists(iso_folder)) { + lg::warn("Could not locate ISO path!"); return false; } + + // Look for iso build info if it's available, otherwise default to ntsc_v1 + const auto version_info = get_version_info_or_default(iso_folder); + + decompiler::Config config; + try { + config = decompiler::read_config_file( + file_util::get_jak_project_dir() / "decompiler/config/jak1/jak1_config.jsonc", + version_info.decomp_config_version, + R"({"decompile_code": false, "find_functions": false, "levels_extract": true, "allowed_objects": []})"); + } catch (const std::exception& e) { + lg::error("Failed to parse config: {}", e.what()); + return false; + } + std::vector dgos, objs; for (const auto& dgo_name : config.dgo_names) { - dgos.push_back(in_folder / dgo_name); + dgos.push_back(iso_folder / dgo_name); } for (const auto& obj_name : config.object_file_names) { - objs.push_back(in_folder / obj_name); + objs.push_back(iso_folder / obj_name); } decompiler::ObjectFileDB db(dgos, fs::path(config.obj_file_name_map_file), objs, {}, {},