From 160f503bc81bffdef6dbaa16eec7c73fccef0eee Mon Sep 17 00:00:00 2001 From: Jack <66967891+ASpoonPlaysGames@users.noreply.github.com> Date: Sat, 7 Sep 2024 21:10:28 +0100 Subject: [PATCH 1/2] Big rpak loading refactor (#766) This reworks how rpaks are loaded, unloaded and tracked. It allows for rpak reloading between map loads, meaning that skins and map overhauls could be enabled and disabled on the fly. Previous methods of loading rpaks still work. --- primedev/core/filesystem/rpakfilesystem.cpp | 543 ++++++++++++++------ primedev/core/filesystem/rpakfilesystem.h | 86 +++- primedev/mods/modmanager.cpp | 49 +- primedev/mods/modmanager.h | 29 +- primedev/scripts/scriptdatatables.cpp | 11 +- 5 files changed, 524 insertions(+), 194 deletions(-) diff --git a/primedev/core/filesystem/rpakfilesystem.cpp b/primedev/core/filesystem/rpakfilesystem.cpp index da72646b9..ebb9085a7 100644 --- a/primedev/core/filesystem/rpakfilesystem.cpp +++ b/primedev/core/filesystem/rpakfilesystem.cpp @@ -2,206 +2,447 @@ #include "mods/modmanager.h" #include "dedicated/dedicated.h" #include "core/tier0.h" +#include "util/utils.h" -AUTOHOOK_INIT() - -// there are more i'm just too lazy to add +#pragma pack(push, 1) struct PakLoadFuncs { - void* unk0[3]; - int (*LoadPakAsync)(const char* pPath, void* unknownSingleton, int flags, void* callback0, void* callback1); - void* unk1[2]; - void* (*UnloadPak)(int iPakHandle, void* callback); - void* unk2[6]; - void* (*LoadFile)(const char* path); // unsure - void* unk3[10]; - void* (*ReadFileAsync)(const char* pPath, void* a2); + void (*InitRpakSystem)(); + void (*AddAssetLoaderWithJobDetails)(/*assetTypeHeader*/ void*, uint32_t, int); + void (*AddAssetLoader)(/*assetTypeHeader*/ void*); + PakHandle (*LoadRpakFileAsync)(const char* pPath, void* allocator, int flags); + void (*LoadRpakFile)(const char*, __int64(__fastcall*)(), __int64, void(__cdecl*)()); + __int64 qword28; + void (*UnloadPak)(PakHandle iPakHandle, void* callback); + __int64 qword38; + __int64 qword40; + __int64 qword48; + __int64 qword50; + FARPROC (*GetDllCallback)(__int16 a1, const CHAR* a2); + __int64 (*GetAssetByHash)(__int64 hash); + __int64 (*GetAssetByName)(const char* a1); + __int64 qword70; + __int64 qword78; + __int64 qword80; + __int64 qword88; + __int64 qword90; + __int64 qword98; + __int64 qwordA0; + __int64 qwordA8; + __int64 qwordB0; + __int64 qwordBB; + void* (*OpenFile)(const char* pPath); + __int64 CloseFile; + __int64 qwordD0; + __int64 FileReadAsync; + __int64 ComplexFileReadAsync; + __int64 GetReadJobState; + __int64 WaitForFileReadJobComplete; + __int64 CancelFileReadJob; + __int64 CancelFileReadJobAsync; + __int64 qword108; }; +static_assert(sizeof(PakLoadFuncs) == 0x110); +#pragma pack(pop) PakLoadFuncs* g_pakLoadApi; - PakLoadManager* g_pPakLoadManager; -void** pUnknownPakLoadSingleton; -int PakLoadManager::LoadPakAsync(const char* pPath, const ePakLoadSource nLoadSource) +static char* pszCurrentMapRpakPath = nullptr; +static PakHandle* piCurrentMapRpakHandle = nullptr; +static PakHandle* piCurrentMapPatchRpakHandle = nullptr; +static /*CModelLoader*/ void** ppModelLoader = nullptr; +static void** rpakMemoryAllocator = nullptr; + +static __int64 (*o_pLoadGametypeSpecificRpaks)(const char* levelName) = nullptr; +static __int64 (**o_pCleanMaterialSystemStuff)() = nullptr; +static __int64 (**o_pCModelLoader_UnreferenceAllModels)(/*CModelLoader*/ void* a1) = nullptr; +static char* (*o_pLoadlevelLoadscreen)(const char* levelName) = nullptr; +static unsigned int (*o_pGetPakPatchNumber)(const char* pPakPath) = nullptr; + +// Marks all mod Paks to be unloaded on next map load. +// Also cleans up any mod Paks that are already unloaded. +void PakLoadManager::UnloadAllModPaks() { - int nHandle = g_pakLoadApi->LoadPakAsync(pPath, *pUnknownPakLoadSingleton, 2, nullptr, nullptr); - - // set the load source of the pak we just loaded - if (nHandle != -1) - GetPakInfo(nHandle)->m_nLoadSource = nLoadSource; - - return nHandle; + NS::log::rpak->info("Reloading RPaks on next map load..."); + for (auto& modPak : m_modPaks) + { + modPak.m_markedForDelete = true; + } + // clean up any paks that are both marked for unload and already unloaded + CleanUpUnloadedPaks(); + SetForceReloadOnMapLoad(true); } -void PakLoadManager::UnloadPak(const int nPakHandle) +// Tracks all Paks related to a mod. +void PakLoadManager::TrackModPaks(Mod& mod) { - g_pakLoadApi->UnloadPak(nPakHandle, nullptr); + const fs::path modPakPath("./" / mod.m_ModDirectory / "paks"); + + for (auto& modRpakEntry : mod.Rpaks) + { + ModPak_t pak; + pak.m_modName = mod.Name; + pak.m_path = (modPakPath / modRpakEntry.m_pakName).string(); + pak.m_pathHash = STR_HASH(pak.m_path); + + pak.m_preload = modRpakEntry.m_preload; + pak.m_dependentPakHash = modRpakEntry.m_dependentPakHash; + pak.m_mapRegex = modRpakEntry.m_loadRegex; + + m_modPaks.push_back(pak); + } } -void PakLoadManager::UnloadMapPaks() +// Untracks all paks that aren't currently loaded and are marked for unload. +void PakLoadManager::CleanUpUnloadedPaks() { - for (auto& pair : m_vLoadedPaks) - if (pair.second.m_nLoadSource == ePakLoadSource::MAP) - UnloadPak(pair.first); + auto fnRemovePredicate = [](ModPak_t& pak) -> bool { return pak.m_markedForDelete && pak.m_handle == PakHandle::INVALID; }; + + m_modPaks.erase(std::remove_if(m_modPaks.begin(), m_modPaks.end(), fnRemovePredicate), m_modPaks.end()); } -LoadedPak* PakLoadManager::TrackLoadedPak(ePakLoadSource nLoadSource, int nPakHandle, size_t nPakNameHash) +// Unloads all paks that are marked for unload. +void PakLoadManager::UnloadMarkedPaks() { - LoadedPak pak; - pak.m_nLoadSource = nLoadSource; - pak.m_nPakHandle = nPakHandle; - pak.m_nPakNameHash = nPakNameHash; + ++m_reentranceCounter; + const ScopeGuard guard([&]() { --m_reentranceCounter; }); - m_vLoadedPaks.insert(std::make_pair(nPakHandle, pak)); - return &m_vLoadedPaks.at(nPakHandle); + (*o_pCModelLoader_UnreferenceAllModels)(*ppModelLoader); + (*o_pCleanMaterialSystemStuff)(); + + for (auto& modPak : m_modPaks) + { + if (modPak.m_handle == PakHandle::INVALID || !modPak.m_markedForDelete) + continue; + + g_pakLoadApi->UnloadPak(modPak.m_handle, *o_pCleanMaterialSystemStuff); + modPak.m_handle = PakHandle::INVALID; + } } -void PakLoadManager::RemoveLoadedPak(int nPakHandle) +// Loads all modded paks for the given map. +void PakLoadManager::LoadModPaksForMap(const char* mapName) { - m_vLoadedPaks.erase(nPakHandle); + ++m_reentranceCounter; + const ScopeGuard guard([&]() { --m_reentranceCounter; }); + + for (auto& modPak : m_modPaks) + { + // don't load paks that are already loaded + if (modPak.m_handle != PakHandle::INVALID) + continue; + std::cmatch matches; + if (!std::regex_match(mapName, matches, modPak.m_mapRegex)) + continue; + + modPak.m_handle = g_pakLoadApi->LoadRpakFileAsync(modPak.m_path.c_str(), *rpakMemoryAllocator, 7); + m_mapPaks.push_back(modPak.m_pathHash); + } } -LoadedPak* PakLoadManager::GetPakInfo(const int nPakHandle) +// Unloads all modded map paks. +void PakLoadManager::UnloadModPaks() { - return &m_vLoadedPaks.at(nPakHandle); + ++m_reentranceCounter; + const ScopeGuard guard([&]() { --m_reentranceCounter; }); + + (*o_pCModelLoader_UnreferenceAllModels)(*ppModelLoader); + (*o_pCleanMaterialSystemStuff)(); + + for (auto& modPak : m_modPaks) + { + for (auto it = m_mapPaks.begin(); it != m_mapPaks.end(); ++it) + { + if (*it != modPak.m_pathHash) + continue; + + m_mapPaks.erase(it, it + 1); + g_pakLoadApi->UnloadPak(modPak.m_handle, *o_pCleanMaterialSystemStuff); + modPak.m_handle = PakHandle::INVALID; + break; + } + } + + // If this has happened, we may have leaked a pak? + // It basically means that none of the entries in m_modPaks matched the hash in m_mapPaks so we didn't end up unloading it + assert_msg(m_mapPaks.size() == 0, "Not all map paks were unloaded?"); } -int PakLoadManager::GetPakHandle(const size_t nPakNameHash) +// Called after a Pak was loaded. +void PakLoadManager::OnPakLoaded(std::string& originalPath, std::string& resultingPath, PakHandle resultingHandle) { - for (auto& pair : m_vLoadedPaks) - if (pair.second.m_nPakNameHash == nPakNameHash) - return pair.first; + if (IsVanillaCall()) + { + // add entry to loaded vanilla rpaks + m_vanillaPaks.emplace_back(originalPath, resultingHandle); + } - return -1; + LoadDependentPaks(resultingPath, resultingHandle); } -int PakLoadManager::GetPakHandle(const char* pPath) +// Called before a Pak was unloaded. +void PakLoadManager::OnPakUnloading(PakHandle handle) { - return GetPakHandle(STR_HASH(pPath)); + UnloadDependentPaks(handle); + + if (IsVanillaCall()) + { + // remove entry from loaded vanilla rpaks + auto fnRemovePredicate = [handle](std::pair& pair) -> bool { return pair.second == handle; }; + + m_vanillaPaks.erase(std::remove_if(m_vanillaPaks.begin(), m_vanillaPaks.end(), fnRemovePredicate), m_vanillaPaks.end()); + + // no need to handle aliasing here, if vanilla wants it gone, it's gone + } + else + { + // note: aliasing is handled the old way, long term todo: move it over to the PakLoadManager + // handle the potential unloading of an aliased vanilla rpak (we aliased it, and we are now unloading the alias, so we should load + // the vanilla one again) + // for (auto& [path, resultingHandle] : m_vanillaPaks) + //{ + // if (resultingHandle != handle) + // continue; + + // // load vanilla rpak + //} + } + + // set handle of the mod pak (if any) that has this handle for proper tracking + for (auto& modPak : m_modPaks) + { + if (modPak.m_handle == handle) + modPak.m_handle = PakHandle::INVALID; + } } -void* PakLoadManager::LoadFile(const char* path) +// Whether the vanilla game has this rpak +static bool VanillaHasPak(const char* pakName) { - return g_pakLoadApi->LoadFile(path); + fs::path originalPath = fs::path("./r2/paks/Win64") / pakName; + unsigned int highestPatch = o_pGetPakPatchNumber(pakName); + if (highestPatch) + { + // add the patch path to the extension + char buf[16]; + snprintf(buf, sizeof(buf), "(%02u).rpak", highestPatch); + // remove the .rpak and add the new suffix + originalPath = originalPath.replace_extension().string() + buf; + } + else + { + originalPath /= pakName; + } + + return fs::exists(originalPath); } -void HandlePakAliases(char** map) +// If vanilla doesn't have an rpak for this path, tries to map it to a modded rpak of the same name. +void PakLoadManager::FixupPakPath(std::string& pakPath) { - // convert the pak being loaded to it's aliased one, e.g. aliasing mp_hub_timeshift => sp_hub_timeshift - for (int64_t i = g_pModManager->m_LoadedMods.size() - 1; i > -1; i--) + if (VanillaHasPak(pakPath.c_str())) + return; + + for (ModPak_t& modPak : m_modPaks) { - Mod* mod = &g_pModManager->m_LoadedMods[i]; - if (!mod->m_bEnabled) + if (modPak.m_markedForDelete) continue; - if (mod->RpakAliases.find(*map) != mod->RpakAliases.end()) + fs::path modPakFilename = fs::path(modPak.m_path).filename(); + if (pakPath == modPakFilename.string()) { - *map = &mod->RpakAliases[*map][0]; + pakPath = modPak.m_path; return; } } } -void LoadPreloadPaks() +// Loads all "Preload" Paks. todo: deprecate Preload. +void PakLoadManager::LoadPreloadPaks() { - // note, loading from ./ is necessary otherwise paks will load from gamedir/r2/paks - for (Mod& mod : g_pModManager->m_LoadedMods) + ++m_reentranceCounter; + const ScopeGuard guard([&]() { --m_reentranceCounter; }); + + for (auto& modPak : m_modPaks) { - if (!mod.m_bEnabled) + if (modPak.m_markedForDelete || modPak.m_handle != PakHandle::INVALID || !modPak.m_preload) continue; - // need to get a relative path of mod to mod folder - fs::path modPakPath("./" / mod.m_ModDirectory / "paks"); + modPak.m_handle = g_pakLoadApi->LoadRpakFileAsync(modPak.m_path.c_str(), *rpakMemoryAllocator, 7); + } +} - for (ModRpakEntry& pak : mod.Rpaks) - if (pak.m_bAutoLoad) - g_pPakLoadManager->LoadPakAsync((modPakPath / pak.m_sPakName).string().c_str(), ePakLoadSource::CONSTANT); +// Causes all "Postload" paks to be loaded again. +void PakLoadManager::ReloadPostloadPaks() +{ + ++m_reentranceCounter; + const ScopeGuard guard([&]() { --m_reentranceCounter; }); + + // pretend that we just loaded all of these vanilla paks + for (auto& [path, handle] : m_vanillaPaks) + { + LoadDependentPaks(path, handle); } } -void LoadPostloadPaks(const char* pPath) +// Wrapper for Pak load API. +void* PakLoadManager::OpenFile(const char* path) { - // note, loading from ./ is necessary otherwise paks will load from gamedir/r2/paks - for (Mod& mod : g_pModManager->m_LoadedMods) + return g_pakLoadApi->OpenFile(path); +} + +// Loads Paks that depend on this Pak. +void PakLoadManager::LoadDependentPaks(std::string& path, PakHandle handle) +{ + ++m_reentranceCounter; + const ScopeGuard guard([&]() { --m_reentranceCounter; }); + + const size_t hash = STR_HASH(path); + for (auto& modPak : m_modPaks) { - if (!mod.m_bEnabled) + if (modPak.m_handle != PakHandle::INVALID) + continue; + if (modPak.m_dependentPakHash != hash) continue; - // need to get a relative path of mod to mod folder - fs::path modPakPath("./" / mod.m_ModDirectory / "paks"); - - for (ModRpakEntry& pak : mod.Rpaks) - if (pak.m_sLoadAfterPak == pPath) - g_pPakLoadManager->LoadPakAsync((modPakPath / pak.m_sPakName).string().c_str(), ePakLoadSource::CONSTANT); + // load pak + modPak.m_handle = g_pakLoadApi->LoadRpakFileAsync(modPak.m_path.c_str(), *rpakMemoryAllocator, 7); + m_dependentPaks.emplace_back(handle, hash); } } -void LoadCustomMapPaks(char** pakName, bool* bNeedToFreePakName) +// Unloads Paks that depend on this Pak. +void PakLoadManager::UnloadDependentPaks(PakHandle handle) { - // whether the vanilla game has this rpak - bool bHasOriginalPak = fs::exists(fs::path("r2/paks/Win64/") / *pakName); + ++m_reentranceCounter; + const ScopeGuard guard([&]() { --m_reentranceCounter; }); - // note, loading from ./ is necessary otherwise paks will load from gamedir/r2/paks - for (Mod& mod : g_pModManager->m_LoadedMods) + auto fnRemovePredicate = [&](std::pair& pair) -> bool { - if (!mod.m_bEnabled) - continue; + if (pair.first != handle) + return false; - // need to get a relative path of mod to mod folder - fs::path modPakPath("./" / mod.m_ModDirectory / "paks"); + for (auto& modPak : m_modPaks) + { + if (modPak.m_pathHash != pair.second || modPak.m_handle == PakHandle::INVALID) + continue; - for (ModRpakEntry& pak : mod.Rpaks) + // unload pak + g_pakLoadApi->UnloadPak(modPak.m_handle, *o_pCleanMaterialSystemStuff); + modPak.m_handle = PakHandle::INVALID; + } + + return true; + }; + m_dependentPaks.erase(std::remove_if(m_dependentPaks.begin(), m_dependentPaks.end(), fnRemovePredicate), m_dependentPaks.end()); +} + +// Handles aliases for rpaks defined in rpak.json, effectively redirecting an rpak load to a different path. +static void HandlePakAliases(std::string& originalPath) +{ + // convert the pak being loaded to its aliased one, e.g. aliasing mp_hub_timeshift => sp_hub_timeshift + for (int64_t i = g_pModManager->m_LoadedMods.size() - 1; i > PakHandle::INVALID; i--) + { + Mod* mod = &g_pModManager->m_LoadedMods[i]; + if (!mod->m_bEnabled) + continue; + + if (mod->RpakAliases.find(originalPath) != mod->RpakAliases.end()) { - if (!pak.m_bAutoLoad && !pak.m_sPakName.compare(*pakName)) - { - // if the game doesn't have the original pak, let it handle loading this one as if it was the one it was loading originally - if (!bHasOriginalPak) - { - std::string path = (modPakPath / pak.m_sPakName).string(); - *pakName = new char[path.size() + 1]; - strcpy(*pakName, &path[0]); - (*pakName)[path.size()] = '\0'; - - bHasOriginalPak = true; - *bNeedToFreePakName = - true; // we can't free this memory until we're done with the pak, so let whatever's calling this deal with it - } - else - g_pPakLoadManager->LoadPakAsync((modPakPath / pak.m_sPakName).string().c_str(), ePakLoadSource::MAP); - } + originalPath = (mod->m_ModDirectory / "paks" / mod->RpakAliases[originalPath]).string(); + return; } } } -// clang-format off -HOOK(LoadPakAsyncHook, LoadPakAsync, -int, __fastcall, (char* pPath, void* unknownSingleton, int flags, void* pCallback0, void* pCallback1)) -// clang-format on +///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static bool (*o_pLoadMapRpaks)(char* mapPath) = nullptr; +static bool h_LoadMapRpaks(char* mapPath) { - HandlePakAliases(&pPath); + // unload all mod rpaks that are marked for unload + g_pPakLoadManager->UnloadMarkedPaks(); + g_pPakLoadManager->CleanUpUnloadedPaks(); - // dont load the pak if it's currently loaded already - size_t nPathHash = STR_HASH(pPath); - if (g_pPakLoadManager->GetPakHandle(nPathHash) != -1) - return -1; + // strip file extension + const std::string mapName = fs::path(mapPath).replace_extension().string(); - bool bNeedToFreePakName = false; + // load mp_common, sp_common etc. + o_pLoadGametypeSpecificRpaks(mapName.c_str()); - static bool bShouldLoadPaks = true; - if (bShouldLoadPaks) + // unload old modded map paks + g_pPakLoadManager->UnloadModPaks(); + // load modded map paks + g_pPakLoadManager->LoadModPaksForMap(mapName.c_str()); + + // don't load/unload anything when going to the lobby, presumably to save load times when going back to the same map + if (!g_pPakLoadManager->GetForceReloadOnMapLoad() && !strcmp("mp_lobby", mapName.c_str())) + return false; + + if (g_pPakLoadManager->GetForceReloadOnMapLoad()) { - // make a copy of the path for comparing to determine whether we should load this pak on dedi, before it could get overwritten by - // LoadCustomMapPaks - std::string originalPath(pPath); + g_pPakLoadManager->LoadPreloadPaks(); + g_pPakLoadManager->ReloadPostloadPaks(); + } - // disable preloading while we're doing this - bShouldLoadPaks = false; + char mapRpakStr[272]; + snprintf(mapRpakStr, 272, "%s.rpak", mapName.c_str()); - LoadPreloadPaks(); - LoadCustomMapPaks(&pPath, &bNeedToFreePakName); + // if level being loaded is the same as current level, do nothing + if (!g_pPakLoadManager->GetForceReloadOnMapLoad() && !strcmp(mapRpakStr, pszCurrentMapRpakPath)) + return true; - bShouldLoadPaks = true; + strcpy(pszCurrentMapRpakPath, mapRpakStr); + + (*o_pCleanMaterialSystemStuff)(); + o_pLoadlevelLoadscreen(mapName.c_str()); + + // unload old map rpaks + PakHandle curHandle = *piCurrentMapRpakHandle; + PakHandle curPatchHandle = *piCurrentMapPatchRpakHandle; + if (curHandle != PakHandle::INVALID) + { + (*o_pCModelLoader_UnreferenceAllModels)(*ppModelLoader); + (*o_pCleanMaterialSystemStuff)(); + g_pakLoadApi->UnloadPak(curHandle, *o_pCleanMaterialSystemStuff); + *piCurrentMapRpakHandle = PakHandle::INVALID; + } + if (curPatchHandle != PakHandle::INVALID) + { + (*o_pCModelLoader_UnreferenceAllModels)(*ppModelLoader); + (*o_pCleanMaterialSystemStuff)(); + g_pakLoadApi->UnloadPak(curPatchHandle, *o_pCleanMaterialSystemStuff); + *piCurrentMapPatchRpakHandle = PakHandle::INVALID; + } + + *piCurrentMapRpakHandle = g_pakLoadApi->LoadRpakFileAsync(mapRpakStr, *rpakMemoryAllocator, 7); + + // load special _patch rpak (seemingly used for dev things?) + char levelPatchRpakStr[272]; + snprintf(levelPatchRpakStr, 272, "%s_patch.rpak", mapName.c_str()); + *piCurrentMapPatchRpakHandle = g_pakLoadApi->LoadRpakFileAsync(levelPatchRpakStr, *rpakMemoryAllocator, 7); + + // we just reloaded the paks, so we don't need to force it again + g_pPakLoadManager->SetForceReloadOnMapLoad(false); + return true; +} + +// clang-format off +HOOK(LoadPakAsyncHook, LoadPakAsync, +PakHandle, __fastcall, (const char* pPath, void* memoryAllocator, int flags)) +// clang-format on +{ + // make a copy of the path for comparing to determine whether we should load this pak on dedi, before it could get overwritten + std::string svOriginalPath(pPath); + + std::string resultingPath(pPath); + HandlePakAliases(resultingPath); + + if (g_pPakLoadManager->IsVanillaCall()) + { + g_pPakLoadManager->LoadPreloadPaks(); + g_pPakLoadManager->FixupPakPath(resultingPath); // do this after custom paks load and in bShouldLoadPaks so we only ever call this on the root pakload call // todo: could probably add some way to flag custom paks to not be loaded on dedicated servers in rpak.json @@ -210,44 +451,27 @@ int, __fastcall, (char* pPath, void* unknownSingleton, int flags, void* pCallbac // sp_ rpaks contain tutorial ghost data // sucks to have to load the entire rpak for that but sp was never meant to be done on dedi if (IsDedicatedServer() && - (CommandLine()->CheckParm("-nopakdedi") || strncmp(&originalPath[0], "common", 6) && strncmp(&originalPath[0], "sp_", 3))) + (CommandLine()->CheckParm("-nopakdedi") || strncmp(&svOriginalPath[0], "common", 6) && strncmp(&svOriginalPath[0], "sp_", 3))) { - if (bNeedToFreePakName) - delete[] pPath; - - NS::log::rpak->info("Not loading pak {} for dedicated server", originalPath); - return -1; + NS::log::rpak->info("Not loading pak {} for dedicated server", svOriginalPath); + return PakHandle::INVALID; } } - int iPakHandle = LoadPakAsync(pPath, unknownSingleton, flags, pCallback0, pCallback1); - NS::log::rpak->info("LoadPakAsync {} {}", pPath, iPakHandle); - - // trak the pak - g_pPakLoadManager->TrackLoadedPak(ePakLoadSource::UNTRACKED, iPakHandle, nPathHash); - LoadPostloadPaks(pPath); + PakHandle iPakHandle = LoadPakAsync(resultingPath.c_str(), memoryAllocator, flags); + NS::log::rpak->info("LoadPakAsync {} {}", resultingPath, iPakHandle); - if (bNeedToFreePakName) - delete[] pPath; + g_pPakLoadManager->OnPakLoaded(svOriginalPath, resultingPath, iPakHandle); return iPakHandle; } // clang-format off HOOK(UnloadPakHook, UnloadPak, -void*, __fastcall, (int nPakHandle, void* pCallback)) +void*, __fastcall, (PakHandle nPakHandle, void* pCallback)) // clang-format on { - // stop tracking the pak - g_pPakLoadManager->RemoveLoadedPak(nPakHandle); - - static bool bShouldUnloadPaks = true; - if (bShouldUnloadPaks) - { - bShouldUnloadPaks = false; - g_pPakLoadManager->UnloadMapPaks(); - bShouldUnloadPaks = true; - } + g_pPakLoadManager->OnPakUnloading(nPakHandle); NS::log::rpak->info("UnloadPak {}", nPakHandle); return UnloadPak(nPakHandle, pCallback); @@ -256,7 +480,7 @@ void*, __fastcall, (int nPakHandle, void* pCallback)) // we hook this exclusively for resolving stbsp paths, but seemingly it's also used for other stuff like vpk, rpak, mprj and starpak loads // tbh this actually might be for memory mapped files or something, would make sense i think // clang-format off -HOOK(ReadFileAsyncHook, ReadFileAsync, +HOOK(OpenFileHook, o_pOpenFile, void*, __fastcall, (const char* pPath, void* pCallback)) // clang-format on { @@ -329,19 +553,34 @@ void*, __fastcall, (const char* pPath, void* pCallback)) NS::log::rpak->info("LoadStreamPak: {}", filename.string()); } - return ReadFileAsync(pPath, pCallback); + return o_pOpenFile(pPath, pCallback); } ON_DLL_LOAD("engine.dll", RpakFilesystem, (CModule module)) { - AUTOHOOK_DISPATCH(); - g_pPakLoadManager = new PakLoadManager; g_pakLoadApi = module.Offset(0x5BED78).Deref().RCast(); - pUnknownPakLoadSingleton = module.Offset(0x7C5E20).RCast(); - LoadPakAsyncHook.Dispatch((LPVOID*)g_pakLoadApi->LoadPakAsync); + LoadPakAsyncHook.Dispatch((LPVOID*)g_pakLoadApi->LoadRpakFileAsync); UnloadPakHook.Dispatch((LPVOID*)g_pakLoadApi->UnloadPak); - ReadFileAsyncHook.Dispatch((LPVOID*)g_pakLoadApi->ReadFileAsync); + OpenFileHook.Dispatch((LPVOID*)g_pakLoadApi->OpenFile); + + pszCurrentMapRpakPath = module.Offset(0x1315C3E0).RCast(); + piCurrentMapRpakHandle = module.Offset(0x7CB5A0).RCast(); + piCurrentMapPatchRpakHandle = module.Offset(0x7CB5A4).RCast(); + ppModelLoader = module.Offset(0x7C4AC0).RCast(); + rpakMemoryAllocator = module.Offset(0x7C5E20).RCast(); + + o_pLoadGametypeSpecificRpaks = module.Offset(0x15AD20).RCast(); + o_pCleanMaterialSystemStuff = module.Offset(0x12A11F00).RCast(); + o_pCModelLoader_UnreferenceAllModels = module.Offset(0x5ED580).RCast(); + o_pLoadlevelLoadscreen = module.Offset(0x15A810).RCast(); + + o_pLoadMapRpaks = module.Offset(0x15A8C0).RCast(); + HookAttach(&(PVOID&)o_pLoadMapRpaks, (PVOID)h_LoadMapRpaks); + + // kinda bad, doing things in rtech in an engine callback but it seems fine for now + CModule rtechModule(GetModuleHandleA("rtech_game.dll")); + o_pGetPakPatchNumber = rtechModule.Offset(0x9A00).RCast(); } diff --git a/primedev/core/filesystem/rpakfilesystem.h b/primedev/core/filesystem/rpakfilesystem.h index bcd57a731..87a41e7b1 100644 --- a/primedev/core/filesystem/rpakfilesystem.h +++ b/primedev/core/filesystem/rpakfilesystem.h @@ -1,39 +1,81 @@ #pragma once -enum class ePakLoadSource -{ - UNTRACKED = -1, // not a pak we loaded, we shouldn't touch this one +#include - CONSTANT, // should be loaded at all times - MAP // loaded from a map, should be unloaded when the map is unloaded +enum PakHandle : int +{ + INVALID = -1, }; -struct LoadedPak +struct ModPak_t { - ePakLoadSource m_nLoadSource; - int m_nPakHandle; - size_t m_nPakNameHash; + std::string m_modName; + + std::string m_path; + size_t m_pathHash = 0; + + // If the map being loaded matches this regex, this pak will be loaded. + std::regex m_mapRegex; + // If a pak with a hash matching this is loaded, this pak will be loaded. + size_t m_dependentPakHash = 0; + // If this is set, this pak will be loaded whenever any other pak is loaded. + bool m_preload = false; + + // If this is set, the Pak will be unloaded on next map load + bool m_markedForDelete = false; + // The current rpak handle associated with this Pak + PakHandle m_handle = PakHandle::INVALID; }; class PakLoadManager { -private: - std::map m_vLoadedPaks {}; - std::unordered_map m_HashToPakHandle {}; - public: - int LoadPakAsync(const char* pPath, const ePakLoadSource nLoadSource); - void UnloadPak(const int nPakHandle); - void UnloadMapPaks(); - void* LoadFile(const char* path); // this is a guess + void UnloadAllModPaks(); + void TrackModPaks(Mod& mod); + + void CleanUpUnloadedPaks(); + void UnloadMarkedPaks(); + + void LoadModPaksForMap(const char* mapName); + void UnloadModPaks(); + + // Whether the current context is a vanilla call to a function, or a modded one + bool IsVanillaCall() const { return m_reentranceCounter == 0; } + // Whether paks will be forced to reload on the next map load + bool GetForceReloadOnMapLoad() const { return m_forceReloadOnMapLoad; } + void SetForceReloadOnMapLoad(bool value) { m_forceReloadOnMapLoad = value; } + + void OnPakLoaded(std::string& originalPath, std::string& resultingPath, PakHandle resultingHandle); + void OnPakUnloading(PakHandle handle); + + void FixupPakPath(std::string& path); + + void LoadPreloadPaks(); + void ReloadPostloadPaks(); + + void* OpenFile(const char* path); + +private: + void LoadDependentPaks(std::string& path, PakHandle handle); + void UnloadDependentPaks(PakHandle handle); - LoadedPak* TrackLoadedPak(ePakLoadSource nLoadSource, int nPakHandle, size_t nPakNameHash); - void RemoveLoadedPak(int nPakHandle); + // All paks that vanilla has attempted to load. (they may have been aliased away) + // Also known as a list of rpaks that the vanilla game would have loaded at this point in time. + std::vector> m_vanillaPaks; - LoadedPak* GetPakInfo(const int nPakHandle); + // All mod Paks that are currently tracked + std::vector m_modPaks; + // Hashes of the currently loaded map mod paks + std::vector m_mapPaks; + // Currently loaded Pak path hashes that depend on a handle to remain loaded (Postload) + std::vector> m_dependentPaks; - int GetPakHandle(const size_t nPakNameHash); - int GetPakHandle(const char* pPath); + // Used to force rpaks to be unloaded and reloaded on the next map load. + // Vanilla behaviour is to not do this when loading into mp_lobby, or loading into the same map you were last in. + bool m_forceReloadOnMapLoad = false; + // Used to track if the current hook call is a vanilla call or not. + // When loading/unloading a mod Pak, increment this before doing so, and decrement afterwards. + int m_reentranceCounter = 0; }; extern PakLoadManager* g_pPakLoadManager; diff --git a/primedev/mods/modmanager.cpp b/primedev/mods/modmanager.cpp index a3e0a5f57..52fc6e8b1 100644 --- a/primedev/mods/modmanager.cpp +++ b/primedev/mods/modmanager.cpp @@ -866,7 +866,9 @@ void ModManager::LoadMods() if (fs::is_regular_file(file) && file.path().extension() == ".rpak") { std::string pakName(file.path().filename().string()); - ModRpakEntry& modPak = mod.Rpaks.emplace_back(); + ModRpakEntry& modPak = mod.Rpaks.emplace_back(mod); + + modPak.m_pakName = pakName; if (!bUseRpakJson) { @@ -874,19 +876,47 @@ void ModManager::LoadMods() } else { - modPak.m_bAutoLoad = + modPak.m_preload = (dRpakJson.HasMember("Preload") && dRpakJson["Preload"].IsObject() && dRpakJson["Preload"].HasMember(pakName) && dRpakJson["Preload"][pakName].IsTrue()); + // only one load method can be used for an rpak. + if (modPak.m_preload) + goto REGISTER_STARPAK; + // postload things if (dRpakJson.HasMember("Postload") && dRpakJson["Postload"].IsObject() && dRpakJson["Postload"].HasMember(pakName)) { - modPak.m_sLoadAfterPak = dRpakJson["Postload"][pakName].GetString(); + modPak.m_dependentPakHash = STR_HASH(dRpakJson["Postload"][pakName].GetString()); + + // only one load method can be used for an rpak. + goto REGISTER_STARPAK; } - } - modPak.m_sPakName = pakName; + // this is the only bit of rpak.json that isn't really deprecated. Even so, it will be moved over to the mod.json + // eventually + if (dRpakJson.HasMember(pakName)) + { + if (!dRpakJson[pakName].IsString()) + { + spdlog::error("Mod {} has invalid rpak.json. Rpak entries must be strings.", mod.Name); + continue; + } + + std::string loadStr = dRpakJson[pakName].GetString(); + try + { + modPak.m_loadRegex = std::regex(loadStr); + } + catch (...) + { + spdlog::error("Mod {} has invalid rpak.json. Malformed regex \"{}\" for {}", mod.Name, loadStr, pakName); + return; + } + } + } + REGISTER_STARPAK: // read header of file and get the starpak paths // this is done here as opposed to on starpak load because multiple rpaks can load a starpak // and there is seemingly no good way to tell which rpak is causing the load of a starpak :/ @@ -926,12 +956,11 @@ void ModManager::LoadMods() } } } - - // not using atm because we need to resolve path to rpak - // if (m_hasLoadedMods && modPak.m_bAutoLoad) - // g_pPakLoadManager->LoadPakAsync(pakName.c_str()); } } + + if (g_pPakLoadManager != nullptr) + g_pPakLoadManager->TrackModPaks(mod); } // read keyvalues paths @@ -1059,6 +1088,8 @@ void ModManager::UnloadMods() fs::remove_all(GetCompiledAssetsPath()); g_CustomAudioManager.ClearAudioOverrides(); + if (g_pPakLoadManager != nullptr) + g_pPakLoadManager->UnloadAllModPaks(); if (!m_bHasEnabledModsCfg) m_EnabledModsCfg.SetObject(); diff --git a/primedev/mods/modmanager.h b/primedev/mods/modmanager.h index 95a8fe121..7859d6184 100644 --- a/primedev/mods/modmanager.h +++ b/primedev/mods/modmanager.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace fs = std::filesystem; @@ -19,6 +20,8 @@ const std::string COMPILED_ASSETS_SUFFIX = "\\runtime\\compiled"; const std::set MODS_BLACKLIST = {"Mod Settings"}; +class Mod; + struct ModConVar { public: @@ -71,9 +74,22 @@ struct ModVPKEntry struct ModRpakEntry { public: - bool m_bAutoLoad; - std::string m_sPakName; - std::string m_sLoadAfterPak; + ModRpakEntry(Mod& parent) + : m_parent(parent) + , m_loadRegex("^thisMatchesNothing^") // discord couldnt give me a funny string + { + } + + Mod& m_parent; + std::string m_pakName; + std::regex m_loadRegex; + + // these exist purely for backwards compatibility, i don't really like them anymore + + // Preload, loads before the first rpak is loaded + bool m_preload = false; + // Postload, this rpak depends on an rpak with this hash + size_t m_dependentPakHash; }; class Mod @@ -120,11 +136,12 @@ class Mod std::string Pdiff; // only need one per mod std::vector Rpaks; - std::unordered_map - RpakAliases; // paks we alias to other rpaks, e.g. to load sp_crashsite paks on the map mp_crashsite - std::vector StarpakPaths; // starpaks that this mod contains + // paks we alias to other rpaks, e.g. to load sp_crashsite paks on the map mp_crashsite + std::unordered_map RpakAliases; + // starpaks that this mod contains // there seems to be no nice way to get the rpak that is causing the load of a starpak? // hashed with STR_HASH + std::vector StarpakPaths; std::unordered_map DependencyConstants; std::vector PluginDependencyConstants; diff --git a/primedev/scripts/scriptdatatables.cpp b/primedev/scripts/scriptdatatables.cpp index c91e16ffb..b3c599212 100644 --- a/primedev/scripts/scriptdatatables.cpp +++ b/primedev/scripts/scriptdatatables.cpp @@ -70,10 +70,11 @@ REPLACE_SQFUNC(GetDataTable, (ScriptContext::UI | ScriptContext::CLIENT | Script g_pSquirrel->raiseerror(sqvm, fmt::format("Asset \"{}\" doesn't start with \"datatable/\"", pAssetName).c_str()); return SQRESULT_ERROR; } - else if (!Cvar_ns_prefer_datatable_from_disk->GetBool() && g_pPakLoadManager->LoadFile(pAssetName)) + else if (!Cvar_ns_prefer_datatable_from_disk->GetBool() && g_pPakLoadManager->OpenFile(pAssetName)) + { return g_pSquirrel->m_funcOriginals["GetDataTable"](sqvm); - // either we prefer disk datatables, or we're loading a datatable that wasn't found in rpak - else + } + else // either we prefer disk datatables, or we're loading a datatable that wasn't found in rpak { std::string sAssetPath(fmt::format("scripts/{}", pAssetName)); @@ -223,7 +224,7 @@ REPLACE_SQFUNC(GetDataTable, (ScriptContext::UI | ScriptContext::CLIENT | Script return SQRESULT_NOTNULL; } // the file doesn't exist on disk, check rpak if we haven't already - else if (Cvar_ns_prefer_datatable_from_disk->GetBool() && g_pPakLoadManager->LoadFile(pAssetName)) + else if (Cvar_ns_prefer_datatable_from_disk->GetBool() && g_pPakLoadManager->OpenFile(pAssetName)) return g_pSquirrel->m_funcOriginals["GetDataTable"](sqvm); // the file doesn't exist at all, error else @@ -750,7 +751,7 @@ std::string DataTableToString(Datatable* datatable) void DumpDatatable(const char* pDatatablePath) { - Datatable* pDatatable = (Datatable*)g_pPakLoadManager->LoadFile(pDatatablePath); + Datatable* pDatatable = (Datatable*)g_pPakLoadManager->OpenFile(pDatatablePath); if (!pDatatable) { spdlog::error("couldn't load datatable {} (rpak containing it may not be loaded?)", pDatatablePath); From 8c546ed68c83b42cdff32d9b848b25ec2cba9c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Raes?= Date: Sun, 8 Sep 2024 00:27:44 +0200 Subject: [PATCH 2/2] Adjust for restructured `verified-mods.json` (#748) Updates the launcher code to deal with adjusted verified mods JSON structure from the default manifest source The idea here is to allow installing mods from other sources than Thunderstore. --- primedev/mods/autodownload/moddownloader.cpp | 42 ++++++++++++-------- primedev/mods/autodownload/moddownloader.h | 19 ++++++--- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/primedev/mods/autodownload/moddownloader.cpp b/primedev/mods/autodownload/moddownloader.cpp index 8e533decb..c20a3adbc 100644 --- a/primedev/mods/autodownload/moddownloader.cpp +++ b/primedev/mods/autodownload/moddownloader.cpp @@ -103,23 +103,23 @@ void ModDownloader::FetchModsListFromAPI() for (auto i = verifiedModsJson.MemberBegin(); i != verifiedModsJson.MemberEnd(); ++i) { // Format testing - if (!i->value.HasMember("DependencyPrefix") || !i->value.HasMember("Versions")) + if (!i->value.HasMember("Repository") || !i->value.HasMember("Versions")) { spdlog::warn("Verified mods manifesto format is unrecognized, skipping loading."); return; } std::string name = i->name.GetString(); - std::string dependency = i->value["DependencyPrefix"].GetString(); - std::unordered_map modVersions; + rapidjson::Value& versions = i->value["Versions"]; assert(versions.IsArray()); for (auto& attribute : versions.GetArray()) { assert(attribute.IsObject()); // Format testing - if (!attribute.HasMember("Version") || !attribute.HasMember("Checksum")) + if (!attribute.HasMember("Version") || !attribute.HasMember("Checksum") || !attribute.HasMember("DownloadLink") || + !attribute.HasMember("Platform")) { spdlog::warn("Verified mods manifesto format is unrecognized, skipping loading."); return; @@ -127,10 +127,14 @@ void ModDownloader::FetchModsListFromAPI() std::string version = attribute["Version"].GetString(); std::string checksum = attribute["Checksum"].GetString(); - modVersions.insert({version, {.checksum = checksum}}); + std::string downloadLink = attribute["DownloadLink"].GetString(); + std::string platformValue = attribute["Platform"].GetString(); + VerifiedModPlatform platform = + platformValue.compare("thunderstore") == 0 ? VerifiedModPlatform::Thunderstore : VerifiedModPlatform::Unknown; + modVersions.insert({version, {.checksum = checksum, .downloadLink = downloadLink, .platform = platform}}); } - VerifiedModDetails modConfig = {.dependencyPrefix = dependency, .versions = modVersions}; + VerifiedModDetails modConfig = {.versions = modVersions}; verifiedMods.insert({name, modConfig}); spdlog::info("==> Loaded configuration for mod \"" + name + "\""); } @@ -164,13 +168,10 @@ int ModDownloader::ModFetchingProgressCallback( return 0; } -std::optional ModDownloader::FetchModFromDistantStore(std::string_view modName, std::string_view modVersion) +std::optional ModDownloader::FetchModFromDistantStore(std::string_view modName, VerifiedModVersion version) { - // Retrieve mod prefix from local mods list, or use mod name as mod prefix if bypass flag is set - std::string modPrefix = strstr(GetCommandLineA(), VERIFICATION_FLAG) ? modName.data() : verifiedMods[modName.data()].dependencyPrefix; - // Build archive distant URI - std::string archiveName = std::format("{}-{}.zip", modPrefix, modVersion.data()); - std::string url = STORE_URL + archiveName; + std::string url = version.downloadLink; + std::string archiveName = fs::path(url).filename().generic_string(); spdlog::info(std::format("Fetching mod archive from {}", url)); // Download destination @@ -390,7 +391,7 @@ int GetModArchiveSize(unzFile file, unz_global_info64 info) return totalSize; } -void ModDownloader::ExtractMod(fs::path modPath) +void ModDownloader::ExtractMod(fs::path modPath, VerifiedModPlatform platform) { unzFile file; std::string name; @@ -428,6 +429,14 @@ void ModDownloader::ExtractMod(fs::path modPath) modState.total = GetModArchiveSize(file, gi); modState.progress = 0; + // Right now, we only know how to extract Thunderstore mods + if (platform != VerifiedModPlatform::Thunderstore) + { + spdlog::error("Failed extracting mod from unknown platform (value: {}).", platform); + modState.state = UNKNOWN_PLATFORM; + return; + } + // Mod directory name (removing the ".zip" fom the archive name) name = modPath.filename().string(); name = name.substr(0, name.length() - 4); @@ -598,8 +607,9 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) }); // Download mod archive - std::string expectedHash = verifiedMods[modName].versions[modVersion].checksum; - std::optional fetchingResult = FetchModFromDistantStore(std::string_view(modName), std::string_view(modVersion)); + VerifiedModVersion fullVersion = verifiedMods[modName].versions[modVersion]; + std::string expectedHash = fullVersion.checksum; + std::optional fetchingResult = FetchModFromDistantStore(std::string_view(modName), fullVersion); if (!fetchingResult.has_value()) { spdlog::error("Something went wrong while fetching archive, aborting."); @@ -615,7 +625,7 @@ void ModDownloader::DownloadMod(std::string modName, std::string modVersion) } // Extract downloaded mod archive - ExtractMod(archiveLocation); + ExtractMod(archiveLocation, fullVersion.platform); }); requestThread.detach(); diff --git a/primedev/mods/autodownload/moddownloader.h b/primedev/mods/autodownload/moddownloader.h index 98fc27ae3..c7a88c19a 100644 --- a/primedev/mods/autodownload/moddownloader.h +++ b/primedev/mods/autodownload/moddownloader.h @@ -5,17 +5,22 @@ class ModDownloader private: const char* VERIFICATION_FLAG = "-disablemodverification"; const char* CUSTOM_MODS_URL_FLAG = "-customverifiedurl="; - const char* STORE_URL = "https://gcdn.thunderstore.io/live/repository/packages/"; const char* DEFAULT_MODS_LIST_URL = "https://raw.githubusercontent.com/R2Northstar/VerifiedMods/main/verified-mods.json"; char* modsListUrl; + enum class VerifiedModPlatform + { + Unknown, + Thunderstore + }; struct VerifiedModVersion { std::string checksum; + std::string downloadLink; + VerifiedModPlatform platform; }; struct VerifiedModDetails { - std::string dependencyPrefix; std::unordered_map versions = {}; }; std::unordered_map verifiedMods = {}; @@ -45,7 +50,7 @@ class ModDownloader * @param modVersion version of the mod to be downloaded * @returns location of the downloaded archive */ - std::optional FetchModFromDistantStore(std::string_view modName, std::string_view modVersion); + std::optional FetchModFromDistantStore(std::string_view modName, VerifiedModVersion modVersion); /** * Tells if a mod archive has not been corrupted. @@ -65,12 +70,13 @@ class ModDownloader * Extracts a mod archive to the game folder. * * This extracts a downloaded mod archive from its original location to the - * current game profile, in the remote mods folder. + * current game profile; the install folder is defined by the platform parameter. * * @param modPath location of the downloaded archive + * @param platform origin of the downloaded archive * @returns nothing */ - void ExtractMod(fs::path modPath); + void ExtractMod(fs::path modPath, VerifiedModPlatform platform); public: ModDownloader(); @@ -131,7 +137,8 @@ class ModDownloader MOD_FETCHING_FAILED, MOD_CORRUPTED, // Downloaded archive checksum does not match verified hash NO_DISK_SPACE_AVAILABLE, - NOT_FOUND // Mod is not currently being auto-downloaded + NOT_FOUND, // Mod is not currently being auto-downloaded + UNKNOWN_PLATFORM }; struct MOD_STATE