From 4b05c6aa9b61014fc8654f128a8dfeaafd3e169e Mon Sep 17 00:00:00 2001 From: JezuzLizard Date: Tue, 12 Dec 2023 19:00:20 -0800 Subject: [PATCH 1/5] Add decompression of animtree rawfiles for T6. --- .../T6/AssetDumpers/AssetDumperRawFile.cpp | 87 ++++++++++++++++++- .../Game/T6/AssetDumpers/AssetDumperRawFile.h | 3 + 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp b/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp index 33947efdb..41c71f29b 100644 --- a/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp +++ b/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp @@ -1,12 +1,87 @@ #include "AssetDumperRawFile.h" +#include +#include + using namespace T6; +namespace fs = std::filesystem; + bool AssetDumperRawFile::ShouldDump(XAssetInfo* asset) { return true; } +void AssetDumperRawFile::DumpAnimtree(AssetDumpingContext& context, XAssetInfo* asset, std::ostream& stream) +{ + const auto* rawFile = asset->Asset(); + + if (rawFile->len <= 4) + { + std::cout << "Invalid len of animtree file \"" << rawFile->name << "\"" << std::endl; + return; + } + + const auto outLen = reinterpret_cast(rawFile->buffer)[0]; + const auto inLen = rawFile->len; + + if (outLen > ANIMTREE_MAX_SIZE) + { + std::cout << "Invalid size of animtree file \"" << rawFile->name << "\": " << outLen << std::endl; + return; + } + + z_stream_s zs{}; + + zs.zalloc = Z_NULL; + zs.zfree = Z_NULL; + zs.opaque = Z_NULL; + zs.avail_in = 0; + zs.next_in = Z_NULL; + + int ret = inflateInit2(&zs, -13); + + if (ret != Z_OK) + { + throw std::runtime_error("Initializing inflate failed"); + } + + zs.next_in = reinterpret_cast(&rawFile->buffer[4]); + zs.avail_in = inLen - 4; + + Bytef buffer[0x1000]; + + size_t writtenSize = 0; + while (zs.avail_in > 0) + { + zs.next_out = buffer; + zs.avail_out = sizeof buffer; + ret = inflate(&zs, Z_SYNC_FLUSH); + + if (ret < 0) + { + std::cout << "Inflate failed for dumping animtree file \"" << rawFile->name << "\"" << std::endl; + inflateEnd(&zs); + return; + } + + const auto inflateOutSize = sizeof buffer - zs.avail_out; + + if (writtenSize + inflateOutSize >= outLen) + { + // Last byte is a \0 byte. Skip it. + stream.write(reinterpret_cast(buffer), inflateOutSize - 1); + } + else + { + stream.write(reinterpret_cast(buffer), inflateOutSize); + } + writtenSize += inflateOutSize; + } + + inflateEnd(&zs); +} + void AssetDumperRawFile::DumpAsset(AssetDumpingContext& context, XAssetInfo* asset) { const auto* rawFile = asset->Asset(); @@ -16,5 +91,15 @@ void AssetDumperRawFile::DumpAsset(AssetDumpingContext& context, XAssetInfobuffer, rawFile->len); + const fs::path rawFilePath(rawFile->name); + const auto extension = rawFilePath.extension().string(); + + if (extension == ".atr") + { + DumpAnimtree(context, asset, stream); + } + else + { + stream.write(rawFile->buffer, rawFile->len); + } } diff --git a/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.h b/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.h index 017d38b68..fcffaee68 100644 --- a/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.h +++ b/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.h @@ -7,6 +7,9 @@ namespace T6 { class AssetDumperRawFile final : public AbstractAssetDumper { + constexpr static size_t ANIMTREE_MAX_SIZE = 0xC000000; + + void DumpAnimtree(AssetDumpingContext& context, XAssetInfo* asset, std::ostream& stream); protected: bool ShouldDump(XAssetInfo* asset) override; void DumpAsset(AssetDumpingContext& context, XAssetInfo* asset) override; From 76a98e65fd3ad1560f90cdd80d685f627e2ad4a6 Mon Sep 17 00:00:00 2001 From: JezuzLizard Date: Tue, 12 Dec 2023 20:59:26 -0800 Subject: [PATCH 2/5] Replace magic numbers with macros from zutil.h. Compress animtrees when linking. --- .../T6/AssetLoaders/AssetLoaderRawFile.cpp | 78 +++++++++++++++++-- .../Game/T6/AssetLoaders/AssetLoaderRawFile.h | 6 ++ .../T6/AssetDumpers/AssetDumperRawFile.cpp | 5 +- 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp index 559f339b1..f7334b67e 100644 --- a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp +++ b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp @@ -3,10 +3,16 @@ #include "Game/T6/T6.h" #include "Pool/GlobalAssetPool.h" +#include +#include #include +#include +#include using namespace T6; +namespace fs = std::filesystem; + void* AssetLoaderRawFile::CreateEmptyAsset(const std::string& assetName, MemoryManager* memory) { auto* rawFile = memory->Create(); @@ -20,13 +26,61 @@ bool AssetLoaderRawFile::CanLoadFromRaw() const return true; } -bool AssetLoaderRawFile::LoadFromRaw( - const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager, Zone* zone) const +bool AssetLoaderRawFile::LoadAnimtree( + const SearchPathOpenFile& file, const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager) { - const auto file = searchPath->Open(assetName); - if (!file.IsOpen()) + const auto uncompressedBuffer = std::make_unique(static_cast(file.m_length + 1)); + file.m_stream->read(uncompressedBuffer.get(), file.m_length); + if (file.m_stream->gcount() != file.m_length) + return false; + uncompressedBuffer[static_cast(file.m_length)] = '\0'; + + const auto compressionBufferSize = static_cast(file.m_length + 1 + sizeof(uint32_t) + COMPRESSED_BUFFER_SIZE_PADDING); + auto* compressedBuffer = static_cast(memory->Alloc(compressionBufferSize)); + + z_stream_s zs{}; + + zs.zalloc = Z_NULL; + zs.zfree = Z_NULL; + zs.opaque = Z_NULL; + zs.avail_in = static_cast(file.m_length + 1); + zs.avail_out = compressionBufferSize; + zs.next_in = reinterpret_cast(uncompressedBuffer.get()); + zs.next_out = reinterpret_cast(&compressedBuffer[sizeof(uint32_t)]); + + int ret = deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -DEF_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); + + if (ret != Z_OK) + { + throw std::runtime_error("Initializing deflate failed"); + } + + ret = deflate(&zs, Z_FINISH); + + if (ret != Z_STREAM_END) + { + std::cout << "Deflate failed for loading animtree file \"" << assetName << "\"" << std::endl; + deflateEnd(&zs); return false; + } + + const auto compressedSize = compressionBufferSize + sizeof(uint32_t) - zs.avail_out; + + reinterpret_cast(compressedBuffer)[0] = static_cast(file.m_length + 1); // outLen + + auto* rawFile = memory->Create(); + rawFile->name = memory->Dup(assetName.c_str()); + rawFile->len = static_cast(compressedSize); + rawFile->buffer = static_cast(compressedBuffer); + + manager->AddAsset(ASSET_TYPE_RAWFILE, assetName, rawFile); + return true; +} + +bool AssetLoaderRawFile::LoadDefault( + const SearchPathOpenFile& file, const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager) +{ auto* rawFile = memory->Create(); rawFile->name = memory->Dup(assetName.c_str()); rawFile->len = static_cast(file.m_length); @@ -39,6 +93,20 @@ bool AssetLoaderRawFile::LoadFromRaw( rawFile->buffer = static_cast(fileBuffer); manager->AddAsset(ASSET_TYPE_RAWFILE, assetName, rawFile); +} - return true; +bool AssetLoaderRawFile::LoadFromRaw( + const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager, Zone* zone) const +{ + const auto file = searchPath->Open(assetName); + if (!file.IsOpen()) + return false; + + const fs::path rawFilePath(assetName); + const auto extension = rawFilePath.extension().string(); + + if (extension == ".atr") + return LoadAnimtree(file, assetName, searchPath, memory, manager); + + return LoadDefault(file, assetName, searchPath, memory, manager); } diff --git a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.h b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.h index d4083e5d3..7e4c6f931 100644 --- a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.h +++ b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.h @@ -8,6 +8,12 @@ namespace T6 { class AssetLoaderRawFile final : public BasicAssetLoader { + static constexpr size_t COMPRESSED_BUFFER_SIZE_PADDING = 64; + + static bool LoadAnimtree( + const SearchPathOpenFile& file, const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager); + static bool LoadDefault( + const SearchPathOpenFile& file, const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager); public: _NODISCARD void* CreateEmptyAsset(const std::string& assetName, MemoryManager* memory) override; _NODISCARD bool CanLoadFromRaw() const override; diff --git a/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp b/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp index 41c71f29b..5440c69f0 100644 --- a/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp +++ b/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp @@ -2,6 +2,7 @@ #include #include +#include using namespace T6; @@ -39,7 +40,7 @@ void AssetDumperRawFile::DumpAnimtree(AssetDumpingContext& context, XAssetInfo(&rawFile->buffer[4]); - zs.avail_in = inLen - 4; + zs.avail_in = inLen - sizeof(uint32_t); Bytef buffer[0x1000]; From bcf9956da22e84796bd015251055bef994cf618a Mon Sep 17 00:00:00 2001 From: JezuzLizard Date: Tue, 12 Dec 2023 21:12:14 -0800 Subject: [PATCH 3/5] Fix warning. --- src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp index f7334b67e..3d987908e 100644 --- a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp +++ b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp @@ -93,6 +93,8 @@ bool AssetLoaderRawFile::LoadDefault( rawFile->buffer = static_cast(fileBuffer); manager->AddAsset(ASSET_TYPE_RAWFILE, assetName, rawFile); + + return true; } bool AssetLoaderRawFile::LoadFromRaw( From 96e67dc1fa59388e98438448e90e3d61b08c97f2 Mon Sep 17 00:00:00 2001 From: JezuzLizard Date: Wed, 13 Dec 2023 12:40:56 -0800 Subject: [PATCH 4/5] Use clang-format. --- src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp | 4 ++-- src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.h | 1 + src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.h | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp index 3d987908e..a4e97edd2 100644 --- a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp +++ b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp @@ -3,9 +3,9 @@ #include "Game/T6/T6.h" #include "Pool/GlobalAssetPool.h" -#include -#include #include +#include +#include #include #include diff --git a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.h b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.h index 7e4c6f931..4b7e0f059 100644 --- a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.h +++ b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.h @@ -14,6 +14,7 @@ namespace T6 const SearchPathOpenFile& file, const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager); static bool LoadDefault( const SearchPathOpenFile& file, const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager); + public: _NODISCARD void* CreateEmptyAsset(const std::string& assetName, MemoryManager* memory) override; _NODISCARD bool CanLoadFromRaw() const override; diff --git a/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.h b/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.h index fcffaee68..1f9f033f1 100644 --- a/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.h +++ b/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.h @@ -10,6 +10,7 @@ namespace T6 constexpr static size_t ANIMTREE_MAX_SIZE = 0xC000000; void DumpAnimtree(AssetDumpingContext& context, XAssetInfo* asset, std::ostream& stream); + protected: bool ShouldDump(XAssetInfo* asset) override; void DumpAsset(AssetDumpingContext& context, XAssetInfo* asset) override; From 8b62bc0bc0fe0b72bbbe1f6d3dfcfccff5fdda17 Mon Sep 17 00:00:00 2001 From: JezuzLizard Date: Thu, 14 Dec 2023 13:48:44 -0800 Subject: [PATCH 5/5] Address review comments. --- .../T6/AssetLoaders/AssetLoaderRawFile.cpp | 11 +++++------ .../T6/AssetDumpers/AssetDumperRawFile.cpp | 18 ++++-------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp index a4e97edd2..0257177f0 100644 --- a/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp +++ b/src/ObjLoading/Game/T6/AssetLoaders/AssetLoaderRawFile.cpp @@ -29,13 +29,12 @@ bool AssetLoaderRawFile::CanLoadFromRaw() const bool AssetLoaderRawFile::LoadAnimtree( const SearchPathOpenFile& file, const std::string& assetName, ISearchPath* searchPath, MemoryManager* memory, IAssetLoadingManager* manager) { - const auto uncompressedBuffer = std::make_unique(static_cast(file.m_length + 1)); + const auto uncompressedBuffer = std::make_unique(static_cast(file.m_length)); file.m_stream->read(uncompressedBuffer.get(), file.m_length); if (file.m_stream->gcount() != file.m_length) return false; - uncompressedBuffer[static_cast(file.m_length)] = '\0'; - const auto compressionBufferSize = static_cast(file.m_length + 1 + sizeof(uint32_t) + COMPRESSED_BUFFER_SIZE_PADDING); + const auto compressionBufferSize = static_cast(file.m_length + sizeof(uint32_t) + COMPRESSED_BUFFER_SIZE_PADDING); auto* compressedBuffer = static_cast(memory->Alloc(compressionBufferSize)); z_stream_s zs{}; @@ -43,7 +42,7 @@ bool AssetLoaderRawFile::LoadAnimtree( zs.zalloc = Z_NULL; zs.zfree = Z_NULL; zs.opaque = Z_NULL; - zs.avail_in = static_cast(file.m_length + 1); + zs.avail_in = static_cast(file.m_length); zs.avail_out = compressionBufferSize; zs.next_in = reinterpret_cast(uncompressedBuffer.get()); zs.next_out = reinterpret_cast(&compressedBuffer[sizeof(uint32_t)]); @@ -59,14 +58,14 @@ bool AssetLoaderRawFile::LoadAnimtree( if (ret != Z_STREAM_END) { - std::cout << "Deflate failed for loading animtree file \"" << assetName << "\"" << std::endl; + std::cerr << "Deflate failed for loading animtree file \"" << assetName << "\"" << std::endl; deflateEnd(&zs); return false; } const auto compressedSize = compressionBufferSize + sizeof(uint32_t) - zs.avail_out; - reinterpret_cast(compressedBuffer)[0] = static_cast(file.m_length + 1); // outLen + reinterpret_cast(compressedBuffer)[0] = static_cast(file.m_length); // outLen auto* rawFile = memory->Create(); rawFile->name = memory->Dup(assetName.c_str()); diff --git a/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp b/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp index 5440c69f0..6a9ebd3c8 100644 --- a/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp +++ b/src/ObjWriting/Game/T6/AssetDumpers/AssetDumperRawFile.cpp @@ -19,7 +19,7 @@ void AssetDumperRawFile::DumpAnimtree(AssetDumpingContext& context, XAssetInfolen <= 4) { - std::cout << "Invalid len of animtree file \"" << rawFile->name << "\"" << std::endl; + std::cerr << "Invalid len of animtree file \"" << rawFile->name << "\"" << std::endl; return; } @@ -28,7 +28,7 @@ void AssetDumperRawFile::DumpAnimtree(AssetDumpingContext& context, XAssetInfo ANIMTREE_MAX_SIZE) { - std::cout << "Invalid size of animtree file \"" << rawFile->name << "\": " << outLen << std::endl; + std::cerr << "Invalid size of animtree file \"" << rawFile->name << "\": " << outLen << std::endl; return; } @@ -52,7 +52,6 @@ void AssetDumperRawFile::DumpAnimtree(AssetDumpingContext& context, XAssetInfo 0) { zs.next_out = buffer; @@ -61,23 +60,14 @@ void AssetDumperRawFile::DumpAnimtree(AssetDumpingContext& context, XAssetInfoname << "\"" << std::endl; + std::cerr << "Inflate failed for dumping animtree file \"" << rawFile->name << "\"" << std::endl; inflateEnd(&zs); return; } const auto inflateOutSize = sizeof buffer - zs.avail_out; - if (writtenSize + inflateOutSize >= outLen) - { - // Last byte is a \0 byte. Skip it. - stream.write(reinterpret_cast(buffer), inflateOutSize - 1); - } - else - { - stream.write(reinterpret_cast(buffer), inflateOutSize); - } - writtenSize += inflateOutSize; + stream.write(reinterpret_cast(buffer), inflateOutSize); } inflateEnd(&zs);