Skip to content

Commit

Permalink
feat(vpkpp): add creation support for most formats
Browse files Browse the repository at this point in the history
  • Loading branch information
craftablescience committed Aug 22, 2024
1 parent 583b78c commit fe25d93
Show file tree
Hide file tree
Showing 58 changed files with 667 additions and 1,134 deletions.
7 changes: 7 additions & 0 deletions include/sourcepp/Macros.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#pragma once

#define SOURCEPP_CONCAT_INNER(a, b) a##b
#define SOURCEPP_CONCAT(a, b) SOURCEPP_CONCAT_INNER(a, b)

/// Adds the current line number to the given base
#define SOURCEPP_UNIQUE_NAME(base) SOURCEPP_CONCAT(base, __LINE__)
23 changes: 3 additions & 20 deletions include/vpkpp/Options.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,19 @@

namespace vpkpp {

/// VPK - Maximum preload data size in bytes
constexpr uint32_t VPK_MAX_PRELOAD_BYTES = 1024;

/// VPK - Chunk size in bytes (default is 200mb)
constexpr uint32_t VPK_DEFAULT_CHUNK_SIZE = 200 * 1024 * 1024;

struct PackFileOptions {
struct BakeOptions {
/// GMA - Write CRCs for files and the overall GMA file when baking
bool gma_writeCRCs = true;

/// VPK - Version (ignored when opening an existing VPK)
uint32_t vpk_version = 2;

/// VPK - If this value is 0, chunks have an unlimited size (not controlled by the library)
/// Chunk size is max 4gb, but since its not useful to have large chunks, try to keep the
/// preferred chunk size around the default
uint32_t vpk_preferredChunkSize = VPK_DEFAULT_CHUNK_SIZE;

/// VPK - Controls generation of per-file MD5 hashes (only for VPK v2)
/// VPK - Generate MD5 hashes for each file (VPK v2 only)
bool vpk_generateMD5Entries = false;

/// ZIP/BSP - The compression method. Check the MZ_COMPRESS_METHOD definitions for valid values. Only accepts STORE currently
uint16_t zip_compressionMethod = 0; // MZ_COMPRESS_METHOD_STORE
};

struct EntryOptions {
/// VPK - Save this entry to the directory VPK
bool vpk_saveToDirectory = false;

/// VPK - The amount in bytes of the file to preload. Maximum is controlled by VPK_MAX_PRELOAD_BYTES
/// VPK - The amount in bytes of the file to preload. Maximum is controlled by VPK_MAX_PRELOAD_BYTES (format/VPK.h)
uint32_t vpk_preloadBytes = 0;
};

Expand Down
73 changes: 38 additions & 35 deletions include/vpkpp/PackFile.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#pragma once

#include <concepts>
#include <functional>
#include <memory>
#include <optional>
Expand All @@ -10,6 +11,7 @@
#include <vector>

#include <sourcepp/math/Integer.h>
#include <sourcepp/Macros.h>
#include <tsl/htrie_map.h>

#include "Attribute.h"
Expand All @@ -27,10 +29,13 @@ constexpr std::string_view EXECUTABLE_EXTENSION2 = ".x86_64"; // | Godot
class PackFile {
public:
/// Accepts the entry's path and metadata
using EntryCallback = std::function<void(const std::string& path, const Entry& entry)>;
template<typename R>
using EntryCallbackBase = std::function<R(const std::string& path, const Entry& entry)>;
using EntryCallback = EntryCallbackBase<void>;
using EntryPredicate = EntryCallbackBase<bool>;

/// Accepts the entry's path and metadata
using EntryPredicate = std::function<bool(const std::string& path, const Entry& entry)>;
// Accepts the entry's path
using EntryCreation = std::function<EntryOptions(const std::string& path)>;

using EntryTrie = tsl::htrie_map<char, Entry>;

Expand All @@ -43,14 +48,14 @@ class PackFile {
virtual ~PackFile() = default;

/// Open a generic pack file. The parser is selected based on the file extension
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, PackFileOptions options = {}, const EntryCallback& callback = nullptr);
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);

/// Returns a sorted list of supported extensions for opening, e.g. {".bsp", ".vpk"}
[[nodiscard]] static std::vector<std::string> getOpenableExtensions();

/// Get the file type of the pack file
[[nodiscard]] PackFileType getType() const;

/// Get the current options of the pack file
[[nodiscard]] PackFileOptions getOptions() const;

/// Returns true if the format has a checksum for each entry
[[nodiscard]] virtual constexpr bool hasEntryChecksums() const {
return false;
Expand Down Expand Up @@ -96,13 +101,19 @@ class PackFile {
}

/// Add a new entry from a file path - the first parameter is the path in the PackFile, the second is the path on disk
void addEntry(const std::string& entryPath, const std::string& filepath, EntryOptions options_);
void addEntry(const std::string& entryPath, const std::string& filepath, EntryOptions options = {});

/// Add a new entry from a buffer
void addEntry(const std::string& path, std::vector<std::byte>&& buffer, EntryOptions options_);
void addEntry(const std::string& path, std::vector<std::byte>&& buffer, EntryOptions options = {});

/// Add a new entry from a buffer
void addEntry(const std::string& path, const std::byte* buffer, uint64_t bufferLen, EntryOptions options_);
void addEntry(const std::string& path, const std::byte* buffer, uint64_t bufferLen, EntryOptions options = {});

/// Adds new entries using the contents of a given directory
void addDirectory(const std::string& entryBaseDir, const std::string& dir, EntryOptions options = {});

/// Adds new entries using the contents of a given directory
void addDirectory(const std::string& entryBaseDir, const std::string& dir, const EntryCreation& creation);

/// Rename an existing entry
virtual bool renameEntry(const std::string& oldPath_, const std::string& newPath_); // NOLINT(*-use-nodiscard)
Expand All @@ -116,8 +127,8 @@ class PackFile {
/// Remove a directory
virtual std::size_t removeDirectory(const std::string& dirName_);

/// If output folder is unspecified, it will overwrite the original
virtual bool bake(const std::string& outputDir_ /*= ""*/, const EntryCallback& callback /*= nullptr*/) = 0;
/// If output folder is an empty string, it will overwrite the original
virtual bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) = 0;

/// Extract the given entry to disk at the given file path
bool extractEntry(const std::string& entryPath, const std::string& filepath) const; // NOLINT(*-use-nodiscard)
Expand Down Expand Up @@ -170,17 +181,14 @@ class PackFile {
/// On Windows, some characters and file names are invalid - this escapes the given entry path
[[nodiscard]] static std::string escapeEntryPathForWrite(const std::string& path);

/// Returns a list of supported extensions, e.g. {".vpk", ".bsp"}
[[nodiscard]] static std::vector<std::string> getSupportedFileTypes();

protected:
PackFile(std::string fullFilePath_, PackFileOptions options_);
explicit PackFile(std::string fullFilePath_);

void runForAllEntriesInternal(const std::function<void(const std::string&, Entry&)>& operation, bool includeUnbaked = true);

[[nodiscard]] std::vector<std::string> verifyEntryChecksumsUsingCRC32() const;

virtual void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options_) = 0;
virtual void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options) = 0;

[[nodiscard]] std::string getBakeOutputDir(const std::string& outputDir) const;

Expand All @@ -194,19 +202,18 @@ class PackFile {

[[nodiscard]] static std::optional<std::vector<std::byte>> readUnbakedEntry(const Entry& entry);

using OpenFactoryFunction = std::function<std::unique_ptr<PackFile>(const std::string& path, const EntryCallback& callback)>;

static std::unordered_map<std::string, std::vector<OpenFactoryFunction>>& getOpenExtensionRegistry();

static const OpenFactoryFunction& registerOpenExtensionForTypeFactory(std::string_view extension, const OpenFactoryFunction& factory);

std::string fullFilePath;

PackFileType type = PackFileType::UNKNOWN;
PackFileOptions options;

EntryTrie entries;
EntryTrie unbakedEntries;

using FactoryFunction = std::function<std::unique_ptr<PackFile>(const std::string& path, PackFileOptions options, const EntryCallback& callback)>;

static std::unordered_map<std::string, std::vector<FactoryFunction>>& getOpenExtensionRegistry();

static const FactoryFunction& registerOpenExtensionForTypeFactory(std::string_view extension, const FactoryFunction& factory);
};

class PackFileReadOnly : public PackFile {
Expand All @@ -218,23 +225,19 @@ class PackFileReadOnly : public PackFile {
[[nodiscard]] explicit operator std::string() const override;

protected:
PackFileReadOnly([[maybe_unused]] [[maybe_unused]] std::string fullFilePath_, PackFileOptions options_);
explicit PackFileReadOnly(std::string fullFilePath_);

void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options_) final;
void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options) final;

bool bake(const std::string& outputDir_ /*= ""*/, const EntryCallback& callback /*= nullptr*/) final;
bool bake(const std::string& outputDir_, BakeOptions options, const EntryCallback& callback) final;
};

} // namespace vpkpp

#define VPKPP_HELPER_CONCAT_INNER(a, b) a ## b
#define VPKPP_HELPER_CONCAT(a, b) VPKPP_HELPER_CONCAT_INNER(a, b)
#define VPKPP_HELPER_UNIQUE_NAME(base) VPKPP_HELPER_CONCAT(base, __LINE__)

#define VPKPP_REGISTER_PACKFILE_OPEN(extension, function) \
static inline const FactoryFunction& VPKPP_HELPER_UNIQUE_NAME(packFileOpenTypeFactoryFunction) = PackFile::registerOpenExtensionForTypeFactory(extension, function)
static inline const OpenFactoryFunction& SOURCEPP_UNIQUE_NAME(packFileOpenTypeFactoryFunction) = PackFile::registerOpenExtensionForTypeFactory(extension, function)

#define VPKPP_REGISTER_PACKFILE_OPEN_EXECUTABLE(function) \
static inline const FactoryFunction& VPKPP_HELPER_UNIQUE_NAME(packFileOpenExecutable0TypeFactoryFunction) = PackFile::registerOpenExtensionForTypeFactory(vpkpp::EXECUTABLE_EXTENSION0, function); \
static inline const FactoryFunction& VPKPP_HELPER_UNIQUE_NAME(packFileOpenExecutable1TypeFactoryFunction) = PackFile::registerOpenExtensionForTypeFactory(vpkpp::EXECUTABLE_EXTENSION1, function); \
static inline const FactoryFunction& VPKPP_HELPER_UNIQUE_NAME(packFileOpenExecutable2TypeFactoryFunction) = PackFile::registerOpenExtensionForTypeFactory(vpkpp::EXECUTABLE_EXTENSION2, function)
static inline const OpenFactoryFunction& SOURCEPP_UNIQUE_NAME(packFileOpenExecutable0TypeFactoryFunction) = PackFile::registerOpenExtensionForTypeFactory(vpkpp::EXECUTABLE_EXTENSION0, function); \
static inline const OpenFactoryFunction& SOURCEPP_UNIQUE_NAME(packFileOpenExecutable1TypeFactoryFunction) = PackFile::registerOpenExtensionForTypeFactory(vpkpp::EXECUTABLE_EXTENSION1, function); \
static inline const OpenFactoryFunction& SOURCEPP_UNIQUE_NAME(packFileOpenExecutable2TypeFactoryFunction) = PackFile::registerOpenExtensionForTypeFactory(vpkpp::EXECUTABLE_EXTENSION2, function)
6 changes: 3 additions & 3 deletions include/vpkpp/format/BSP.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,18 @@ constexpr std::string_view BSP_EXTENSION = ".bsp";
class BSP : public ZIP, private bsppp::BSP {
public:
/// Open a BSP file
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, PackFileOptions options = {}, const EntryCallback& callback = nullptr);
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);

[[nodiscard]] constexpr bool isCaseSensitive() const noexcept override {
return false;
}

bool bake(const std::string& outputDir_ /*= ""*/, const EntryCallback& callback /*= nullptr*/) override;
bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) override;

[[nodiscard]] explicit operator std::string() const override;

protected:
BSP(const std::string& fullFilePath_, PackFileOptions options_);
explicit BSP(const std::string& fullFilePath_);

const std::string tempBSPPakLumpPath;

Expand Down
11 changes: 7 additions & 4 deletions include/vpkpp/format/FPX.h
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,16 @@ constexpr std::string_view FPX_EXTENSION = ".fpx";

class FPX : public VPK {
public:
/// Open a directory VPK file
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, PackFileOptions options = {}, const EntryCallback& callback = nullptr);
/// Create a new directory FPX file - should end in "_dir.fpx"! This is not enforced but STRONGLY recommended
static std::unique_ptr<PackFile> create(const std::string& path);

/// Open an FPX file
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);

protected:
FPX(const std::string& fullFilePath_, PackFileOptions options_);
explicit FPX(const std::string& fullFilePath_);

[[nodiscard]] static std::unique_ptr<PackFile> openInternal(const std::string& path, PackFileOptions options = {}, const EntryCallback& callback = nullptr);
[[nodiscard]] static std::unique_ptr<PackFile> openInternal(const std::string& path, const EntryCallback& callback = nullptr);

private:
using VPK::generateKeyPairFiles;
Expand Down
4 changes: 2 additions & 2 deletions include/vpkpp/format/GCF.h
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ class GCF : public PackFileReadOnly {
#pragma pack(pop)

public:
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, PackFileOptions options = {}, const EntryCallback& callback = nullptr);
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);

[[nodiscard]] constexpr bool hasEntryChecksums() const override {
return true;
Expand All @@ -125,7 +125,7 @@ class GCF : public PackFileReadOnly {
[[nodiscard]] explicit operator std::string() const override;

protected:
GCF(const std::string& fullFilePath_, PackFileOptions options_);
explicit GCF(const std::string& fullFilePath_);

Header header{};
BlockHeader blockheader{};
Expand Down
8 changes: 4 additions & 4 deletions include/vpkpp/format/GMA.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class GMA : public PackFile {

public:
/// Open a GMA file
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, PackFileOptions options = {}, const EntryCallback& callback = nullptr);
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);

[[nodiscard]] constexpr bool hasEntryChecksums() const override {
return true;
Expand All @@ -39,16 +39,16 @@ class GMA : public PackFile {

[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;

bool bake(const std::string& outputDir_ /*= ""*/, const EntryCallback& callback /*= nullptr*/) override;
bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) override;

[[nodiscard]] Attribute getSupportedEntryAttributes() const override;

[[nodiscard]] explicit operator std::string() const override;

protected:
GMA(const std::string& fullFilePath_, PackFileOptions options_);
explicit GMA(const std::string& fullFilePath_);

void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options_) override;
void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options) override;

Header header{};

Expand Down
11 changes: 7 additions & 4 deletions include/vpkpp/format/PAK.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,22 @@ constexpr std::string_view PAK_EXTENSION = ".pak";

class PAK : public PackFile {
public:
/// Create a PAK file
static std::unique_ptr<PackFile> create(const std::string& path);

/// Open a PAK file
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, PackFileOptions options = {}, const EntryCallback& callback = nullptr);
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);

[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;

bool bake(const std::string& outputDir_ /*= ""*/, const EntryCallback& callback /*= nullptr*/) override;
bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) override;

[[nodiscard]] Attribute getSupportedEntryAttributes() const override;

protected:
PAK(const std::string& fullFilePath_, PackFileOptions options_);
explicit PAK(const std::string& fullFilePath_);

void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options_) override;
void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options) override;

private:
VPKPP_REGISTER_PACKFILE_OPEN(PAK_EXTENSION, &PAK::open);
Expand Down
11 changes: 7 additions & 4 deletions include/vpkpp/format/PCK.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,25 +27,28 @@ class PCK : public PackFile {
};

public:
/// Create a new PCK file
static std::unique_ptr<PackFile> create(const std::string& path, uint32_t version = 2, uint32_t godotMajorVersion = 0, uint32_t godotMinorVersion = 0, uint32_t godotPatchVersion = 0);

/// Open a PCK file (potentially embedded in an executable)
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, PackFileOptions options = {}, const EntryCallback& callback = nullptr);
[[nodiscard]] static std::unique_ptr<PackFile> open(const std::string& path, const EntryCallback& callback = nullptr);

[[nodiscard]] constexpr bool isCaseSensitive() const noexcept override {
return true;
}

[[nodiscard]] std::optional<std::vector<std::byte>> readEntry(const std::string& path_) const override;

bool bake(const std::string& outputDir_ /*= ""*/, const EntryCallback& callback /*= nullptr*/) override;
bool bake(const std::string& outputDir_ /*= ""*/, BakeOptions options /*= {}*/, const EntryCallback& callback /*= nullptr*/) override;

[[nodiscard]] Attribute getSupportedEntryAttributes() const override;

[[nodiscard]] explicit operator std::string() const override;

protected:
PCK(const std::string& fullFilePath_, PackFileOptions options_);
explicit PCK(const std::string& fullFilePath_);

void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options_) override;
void addEntryInternal(Entry& entry, const std::string& path, std::vector<std::byte>& buffer, EntryOptions options) override;

Header header{};

Expand Down
Loading

0 comments on commit fe25d93

Please sign in to comment.