feat: V2 save format

  Implement new SQLite-based save format. New worlds can be configured
  to use V2. Existing worlds can be converted from the main menu.

  Existing V1 worlds should load exactly the same as before until
  converted.

Signed-off-by: David Li <[email protected]>

.github/workflows/clang-tidy.yml

@@ -50,7 +50,8 @@ jobs:
           sudo apt-get install \
             cmake gettext ninja-build mold ccache jq \
             clang-18 libclang-18-dev llvm-18 llvm-18-dev clang-tidy-18 \
-            libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev libpulse-dev libflac-dev
+            libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev libsdl2-mixer-dev libpulse-dev libflac-dev \
+            sqlite3 libsqlite3-dev zlib1g-dev
 
       - name: ensure clang 18 is installed
         run: |

.github/workflows/manual-release.yml

@@ -233,11 +233,11 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install libncursesw5-dev libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev \
-            libsdl2-mixer-dev libpulse-dev ccache gettext parallel
+            libsdl2-mixer-dev libpulse-dev ccache gettext parallel libsqlite3-dev zlib1g-dev
       - name: Install dependencies (mac)
         if: runner.os == 'macOS'
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install sdl2 sdl2_image sdl2_ttf sdl2_mixer gettext ccache parallel
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install sdl2 sdl2_image sdl2_ttf sdl2_mixer gettext ccache parallel sqlite3 zlib
           python3 -m venv ./venv
           source ./venv/bin/activate
           pip3 install mac_alias==2.2.0 dmgbuild==1.6.1 biplist

.github/workflows/matrix.yml

@@ -156,7 +156,7 @@ jobs:
         if: ${{ env.SKIP == 'false' && runner.os == 'Linux' }}
         run: |
           sudo apt-get update
-          sudo apt-get install libncursesw5-dev ccache gettext parallel
+          sudo apt-get install libncursesw5-dev ccache gettext parallel libsqlite3-dev zlib1g-dev
           sudo locale-gen en_US.UTF-8 fr_FR.UTF-8 ru_RU.UTF-8
 
       - name: install SDL2 dependencies (ubuntu)
@@ -189,7 +189,7 @@ jobs:
           # Ensure that it is actually needed version
           ${{ matrix.compiler }} --version | grep -q -E "${{ matrix.grep_clang_version_rxp }}"
 
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel sqlite3 zlib
 
       # problem matchers are only ran on GCC to reduce error spam
       - name: add problem matcher

.github/workflows/msys2-cmake.yml

@@ -55,6 +55,8 @@ jobs:
           mingw-w64-x86_64-SDL2_ttf
           mingw-w64-x86_64-toolchain
           mingw-w64-x86_64-cmake
+          mingw-w64-x86_64-sqlite3
+          mingw-w64-x86_64-zstd
 
     - name: Create build directory
       run: mkdir build

.github/workflows/release.yml

@@ -263,7 +263,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install libncursesw5-dev libsdl2-dev libsdl2-ttf-dev libsdl2-image-dev \
-            libsdl2-mixer-dev libpulse-dev ccache gettext parallel
+            libsdl2-mixer-dev libpulse-dev ccache gettext parallel libsqlite3-dev zlib1g-dev
 
       - name: Install runtime dependencies (mac)
         if: runner.os == 'macOS'
@@ -276,7 +276,7 @@ jobs:
       - name: Install build dependencies (mac)
         if: runner.os == 'macOS'
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel
+          HOMEBREW_NO_AUTO_UPDATE=yes HOMEBREW_NO_INSTALL_CLEANUP=yes brew install gettext ccache parallel sqlite3 zlib
           python3 -m venv ./venv
           source ./venv/bin/activate
           pip3 install mac_alias==2.2.0 dmgbuild==1.6.1 biplist

CMakeLists.txt

@@ -113,6 +113,8 @@ include(CheckCXXCompilerFlag)
 #SET(CMAKE_SHARED_LIBRARY_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CXX_FLAGS} -m32")
 
 find_package(PkgConfig)
+find_package(SQLite3)
+find_package(ZLIB)
 if (NOT DYNAMIC_LINKING)
     if(NOT MSVC)
         set(CMAKE_FIND_LIBRARY_SUFFIXES ".a;.dll.a")

data/raw/keybindings/keybindings.json

@@ -744,6 +744,13 @@
     "name": "Pick random world name",
     "bindings": [ { "input_method": "keyboard", "key": "*" } ]
   },
+  {
+    "type": "keybinding",
+    "id": "TOGGLE_V2_SAVE_FORMAT",
+    "category": "WORLDGEN_CONFIRM_DIALOG",
+    "name": "Toggle new world save format",
+    "bindings": [ { "input_method": "keyboard", "key": "=" } ]
+  },
   {
     "type": "keybinding",
     "id": "QUIT",

doc/src/content/docs/en/dev/guides/building/makefile.md

@@ -58,7 +58,7 @@ your distro packages libraries and their development files separately (e.g. Debi
 
 Rough list based on building on Arch:
 
-- General: `gcc-libs`, `glibc`, `zlib`, `bzip2`
+- General: `gcc-libs`, `glibc`, `zlib`, `bzip2`, `sqlite3`
 - Optional: `intltool`
 - Curses: `ncurses`
 - Tiles: `sdl2`, `sdl2_image`, `sdl2_ttf`, `sdl2_mixer`, `freetype2`
@@ -137,7 +137,7 @@ Dependencies:
 Install:
 
 ```sh
-sudo apt-get install libncurses5-dev libncursesw5-dev build-essential astyle
+sudo apt-get install libncurses5-dev libncursesw5-dev build-essential astyle libsqlite3-dev zlib1g-dev
 ```
 
 ### Building

msvc-full-features/vcpkg.json

@@ -11,7 +11,9 @@
           "name": "sdl2-mixer",
           "features": [ "libflac", "mpg123", "libmodplug" ]
         },
-        "sdl2-ttf"
+        "sdl2-ttf",
+        "sqlite3",
+        "zlib"
     ],
     "builtin-baseline": "c9aba300923c8ec0ab190e2bff23085209925c97",
     "vcpkg-configuration": {

src/CMakeLists.txt

@@ -61,6 +61,8 @@ if (TILES)
 
     add_dependencies(cataclysm-bn-tiles-common get_version)
 
+    target_link_libraries(cataclysm-bn-tiles-common PUBLIC ZLIB::ZLIB)
+    target_link_libraries(cataclysm-bn-tiles-common PUBLIC SQLite::SQLite3)
     target_link_libraries(cataclysm-bn-tiles PRIVATE cataclysm-bn-tiles-common)
     target_compile_definitions(cataclysm-bn-tiles-common PUBLIC TILES )
 
@@ -176,9 +178,12 @@ if (CURSES)
     endif ()
 
     add_dependencies(cataclysm-bn-common get_version)
+
     target_link_libraries(cataclysm-bn PRIVATE cataclysm-bn-common)
 
     target_include_directories(cataclysm-bn-common PUBLIC ${CURSES_INCLUDE_DIR})
+    target_link_libraries(cataclysm-bn-common PUBLIC ZLIB::ZLIB)
+    target_link_libraries(cataclysm-bn-common PUBLIC SQLite::SQLite3)
     target_link_libraries(cataclysm-bn-common PUBLIC ${CURSES_LIBRARIES})
 
     if (CMAKE_USE_PTHREADS_INIT)

src/compress.cpp

@@ -0,0 +1,53 @@
+#include "compress.h"
+
+#include <zlib.h>
+#include <vector>
+#include <string>
+#include <stdexcept>
+#include <cstddef>
+
+void zlib_compress( const std::string &input, std::vector<std::byte> &output )
+{
+    uLongf compressedSize = compressBound( input.size() );
+    output.resize( compressedSize );
+
+    int result = compress2(
+                     reinterpret_cast<Bytef *>( output.data() ),
+                     &compressedSize,
+                     reinterpret_cast<const Bytef *>( input.data() ),
+                     input.size(),
+                     Z_BEST_SPEED
+                 );
+
+    if( result != Z_OK ) {
+        throw std::runtime_error( "Zlib compression error" );
+    }
+
+    output.resize( compressedSize );
+}
+
+void zlib_decompress( const void *compressed_data, int compressed_size, std::string &output )
+{
+    // We need to guess at the decompressed size - we expect things to compress fairly well.
+    uLongf decompressedSize = compressed_size * 8;
+    output.resize( decompressedSize );
+
+    int result;
+    do {
+        result = uncompress(
+                     reinterpret_cast<Bytef *>( output.data() ),
+                     &decompressedSize,
+                     reinterpret_cast<const Bytef *>( compressed_data ),
+                     compressed_size
+                 );
+
+        if( result == Z_BUF_ERROR ) {
+            decompressedSize *= 2; // Double the buffer size and retry
+            output.resize( decompressedSize );
+        } else if( result != Z_OK ) {
+            throw std::runtime_error( "Zlib decompression failed" );
+        }
+    } while( result == Z_BUF_ERROR );
+
+    output.resize( decompressedSize );
+}

src/compress.h

@@ -0,0 +1,12 @@
+#pragma once
+#ifndef CATA_SRC_COMPRESS_H
+#define CATA_SRC_COMPRESS_H
+
+#include <string>
+
+#include "fstream_utils.h"
+
+void zlib_compress( const std::string &input, std::vector<std::byte> &output );
+void zlib_decompress( const void *compressed_data, int compressed_size, std::string &output );
+
+#endif // CATA_SRC_COMPRESS_H

src/main_menu.cpp

@@ -500,6 +500,7 @@ void main_menu::init_strings()
     vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Character to Template" ) );
     vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Reset World" ) );
     vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Delete World" ) );
+    vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Convert to V2 Save Format" ) );
     vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "<= Return" ) );
 
     vWorldHotkeys = { 'm', 'e', 's', 't', 'r', 'd', 'q' };
@@ -1061,7 +1062,21 @@ void main_menu::world_tab( const std::string &worldname )
         }
     };
 
+    auto convert_v2 = [this, &worldname]() {
+        world_generator->set_active_world( nullptr );
+        savegames.clear();
+        MAPBUFFER.clear();
+        overmap_buffer.clear();
+        world_generator->convert_to_v2( worldname );
+    };
+
     switch( opt_val ) {
+        case 6: // Convert to V2 Save Format
+            if( query_yn(
+                    _( "Convert to V2 Save Format? A backup will be created. Conversion may take several minutes." ) ) ) {
+                convert_v2();
+            }
+            break;
         case 5: // Delete World
             if( query_yn( _( "Delete the world and all saves within?" ) ) ) {
                 clear_world( true );