Rewind ZSTD support

Common/Serialize/Serializer.h

@@ -255,7 +255,6 @@ class CChunkFileReader
 		}
 	}
 
-
 	// Load file template
 	template<class T>
 	static Error Load(const Path &filename, std::string *gitVersion, T& _class, std::string *failureReason)

Core/SaveState.cpp

@@ -20,6 +20,8 @@
 #include <thread>
 #include <mutex>
 
+#include <zstd.h>
+
 #include "Common/Data/Text/I18n.h"
 #include "Common/Thread/ThreadUtil.h"
 #include "Common/Data/Text/Parsers.h"
@@ -100,6 +102,17 @@ namespace SaveState
 		return CChunkFileReader::LoadPtr(&data[0], state, errorString);
 	}
 
+	struct StateBuffer {
+		void Clear() {
+			zstd_compressed.clear();
+			decompressed_size = 0;
+			compressed_size = 0;
+		}
+		std::vector<u8> zstd_compressed;
+		size_t decompressed_size = 0;
+		size_t compressed_size = 0;
+	};
+
 	// This ring buffer of states is for rewind save states, which are kept in RAM.
 	// Save states are compressed against one of two reference saves (bases_), and the reference
 	// is switched to a fresh save every N saves, where N is BASE_USAGE_INTERVAL.
@@ -151,7 +164,7 @@ namespace SaveState
 			if (err == CChunkFileReader::ERROR_NONE)
 				ScheduleCompress(&states_[n], compressBuffer, &bases_[base_]);
 			else
-				states_[n].clear();
+				states_[n].zstd_compressed.clear();
 
 			baseMapping_[n] = base_;
 			return err;
@@ -166,81 +179,135 @@ namespace SaveState
 				return CChunkFileReader::ERROR_BAD_FILE;
 
 			int n = (--next_ + size_) % size_;
-			if (states_[n].empty())
+			if (states_[n].zstd_compressed.empty())
 				return CChunkFileReader::ERROR_BAD_FILE;
 
 			static std::vector<u8> buffer;
-			LockedDecompress(buffer, states_[n], bases_[baseMapping_[n]]);
-			CChunkFileReader::Error error = LoadFromRam(buffer, errorString);
-			rewindLastTime_ = time_now_d();
-			return error;
+			if (LockedDecompress(buffer, states_[n], bases_[baseMapping_[n]])) {
+				CChunkFileReader::Error error = LoadFromRam(buffer, errorString);
+				if (error == CChunkFileReader::ERROR_NONE) {
+					INFO_LOG(SAVESTATE, "Rewinding to recent savestate snapshot (%d bytes compressed)", states_[n].zstd_compressed.size());
+					rewindLastTime_ = time_now_d();
+				}
+				return error;
+			} else {
+				WARN_LOG(SAVESTATE, "Failed to load rewind savestate");
+				// Unclear what CChunkFileReader error code we should pass in this case, which I'm not sure will
+				// happen in practice barring memory corruption.
+			}
+			return CChunkFileReader::ERROR_NONE;
 		}
 
-		void ScheduleCompress(std::vector<u8> *result, const std::vector<u8> *state, const std::vector<u8> *base)
+		void ScheduleCompress(StateBuffer *result, const std::vector<u8> *state, const std::vector<u8> *base)
 		{
 			if (compressThread_.joinable())
 				compressThread_.join();
 			compressThread_ = std::thread([=]{
 				SetCurrentThreadName("SaveStateCompress");
 
 				// Should do no I/O, so no JNI thread context needed.
-				Compress(*result, *state, *base);
+				Compress(result, *state, *base);
 			});
 		}
 
-		void Compress(std::vector<u8> &result, const std::vector<u8> &state, const std::vector<u8> &base)
+		const bool USE_XOR = false;
+
+		void Compress(StateBuffer *result, const std::vector<u8> &state, const std::vector<u8> &base)
 		{
 			std::lock_guard<std::mutex> guard(lock_);
 			// Bail if we were cleared before locking.
 			if (first_ == 0 && next_ == 0)
 				return;
 
 			double start_time = time_now_d();
-			result.clear();
-			result.reserve(512 * 1024);
-			for (size_t i = 0; i < state.size(); i += BLOCK_SIZE)
-			{
-				int blockSize = std::min(BLOCK_SIZE, (int)(state.size() - i));
-				if (i + blockSize > base.size() || memcmp(&state[i], &base[i], blockSize) != 0)
+			std::vector<u8> compressed;
+			if (USE_XOR) {
+				compressed.resize(state.size());
+				for (size_t i = 0; i < state.size(); i++) {
+					if (i >= base.size()) {
+						compressed[i] = state[i];
+					} else {
+						compressed[i] = base[i] ^ state[i];
+					}
+				}
+			} else {
+				compressed.reserve(512 * 1024);
+				for (size_t i = 0; i < state.size(); i += BLOCK_SIZE)
 				{
-					result.push_back(1);
-					result.insert(result.end(), state.begin() + i, state.begin() + i + blockSize);
+					int blockSize = std::min(BLOCK_SIZE, (int)(state.size() - i));
+					if (i + blockSize > base.size() || memcmp(&state[i], &base[i], blockSize) != 0)
+					{
+						compressed.push_back(1);
+						compressed.insert(compressed.end(), state.begin() + i, state.begin() + i + blockSize);
+					} else {
+						compressed.push_back(0);
+					}
 				}
-				else
-					result.push_back(0);
 			}
 
 			double taken_s = time_now_d() - start_time;
-			DEBUG_LOG(SAVESTATE, "Rewind: Compressed save from %d bytes to %d in %0.2f ms.", (int)state.size(), (int)result.size(), taken_s * 1000.0);
+			DEBUG_LOG(SAVESTATE, "Rewind: Compressed save from %d bytes to %d in %0.2f ms.", (int)state.size(), (int)compressed.size(), taken_s * 1000.0);
+
+			// Temporarily allocate a buffer to do compression in.
+			size_t compressCapacity = ZSTD_compressBound(compressed.size());
+			result->zstd_compressed.resize(compressCapacity);
+			result->compressed_size = ZSTD_compress(&result->zstd_compressed[0], compressCapacity, compressed.data(), compressed.size(), 1);
+			if (result->compressed_size) {
+				result->zstd_compressed.resize(result->compressed_size);
+				result->decompressed_size = compressed.size();
+			}
+
+			double zstd_s = time_now_d() - start_time - taken_s;
+			DEBUG_LOG(SAVESTATE, "Rewind: ZSTD compressed to %d in %0.2f ms.", (int)result->compressed_size, zstd_s * 1000.0);
 		}
 
-		void LockedDecompress(std::vector<u8> &result, const std::vector<u8> &compressed, const std::vector<u8> &base)
+		bool LockedDecompress(std::vector<u8> &result, const StateBuffer &buffer, const std::vector<u8> &base)
 		{
 			result.clear();
 			result.reserve(base.size());
 			auto basePos = base.begin();
-			for (size_t i = 0; i < compressed.size(); )
-			{
-				if (compressed[i] == 0)
-				{
-					++i;
-					int blockSize = std::min(BLOCK_SIZE, (int)(base.size() - result.size()));
-					result.insert(result.end(), basePos, basePos + blockSize);
-					basePos += blockSize;
+
+			// OK, zstd decompress first.
+			std::vector<u8> compressed = std::vector<u8>(buffer.decompressed_size, 0);
+			size_t retval = ZSTD_decompress(&compressed[0], compressed.size(), buffer.zstd_compressed.data(), buffer.zstd_compressed.size());
+			if (ZSTD_isError(retval)) {
+				WARN_LOG(SAVESTATE, "Failed to decompress zstd-compressed rewind savestate");
+				return false;
+			}
+
+			if (USE_XOR) {
+				result.resize(compressed.size());
+				for (size_t i = 0; i < compressed.size(); i++) {
+					if (i < base.size()) {
+						result[i] = compressed[i] ^ base[i];
+					} else {
+						result[i] = compressed[i];
+					}
 				}
-				else
-				{
-					++i;
-					int blockSize = std::min(BLOCK_SIZE, (int)(compressed.size() - i));
-					result.insert(result.end(), compressed.begin() + i, compressed.begin() + i + blockSize);
-					i += blockSize;
-					// This check is to avoid advancing basePos out of range, which MSVC catches.
-					// When this happens, we're at the end of decoding anyway.
-					if (base.end() - basePos >= blockSize) {
+			} else {
+				for (size_t i = 0; i < compressed.size(); ) {
+					if (compressed[i] == 0) {
+						++i;
+						int blockSize = std::min(BLOCK_SIZE, (int)(base.size() - result.size()));
+						_dbg_assert_(blockSize >= 0);
+						result.insert(result.end(), basePos, basePos + blockSize);
 						basePos += blockSize;
+					} else {
+						++i;
+						int blockSize = std::min(BLOCK_SIZE, (int)(compressed.size() - i));
+						if (blockSize > 0) {
+							result.insert(result.end(), compressed.begin() + i, compressed.begin() + i + blockSize);
+							i += blockSize;
+							// This check is to avoid advancing basePos out of range, which MSVC catches.
+							// When this happens, we're at the end of decoding anyway.
+							if (base.end() - basePos >= blockSize) {
+								basePos += blockSize;
+							}
+						}
 					}
 				}
 			}
+			return true;
 		}
 
 		void Clear()
@@ -258,7 +325,7 @@ namespace SaveState
 			baseMapping_.clear();
 			baseMapping_.resize(size_);
 			for (auto &s : states_) {
-				s.clear();
+				s.Clear();
 			}
 			buffer_.clear();
 			base_ = -1;
@@ -297,14 +364,12 @@ namespace SaveState
 		// TODO: Instead, based on size of compressed state?
 		const int BASE_USAGE_INTERVAL = 15;
 
-		typedef std::vector<u8> StateBuffer;
-
 		int first_ = 0;
 		int next_ = 0;
 		int size_;
 
 		std::vector<StateBuffer> states_;
-		StateBuffer bases_[2];
+		std::vector<u8> bases_[2];
 		std::vector<int> baseMapping_;
 		std::mutex lock_;
 		std::thread compressThread_;
@@ -313,7 +378,7 @@ namespace SaveState
 		int base_ = -1;
 		int baseUsage_ = 0;
 
-		double rewindLastTime_ = 0.0f;
+		double rewindLastTime_ = 0.0;
 	};
 
 	static bool needsProcess = false;
@@ -1019,7 +1084,6 @@ namespace SaveState
 				break;
 
 			case SAVESTATE_REWIND:
-				INFO_LOG(SAVESTATE, "Rewinding to recent savestate snapshot");
 				result = rewindStates.Restore(&errorString);
 				if (result == CChunkFileReader::ERROR_NONE) {
 					callbackMessage = sc->T("Loaded State");

assets/lang/az_AZ.ini

@@ -1285,7 +1285,7 @@ Record Display = Record display
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Save path in installed.txt = Save path in installed.txt
 Save path in My Documents = Save path in My Documents
 Savestate Slot = Savestate slot

assets/lang/bg_BG.ini

@@ -1285,7 +1285,7 @@ Record Display = Record display
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Възстанови първоначалните настройки на PPSSPP
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind snapshot честота („яде“ памет)
+Rewind Snapshot Interval = Rewind snapshot честота
 Save path in installed.txt = Save path in installed.txt
 Save path in My Documents = Save path in My Documents
 Savestate Slot = слот за запазено състояние

assets/lang/ca_ES.ini

@@ -1285,7 +1285,7 @@ Record Display = Record display
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Save path in installed.txt = Save path in installed.txt
 Save path in My Documents = Save path in My Documents
 Savestate Slot = Savestate slot

assets/lang/cz_CZ.ini

@@ -1285,7 +1285,7 @@ Record Display = Record display
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Obnovit výchozí nastavení PPSSPP
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Četnost snímků přetočení (žrout paměti)
+Rewind Snapshot Interval = Četnost snímků přetočení
 Save path in installed.txt = Cesta pro uložení dat je installed.txt
 Save path in My Documents = Cesta pro uložení dat je v Dokumentech
 Savestate Slot = Pozice uložené hry

assets/lang/da_DK.ini

@@ -1285,7 +1285,7 @@ Record Display = Optag skærm
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Sæt PPSSPP's indstillinger tilbage til standard
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Tilbagespol snapshot frekvens (mem hog)
+Rewind Snapshot Interval = Tilbagespol snapshot interval
 Save path in installed.txt = Gem sti i installed.txt
 Save path in My Documents = Gem sti i Mine Dokumenter
 Savestate Slot = Lagerplads for spil-status

assets/lang/de_DE.ini

@@ -1285,7 +1285,7 @@ Record Display = Bildschirm aufzeichnen
 Reset Recording on Save/Load State = Zurücksetzen der Aufnahme bei Laden/Speichern eines Standes
 Restore Default Settings = Auf Standardeinstellungen zurücksetzen
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Zurückspulen-Snapshot Frequenz (Speicherfresser)
+Rewind Snapshot Interval = Zurückspulen-Snapshot Frequenz
 Save path in installed.txt = Speicherpfad in installed.txt
 Save path in My Documents = Speicherpfad in "Meine Dokumente"
 Savestate Slot = Speicherplatz

assets/lang/dr_ID.ini

@@ -1285,7 +1285,7 @@ Record Display = Record display
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Save path in installed.txt = Save path in installed.txt
 Save path in My Documents = Save path in My Documents
 Savestate Slot = Savestate slot

assets/lang/en_US.ini

@@ -1299,7 +1299,7 @@ Record Display = Record display
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Save path in installed.txt = Save path in installed.txt
 Save path in My Documents = Save path in My Documents
 Savestate Slot = Savestate slot

assets/lang/es_LA.ini

@@ -1287,7 +1287,7 @@ Record Display = Grabar pantalla
 Reset Recording on Save/Load State = Reiniciar grabación al abrir/guardar estados
 Restore Default Settings = Reestablecer ajustes
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Frecuencia de rebobinado\n(consume memoria)
+Rewind Snapshot Interval = Intervalo de rebobinado
 Save path in installed.txt = Guardar ruta en installed.txt
 Save path in My Documents = Guardar ruta en "C:/Users/(tu usuario)/Documents"
 Savestate Slot = Ranura de estado guardado

assets/lang/fa_IR.ini

@@ -1285,7 +1285,7 @@ Record Display = ‎ضبط صفحه
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = ‎به حالت اولیه PPSSPP بازگشت تنظیمات
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = ‎تعداد فریم ذخیره شده برای به عقب رفتن (مصرف زیاد رم)
+Rewind Snapshot Interval = ‎تعداد فریم ذخیره شده برای به عقب رفتن
 Save path in installed.txt = Save path in installed.txt
 Save path in My Documents = ذخیره در داکیومنت ها
 Savestate Slot = Savestate slot

assets/lang/fi_FI.ini

@@ -1285,7 +1285,7 @@ Record Display = Nauhoita näyttöä
 Reset Recording on Save/Load State = Nollaa nauhoitus tallennettaessa/ladattaessa tila
 Restore Default Settings = Palauta PPSSPP:n oletusasetukset
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Pikakelaa tilannevedosten välit (muistisyöppö)
+Rewind Snapshot Interval = Pikakelaa tilannevedosten välit
 Save path in installed.txt = Tallennuskansio installed.txt-tiedostossa
 Save path in My Documents = Tallennuskansio Omat tiedostot -kansiossa
 Savestate Slot = Tilatallennuksen lohko

assets/lang/gl_ES.ini

@@ -1285,7 +1285,7 @@ Record Display = Record display
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Reestablecer axustes
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Frecuencia de rebobinado de instantánea (mem hog)
+Rewind Snapshot Interval = Frecuencia de rebobinado de instantánea
 Save path in installed.txt = Carpeta de gardado en installed.txt
 Save path in My Documents = Carpeta de gardado en Meus documentos
 Savestate Slot = Ranura de estado gardado

assets/lang/gr_EL.ini

@@ -1285,7 +1285,7 @@ Record Display = Καταγραφή Εικόνας
 Reset Recording on Save/Load State = Επαναφορά της εγγραφής κατή την Αποθήκευση/Φόρτωση σημείου αποθήκευσης
 Restore Default Settings = Επαναφορά προεπιλεγμένων ρυθμίσεων του PPSSPP
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Συχνότητα Αντιστροφής Στιγμιότυπου (mem hog)
+Rewind Snapshot Interval = Συχνότητα Αντιστροφής Στιγμιότυπου
 Save path in installed.txt = Αποθήκευση διαδρομής στο installed.txt
 Save path in My Documents = Αποθήκευση διαδρομής στα Έγγραφα μου
 Savestate Slot = Slot Σημείου Αποθήκευσης

assets/lang/he_IL.ini

@@ -1285,7 +1285,7 @@ Record Display = Record display
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Save path in installed.txt = Save path in installed.txt
 Save path in My Documents = Save path in My Documents
 Savestate Slot = Savestate slot

assets/lang/he_IL_invert.ini

@@ -1285,7 +1285,7 @@ Record Display = Record display
 Reset Recording on Save/Load State = Reset recording on Save/Load state
 Restore Default Settings = Restore PPSSPP's settings to default
 RetroAchievements = RetroAchievements
-Rewind Snapshot Interval = Rewind Snapshot Interval (mem hog)
+Rewind Snapshot Interval = Rewind Snapshot Interval
 Save path in installed.txt = Save path in installed.txt
 Save path in My Documents = Save path in My Documents
 Savestate Slot = Savestate slot