From a33a1b63077818c9c0350e8a69a7a4e390599f03 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:32:35 -0500 Subject: [PATCH 01/86] added check for L3 and R3 buttons for older controllers --- Cores/Atari800/Sources/PVAtari800Bridge/PVAtari800Bridge.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cores/Atari800/Sources/PVAtari800Bridge/PVAtari800Bridge.m b/Cores/Atari800/Sources/PVAtari800Bridge/PVAtari800Bridge.m index 4cbc42270b..d06378fb9a 100644 --- a/Cores/Atari800/Sources/PVAtari800Bridge/PVAtari800Bridge.m +++ b/Cores/Atari800/Sources/PVAtari800Bridge/PVAtari800Bridge.m @@ -733,10 +733,10 @@ - (void)pollControllers { } else if (gamepad.rightThumbstick.down.isPressed) { //4 button INPUT_key_code = AKEY_5200_4; - } else if (gamepad.leftThumbstickButton.isPressed) { + } else if (gamepad.leftThumbstickButton != nil && gamepad.leftThumbstickButton.isPressed) { //5 button INPUT_key_code = AKEY_5200_5; - } else if (gamepad.rightThumbstickButton.isPressed) { + } else if (gamepad.rightThumbstickButton != nil && gamepad.rightThumbstickButton.isPressed) { //6 button INPUT_key_code = AKEY_5200_6; } else if (gamepad.buttonX.isPressed) { From efe7ae6160fd824dfb60533c08910bc75ae8779b Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:32:57 -0500 Subject: [PATCH 02/86] updated minimum to 16.0 --- Cores/Gearcoleco/PVGearcoleco.xcodeproj/project.pbxproj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cores/Gearcoleco/PVGearcoleco.xcodeproj/project.pbxproj b/Cores/Gearcoleco/PVGearcoleco.xcodeproj/project.pbxproj index ff1b82f872..b2174c1260 100644 --- a/Cores/Gearcoleco/PVGearcoleco.xcodeproj/project.pbxproj +++ b/Cores/Gearcoleco/PVGearcoleco.xcodeproj/project.pbxproj @@ -577,7 +577,7 @@ GCC_WARN_INHIBIT_ALL_WARNINGS = YES; INFOPLIST_FILE = "$(SRCROOT)/PVGearcoleco/Info.plist"; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; - IPHONEOS_DEPLOYMENT_TARGET = 15.6; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -837,7 +837,7 @@ GCC_WARN_INHIBIT_ALL_WARNINGS = YES; INFOPLIST_FILE = "$(SRCROOT)/PVGearcoleco/Info.plist"; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; - IPHONEOS_DEPLOYMENT_TARGET = 15.6; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -871,7 +871,7 @@ GCC_WARN_INHIBIT_ALL_WARNINGS = YES; INFOPLIST_FILE = "$(SRCROOT)/PVGearcoleco/Info.plist"; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; - IPHONEOS_DEPLOYMENT_TARGET = 15.6; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", From be7da848999f0555670108ebc46b9dabcf7e3e4a Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 13 Jan 2025 23:14:29 -0500 Subject: [PATCH 03/86] updated to use variables for app groups and icloud container --- Extensions/Spotlight/Spotlight.entitlements | 2 +- Extensions/TopShelf/TopShelf.entitlements | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Extensions/Spotlight/Spotlight.entitlements b/Extensions/Spotlight/Spotlight.entitlements index 267cbe6e36..d81f3d833e 100644 --- a/Extensions/Spotlight/Spotlight.entitlements +++ b/Extensions/Spotlight/Spotlight.entitlements @@ -6,7 +6,7 @@ com.apple.security.application-groups - group.org.provenance-emu.provenance + $(APP_GROUP_IDENTIFIER) com.apple.security.assets.pictures.read-only diff --git a/Extensions/TopShelf/TopShelf.entitlements b/Extensions/TopShelf/TopShelf.entitlements index 5ae8701f1b..7c83d2b2dd 100644 --- a/Extensions/TopShelf/TopShelf.entitlements +++ b/Extensions/TopShelf/TopShelf.entitlements @@ -6,7 +6,7 @@ development com.apple.developer.icloud-container-identifiers - iCloud.org.provenance.provenance + $(ICLOUD_CONTAINER_IDENTIFIER) com.apple.developer.icloud-services @@ -16,7 +16,7 @@ $(TeamIdentifierPrefix)$(CFBundleIdentifier) com.apple.security.application-groups - group.org.provenance-emu.provenance + $(APP_GROUP_IDENTIFIER) From c6224c95edb40f7537b04bf6c5229c5b9b4a43ac Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 13 Jan 2025 23:42:55 -0500 Subject: [PATCH 04/86] added todo to test without .icloud relative --- .../Services/GameImporter/GameImporterDatabaseService.swift | 6 +++--- .../Conversion/Extensions/SaveState+PVSaveState.swift | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index 3e9d95b146..1985b2517a 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -216,7 +216,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } if PVMediaCache.fileExists(forKey: url) { if let localURL = PVMediaCache.filePath(forKey: url) { - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) + let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud)//TODO: test without .iCloud game.originalArtworkFile = file return game } @@ -240,7 +240,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { if let artwork = NSImage(data: data) { do { let localURL = try PVMediaCache.writeImage(toDisk: artwork, withKey: url) - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) + let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud)//TODO: test without .iCloud game.originalArtworkFile = file } catch { ELOG("\(error.localizedDescription)") } } @@ -248,7 +248,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { if let artwork = UIImage(data: data) { do { let localURL = try PVMediaCache.writeImage(toDisk: artwork, withKey: url) - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) + let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud)//TODO: test without .iCloud game.originalArtworkFile = file } catch { ELOG("\(error.localizedDescription)") } } diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift index 149fbc10f7..a44e078d9a 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift @@ -75,7 +75,7 @@ extension SaveState: RealmRepresentable { let dir = path.deletingLastPathComponent() let imagePath = dir.appendingPathComponent(image.fileName) DLOG("path: \(imagePath)") - object.image = PVImageFile(withURL: imagePath, relativeRoot: .iCloud) + object.image = PVImageFile(withURL: imagePath, relativeRoot: .iCloud)//TODO: test without .iCloud } object.isAutosave = isAutosave } From bd7eb3441fed4dbcbe49da468b9b7c4f9a459186 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 13 Jan 2025 23:43:22 -0500 Subject: [PATCH 05/86] added todo to test without .icloud relative --- .../PVEmulatorVC/PVEmulatorViewController+Saves.swift | 4 ++-- .../PVEmulatorVC/PVEmulatorViewController+ScreenShot.swift | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+Saves.swift b/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+Saves.swift index 223998f1bf..b347a514ba 100644 --- a/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+Saves.swift +++ b/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+Saves.swift @@ -95,14 +95,14 @@ public extension PVEmulatorViewController { do { try jpegData.write(to: imageURL) // try RomDatabase.sharedInstance.writeTransaction { - // let newFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud) + // let newFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud)//TODO: test without .iCloud // game.screenShots.append(newFile) // } } catch { presentError("Unable to write image to disk, error: \(error.localizedDescription)", source: self.view) } - imageFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud) + imageFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud)//TODO: test without .iCloud } } diff --git a/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+ScreenShot.swift b/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+ScreenShot.swift index 537f5ef1f9..d75db6ffd7 100644 --- a/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+ScreenShot.swift +++ b/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+ScreenShot.swift @@ -52,7 +52,7 @@ extension PVEmulatorViewController { try pngData.write(to: imageURL) RomDatabase.sharedInstance.asyncWriteTransaction { self.game.realm?.refresh() - let newFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud) + let newFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud)//TODO: test without .iCloud self.game.screenShots.append(newFile) } } catch { From a852aeaeb3ca8c03d47e3d30c9102c48e402a04c Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 13 Jan 2025 23:44:04 -0500 Subject: [PATCH 06/86] fix for icloud URL paths that duplicates the root directories --- .../RealmPlatform/Entities/Files/PVFile.swift | 121 +++++++++++++++++- 1 file changed, 117 insertions(+), 4 deletions(-) diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift index 8384d54516..69f78206d9 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift @@ -78,27 +78,140 @@ public extension PVFile { var url: URL { get { + let url2 = urlUpdate + print("url2=\(url2)\tpartialPath=\(partialPath)") if partialPath.contains("iCloud") || partialPath.contains("private") { var pathComponents = (partialPath as NSString).pathComponents pathComponents.removeFirst() let path = pathComponents.joined(separator: "/") let isDocumentsDir = path.contains("Documents") - + if isDocumentsDir { let iCloudBase = URL.iCloudContainerDirectory let url = (iCloudBase ?? RelativeRoot.documentsDirectory).appendingPathComponent(path) - return url + print("url:\(url)") + return url2 } else { if let iCloudBase = URL.iCloudDocumentsDirectory { - return iCloudBase.appendingPathComponent(path) + let appendedICloudBase = iCloudBase.appendingPathComponent(path) + print("appendedICloudBase:\(appendedICloudBase))") + return url2 } else { - return RelativeRoot.documentsDirectory.appendingPathComponent(path) + let appendedRelativeRoot = RelativeRoot.documentsDirectory.appendingPathComponent(path) + print("appendedRelativeRoot:\(appendedRelativeRoot)") + return url2 } } } let root = relativeRoot let resolvedURL = root.appendingPath(partialPath) + print("resolvedURL:\(resolvedURL))") + return url2 + } + } + var urlUpdate:URL { + get { + print("relativeRoot=\(relativeRoot)\tpartialPath=\(partialPath)") + let pathSuffix: String + if let bundleIdentifier = Bundle.main.bundleIdentifier { + print("Bundle Identifier: \(bundleIdentifier)") + let bundleComponents = bundleIdentifier.split(separator: ".") + print("bundleComponents=\(bundleComponents)") + let joined = bundleComponents.joined(separator: "~") + print("joined=\(joined)") + pathSuffix = joined + } else { + pathSuffix = "org~provenance-emu~provenance" + } + let privateDirectory = "private" + if partialPath.contains("/iCloud~\(pathSuffix)/") {//&& partialPath.hasPrefix("\(privateDirectory)/") { + let completePath: String + let filePrefix = "file:///" + if !partialPath.hasPrefix(filePrefix) { + completePath = "\(filePrefix)\(partialPath)" + } else { + completePath = partialPath + } + if let urlPath = URL(string: completePath) { + print("urlPath=\(urlPath)") + return urlPath + } + + var pathComponents = (partialPath as NSString).pathComponents + print("pathComponents=\(pathComponents)") + //["private", "var", "mobile", "Library", "Mobile Documents", "iCloud~\(pathSuffix)", "Documents"] + let mobileDocumentsEncoded = "Mobile%20Documents" + let mobileDocumentsDecoded = "Mobile Documents" + let directoryPath: String + + if let iCloudDocumentsDirectoryContainer = URL.iCloudDocumentsDirectory { + var tmp = "\(iCloudDocumentsDirectoryContainer)" + let filePrefix = "file://" + if tmp.hasPrefix(filePrefix) { + tmp = tmp.replacingOccurrences(of: filePrefix, with: "") + } + directoryPath = tmp + } else { + directoryPath = "\(privateDirectory)/var/mobile/Library/\(mobileDocumentsDecoded)/\(mobileDocumentsDecoded)/iCloud~\(pathSuffix)" + } + print("directoryPath=\(directoryPath)") + var prefixes = directoryPath.split(separator: "/") + let mobileDocumentsEncodedSub = mobileDocumentsEncoded.prefix(mobileDocumentsEncoded.count) + //we also add an encoded one. + if !prefixes.contains(mobileDocumentsEncodedSub) { + prefixes.append(mobileDocumentsEncodedSub) + } + let mobileDocumentsDecodedSub = mobileDocumentsDecoded.prefix(mobileDocumentsDecoded.count) + //we also add a decoded one. + if !prefixes.contains(mobileDocumentsDecodedSub) { + prefixes.append(mobileDocumentsDecodedSub) + } + print("prefixes=\(prefixes)") + while prefixes.contains(where: {String($0) == pathComponents.first}) { + /* + Action Button Pressed 1706495469592 Optional(1706495461875) + relativeRoot=documents partialPath=private/var/mobile/Library/Mobile Documents/iCloud~com~pqskapps~provenance/Documents/Save States/Gremlins (USA).a52/DC271E475B4766E80151F1DA5B764E52.728185244.058265.svs + pathComponents=["private", "var", "mobile", "Library", "Mobile Documents", "iCloud~com~pqskapps~provenance", "Documents", "Save States", "Gremlins (USA).a52", "DC271E475B4766E80151F1DA5B764E52.728185244.058265.svs"] + pathComponentslremoveFirst()=["var", "mobile", "Library", "Mobile Documents", "iCloud~com~pqskapps~provenance", "Documents", "Save States", "Gremlins (USA).a52", "DC271E475B4766E80151F1DA5B764E52.728185244.058265.svs"] + path=var/mobile/Library/Mobile Documents/iCloud~com~pqskapps~provenance/Documents/Save States/Gremlins (USA).a52/DC271E475B4766E80151F1DA5B764E52.728185244.058265.svs + PVEmulatorConfiguration.iCloudContainerDirectory=Optional(file:///private/var/mobile/Library/Mobile%20Documents/iCloud~com~pqskapps~provenance/) + PVEmulatorConfiguration.iCloudDocumentsDirectory=Optional(file:///private/var/mobile/Library/Mobile%20Documents/iCloud~com~pqskapps~provenance/Documents/) + iCloudBase=Optional(file:///private/var/mobile/Library/Mobile%20Documents/iCloud~com~pqskapps~provenance/) + url=file:///private/var/mobile/Library/Mobile%20Documents/iCloud~com~pqskapps~provenance/var/mobile/Library/Mobile%20Documents/iCloud~com~pqskapps~provenance/Documents/Save%20States/Gremlins%20(USA).a52/DC271E475B4766E80151F1DA5B764E52.728185244.058265.svs + */ + pathComponents.removeFirst() + print("pathComponentslremoveFirst()=\(pathComponents)") + } + let path = pathComponents.joined(separator: "/") + print("path=\(path)") + print("PVEmulatorConfiguration.iCloudContainerDirectory=\(String(describing: URL.iCloudContainerDirectory))") + print("PVEmulatorConfiguration.iCloudDocumentsDirectory=\(String(describing: URL.iCloudDocumentsDirectory))") + let iCloudBase = path.contains("Documents") ? URL.iCloudContainerDirectory : URL.iCloudDocumentsDirectory + print("iCloudBase=\(String(describing: iCloudBase))") + let url = (iCloudBase ?? RelativeRoot.documentsDirectory).appendingPathComponent(path) + print("url=\(url)") + return url + } + let root = relativeRoot + print("root=\(root)") + var actualPartialPath = partialPath + print("actualPartialPath=\(actualPartialPath)") + if partialPath.hasPrefix(privateDirectory) { + var tmp = partialPath.split(separator: "/") + tmp.removeFirst() + actualPartialPath = tmp.joined(separator: "/") + } + print("actualPartialPath=\(actualPartialPath)") + let resolvedURL = root.appendingPath(actualPartialPath) + print("resolvedURL=\(resolvedURL)") return resolvedURL + /* + relativeRoot=iCloud partialPath=var/mobile/Containers/Data/Application/B8153B85-9BB5-44B6-A189-FDE9D8ABC29C/Documents/PVCache/F62D5AA941BB70E1913B787A65CD7EFC + Bundle Identifier: com.pqskapps.provenance + bundleComponents=["com", "pqskapps", "provenance"] + joined=com~pqskapps~provenance + resolvedURL=file:///private/var/mobile/Library/Mobile%20Documents/iCloud~com~pqskapps~provenance/Documents/var/mobile/Containers/Data/Application/B8153B85-9BB5-44B6-A189-FDE9D8ABC29C/Documents/PVCache/F62D5AA941BB70E1913B787A65CD7EFC + */ } } From c5ee69eba8c110cefeea9e211322c72f4b4bd021 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 13 Jan 2025 23:44:35 -0500 Subject: [PATCH 07/86] added a check for nil --- PVUI/Sources/PVSwiftUI/Components/GameContextMenu.swift | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/PVUI/Sources/PVSwiftUI/Components/GameContextMenu.swift b/PVUI/Sources/PVSwiftUI/Components/GameContextMenu.swift index 06250bf806..1d49551173 100644 --- a/PVUI/Sources/PVSwiftUI/Components/GameContextMenu.swift +++ b/PVUI/Sources/PVSwiftUI/Components/GameContextMenu.swift @@ -39,9 +39,11 @@ struct GameContextMenu: View { self.contextMenuDelegate = contextMenuDelegate // Initialize computed properties - _availableCores = State(initialValue: game.system.cores.filter { - !(AppState.shared.isAppStore && $0.appStoreDisabled) - }) + if let currentSystem = game.system { + _availableCores = State(initialValue: currentSystem.cores.filter { + !(AppState.shared.isAppStore && $0.appStoreDisabled) + }) + } _hasSaveStates = State(initialValue: !game.saveStates.isEmpty) } From f5fad8b11b995774bd1ea73b746a3bbfe63f1b29 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 13 Jan 2025 23:45:50 -0500 Subject: [PATCH 08/86] added comment of encoded space to review --- PVLibrary/Sources/PVFileSystem/Paths.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/PVLibrary/Sources/PVFileSystem/Paths.swift b/PVLibrary/Sources/PVFileSystem/Paths.swift index 7d730dd754..1172640ab8 100644 --- a/PVLibrary/Sources/PVFileSystem/Paths.swift +++ b/PVLibrary/Sources/PVFileSystem/Paths.swift @@ -78,11 +78,11 @@ public extension URL { public struct Paths { public struct Legacy { - public static var batterySavesPath: URL { + public static var batterySavesPath: URL {//TODO: review encoded space %20 URL.documentsPath.appendingPathComponent("Battery States", isDirectory: true) } - public static var saveSavesPath: URL { + public static var saveSavesPath: URL {//TODO: review encoded space %20 URL.documentsPath.appendingPathComponent("Save States", isDirectory: true) } @@ -105,12 +105,12 @@ public struct Paths { }} /// Should be called on BG Thread, iCloud blocks - public static var batterySavesPath: URL { get { + public static var batterySavesPath: URL { get {//TODO: review encoded space %20 return URL.documentsiCloudOrLocalPath.appendingPathComponent("Battery States", isDirectory: true) }} /// Should be called on BG Thread, iCloud blocks - public static var saveSavesPath: URL { get { + public static var saveSavesPath: URL { get {//TODO: review encoded space %20 return URL.documentsiCloudOrLocalPath.appendingPathComponent("Save States", isDirectory: true) }} From 2a8ac49a34c9257e9d49a8dacb3106c2521937d7 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 13 Jan 2025 23:47:13 -0500 Subject: [PATCH 09/86] removed duplicate entry --- ProvenanceTV/ProvenanceTV-AppStore.entitlements | 1 - 1 file changed, 1 deletion(-) diff --git a/ProvenanceTV/ProvenanceTV-AppStore.entitlements b/ProvenanceTV/ProvenanceTV-AppStore.entitlements index 8fcc57498a..5b27db4542 100644 --- a/ProvenanceTV/ProvenanceTV-AppStore.entitlements +++ b/ProvenanceTV/ProvenanceTV-AppStore.entitlements @@ -26,7 +26,6 @@ com.apple.security.application-groups $(APP_GROUP_IDENTIFIER) - group.org.provenance-emu keychain-access-groups From 263da3bf123a2484f3c3d0b2d6d167cf93a9928a Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 18 Jan 2025 16:48:27 -0500 Subject: [PATCH 10/86] updated project file to use build variables --- Provenance.xcodeproj/project.pbxproj | 144 +++++++++++++-------------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/Provenance.xcodeproj/project.pbxproj b/Provenance.xcodeproj/project.pbxproj index 246be55b1d..836dacee15 100644 --- a/Provenance.xcodeproj/project.pbxproj +++ b/Provenance.xcodeproj/project.pbxproj @@ -6922,7 +6922,7 @@ CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = dwarf; DEVELOPMENT_ASSET_PATHS = "\"Provenance Mini Watch App/Preview Content\""; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_PREVIEWS = YES; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -6939,7 +6939,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini.watchkitapp"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini.watchkitapp"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; SKIP_INSTALL = YES; @@ -6974,7 +6974,7 @@ CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; DEVELOPMENT_ASSET_PATHS = "\"Provenance Mini Watch App/Preview Content\""; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_PREVIEWS = YES; ENABLE_USER_SCRIPT_SANDBOXING = YES; @@ -6992,7 +6992,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini.watchkitapp"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini.watchkitapp"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; SKIP_INSTALL = YES; @@ -7025,7 +7025,7 @@ CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; DEVELOPMENT_ASSET_PATHS = "\"Provenance Mini Watch App/Preview Content\""; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_PREVIEWS = YES; ENABLE_USER_SCRIPT_SANDBOXING = YES; @@ -7043,7 +7043,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini.watchkitapp"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini.watchkitapp"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; SKIP_INSTALL = YES; @@ -7071,7 +7071,7 @@ CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = dwarf; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; @@ -7081,7 +7081,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; @@ -7108,7 +7108,7 @@ COPY_PHASE_STRIP = NO; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -7119,7 +7119,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SWIFT_VERSION = 5.0; @@ -7144,7 +7144,7 @@ COPY_PHASE_STRIP = NO; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -7155,7 +7155,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SWIFT_VERSION = 5.0; @@ -7180,7 +7180,7 @@ CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = dwarf; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; @@ -7189,7 +7189,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini-Watch-AppTests"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini-Watch-AppTests"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; @@ -7221,7 +7221,7 @@ COPY_PHASE_STRIP = NO; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -7231,7 +7231,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini-Watch-AppTests"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini-Watch-AppTests"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; SWIFT_EMIT_LOC_STRINGS = NO; @@ -7261,7 +7261,7 @@ COPY_PHASE_STRIP = NO; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -7271,7 +7271,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini-Watch-AppTests"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini-Watch-AppTests"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; SWIFT_EMIT_LOC_STRINGS = NO; @@ -7299,7 +7299,7 @@ CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = dwarf; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; @@ -7308,7 +7308,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini-Watch-AppUITests"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini-Watch-AppUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; @@ -7339,7 +7339,7 @@ COPY_PHASE_STRIP = NO; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -7349,7 +7349,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini-Watch-AppUITests"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini-Watch-AppUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; SWIFT_EMIT_LOC_STRINGS = NO; @@ -7378,7 +7378,7 @@ COPY_PHASE_STRIP = NO; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -7388,7 +7388,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.Provenance-Mini-Watch-AppUITests"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_PREFIX).Provenance-Mini-Watch-AppUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; SWIFT_EMIT_LOC_STRINGS = NO; @@ -7454,7 +7454,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; DEBUG_INFORMATION_FORMAT = dwarf; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; "ENABLE_HARDENED_RUNTIME[sdk=macosx*]" = YES; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -7529,7 +7529,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; "ENABLE_HARDENED_RUNTIME[sdk=macosx*]" = YES; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -7603,7 +7603,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; "ENABLE_HARDENED_RUNTIME[sdk=macosx*]" = YES; FRAMEWORK_SEARCH_PATHS = ( "$(inherited)", @@ -7702,7 +7702,7 @@ ); MTL_ENABLE_DEBUG_INFO = YES; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.lite"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE))"; PRODUCT_NAME = Provenance; PROVISIONING_PROFILE_SPECIFIER = ""; SDKROOT = appletvos; @@ -7758,7 +7758,7 @@ MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.lite"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; PRODUCT_NAME = Provenance; PROVISIONING_PROFILE_SPECIFIER = ""; SDKROOT = appletvos; @@ -7814,7 +7814,7 @@ ); MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.lite"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; PRODUCT_NAME = Provenance; PROVISIONING_PROFILE_SPECIFIER = ""; SDKROOT = appletvos; @@ -7860,7 +7860,7 @@ ); MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.ThumbnailExtension"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).ThumbnailExtension"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SKIP_INSTALL = YES; @@ -7910,7 +7910,7 @@ ); MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.ThumbnailExtension"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).ThumbnailExtension"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SKIP_INSTALL = YES; @@ -7959,7 +7959,7 @@ ); MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.ThumbnailExtension"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).ThumbnailExtension"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SKIP_INSTALL = YES; @@ -7990,7 +7990,7 @@ CURRENT_PROJECT_VERSION = 1; DEAD_CODE_STRIPPING = YES; DEBUG_INFORMATION_FORMAT = dwarf; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_HARDENED_RUNTIME = YES; GCC_C_LANGUAGE_STANDARD = gnu11; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; @@ -8008,7 +8008,7 @@ MACOSX_DEPLOYMENT_TARGET = 15.4; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.SpotlightImportExtension"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).SpotlightImportExtension"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SKIP_INSTALL = YES; @@ -8042,7 +8042,7 @@ CURRENT_PROJECT_VERSION = 1; DEAD_CODE_STRIPPING = YES; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_HARDENED_RUNTIME = YES; ENABLE_NS_ASSERTIONS = NO; GCC_C_LANGUAGE_STANDARD = gnu11; @@ -8061,7 +8061,7 @@ MACOSX_DEPLOYMENT_TARGET = 15.4; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.SpotlightImportExtension"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).SpotlightImportExtension"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SKIP_INSTALL = YES; @@ -8094,7 +8094,7 @@ CURRENT_PROJECT_VERSION = 1; DEAD_CODE_STRIPPING = YES; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_HARDENED_RUNTIME = YES; ENABLE_NS_ASSERTIONS = NO; GCC_C_LANGUAGE_STANDARD = gnu11; @@ -8113,7 +8113,7 @@ MACOSX_DEPLOYMENT_TARGET = 15.4; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.SpotlightImportExtension"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).SpotlightImportExtension"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SKIP_INSTALL = YES; @@ -8163,7 +8163,7 @@ ); MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.WidgetExtension"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).WidgetExtension"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SKIP_INSTALL = YES; @@ -8215,7 +8215,7 @@ ); MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.WidgetExtension"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).WidgetExtension"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SKIP_INSTALL = YES; @@ -8266,7 +8266,7 @@ ); MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.WidgetExtension"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).WidgetExtension"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SKIP_INSTALL = YES; @@ -8978,7 +8978,7 @@ CODE_SIGN_STYLE = Automatic; DEAD_CODE_STRIPPING = YES; DEBUG_INFORMATION_FORMAT = dwarf; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_HARDENED_RUNTIME = YES; "ENABLE_HARDENED_RUNTIME[sdk=macosx*]" = YES; FRAMEWORK_SEARCH_PATHS = ( @@ -9076,7 +9076,7 @@ CODE_SIGN_STYLE = Automatic; DEAD_CODE_STRIPPING = YES; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_HARDENED_RUNTIME = YES; "ENABLE_HARDENED_RUNTIME[sdk=macosx*]" = YES; FRAMEWORK_SEARCH_PATHS = ( @@ -9174,7 +9174,7 @@ CODE_SIGN_STYLE = Automatic; DEAD_CODE_STRIPPING = YES; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_HARDENED_RUNTIME = YES; "ENABLE_HARDENED_RUNTIME[sdk=macosx*]" = YES; FRAMEWORK_SEARCH_PATHS = ( @@ -9295,7 +9295,7 @@ CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = dwarf; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; @@ -9313,7 +9313,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.ROM-File-Provider"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).ROM-File-Provider"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SKIP_INSTALL = YES; @@ -9345,7 +9345,7 @@ COPY_PHASE_STRIP = NO; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -9364,7 +9364,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.ROM-File-Provider"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).ROM-File-Provider"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SKIP_INSTALL = YES; @@ -9394,7 +9394,7 @@ COPY_PHASE_STRIP = NO; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -9413,7 +9413,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.ROM-File-Provider"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).ROM-File-Provider"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SKIP_INSTALL = YES; @@ -9441,7 +9441,7 @@ CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = dwarf; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; @@ -9459,7 +9459,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.ROM-File-ProviderUI"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).ROM-File-ProviderUI"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SKIP_INSTALL = YES; @@ -9490,7 +9490,7 @@ COPY_PHASE_STRIP = NO; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -9509,7 +9509,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.ROM-File-ProviderUI"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).ROM-File-ProviderUI"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SKIP_INSTALL = YES; @@ -9538,7 +9538,7 @@ COPY_PHASE_STRIP = NO; CURRENT_PROJECT_VERSION = 1; DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - DEVELOPMENT_TEAM = S32Z3HMYVQ; + DEVELOPMENT_TEAM = "$(DEVELOPMENT_TEAM)"; ENABLE_NS_ASSERTIONS = NO; ENABLE_USER_SCRIPT_SANDBOXING = YES; GCC_C_LANGUAGE_STANDARD = gnu17; @@ -9557,7 +9557,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.ROM-File-ProviderUI"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).ROM-File-ProviderUI"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; SKIP_INSTALL = YES; @@ -9598,7 +9598,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.stickers"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).stickers"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER_MAC = ""; SDKROOT = iphoneos; @@ -9641,7 +9641,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.stickers"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).stickers"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER_MAC = ""; SDKROOT = iphoneos; @@ -9684,7 +9684,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.provenance.stickers"; + PRODUCT_BUNDLE_IDENTIFIER = "$(PRODUCT_BUNDLE_IDENTIFIER).stickers"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER_MAC = ""; SDKROOT = iphoneos; @@ -9830,7 +9830,7 @@ MACOSX_DEPLOYMENT_TARGET = 11.0; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.ThumbnailExtensionMacOS"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_IDENTIFIER).ThumbnailExtensionMacOS"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SDKROOT = macosx; @@ -9877,7 +9877,7 @@ MACOSX_DEPLOYMENT_TARGET = 11.0; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.ThumbnailExtensionMacOS"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_IDENTIFIER).ThumbnailExtensionMacOS"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SDKROOT = macosx; @@ -9923,7 +9923,7 @@ MACOSX_DEPLOYMENT_TARGET = 11.0; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = "org.provenance-emu.ThumbnailExtensionMacOS"; + PRODUCT_BUNDLE_IDENTIFIER = "$(ORG_IDENTIFIER).ThumbnailExtensionMacOS"; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; SDKROOT = macosx; @@ -9983,9 +9983,9 @@ MTL_FAST_MATH = YES; OTHER_CFLAGS = "-Wno-deprecated-declarations"; OTHER_LDFLAGS = "$(inherited)"; - "PRODUCT_BUNDLE_IDENTIFIER[sdk=appletvos*]" = "org.provenance-emu.provenance.lite"; - "PRODUCT_BUNDLE_IDENTIFIER[sdk=iphoneos*]" = "org.provenance-emu.provenance.lite"; - "PRODUCT_BUNDLE_IDENTIFIER[sdk=macosx*]" = "org.provenance-emu.provenance.lite"; + "PRODUCT_BUNDLE_IDENTIFIER[sdk=appletvos*]" = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; + "PRODUCT_BUNDLE_IDENTIFIER[sdk=iphoneos*]" = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; + "PRODUCT_BUNDLE_IDENTIFIER[sdk=macosx*]" = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; PRODUCT_MODULE_NAME = Provenance; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -10061,9 +10061,9 @@ "-Wno-deprecated-declarations", ); OTHER_LDFLAGS = "$(inherited)"; - "PRODUCT_BUNDLE_IDENTIFIER[sdk=appletvos*]" = "org.provenance-emu.provenance.lite"; - "PRODUCT_BUNDLE_IDENTIFIER[sdk=iphoneos*]" = "org.provenance-emu.provenance.lite"; - "PRODUCT_BUNDLE_IDENTIFIER[sdk=macosx*]" = "org.provenance-emu.provenance.lite"; + "PRODUCT_BUNDLE_IDENTIFIER[sdk=appletvos*]" = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; + "PRODUCT_BUNDLE_IDENTIFIER[sdk=iphoneos*]" = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; + "PRODUCT_BUNDLE_IDENTIFIER[sdk=macosx*]" = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; PRODUCT_MODULE_NAME = Provenance; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -10137,9 +10137,9 @@ "-Wno-deprecated-declarations", ); OTHER_LDFLAGS = "$(inherited)"; - "PRODUCT_BUNDLE_IDENTIFIER[sdk=appletvos*]" = "org.provenance-emu.provenance.lite"; - "PRODUCT_BUNDLE_IDENTIFIER[sdk=iphoneos*]" = "org.provenance-emu.provenance.lite"; - "PRODUCT_BUNDLE_IDENTIFIER[sdk=macosx*]" = "org.provenance-emu.provenance.lite"; + "PRODUCT_BUNDLE_IDENTIFIER[sdk=appletvos*]" = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; + "PRODUCT_BUNDLE_IDENTIFIER[sdk=iphoneos*]" = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; + "PRODUCT_BUNDLE_IDENTIFIER[sdk=macosx*]" = "$(PRODUCT_BUNDLE_IDENTIFIER_LITE)"; PRODUCT_MODULE_NAME = Provenance; PRODUCT_NAME = "$(TARGET_NAME)"; PROVISIONING_PROFILE_SPECIFIER = ""; From f6c155841a420dd7486e7b23d9f26e2f5e21b364 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 18 Jan 2025 23:51:37 -0500 Subject: [PATCH 11/86] removed TODOs --- PVLibrary/Sources/PVFileSystem/Paths.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/PVLibrary/Sources/PVFileSystem/Paths.swift b/PVLibrary/Sources/PVFileSystem/Paths.swift index 1172640ab8..7d730dd754 100644 --- a/PVLibrary/Sources/PVFileSystem/Paths.swift +++ b/PVLibrary/Sources/PVFileSystem/Paths.swift @@ -78,11 +78,11 @@ public extension URL { public struct Paths { public struct Legacy { - public static var batterySavesPath: URL {//TODO: review encoded space %20 + public static var batterySavesPath: URL { URL.documentsPath.appendingPathComponent("Battery States", isDirectory: true) } - public static var saveSavesPath: URL {//TODO: review encoded space %20 + public static var saveSavesPath: URL { URL.documentsPath.appendingPathComponent("Save States", isDirectory: true) } @@ -105,12 +105,12 @@ public struct Paths { }} /// Should be called on BG Thread, iCloud blocks - public static var batterySavesPath: URL { get {//TODO: review encoded space %20 + public static var batterySavesPath: URL { get { return URL.documentsiCloudOrLocalPath.appendingPathComponent("Battery States", isDirectory: true) }} /// Should be called on BG Thread, iCloud blocks - public static var saveSavesPath: URL { get {//TODO: review encoded space %20 + public static var saveSavesPath: URL { get { return URL.documentsiCloudOrLocalPath.appendingPathComponent("Save States", isDirectory: true) }} From a788cabed161fa87f5d6df9ad7c5c960a8aa0d8b Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 20 Jan 2025 00:25:18 -0500 Subject: [PATCH 12/86] added notification when ROM db finishes initializing; updated cloudsync to find undownloaded files and download them and send a notification so roms can be imported --- .../Database/Realm Database/RomDatabase.swift | 2 + .../Importer/iCloud/iCloudSync.swift | 183 +++++++++++++----- Provenance/Main UI/PVAppDelegate.swift | 12 +- 3 files changed, 147 insertions(+), 50 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift index 1b81bd4e7d..20c7f78d96 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift @@ -328,6 +328,8 @@ public final class RomDatabase { ILOG("Database initialization completed") databaseInitialized = true + NotificationCenter.default.post(name: Notification.Name("kRomDatabaseInitialized"), object: nil, userInfo: nil) + } else { ILOG("Database already initialized") } diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 18f5d5daba..2d7b153fee 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -31,6 +31,10 @@ public protocol Container { var containerURL: URL? { get } } +extension Notification.Name { + static let cloudDataDownloaded = Notification.Name("kCloudDataDownloaded") +} + extension Container { public var containerURL: URL? { get { return URL.iCloudContainerDirectory }} var documentsURL: URL? { get { return URL.iCloudDocumentsDirectory }} @@ -44,8 +48,9 @@ public protocol SyncFileToiCloud: Container { } public protocol iCloudTypeSyncer: Container { + var newFiles: Set { get } + var directory: String { get } var metadataQuery: NSMetadataQuery { get } - var metadataQueryPredicate: NSPredicate { get } // ex NSPredicate(format: "%K like 'PHOTO*'", NSMetadataItemFSNameKey) func loadAllFromICloud() -> Completable func removeAllFromICloud() -> Completable @@ -63,33 +68,33 @@ final class NotificationObserver { observer = center.addObserver(forName: name, object: object, queue: queue, using: block) } - deinit { - center.removeObserver(observer, name: name, object: object) + deinit {//because this was created inline, deinit gets called right away. does this ever need to be removed? shouldn't this be in the lifetime of the application? + //center.removeObserver(observer, name: name, object: object) } } extension iCloudTypeSyncer { public func loadAllFromICloud() -> Completable { return Completable.create { completable in - Task { - guard self.containerURL != nil else { - completable(.error(SyncError.noUbiquityURL)) - return Disposables.create {} - } + //body mustn't run in a task, otherwise the NotificationObserver closure won't get called + guard containerURL != nil + else { + completable(.error(SyncError.noUbiquityURL)) return Disposables.create {} } - // metadataQuery = NSMetadataQuery() + var tmp = newFiles + tmp.removeAll() self.metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] - self.metadataQuery.predicate = self.metadataQueryPredicate - + metadataQuery.predicate = NSPredicate(format: "%K CONTAINS[c] %@", NSMetadataItemPathKey, "/Documents/\(directory)/") + let _: NotificationObserver = .init( forName: Notification.Name.NSMetadataQueryDidFinishGathering, object: self.metadataQuery, - queue: nil) {[self] notification in + queue: nil) { notification in self.queryFinished(notification: notification) completable(.completed) } - + self.metadataQuery.start() return Disposables.create {} } @@ -107,7 +112,7 @@ extension iCloudTypeSyncer { } // metadataQuery = NSMetadataQuery() self.metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] - self.metadataQuery.predicate = self.metadataQueryPredicate +// self.metadataQuery.predicate = self.metadataQueryPredicate let token: NSObjectProtocol? = NotificationCenter.default.addObserver( forName: Notification.Name.NSMetadataQueryDidFinishGathering, @@ -154,25 +159,47 @@ extension iCloudTypeSyncer { } func queryFinished(notification: Notification) { - let mq = notification.object as! NSMetadataQuery - mq.disableUpdates() - mq.stop() - - // for i in 0 ..< mq.resultCount { - // let result = mq.result(at: i) as! NSMetadataItem - // let name = result.value(forAttribute: NSMetadataItemFSNameKey) as! String - // let url = result.value(forAttribute: NSMetadataItemURLKey) as! URL - // TODO: Some kind of observable rx? - // let document: Self.Type! = DocumentPhoto(fileURL: url) - // document?.open(completionHandler: {(success) -> Void in - // - // if (success) { - // print("Image loaded with name \(name)") - // self.cells.append(document.image) - // self.collectionView.reloadData() - // } - // }) - // } + //TODO: update so this is generic for all downloads + guard type(of: self) != SaveStateSyncer.self + else { + return + } + + guard let query = notification.object as? NSMetadataQuery + else { + return + } + let fileManager = FileManager.default + let isMainThread = Thread.isMainThread + print("isMainThread:\(isMainThread)") + var files: [URL] = [] + //accessing results automatically pauses updates and resumes after deallocated + for item in query.results { + if let fileItem = item as? NSMetadataItem, + let file = fileItem.value(forAttribute: NSMetadataItemURLKey) as? URL, + let downloadStatus = fileItem.value(forAttribute: NSMetadataUbiquitousItemDownloadingStatusKey) as? String, + downloadStatus == NSMetadataUbiquitousItemDownloadingStatusNotDownloaded { + print("Found file: \(String(describing: file)), download status: \(downloadStatus)") + files.append(file) + + do { + try fileManager.startDownloadingUbiquitousItem(at: file) + //TODO: we have to wait until the files are downloaded, we can just create a queue and then just loop until the queue is empty + print("Download started for: \(file.lastPathComponent)") + } catch { + print("Failed to start download: \(error)") + } + } + } + var downloadedFiles = newFiles + while downloadedFiles.count != files.count { + for file in files { + if fileManager.fileExists(atPath: file.path) { + downloadedFiles.insert(file) + } + } + } + } } @@ -281,7 +308,9 @@ enum iCloudError: Error { } public enum iCloudSync { - static var disposeBag: DisposeBag? + //TODO: move bags to each class + static var disposeBagSaveState: DisposeBag? + static var disposeBagRoms: DisposeBag? public static func initICloudDocuments() { let fm = FileManager.default if let currentiCloudToken = fm.ubiquityIdentityToken { @@ -296,27 +325,37 @@ public enum iCloudSync { } let saveStateSyncer = SaveStateSyncer() - let disposeBag = DisposeBag() - self.disposeBag = disposeBag + let currentDisposeBagSaveState = DisposeBag() + self.disposeBagSaveState = currentDisposeBagSaveState saveStateSyncer.loadAllFromICloud() .observe(on: MainScheduler.instance) .subscribe(onCompleted: { importNewSaves() - self.disposeBag = nil }) { error in - ELOG("\(error.localizedDescription)") - }.disposed(by: disposeBag) + ELOG(error.localizedDescription) + }.disposed(by: currentDisposeBagSaveState) + let romsSyncer = RomsSyncer() + let currentDisposeBagRoms = DisposeBag() + disposeBagRoms = currentDisposeBagRoms + romsSyncer.loadAllFromICloud() + .observe(on: MainScheduler.instance) + .subscribe(onCompleted: { + romsSyncer.handleNewRomFiles() + }) { error in + ELOG(error.localizedDescription) + }.disposed(by: currentDisposeBagRoms) } - + +//TODO: prolly this should be in SaveStateSyncer public static func importNewSaves() { - if !RomDatabase.databaseInitialized { - // Keep trying // TODO: Add a notification for this - // instead of dumb loop - DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { - self.importNewSaves() - } + guard RomDatabase.databaseInitialized + else { return } + + defer { + disposeBagSaveState = nil + } Task { let savesDirectory = Paths.saveSavesPath @@ -440,8 +479,54 @@ public enum iCloudSync { } class SaveStateSyncer: iCloudTypeSyncer { - public var metadataQuery: NSMetadataQuery = .init() - public var metadataQueryPredicate: NSPredicate { - return NSPredicate(format: "%K CONTAINS[c] 'Save States'", NSMetadataItemPathKey) + var metadataQuery: NSMetadataQuery = .init() + var newFiles: Set = [] + init() { + NotificationCenter.default.addObserver(self, selector: #selector(wrapper), name: .cloudDataDownloaded, object: nil) + } + deinit { + print("dying") + } + + @objc + func wrapper() { + iCloudSync.importNewSaves() + } + + var directory: String { + "Save States" + } +} + +class RomsSyncer: iCloudTypeSyncer { + var metadataQuery: NSMetadataQuery = .init() + var newFiles: Set = [] + + init() { + NotificationCenter.default.addObserver(self, selector: #selector(handleNewRomFiles), name: .cloudDataDownloaded, object: nil) + } + deinit { + print("dying") + } + + var directory: String { + "ROMs" + } + + /// sends a notification that rom files are ready to e + @objc + func handleNewRomFiles() { + guard !newFiles.isEmpty + else { + return + } + + guard RomDatabase.databaseInitialized + else { + return + } + + NotificationCenter.default.post(name: .cloudDataDownloaded, object: nil, userInfo: ["kCloudDataDownloaded": newFiles]) + iCloudSync.disposeBagRoms = nil } } diff --git a/Provenance/Main UI/PVAppDelegate.swift b/Provenance/Main UI/PVAppDelegate.swift index f421b7d4a1..433b913dc4 100644 --- a/Provenance/Main UI/PVAppDelegate.swift +++ b/Provenance/Main UI/PVAppDelegate.swift @@ -346,13 +346,23 @@ final class PVAppDelegate: UIResponder, GameLaunchingAppDelegate, UIApplicationD let useiCloud = Defaults[.iCloudSync] && URL.supportsICloud if useiCloud { DispatchQueue.main.async { + NotificationCenter.default.addObserver(self, selector: #selector(self.cloudDataDownloaded(notification:)), name: Notification.Name("kCloudDataDownloaded"), object: nil) iCloudSync.initICloudDocuments() - iCloudSync.importNewSaves() } } } } + @objc func cloudDataDownloaded(notification: Notification) { + guard let files = notification.userInfo?["kCloudDataDownloaded"] as? [URL], + let gameImporter = appState?.gameImporter + else { + return + } + gameImporter.addImports(forPaths: files) + gameImporter.startProcessing() + } + var currentThemeObservation: Any? // AnyCancellable? var userInterfaceStyleObservation: Any? var oldPalette: (any UXThemePalette)? From 983534c659be5fe9025d15950e81fb2bed8e7edd Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 20 Jan 2025 22:01:58 -0500 Subject: [PATCH 13/86] updated type --- Provenance/Main UI/PVAppDelegate.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Provenance/Main UI/PVAppDelegate.swift b/Provenance/Main UI/PVAppDelegate.swift index 433b913dc4..19afabe6af 100644 --- a/Provenance/Main UI/PVAppDelegate.swift +++ b/Provenance/Main UI/PVAppDelegate.swift @@ -354,7 +354,7 @@ final class PVAppDelegate: UIResponder, GameLaunchingAppDelegate, UIApplicationD } @objc func cloudDataDownloaded(notification: Notification) { - guard let files = notification.userInfo?["kCloudDataDownloaded"] as? [URL], + guard let files = notification.userInfo?["kCloudDataDownloaded"] as? Set, let gameImporter = appState?.gameImporter else { return From 002cc14ebcbd958635e7e877b803a822f393c70e Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 20 Jan 2025 22:05:35 -0500 Subject: [PATCH 14/86] reverted --- Provenance/Main UI/PVAppDelegate.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Provenance/Main UI/PVAppDelegate.swift b/Provenance/Main UI/PVAppDelegate.swift index 19afabe6af..433b913dc4 100644 --- a/Provenance/Main UI/PVAppDelegate.swift +++ b/Provenance/Main UI/PVAppDelegate.swift @@ -354,7 +354,7 @@ final class PVAppDelegate: UIResponder, GameLaunchingAppDelegate, UIApplicationD } @objc func cloudDataDownloaded(notification: Notification) { - guard let files = notification.userInfo?["kCloudDataDownloaded"] as? Set, + guard let files = notification.userInfo?["kCloudDataDownloaded"] as? [URL], let gameImporter = appState?.gameImporter else { return From 880df3543df7930dd5db385d5c36e877174652be Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Wed, 22 Jan 2025 23:05:08 -0500 Subject: [PATCH 15/86] updated to send notiifcations when new icloud files have been downloaded; fixed wrong notification observing --- .../Importer/iCloud/iCloudSync.swift | 71 +++++++++++++------ 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 2d7b153fee..30bbec93b8 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -33,6 +33,8 @@ public protocol Container { extension Notification.Name { static let cloudDataDownloaded = Notification.Name("kCloudDataDownloaded") + static let newCloudFiles = Notification.Name("kNewCloudFiles") + static let romDatabaseInitialized = Notification.Name("kRomDatabaseInitialized") } extension Container { @@ -48,12 +50,15 @@ public protocol SyncFileToiCloud: Container { } public protocol iCloudTypeSyncer: Container { - var newFiles: Set { get } + var newFiles: Set { get set } var directory: String { get } var metadataQuery: NSMetadataQuery { get } func loadAllFromICloud() -> Completable func removeAllFromICloud() -> Completable + mutating func clearNewFiles() + mutating func insertNewFile(_ file: URL) + mutating func setNewFiles(_ files: Set) } final class NotificationObserver { @@ -74,7 +79,19 @@ final class NotificationObserver { } extension iCloudTypeSyncer { - public func loadAllFromICloud() -> Completable { + mutating func setNewFiles(_ files: Set) { + newFiles = files + } + + mutating func clearNewFiles() { + newFiles.removeAll() + } + + mutating func insertNewFile(_ file: URL) { + newFiles.insert(file) + } + + func loadAllFromICloud() -> Completable { return Completable.create { completable in //body mustn't run in a task, otherwise the NotificationObserver closure won't get called guard containerURL != nil @@ -82,13 +99,11 @@ extension iCloudTypeSyncer { completable(.error(SyncError.noUbiquityURL)) return Disposables.create {} } - var tmp = newFiles - tmp.removeAll() self.metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] metadataQuery.predicate = NSPredicate(format: "%K CONTAINS[c] %@", NSMetadataItemPathKey, "/Documents/\(directory)/") let _: NotificationObserver = .init( - forName: Notification.Name.NSMetadataQueryDidFinishGathering, + forName: .NSMetadataQueryDidFinishGathering, object: self.metadataQuery, queue: nil) { notification in self.queryFinished(notification: notification) @@ -159,20 +174,16 @@ extension iCloudTypeSyncer { } func queryFinished(notification: Notification) { - //TODO: update so this is generic for all downloads - guard type(of: self) != SaveStateSyncer.self - else { - return - } - guard let query = notification.object as? NSMetadataQuery else { return } let fileManager = FileManager.default - let isMainThread = Thread.isMainThread - print("isMainThread:\(isMainThread)") var files: [URL] = [] + query.disableUpdates() + defer { + query.enableUpdates() + } //accessing results automatically pauses updates and resumes after deallocated for item in query.results { if let fileItem = item as? NSMetadataItem, @@ -191,7 +202,7 @@ extension iCloudTypeSyncer { } } } - var downloadedFiles = newFiles + var downloadedFiles = Set() while downloadedFiles.count != files.count { for file in files { if fileManager.fileExists(atPath: file.path) { @@ -199,7 +210,9 @@ extension iCloudTypeSyncer { } } } - + let name = Notification.Name.newCloudFiles + print("downloadedFiles: \(downloadedFiles.count)") + NotificationCenter.default.post(name: name, object: nil, userInfo: [name.rawValue: downloadedFiles]) } } @@ -324,7 +337,7 @@ public enum iCloudSync { UserDefaults.standard.removeObject(forKey: UbiquityIdentityTokenKey) } - let saveStateSyncer = SaveStateSyncer() + var saveStateSyncer = SaveStateSyncer() let currentDisposeBagSaveState = DisposeBag() self.disposeBagSaveState = currentDisposeBagSaveState saveStateSyncer.loadAllFromICloud() @@ -334,7 +347,7 @@ public enum iCloudSync { }) { error in ELOG(error.localizedDescription) }.disposed(by: currentDisposeBagSaveState) - let romsSyncer = RomsSyncer() + var romsSyncer = RomsSyncer() let currentDisposeBagRoms = DisposeBag() disposeBagRoms = currentDisposeBagRoms romsSyncer.loadAllFromICloud() @@ -482,7 +495,7 @@ class SaveStateSyncer: iCloudTypeSyncer { var metadataQuery: NSMetadataQuery = .init() var newFiles: Set = [] init() { - NotificationCenter.default.addObserver(self, selector: #selector(wrapper), name: .cloudDataDownloaded, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(wrapper), name: .romDatabaseInitialized, object: nil) } deinit { print("dying") @@ -503,7 +516,8 @@ class RomsSyncer: iCloudTypeSyncer { var newFiles: Set = [] init() { - NotificationCenter.default.addObserver(self, selector: #selector(handleNewRomFiles), name: .cloudDataDownloaded, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(handleNewRomFiles), name: .romDatabaseInitialized, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(handleNewFiles(_:)), name: .newCloudFiles, object: nil) } deinit { print("dying") @@ -513,6 +527,15 @@ class RomsSyncer: iCloudTypeSyncer { "ROMs" } + @objc + func handleNewFiles(_ notification: Notification) { + guard let downloadedFiles = notification.userInfo?[Notification.Name.newCloudFiles.rawValue] as? Set + else { + return + } + newFiles = downloadedFiles + } + /// sends a notification that rom files are ready to e @objc func handleNewRomFiles() { @@ -526,7 +549,13 @@ class RomsSyncer: iCloudTypeSyncer { return } - NotificationCenter.default.post(name: .cloudDataDownloaded, object: nil, userInfo: ["kCloudDataDownloaded": newFiles]) - iCloudSync.disposeBagRoms = nil + var converted = [URL]() + for item in newFiles { + converted.append(item) + } + newFiles.removeAll() + let name = Notification.Name.cloudDataDownloaded + NotificationCenter.default.post(name: name, object: nil, userInfo: [name.rawValue: converted]) +// iCloudSync.disposeBagRoms = nil } } From 7b4cafaa0754700d77c3d6324cb68c6dd82dd52e Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Wed, 22 Jan 2025 23:05:22 -0500 Subject: [PATCH 16/86] updated to use notification name --- .../Sources/PVLibrary/Database/Realm Database/RomDatabase.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift index 20c7f78d96..e575a94fba 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift @@ -328,7 +328,7 @@ public final class RomDatabase { ILOG("Database initialization completed") databaseInitialized = true - NotificationCenter.default.post(name: Notification.Name("kRomDatabaseInitialized"), object: nil, userInfo: nil) + NotificationCenter.default.post(name: .romDatabaseInitialized, object: nil, userInfo: nil) } else { ILOG("Database already initialized") From d2c820ae22551c6b52544a1628da5cc33cab3fb9 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Wed, 22 Jan 2025 23:06:07 -0500 Subject: [PATCH 17/86] reverted --- .../Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 30bbec93b8..dc353fd7f5 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -180,10 +180,11 @@ extension iCloudTypeSyncer { } let fileManager = FileManager.default var files: [URL] = [] - query.disableUpdates() - defer { - query.enableUpdates() - } + //TODO: don't think we need this +// query.disableUpdates() +// defer { +// query.enableUpdates() +// } //accessing results automatically pauses updates and resumes after deallocated for item in query.results { if let fileItem = item as? NSMetadataItem, From cf81ec72f2a68346e4d4c0f20dd3714a4d5c5510 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Fri, 24 Jan 2025 08:04:44 -0500 Subject: [PATCH 18/86] reverted TODOs --- .../Services/GameImporter/GameImporterDatabaseService.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index 1985b2517a..3e9d95b146 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -216,7 +216,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } if PVMediaCache.fileExists(forKey: url) { if let localURL = PVMediaCache.filePath(forKey: url) { - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud)//TODO: test without .iCloud + let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) game.originalArtworkFile = file return game } @@ -240,7 +240,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { if let artwork = NSImage(data: data) { do { let localURL = try PVMediaCache.writeImage(toDisk: artwork, withKey: url) - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud)//TODO: test without .iCloud + let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) game.originalArtworkFile = file } catch { ELOG("\(error.localizedDescription)") } } @@ -248,7 +248,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { if let artwork = UIImage(data: data) { do { let localURL = try PVMediaCache.writeImage(toDisk: artwork, withKey: url) - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud)//TODO: test without .iCloud + let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) game.originalArtworkFile = file } catch { ELOG("\(error.localizedDescription)") } } From c517f8c0f6a262894fa3b9d523a465a3dbe0d4d1 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Fri, 24 Jan 2025 09:02:42 -0500 Subject: [PATCH 19/86] added downloading of battery saves, bios and screenshots; added todo on crash that occurs when realm is being used by the different iCloud syncing that causes the crash --- .../Importer/iCloud/iCloudSync.swift | 76 +++++++++++++++---- Provenance/Main UI/PVAppDelegate.swift | 2 +- 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index dc353fd7f5..965d8313df 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -323,8 +323,7 @@ enum iCloudError: Error { public enum iCloudSync { //TODO: move bags to each class - static var disposeBagSaveState: DisposeBag? - static var disposeBagRoms: DisposeBag? + static var disposeBag: DisposeBag? public static func initICloudDocuments() { let fm = FileManager.default if let currentiCloudToken = fm.ubiquityIdentityToken { @@ -337,27 +336,49 @@ public enum iCloudSync { } else { UserDefaults.standard.removeObject(forKey: UbiquityIdentityTokenKey) } - + //TODO: there is an issue when all of the icloud files are downloaded and during the importing of Realm + Save States there's a clash. I think we need to import ROMs first, then BIOS (if necessary) and then save states, everything else doesn't need to be imported onto the db (at least from what I understand) var saveStateSyncer = SaveStateSyncer() - let currentDisposeBagSaveState = DisposeBag() - self.disposeBagSaveState = currentDisposeBagSaveState + let disposeBag = DisposeBag() + self.disposeBag = disposeBag saveStateSyncer.loadAllFromICloud() .observe(on: MainScheduler.instance) .subscribe(onCompleted: { importNewSaves() }) { error in ELOG(error.localizedDescription) - }.disposed(by: currentDisposeBagSaveState) + }.disposed(by: disposeBag) var romsSyncer = RomsSyncer() - let currentDisposeBagRoms = DisposeBag() - disposeBagRoms = currentDisposeBagRoms romsSyncer.loadAllFromICloud() .observe(on: MainScheduler.instance) .subscribe(onCompleted: { romsSyncer.handleNewRomFiles() }) { error in ELOG(error.localizedDescription) - }.disposed(by: currentDisposeBagRoms) + }.disposed(by: disposeBag) + let biosSyncer = BiosSyncer() + biosSyncer.loadAllFromICloud() + .observe(on: MainScheduler.instance) + .subscribe(onCompleted: { + //TODO: anything? + }) { error in + ELOG(error.localizedDescription) + }.disposed(by: disposeBag) + let batterySaveSyncer = BatterySavesSyncer() + batterySaveSyncer.loadAllFromICloud() + .observe(on: MainScheduler.instance) + .subscribe(onCompleted: { + //TODO: anything? + }) { error in + ELOG(error.localizedDescription) + }.disposed(by: disposeBag) + let screenshotsSyncer = ScreenshotsSyncer() + screenshotsSyncer.loadAllFromICloud() + .observe(on: MainScheduler.instance) + .subscribe(onCompleted: { + //TODO: anything? + }) { error in + ELOG(error.localizedDescription) + }.disposed(by: disposeBag) } //TODO: prolly this should be in SaveStateSyncer @@ -366,10 +387,6 @@ public enum iCloudSync { else { return } - - defer { - disposeBagSaveState = nil - } Task { let savesDirectory = Paths.saveSavesPath @@ -491,7 +508,7 @@ public enum iCloudSync { } } } - +//TODO: perhaps 1 generic class since a lot of this code is similar and move the extension onto generic class. we could just add a protocol delegate dependency for ROMs and SaveState classes that does specific code class SaveStateSyncer: iCloudTypeSyncer { var metadataQuery: NSMetadataQuery = .init() var newFiles: Set = [] @@ -530,7 +547,7 @@ class RomsSyncer: iCloudTypeSyncer { @objc func handleNewFiles(_ notification: Notification) { - guard let downloadedFiles = notification.userInfo?[Notification.Name.newCloudFiles.rawValue] as? Set + guard let downloadedFiles = notification.userInfo?[notification.name.rawValue] as? Set else { return } @@ -540,6 +557,7 @@ class RomsSyncer: iCloudTypeSyncer { /// sends a notification that rom files are ready to e @objc func handleNewRomFiles() { + //TODO: do we wait until the bios has been downloaded? guard !newFiles.isEmpty else { return @@ -557,6 +575,32 @@ class RomsSyncer: iCloudTypeSyncer { newFiles.removeAll() let name = Notification.Name.cloudDataDownloaded NotificationCenter.default.post(name: name, object: nil, userInfo: [name.rawValue: converted]) -// iCloudSync.disposeBagRoms = nil + } +} + +class BiosSyncer: iCloudTypeSyncer { + var metadataQuery: NSMetadataQuery = .init() + var newFiles: Set = [] + + var directory: String { + "BIOS" + } +} + +class BatterySavesSyncer: iCloudTypeSyncer { + var metadataQuery: NSMetadataQuery = .init() + var newFiles: Set = [] + + var directory: String { + "Battery Saves" + } +} + +class ScreenshotsSyncer: iCloudTypeSyncer { + var metadataQuery: NSMetadataQuery = .init() + var newFiles: Set = [] + //TODO: I think a base class that accepts the directory in the initializer may be better than this + var directory: String { + "Screenshots" } } diff --git a/Provenance/Main UI/PVAppDelegate.swift b/Provenance/Main UI/PVAppDelegate.swift index 433b913dc4..5199e60cf1 100644 --- a/Provenance/Main UI/PVAppDelegate.swift +++ b/Provenance/Main UI/PVAppDelegate.swift @@ -354,7 +354,7 @@ final class PVAppDelegate: UIResponder, GameLaunchingAppDelegate, UIApplicationD } @objc func cloudDataDownloaded(notification: Notification) { - guard let files = notification.userInfo?["kCloudDataDownloaded"] as? [URL], + guard let files = notification.userInfo?[notification.name.rawValue] as? [URL], let gameImporter = appState?.gameImporter else { return From 676d579fb6edf107bb6b014a4f6daf40fa0912d4 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Fri, 24 Jan 2025 23:32:25 -0500 Subject: [PATCH 20/86] refactored notification names to notification.swift and updated naming convention; removed importing from app delegate; added TODOs --- .../PVLibrary/Database/Notifications.swift | 3 + .../Database/Realm Database/RomDatabase.swift | 2 +- .../Services/GameImporter/GameImporter.swift | 1 + .../Importer/iCloud/iCloudSync.swift | 65 +++++++++++-------- Provenance/Main UI/PVAppDelegate.swift | 11 ---- 5 files changed, 42 insertions(+), 40 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Database/Notifications.swift b/PVLibrary/Sources/PVLibrary/Database/Notifications.swift index 0c2b7045a5..6989e19653 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Notifications.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Notifications.swift @@ -10,4 +10,7 @@ import Foundation public extension Notification.Name { static let DatabaseMigrationStarted = Notification.Name("DatabaseMigrarionStarted") static let DatabaseMigrationFinished = Notification.Name("DatabaseMigrarionFinished") + static let NewCloudFilesAvailable = Notification.Name("NewCloudFilesAvailable") + static let RomDatabaseInitialized = Notification.Name("RomDatabaseInitialized") + static let RomsFinishedImporting = Notification.Name("RomsFinishedImporting") } diff --git a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift index e575a94fba..5bdc0de1a2 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift @@ -328,7 +328,7 @@ public final class RomDatabase { ILOG("Database initialization completed") databaseInitialized = true - NotificationCenter.default.post(name: .romDatabaseInitialized, object: nil, userInfo: nil) + NotificationCenter.default.post(name: .RomDatabaseInitialized, object: nil, userInfo: nil) } else { ILOG("Database already initialized") diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 141900c328..8ae45709ce 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -680,6 +680,7 @@ public final class GameImporter: GameImporting, ObservableObject { self.processingState = .idle } ILOG("GameImportQueue - processQueue complete Import Processing") + NotificationCenter.default.post(name: .RomsFinishedImporting, object: nil) } // Process a single ImportItem and update its status diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 965d8313df..b8553f634b 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -31,12 +31,6 @@ public protocol Container { var containerURL: URL? { get } } -extension Notification.Name { - static let cloudDataDownloaded = Notification.Name("kCloudDataDownloaded") - static let newCloudFiles = Notification.Name("kNewCloudFiles") - static let romDatabaseInitialized = Notification.Name("kRomDatabaseInitialized") -} - extension Container { public var containerURL: URL? { get { return URL.iCloudContainerDirectory }} var documentsURL: URL? { get { return URL.iCloudDocumentsDirectory }} @@ -109,6 +103,7 @@ extension iCloudTypeSyncer { self.queryFinished(notification: notification) completable(.completed) } + //TODO: listen for updates self.metadataQuery.start() return Disposables.create {} @@ -174,7 +169,7 @@ extension iCloudTypeSyncer { } func queryFinished(notification: Notification) { - guard let query = notification.object as? NSMetadataQuery + guard (notification.object as? NSMetadataQuery) == metadataQuery else { return } @@ -186,7 +181,7 @@ extension iCloudTypeSyncer { // query.enableUpdates() // } //accessing results automatically pauses updates and resumes after deallocated - for item in query.results { + for item in metadataQuery.results { if let fileItem = item as? NSMetadataItem, let file = fileItem.value(forAttribute: NSMetadataItemURLKey) as? URL, let downloadStatus = fileItem.value(forAttribute: NSMetadataUbiquitousItemDownloadingStatusKey) as? String, @@ -196,7 +191,6 @@ extension iCloudTypeSyncer { do { try fileManager.startDownloadingUbiquitousItem(at: file) - //TODO: we have to wait until the files are downloaded, we can just create a queue and then just loop until the queue is empty print("Download started for: \(file.lastPathComponent)") } catch { print("Failed to start download: \(error)") @@ -204,6 +198,7 @@ extension iCloudTypeSyncer { } } var downloadedFiles = Set() + //we wait for all files to finish downloaded from iCloud while downloadedFiles.count != files.count { for file in files { if fileManager.fileExists(atPath: file.path) { @@ -211,9 +206,9 @@ extension iCloudTypeSyncer { } } } - let name = Notification.Name.newCloudFiles + let name = Notification.Name.NewCloudFilesAvailable print("downloadedFiles: \(downloadedFiles.count)") - NotificationCenter.default.post(name: name, object: nil, userInfo: [name.rawValue: downloadedFiles]) + NotificationCenter.default.post(name: name, object: self, userInfo: [name.rawValue: downloadedFiles]) } } @@ -322,7 +317,6 @@ enum iCloudError: Error { } public enum iCloudSync { - //TODO: move bags to each class static var disposeBag: DisposeBag? public static func initICloudDocuments() { let fm = FileManager.default @@ -336,14 +330,15 @@ public enum iCloudSync { } else { UserDefaults.standard.removeObject(forKey: UbiquityIdentityTokenKey) } - //TODO: there is an issue when all of the icloud files are downloaded and during the importing of Realm + Save States there's a clash. I think we need to import ROMs first, then BIOS (if necessary) and then save states, everything else doesn't need to be imported onto the db (at least from what I understand) + //TODO: there is an issue when all of the icloud files are downloaded and during the importing of ROMs + Save States there's a clash. I think we need to import ROMs first, then BIOS (if necessary) and then save states, everything else doesn't need to be imported onto the db (at least from what I understand) + //TODO: pause when a game starts so we don't interfere with the game and continue listening when no game is running var saveStateSyncer = SaveStateSyncer() let disposeBag = DisposeBag() self.disposeBag = disposeBag saveStateSyncer.loadAllFromICloud() .observe(on: MainScheduler.instance) .subscribe(onCompleted: { - importNewSaves() +// importNewSaves() }) { error in ELOG(error.localizedDescription) }.disposed(by: disposeBag) @@ -381,13 +376,13 @@ public enum iCloudSync { }.disposed(by: disposeBag) } -//TODO: prolly this should be in SaveStateSyncer + //TODO: this function should be refactored onto SaveStateSyncer class public static func importNewSaves() { guard RomDatabase.databaseInitialized else { return } - + //TODO: files should already been downloaded by now. use newFiles from SaveStateSyncer Task { let savesDirectory = Paths.saveSavesPath let legacySavesDirectory = Paths.Legacy.saveSavesPath @@ -512,16 +507,31 @@ public enum iCloudSync { class SaveStateSyncer: iCloudTypeSyncer { var metadataQuery: NSMetadataQuery = .init() var newFiles: Set = [] + var areRomsDownloaded = false init() { - NotificationCenter.default.addObserver(self, selector: #selector(wrapper), name: .romDatabaseInitialized, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(wrapperImportSaves), name: .RomDatabaseInitialized, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(romsFinishedImporting), name: .RomsFinishedImporting, object: nil) } deinit { print("dying") } @objc - func wrapper() { - iCloudSync.importNewSaves() + func romsFinishedImporting() { + //TODO: this should be reset somehow + areRomsDownloaded = true + wrapperImportSaves() + } + + @objc + func wrapperImportSaves() { + //TODO: fix logic. we need to know if there are ROMs to download, if no, then we do the importing of saves + guard areRomsDownloaded + else { + return + } + //TODO: fix, importing saves is crashing + //iCloudSync.importNewSaves() } var directory: String { @@ -532,10 +542,11 @@ class SaveStateSyncer: iCloudTypeSyncer { class RomsSyncer: iCloudTypeSyncer { var metadataQuery: NSMetadataQuery = .init() var newFiles: Set = [] + let gameImporter = GameImporter.shared init() { - NotificationCenter.default.addObserver(self, selector: #selector(handleNewRomFiles), name: .romDatabaseInitialized, object: nil) - NotificationCenter.default.addObserver(self, selector: #selector(handleNewFiles(_:)), name: .newCloudFiles, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(handleNewRomFiles), name: .RomDatabaseInitialized, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(handleNewFiles(_:)), name: .NewCloudFilesAvailable, object: self) } deinit { print("dying") @@ -547,7 +558,8 @@ class RomsSyncer: iCloudTypeSyncer { @objc func handleNewFiles(_ notification: Notification) { - guard let downloadedFiles = notification.userInfo?[notification.name.rawValue] as? Set + guard let downloadedFiles = notification.userInfo?[notification.name.rawValue] as? Set, + (notification.object as? RomsSyncer) === self else { return } @@ -568,13 +580,10 @@ class RomsSyncer: iCloudTypeSyncer { return } - var converted = [URL]() - for item in newFiles { - converted.append(item) - } + var converted = [URL](newFiles) newFiles.removeAll() - let name = Notification.Name.cloudDataDownloaded - NotificationCenter.default.post(name: name, object: nil, userInfo: [name.rawValue: converted]) + gameImporter.addImports(forPaths: converted) + gameImporter.startProcessing() } } diff --git a/Provenance/Main UI/PVAppDelegate.swift b/Provenance/Main UI/PVAppDelegate.swift index 5199e60cf1..305a4d537d 100644 --- a/Provenance/Main UI/PVAppDelegate.swift +++ b/Provenance/Main UI/PVAppDelegate.swift @@ -346,23 +346,12 @@ final class PVAppDelegate: UIResponder, GameLaunchingAppDelegate, UIApplicationD let useiCloud = Defaults[.iCloudSync] && URL.supportsICloud if useiCloud { DispatchQueue.main.async { - NotificationCenter.default.addObserver(self, selector: #selector(self.cloudDataDownloaded(notification:)), name: Notification.Name("kCloudDataDownloaded"), object: nil) iCloudSync.initICloudDocuments() } } } } - @objc func cloudDataDownloaded(notification: Notification) { - guard let files = notification.userInfo?[notification.name.rawValue] as? [URL], - let gameImporter = appState?.gameImporter - else { - return - } - gameImporter.addImports(forPaths: files) - gameImporter.startProcessing() - } - var currentThemeObservation: Any? // AnyCancellable? var userInterfaceStyleObservation: Any? var oldPalette: (any UXThemePalette)? From b6443a997c0c1c4af80c595762c149bd2190d5a2 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Fri, 24 Jan 2025 23:38:52 -0500 Subject: [PATCH 21/86] reverted --- .../Conversion/Extensions/SaveState+PVSaveState.swift | 2 +- .../PVEmulatorVC/PVEmulatorViewController+Saves.swift | 4 ++-- .../PVEmulatorVC/PVEmulatorViewController+ScreenShot.swift | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift index a44e078d9a..149fbc10f7 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift @@ -75,7 +75,7 @@ extension SaveState: RealmRepresentable { let dir = path.deletingLastPathComponent() let imagePath = dir.appendingPathComponent(image.fileName) DLOG("path: \(imagePath)") - object.image = PVImageFile(withURL: imagePath, relativeRoot: .iCloud)//TODO: test without .iCloud + object.image = PVImageFile(withURL: imagePath, relativeRoot: .iCloud) } object.isAutosave = isAutosave } diff --git a/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+Saves.swift b/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+Saves.swift index b347a514ba..223998f1bf 100644 --- a/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+Saves.swift +++ b/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+Saves.swift @@ -95,14 +95,14 @@ public extension PVEmulatorViewController { do { try jpegData.write(to: imageURL) // try RomDatabase.sharedInstance.writeTransaction { - // let newFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud)//TODO: test without .iCloud + // let newFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud) // game.screenShots.append(newFile) // } } catch { presentError("Unable to write image to disk, error: \(error.localizedDescription)", source: self.view) } - imageFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud)//TODO: test without .iCloud + imageFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud) } } diff --git a/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+ScreenShot.swift b/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+ScreenShot.swift index d75db6ffd7..537f5ef1f9 100644 --- a/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+ScreenShot.swift +++ b/PVUI/Sources/PVUIBase/PVEmulatorVC/PVEmulatorViewController+ScreenShot.swift @@ -52,7 +52,7 @@ extension PVEmulatorViewController { try pngData.write(to: imageURL) RomDatabase.sharedInstance.asyncWriteTransaction { self.game.realm?.refresh() - let newFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud)//TODO: test without .iCloud + let newFile = PVImageFile(withURL: imageURL, relativeRoot: .iCloud) self.game.screenShots.append(newFile) } } catch { From cfe6e98bfae96902fd026c566f10cfe271d9a3fd Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 25 Jan 2025 00:17:55 -0500 Subject: [PATCH 22/86] updated to use dlog --- .../Importer/iCloud/iCloudSync.swift | 150 +++++++----------- 1 file changed, 58 insertions(+), 92 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index b8553f634b..d8d8fdc890 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -186,14 +186,14 @@ extension iCloudTypeSyncer { let file = fileItem.value(forAttribute: NSMetadataItemURLKey) as? URL, let downloadStatus = fileItem.value(forAttribute: NSMetadataUbiquitousItemDownloadingStatusKey) as? String, downloadStatus == NSMetadataUbiquitousItemDownloadingStatusNotDownloaded { - print("Found file: \(String(describing: file)), download status: \(downloadStatus)") + DLOG("Found file: \(String(describing: file)), download status: \(downloadStatus)") files.append(file) do { try fileManager.startDownloadingUbiquitousItem(at: file) - print("Download started for: \(file.lastPathComponent)") + DLOG("Download started for: \(file.lastPathComponent)") } catch { - print("Failed to start download: \(error)") + DLOG("Failed to start download: \(error)") } } } @@ -207,7 +207,7 @@ extension iCloudTypeSyncer { } } let name = Notification.Name.NewCloudFilesAvailable - print("downloadedFiles: \(downloadedFiles.count)") + DLOG("downloadedFiles: \(downloadedFiles.count)") NotificationCenter.default.post(name: name, object: self, userInfo: [name.rawValue: downloadedFiles]) } } @@ -375,69 +375,70 @@ public enum iCloudSync { ELOG(error.localizedDescription) }.disposed(by: disposeBag) } +} +//TODO: perhaps 1 generic class since a lot of this code is similar and move the extension onto generic class. we could just add a protocol delegate dependency for ROMs and SaveState classes that does specific code +class SaveStateSyncer: iCloudTypeSyncer { + var metadataQuery: NSMetadataQuery = .init() + var newFiles: Set = [] + var areRomsDownloaded = false + init() { + NotificationCenter.default.addObserver(self, selector: #selector(wrapperImportSaves), name: .RomDatabaseInitialized, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(romsFinishedImporting), name: .RomsFinishedImporting, object: nil) + NotificationCenter.default.addObserver(self, selector: #selector(handleNewFiles(_:)), name: .NewCloudFilesAvailable, object: self) + } + deinit { + DLOG("dying") + } - //TODO: this function should be refactored onto SaveStateSyncer class - public static func importNewSaves() { + @objc + func romsFinishedImporting() { + //TODO: this should be reset somehow + areRomsDownloaded = true + wrapperImportSaves() + } + + @objc + func wrapperImportSaves() { + //TODO: fix logic. we need to know if there are ROMs to download, if no, then we do the importing of saves + guard areRomsDownloaded + else { + return + } + //TODO: fix, importing saves is crashing + importNewSaves() + } + + @objc + func handleNewFiles(_ notification: Notification) { + guard let downloadedFiles = notification.userInfo?[notification.name.rawValue] as? Set, + (notification.object as? RomsSyncer) === self + else { + return + } + newFiles = downloadedFiles + } + + var directory: String { + "Save States" + } + + func importNewSaves() { guard RomDatabase.databaseInitialized else { return } - //TODO: files should already been downloaded by now. use newFiles from SaveStateSyncer + guard !newFiles.isEmpty + else { + return + } Task { - let savesDirectory = Paths.saveSavesPath - let legacySavesDirectory = Paths.Legacy.saveSavesPath - let fm = FileManager.default - guard let subDirs = try? fm.contentsOfDirectory(at: savesDirectory, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) else { - ELOG("Failed to read saves path: \(savesDirectory.path)") - return - } - - let saveFiles = subDirs.compactMap { - try? fm.contentsOfDirectory(at: $0, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) + let saveFiles = newFiles.compactMap { + try? FileManager.default.contentsOfDirectory(at: $0, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) }.joined() let jsonFiles = saveFiles.filter { $0.pathExtension == "json" } let jsonDecorder = JSONDecoder() jsonDecorder.dataDecodingStrategy = .deferredToData - let legacySubDirs: [URL]? - do { - legacySubDirs = try fm.contentsOfDirectory(at: legacySavesDirectory, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) - } catch { - ELOG("\(error.localizedDescription)") - legacySubDirs = nil - } - - await legacySubDirs?.asyncForEach { - do { - let destinationURL = Paths.saveSavesPath.appendingPathComponent($0.lastPathComponent, isDirectory: true) - if !fm.isUbiquitousItem(at: destinationURL) { - try fm.setUbiquitous(true, - itemAt: $0, - destinationURL: destinationURL) - } else { - // var resultURL: NSURL? - // try fm.replaceItem(at: destinationURL, withItemAt: $0, backupItemName: nil, resultingItemURL: &resultURL) - // try fm.evictUbiquitousItem(at: destinationURL) - try fm.startDownloadingUbiquitousItem(at: destinationURL) - } - } catch { - ELOG("Error: \(error)") - } - } - // let saves = realm.objects(PVSaveState.self) - // saves.forEach { - // fm.setUbiquitous(true, itemAt: $0.file.url, destinationURL: Paths.saveSavesPath.appendingPathComponent($0.game.file.fileNameWithoutExtension, isDirectory: true).app) - // } - Task.detached { - jsonFiles.forEach { json in - do { - try FileManager.default.startDownloadingUbiquitousItem(at: json) - } catch { - ELOG("Download error: " + error.localizedDescription) - } - } - } - Task.detached { // @MainActor in await jsonFiles.concurrentForEach { @MainActor json in let realm = try! await Realm() @@ -503,41 +504,6 @@ public enum iCloudSync { } } } -//TODO: perhaps 1 generic class since a lot of this code is similar and move the extension onto generic class. we could just add a protocol delegate dependency for ROMs and SaveState classes that does specific code -class SaveStateSyncer: iCloudTypeSyncer { - var metadataQuery: NSMetadataQuery = .init() - var newFiles: Set = [] - var areRomsDownloaded = false - init() { - NotificationCenter.default.addObserver(self, selector: #selector(wrapperImportSaves), name: .RomDatabaseInitialized, object: nil) - NotificationCenter.default.addObserver(self, selector: #selector(romsFinishedImporting), name: .RomsFinishedImporting, object: nil) - } - deinit { - print("dying") - } - - @objc - func romsFinishedImporting() { - //TODO: this should be reset somehow - areRomsDownloaded = true - wrapperImportSaves() - } - - @objc - func wrapperImportSaves() { - //TODO: fix logic. we need to know if there are ROMs to download, if no, then we do the importing of saves - guard areRomsDownloaded - else { - return - } - //TODO: fix, importing saves is crashing - //iCloudSync.importNewSaves() - } - - var directory: String { - "Save States" - } -} class RomsSyncer: iCloudTypeSyncer { var metadataQuery: NSMetadataQuery = .init() @@ -549,7 +515,7 @@ class RomsSyncer: iCloudTypeSyncer { NotificationCenter.default.addObserver(self, selector: #selector(handleNewFiles(_:)), name: .NewCloudFilesAvailable, object: self) } deinit { - print("dying") + DLOG("dying") } var directory: String { From bf02d64e8627c761315d06ec145718d5020c8e9f Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 25 Jan 2025 01:55:14 -0500 Subject: [PATCH 23/86] added more logs; remove tasks that were causing synchronization issues when saving save states on realm --- .../PVLibrary/Importer/iCloud/iCloudSync.swift | 17 +++++++---------- .../Extensions/SaveState+PVSaveState.swift | 4 ++-- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index d8d8fdc890..8be7a6fd78 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -94,6 +94,7 @@ extension iCloudTypeSyncer { return Disposables.create {} } self.metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] + DLOG("directory: \(directory)") metadataQuery.predicate = NSPredicate(format: "%K CONTAINS[c] %@", NSMetadataItemPathKey, "/Documents/\(directory)/") let _: NotificationObserver = .init( @@ -169,6 +170,7 @@ extension iCloudTypeSyncer { } func queryFinished(notification: Notification) { + DLOG("directory: \(directory)") guard (notification.object as? NSMetadataQuery) == metadataQuery else { return @@ -410,8 +412,7 @@ class SaveStateSyncer: iCloudTypeSyncer { @objc func handleNewFiles(_ notification: Notification) { - guard let downloadedFiles = notification.userInfo?[notification.name.rawValue] as? Set, - (notification.object as? RomsSyncer) === self + guard let downloadedFiles = notification.userInfo?[notification.name.rawValue] as? Set else { return } @@ -432,14 +433,11 @@ class SaveStateSyncer: iCloudTypeSyncer { return } Task { - let saveFiles = newFiles.compactMap { - try? FileManager.default.contentsOfDirectory(at: $0, includingPropertiesForKeys: nil, options: .skipsHiddenFiles) - }.joined() - let jsonFiles = saveFiles.filter { $0.pathExtension == "json" } + let jsonFiles = newFiles.filter { $0.pathExtension == "json" } let jsonDecorder = JSONDecoder() jsonDecorder.dataDecodingStrategy = .deferredToData - Task.detached { // @MainActor in + //Task.detached { // @MainActor in//can we re-add this with the change of not adding a task on the PVSave asRealm() function? await jsonFiles.concurrentForEach { @MainActor json in let realm = try! await Realm() do { @@ -500,7 +498,7 @@ class SaveStateSyncer: iCloudTypeSyncer { return } } - } + //} } } } @@ -524,8 +522,7 @@ class RomsSyncer: iCloudTypeSyncer { @objc func handleNewFiles(_ notification: Notification) { - guard let downloadedFiles = notification.userInfo?[notification.name.rawValue] as? Set, - (notification.object as? RomsSyncer) === self + guard let downloadedFiles = notification.userInfo?[notification.name.rawValue] as? Set else { return } diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift index 149fbc10f7..568b6381f3 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift @@ -64,7 +64,7 @@ extension SaveState: RealmRepresentable { object.core = core.asRealm() } - Task { +// Task {//is this task needed? It's causing issues when downloading icloud files. let path = game.file.fileName.saveStatePath.appendingPathComponent(file.fileName) object.file = PVFile(withURL: path) DLOG("file path: \(path)") @@ -78,7 +78,7 @@ extension SaveState: RealmRepresentable { object.image = PVImageFile(withURL: imagePath, relativeRoot: .iCloud) } object.isAutosave = isAutosave - } +// } } } } From c0520d45c9efec85df76765ff06153e3a392c2c3 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 25 Jan 2025 18:31:59 -0500 Subject: [PATCH 24/86] added todo; remove task because it was causing crashes when syncing with iCloud due to realm --- .../Services/GameImporter/GameImporter.swift | 1 + .../Extensions/SaveState+PVSaveState.swift | 28 +++++++++---------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 8ae45709ce..aa1f359369 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -680,6 +680,7 @@ public final class GameImporter: GameImporting, ObservableObject { self.processingState = .idle } ILOG("GameImportQueue - processQueue complete Import Processing") + //TODO: this doesn't appear to be needed anymore. NotificationCenter.default.post(name: .RomsFinishedImporting, object: nil) } diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift index 568b6381f3..83aa74e8eb 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift @@ -64,21 +64,19 @@ extension SaveState: RealmRepresentable { object.core = core.asRealm() } -// Task {//is this task needed? It's causing issues when downloading icloud files. - let path = game.file.fileName.saveStatePath.appendingPathComponent(file.fileName) - object.file = PVFile(withURL: path) - DLOG("file path: \(path)") - - object.date = date - object.lastOpened = lastOpened - if let image = image { - let dir = path.deletingLastPathComponent() - let imagePath = dir.appendingPathComponent(image.fileName) - DLOG("path: \(imagePath)") - object.image = PVImageFile(withURL: imagePath, relativeRoot: .iCloud) - } - object.isAutosave = isAutosave -// } + let path = game.file.fileName.saveStatePath.appendingPathComponent(file.fileName) + object.file = PVFile(withURL: path) + DLOG("file path: \(path)") + + object.date = date + object.lastOpened = lastOpened + if let image = image { + let dir = path.deletingLastPathComponent() + let imagePath = dir.appendingPathComponent(image.fileName) + DLOG("path: \(imagePath)") + object.image = PVImageFile(withURL: imagePath, relativeRoot: .iCloud) + } + object.isAutosave = isAutosave } } } From 7d7e0bd1291afe7a9860646951865afb929ce989 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 25 Jan 2025 20:40:32 -0500 Subject: [PATCH 25/86] removed unused functions; refacored to wait for signal that file finishes downloading insetad of checking in a loop; updated to import only roms and just download any extra files that are part of the ROMs directory for each core; updated to make the syncers live for the lifetime of the application; added concurrency for downloading files to speed things up; refactored saved states code to just work directly with downloaded files; removed unused events --- .../Importer/iCloud/iCloudSync.swift | 336 ++++++++++++------ 1 file changed, 220 insertions(+), 116 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 8be7a6fd78..c8a9c9141f 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -44,15 +44,14 @@ public protocol SyncFileToiCloud: Container { } public protocol iCloudTypeSyncer: Container { - var newFiles: Set { get set } var directory: String { get } var metadataQuery: NSMetadataQuery { get } - func loadAllFromICloud() -> Completable + func loadAllFromICloud(iterationComplete: @escaping () -> Void) -> Completable func removeAllFromICloud() -> Completable - mutating func clearNewFiles() - mutating func insertNewFile(_ file: URL) - mutating func setNewFiles(_ files: Set) + func insertDownloadingFile(_ file: URL) + func insertDownloadedFile(_ file: URL) + func setNewCloudFilesAvailable() } final class NotificationObserver { @@ -73,19 +72,19 @@ final class NotificationObserver { } extension iCloudTypeSyncer { - mutating func setNewFiles(_ files: Set) { - newFiles = files + func insertDownloadingFile(_ file: URL) { + //no-op } - mutating func clearNewFiles() { - newFiles.removeAll() + func insertDownloadedFile(_ file: URL) { + //no-op } - mutating func insertNewFile(_ file: URL) { - newFiles.insert(file) + func setNewCloudFilesAvailable() { + //no-op } - func loadAllFromICloud() -> Completable { + func loadAllFromICloud(iterationComplete: @escaping () -> Void) -> Completable { return Completable.create { completable in //body mustn't run in a task, otherwise the NotificationObserver closure won't get called guard containerURL != nil @@ -96,15 +95,28 @@ extension iCloudTypeSyncer { self.metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] DLOG("directory: \(directory)") metadataQuery.predicate = NSPredicate(format: "%K CONTAINS[c] %@", NSMetadataItemPathKey, "/Documents/\(directory)/") - + //TODO: update to use Publishers.MergeMany let _: NotificationObserver = .init( forName: .NSMetadataQueryDidFinishGathering, object: self.metadataQuery, queue: nil) { notification in - self.queryFinished(notification: notification) - completable(.completed) + Task { + await queryFinished(notification: notification) +// completable(.completed) + iterationComplete() + } + } + //listen for deletions and new files. what about conflicts? + let _: NotificationObserver = .init( + forName: .NSMetadataQueryDidUpdate, + object: self.metadataQuery, + queue: nil) { notification in + Task { + await queryFinished(notification: notification) +// completable(.completed) + iterationComplete() + } } - //TODO: listen for updates self.metadataQuery.start() return Disposables.create {} @@ -168,49 +180,103 @@ extension iCloudTypeSyncer { } } } + + func queryFinished(notification: Notification) async { + DLOG("directory: \(directory)") + guard (notification.object as? NSMetadataQuery) == metadataQuery + else { + return + } + let fileManager = FileManager.default + var files: Set = [] + var filesDownloaded: Set = [] + let queue = DispatchQueue(label: "org.provenance-emu.provenance.newFiles") + + //accessing results automatically pauses updates and resumes after deallocated + await metadataQuery.results.concurrentForEach { item in + if let fileItem = item as? NSMetadataItem, + let file = fileItem.value(forAttribute: NSMetadataItemURLKey) as? URL, + let isDirectory = try? file.resourceValues(forKeys: [.isDirectoryKey]).isDirectory, + !isDirectory,//we only + let downloadStatus = fileItem.value(forAttribute: NSMetadataUbiquitousItemDownloadingStatusKey) as? String { + DLOG("Found: \(file), download status: \(downloadStatus)") + switch downloadStatus { + case NSMetadataUbiquitousItemDownloadingStatusNotDownloaded: + do { + try fileManager.startDownloadingUbiquitousItem(at: file) + queue.sync { + files.insert(file) + insertDownloadingFile(file) + } + DLOG("Download started for: \(file.lastPathComponent)") + } catch { + DLOG("Failed to start download: \(error)") + } + case NSMetadataUbiquitousItemDownloadingStatusCurrent: + DLOG("item up to date: \(file)") + queue.sync { + filesDownloaded.insert(file) + insertDownloadedFile(file) + } + default: DLOG("\(file.lastPathComponent): download status: \(downloadStatus)") + } + } + } + //TODO: for ROMs and saves, perhaps we need to store the downloaded files that need to be process in the case of a crash or the user puts the app in the background. + setNewCloudFilesAvailable() + DLOG("\(directory): current iteration: files pending to be downloaded: \(files.count), files downloaded : \(filesDownloaded.count)") + } - func queryFinished(notification: Notification) { + func queryFinished2(notification: Notification) async { DLOG("directory: \(directory)") guard (notification.object as? NSMetadataQuery) == metadataQuery else { return } let fileManager = FileManager.default - var files: [URL] = [] + var files: Set = [] //TODO: don't think we need this // query.disableUpdates() // defer { // query.enableUpdates() // } + let queue = DispatchQueue(label: "org.provenance-emu.provenance.newFiles") + //accessing results automatically pauses updates and resumes after deallocated - for item in metadataQuery.results { + await metadataQuery.results.concurrentForEach { item in if let fileItem = item as? NSMetadataItem, let file = fileItem.value(forAttribute: NSMetadataItemURLKey) as? URL, let downloadStatus = fileItem.value(forAttribute: NSMetadataUbiquitousItemDownloadingStatusKey) as? String, downloadStatus == NSMetadataUbiquitousItemDownloadingStatusNotDownloaded { DLOG("Found file: \(String(describing: file)), download status: \(downloadStatus)") - files.append(file) - do { try fileManager.startDownloadingUbiquitousItem(at: file) + queue.sync { + files.insert(file) + insertDownloadingFile(file) + } DLOG("Download started for: \(file.lastPathComponent)") } catch { DLOG("Failed to start download: \(error)") } } } - var downloadedFiles = Set() + //TODO: for ROMs and saves, perhaps we need to store the downloaded files that need to be process in the case of a crash or the user puts the app in the background. //we wait for all files to finish downloaded from iCloud - while downloadedFiles.count != files.count { - for file in files { + let count = files.count + var pendingToDownload = files + while !pendingToDownload.isEmpty { + await pendingToDownload.concurrentForEach { file in if fileManager.fileExists(atPath: file.path) { - downloadedFiles.insert(file) + queue.sync { + files.remove(file) + } } } + pendingToDownload = files } - let name = Notification.Name.NewCloudFilesAvailable - DLOG("downloadedFiles: \(downloadedFiles.count)") - NotificationCenter.default.post(name: name, object: self, userInfo: [name.rawValue: downloadedFiles]) + setNewCloudFilesAvailable() + DLOG("\(directory): downloaded files: \(count)") } } @@ -319,7 +385,9 @@ enum iCloudError: Error { } public enum iCloudSync { - static var disposeBag: DisposeBag? + static let disposeBag: DisposeBag = DisposeBag() + static let saveStateSyncer = SaveStateSyncer() + static let romsSyncer = RomsSyncer() public static func initICloudDocuments() { let fm = FileManager.default if let currentiCloudToken = fm.ubiquityIdentityToken { @@ -332,95 +400,92 @@ public enum iCloudSync { } else { UserDefaults.standard.removeObject(forKey: UbiquityIdentityTokenKey) } - //TODO: there is an issue when all of the icloud files are downloaded and during the importing of ROMs + Save States there's a clash. I think we need to import ROMs first, then BIOS (if necessary) and then save states, everything else doesn't need to be imported onto the db (at least from what I understand) - //TODO: pause when a game starts so we don't interfere with the game and continue listening when no game is running - var saveStateSyncer = SaveStateSyncer() - let disposeBag = DisposeBag() - self.disposeBag = disposeBag - saveStateSyncer.loadAllFromICloud() - .observe(on: MainScheduler.instance) - .subscribe(onCompleted: { -// importNewSaves() - }) { error in - ELOG(error.localizedDescription) - }.disposed(by: disposeBag) - var romsSyncer = RomsSyncer() - romsSyncer.loadAllFromICloud() - .observe(on: MainScheduler.instance) - .subscribe(onCompleted: { - romsSyncer.handleNewRomFiles() - }) { error in - ELOG(error.localizedDescription) - }.disposed(by: disposeBag) + + //TODO: should we pause when a game starts so we don't interfere with the game and continue listening when no game is running? + saveStateSyncer.loadAllFromICloud() { + saveStateSyncer.importNewSaves() + }.observe(on: MainScheduler.instance) + .subscribe(onCompleted: { + }, onError: { error in + ELOG(error.localizedDescription) + }) { + DLOG("disposing saveStateSyncer") + }.disposed(by: disposeBag) + romsSyncer.loadAllFromICloud() { + romsSyncer.handleNewRomFiles() + }.observe(on: MainScheduler.instance) + .subscribe(onCompleted: { + }, onError: { error in + ELOG(error.localizedDescription) + }){ + DLOG("disposing romsSyncer") + }.disposed(by: disposeBag) let biosSyncer = BiosSyncer() - biosSyncer.loadAllFromICloud() - .observe(on: MainScheduler.instance) - .subscribe(onCompleted: { - //TODO: anything? - }) { error in - ELOG(error.localizedDescription) - }.disposed(by: disposeBag) + biosSyncer.loadAllFromICloud() { + + }.observe(on: MainScheduler.instance) + .subscribe(onCompleted: { + //no-op because nothing needs to be imported into the db + }) { error in + ELOG(error.localizedDescription) + }.disposed(by: disposeBag) let batterySaveSyncer = BatterySavesSyncer() - batterySaveSyncer.loadAllFromICloud() - .observe(on: MainScheduler.instance) - .subscribe(onCompleted: { - //TODO: anything? - }) { error in - ELOG(error.localizedDescription) - }.disposed(by: disposeBag) + batterySaveSyncer.loadAllFromICloud() { + + }.observe(on: MainScheduler.instance) + .subscribe(onCompleted: { + //no-op because nothing needs to be imported into the db + }) { error in + ELOG(error.localizedDescription) + }.disposed(by: disposeBag) let screenshotsSyncer = ScreenshotsSyncer() - screenshotsSyncer.loadAllFromICloud() - .observe(on: MainScheduler.instance) - .subscribe(onCompleted: { - //TODO: anything? - }) { error in - ELOG(error.localizedDescription) - }.disposed(by: disposeBag) + screenshotsSyncer.loadAllFromICloud() { + }.observe(on: MainScheduler.instance) + .subscribe(onCompleted: { + //no-op because nothing needs to be imported into the db + }) { error in + ELOG(error.localizedDescription) + }.disposed(by: disposeBag) } } //TODO: perhaps 1 generic class since a lot of this code is similar and move the extension onto generic class. we could just add a protocol delegate dependency for ROMs and SaveState classes that does specific code class SaveStateSyncer: iCloudTypeSyncer { var metadataQuery: NSMetadataQuery = .init() + var pendingFilesToDownload: Set = [] var newFiles: Set = [] - var areRomsDownloaded = false + var didFinishDownloadingAllFiles = false init() { - NotificationCenter.default.addObserver(self, selector: #selector(wrapperImportSaves), name: .RomDatabaseInitialized, object: nil) - NotificationCenter.default.addObserver(self, selector: #selector(romsFinishedImporting), name: .RomsFinishedImporting, object: nil) - NotificationCenter.default.addObserver(self, selector: #selector(handleNewFiles(_:)), name: .NewCloudFilesAvailable, object: self) + NotificationCenter.default.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil, using: importNewSaves) } deinit { DLOG("dying") } - @objc - func romsFinishedImporting() { - //TODO: this should be reset somehow - areRomsDownloaded = true - wrapperImportSaves() + func setNewCloudFilesAvailable() { + didFinishDownloadingAllFiles = pendingFilesToDownload.isEmpty } - @objc - func wrapperImportSaves() { - //TODO: fix logic. we need to know if there are ROMs to download, if no, then we do the importing of saves - guard areRomsDownloaded - else { - return - } - //TODO: fix, importing saves is crashing - importNewSaves() + var directory: String { + "Save States" } - @objc - func handleNewFiles(_ notification: Notification) { - guard let downloadedFiles = notification.userInfo?[notification.name.rawValue] as? Set + func insertDownloadingFile(_ file: URL) { + pendingFilesToDownload.insert(file.absoluteString) + } + + func insertDownloadedFile(_ file: URL) { + guard let _ = pendingFilesToDownload.remove(file.absoluteString), + "json".caseInsensitiveCompare(file.pathExtension) == .orderedSame else { return } - newFiles = downloadedFiles + DLOG("downloaded save file: \(file.lastPathComponent)") + newFiles.insert(file) } - var directory: String { - "Save States" + func importNewSaves(_ notification: Notification) { + NotificationCenter.default.removeObserver(self) + importNewSaves() } func importNewSaves() { @@ -432,12 +497,18 @@ class SaveStateSyncer: iCloudTypeSyncer { else { return } + guard didFinishDownloadingAllFiles + else { + return + } + didFinishDownloadingAllFiles = false + let jsonFiles = newFiles + newFiles.removeAll() Task { - let jsonFiles = newFiles.filter { $0.pathExtension == "json" } let jsonDecorder = JSONDecoder() jsonDecorder.dataDecodingStrategy = .deferredToData - //Task.detached { // @MainActor in//can we re-add this with the change of not adding a task on the PVSave asRealm() function? + Task.detached { // @MainActor in await jsonFiles.concurrentForEach { @MainActor json in let realm = try! await Realm() do { @@ -498,20 +569,21 @@ class SaveStateSyncer: iCloudTypeSyncer { return } } - //} + } } } } class RomsSyncer: iCloudTypeSyncer { var metadataQuery: NSMetadataQuery = .init() - var newFiles: Set = [] let gameImporter = GameImporter.shared - + var didFinishDownloadingAllFiles = false + var newFiles: Set = [] + var pendingFilesToDownload: Set = [] init() { - NotificationCenter.default.addObserver(self, selector: #selector(handleNewRomFiles), name: .RomDatabaseInitialized, object: nil) - NotificationCenter.default.addObserver(self, selector: #selector(handleNewFiles(_:)), name: .NewCloudFilesAvailable, object: self) + NotificationCenter.default.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil, using: handleNewRomFiles) } + deinit { DLOG("dying") } @@ -520,59 +592,91 @@ class RomsSyncer: iCloudTypeSyncer { "ROMs" } - @objc - func handleNewFiles(_ notification: Notification) { - guard let downloadedFiles = notification.userInfo?[notification.name.rawValue] as? Set + func setNewCloudFilesAvailable() { + didFinishDownloadingAllFiles = pendingFilesToDownload.isEmpty + } + + /// sends a notification that rom files are ready to e + func handleNewRomFiles(_ notification: Notification) { + NotificationCenter.default.removeObserver(self) + handleNewRomFiles() + } + + func insertDownloadingFile(_ file: URL) { + pendingFilesToDownload.insert(file.absoluteString) + } + + func insertDownloadedFile(_ file: URL) { + guard let _ = pendingFilesToDownload.remove(file.absoluteString) else { return } - newFiles = downloadedFiles + + let parentDirectory = file.deletingLastPathComponent().lastPathComponent + DLOG("adding file to game import queue: \(file), parent directory: \(parentDirectory)") + //we should only add to the import queue files that are actual ROMs, anything else can be ignored. + guard parentDirectory.range(of: "com.provenance.", + options: [.caseInsensitive, .anchored]) != nil + else { + return + } + + newFiles.insert(file) } - /// sends a notification that rom files are ready to e - @objc func handleNewRomFiles() { - //TODO: do we wait until the bios has been downloaded? + guard RomDatabase.databaseInitialized + else { + return + } guard !newFiles.isEmpty else { return } - - guard RomDatabase.databaseInitialized + guard didFinishDownloadingAllFiles else { return } - - var converted = [URL](newFiles) + didFinishDownloadingAllFiles = false + let importPaths = [URL](newFiles) newFiles.removeAll() - gameImporter.addImports(forPaths: converted) + gameImporter.addImports(forPaths: importPaths) gameImporter.startProcessing() } } class BiosSyncer: iCloudTypeSyncer { var metadataQuery: NSMetadataQuery = .init() - var newFiles: Set = [] var directory: String { "BIOS" } + + deinit { + DLOG("dying") + } } class BatterySavesSyncer: iCloudTypeSyncer { var metadataQuery: NSMetadataQuery = .init() - var newFiles: Set = [] var directory: String { "Battery Saves" } + + deinit { + DLOG("dying") + } } class ScreenshotsSyncer: iCloudTypeSyncer { var metadataQuery: NSMetadataQuery = .init() - var newFiles: Set = [] //TODO: I think a base class that accepts the directory in the initializer may be better than this var directory: String { "Screenshots" } + + deinit { + DLOG("dying") + } } From 7d090bf19f94950937abd85fa3fc56391ba7dcd4 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 25 Jan 2025 20:44:08 -0500 Subject: [PATCH 26/86] removed unused function --- .../Importer/iCloud/iCloudSync.swift | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index c8a9c9141f..4dbbdeff1b 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -226,58 +226,6 @@ extension iCloudTypeSyncer { setNewCloudFilesAvailable() DLOG("\(directory): current iteration: files pending to be downloaded: \(files.count), files downloaded : \(filesDownloaded.count)") } - - func queryFinished2(notification: Notification) async { - DLOG("directory: \(directory)") - guard (notification.object as? NSMetadataQuery) == metadataQuery - else { - return - } - let fileManager = FileManager.default - var files: Set = [] - //TODO: don't think we need this -// query.disableUpdates() -// defer { -// query.enableUpdates() -// } - let queue = DispatchQueue(label: "org.provenance-emu.provenance.newFiles") - - //accessing results automatically pauses updates and resumes after deallocated - await metadataQuery.results.concurrentForEach { item in - if let fileItem = item as? NSMetadataItem, - let file = fileItem.value(forAttribute: NSMetadataItemURLKey) as? URL, - let downloadStatus = fileItem.value(forAttribute: NSMetadataUbiquitousItemDownloadingStatusKey) as? String, - downloadStatus == NSMetadataUbiquitousItemDownloadingStatusNotDownloaded { - DLOG("Found file: \(String(describing: file)), download status: \(downloadStatus)") - do { - try fileManager.startDownloadingUbiquitousItem(at: file) - queue.sync { - files.insert(file) - insertDownloadingFile(file) - } - DLOG("Download started for: \(file.lastPathComponent)") - } catch { - DLOG("Failed to start download: \(error)") - } - } - } - //TODO: for ROMs and saves, perhaps we need to store the downloaded files that need to be process in the case of a crash or the user puts the app in the background. - //we wait for all files to finish downloaded from iCloud - let count = files.count - var pendingToDownload = files - while !pendingToDownload.isEmpty { - await pendingToDownload.concurrentForEach { file in - if fileManager.fileExists(atPath: file.path) { - queue.sync { - files.remove(file) - } - } - } - pendingToDownload = files - } - setNewCloudFilesAvailable() - DLOG("\(directory): downloaded files: \(count)") - } } extension SyncFileToiCloud where Self: LocalFileInfoProvider { From a8931cba1eaaa8d4b6df04314e06af693d9f44b7 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 26 Jan 2025 01:23:59 -0500 Subject: [PATCH 27/86] added deleting of ROMs when file is deleted on icloud --- .../Importer/iCloud/iCloudSync.swift | 51 +++++++++++++++++-- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 4dbbdeff1b..570c79e5da 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -51,6 +51,7 @@ public protocol iCloudTypeSyncer: Container { func removeAllFromICloud() -> Completable func insertDownloadingFile(_ file: URL) func insertDownloadedFile(_ file: URL) + func deleteFromDatastore(_ file: URL) func setNewCloudFilesAvailable() } @@ -84,6 +85,10 @@ extension iCloudTypeSyncer { //no-op } + func deleteFromDatastore(_ file: URL) { + //no-op + } + func loadAllFromICloud(iterationComplete: @escaping () -> Void) -> Completable { return Completable.create { completable in //body mustn't run in a task, otherwise the NotificationObserver closure won't get called @@ -191,6 +196,8 @@ extension iCloudTypeSyncer { var files: Set = [] var filesDownloaded: Set = [] let queue = DispatchQueue(label: "org.provenance-emu.provenance.newFiles") + let removedObjects = notification.userInfo?[NSMetadataQueryUpdateRemovedItemsKey] + DLOG("\(directory): removedObjects: \(removedObjects)") //accessing results automatically pauses updates and resumes after deallocated await metadataQuery.results.concurrentForEach { item in @@ -214,9 +221,14 @@ extension iCloudTypeSyncer { } case NSMetadataUbiquitousItemDownloadingStatusCurrent: DLOG("item up to date: \(file)") - queue.sync { - filesDownloaded.insert(file) - insertDownloadedFile(file) + if !fileManager.fileExists(atPath: file.path) { + DLOG("file DELETED from iCloud: \(file)") + deleteFromDatastore(file) + } else { + queue.sync { + filesDownloaded.insert(file) + insertDownloadedFile(file) + } } default: DLOG("\(file.lastPathComponent): download status: \(downloadStatus)") } @@ -410,7 +422,7 @@ class SaveStateSyncer: iCloudTypeSyncer { } func setNewCloudFilesAvailable() { - didFinishDownloadingAllFiles = pendingFilesToDownload.isEmpty + didFinishDownloadingAllFiles = pendingFilesToDownload.isEmpty && !newFiles.isEmpty } var directory: String { @@ -528,6 +540,7 @@ class RomsSyncer: iCloudTypeSyncer { var didFinishDownloadingAllFiles = false var newFiles: Set = [] var pendingFilesToDownload: Set = [] + init() { NotificationCenter.default.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil, using: handleNewRomFiles) } @@ -541,7 +554,7 @@ class RomsSyncer: iCloudTypeSyncer { } func setNewCloudFilesAvailable() { - didFinishDownloadingAllFiles = pendingFilesToDownload.isEmpty + didFinishDownloadingAllFiles = pendingFilesToDownload.isEmpty && !newFiles.isEmpty } /// sends a notification that rom files are ready to e @@ -572,6 +585,34 @@ class RomsSyncer: iCloudTypeSyncer { newFiles.insert(file) } + func deleteFromDatastore(_ file: URL) { + guard let fileName = file.lastPathComponent.removingPercentEncoding, + let parentDirectory = file.deletingLastPathComponent().lastPathComponent.removingPercentEncoding + else { + return + } + do { + let realm = try Realm() + let romPath = "\(parentDirectory)/\(fileName)" + DLOG("attempting to query PVGame by romPath: \(romPath)") + let results = realm.objects(PVGame.self).filter(NSPredicate(format: "\(NSExpression(forKeyPath: \PVGame.romPath.self).keyPath) == %@", romPath)) + guard let game: PVGame = results.first + else { + return + } + + try realm.write { + game.saveStates.forEach { try? $0.delete() } + game.cheats.forEach { try? $0.delete() } + game.recentPlays.forEach { try? $0.delete() } + game.screenShots.forEach { try? $0.delete() } + realm.delete(game) + } + } catch { + ELOG(error.localizedDescription) + } + } + func handleNewRomFiles() { guard RomDatabase.databaseInitialized else { From 82ad54e5ab41f9e1489d54673ec51dc19cda2ae3 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 26 Jan 2025 01:37:37 -0500 Subject: [PATCH 28/86] added todo for deleting savestate --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 570c79e5da..d4584fd4fa 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -443,6 +443,11 @@ class SaveStateSyncer: iCloudTypeSyncer { newFiles.insert(file) } + func deleteFromDatastore(_ file: URL) { +// PVSaveState + //TODO: delete from database + } + func importNewSaves(_ notification: Notification) { NotificationCenter.default.removeObserver(self) importNewSaves() From ed2a2ad738a6f7d6d462dcdbb78049381b0bb756 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 26 Jan 2025 20:56:48 -0500 Subject: [PATCH 29/86] added todo --- .../Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift index 69f78206d9..f012c1993d 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift @@ -112,6 +112,7 @@ public extension PVFile { var urlUpdate:URL { get { print("relativeRoot=\(relativeRoot)\tpartialPath=\(partialPath)") + //TODO: lazy load this so it's only done once let pathSuffix: String if let bundleIdentifier = Bundle.main.bundleIdentifier { print("Bundle Identifier: \(bundleIdentifier)") From 07f1b5840838f6aefa0fbfdef590f0446703f6fe Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 26 Jan 2025 20:57:11 -0500 Subject: [PATCH 30/86] removed field --- PVLibrary/Sources/PVLibrary/Database/Notifications.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/PVLibrary/Sources/PVLibrary/Database/Notifications.swift b/PVLibrary/Sources/PVLibrary/Database/Notifications.swift index 6989e19653..95cee7c5fe 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Notifications.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Notifications.swift @@ -10,7 +10,6 @@ import Foundation public extension Notification.Name { static let DatabaseMigrationStarted = Notification.Name("DatabaseMigrarionStarted") static let DatabaseMigrationFinished = Notification.Name("DatabaseMigrarionFinished") - static let NewCloudFilesAvailable = Notification.Name("NewCloudFilesAvailable") static let RomDatabaseInitialized = Notification.Name("RomDatabaseInitialized") static let RomsFinishedImporting = Notification.Name("RomsFinishedImporting") } From e7d416ea9bdd4f783a3a361ae6d54ba07f4aa27f Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 26 Jan 2025 21:34:49 -0500 Subject: [PATCH 31/86] refactored to listen for changes on icloud sync switch and init icloud when the flag is on and remove observers when off; --- .../Importer/iCloud/iCloudSync.swift | 379 ++++++++---------- Provenance/Main UI/PVAppDelegate.swift | 9 +- 2 files changed, 178 insertions(+), 210 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index d4584fd4fa..5fe22ada14 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -47,8 +47,7 @@ public protocol iCloudTypeSyncer: Container { var directory: String { get } var metadataQuery: NSMetadataQuery { get } - func loadAllFromICloud(iterationComplete: @escaping () -> Void) -> Completable - func removeAllFromICloud() -> Completable + func loadAllFromICloud(iterationComplete: (() -> Void)?) -> Completable func insertDownloadingFile(_ file: URL) func insertDownloadedFile(_ file: URL) func deleteFromDatastore(_ file: URL) @@ -72,7 +71,23 @@ final class NotificationObserver { } } -extension iCloudTypeSyncer { +class iCloudContainerSyncer: iCloudTypeSyncer { + lazy var pendingFilesToDownload: Set = [] + lazy var newFiles: Set = [] + let directory: String + + init(directory: String) { + self.directory = directory + } + + deinit { + metadataQuery.disableUpdates() + metadataQuery.stop() + DLOG("dying") + } + + var metadataQuery: NSMetadataQuery = .init() + func insertDownloadingFile(_ file: URL) { //no-op } @@ -89,100 +104,45 @@ extension iCloudTypeSyncer { //no-op } - func loadAllFromICloud(iterationComplete: @escaping () -> Void) -> Completable { - return Completable.create { completable in - //body mustn't run in a task, otherwise the NotificationObserver closure won't get called - guard containerURL != nil - else { - completable(.error(SyncError.noUbiquityURL)) - return Disposables.create {} - } - self.metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] - DLOG("directory: \(directory)") - metadataQuery.predicate = NSPredicate(format: "%K CONTAINS[c] %@", NSMetadataItemPathKey, "/Documents/\(directory)/") - //TODO: update to use Publishers.MergeMany - let _: NotificationObserver = .init( - forName: .NSMetadataQueryDidFinishGathering, - object: self.metadataQuery, - queue: nil) { notification in - Task { - await queryFinished(notification: notification) -// completable(.completed) - iterationComplete() - } - } - //listen for deletions and new files. what about conflicts? - let _: NotificationObserver = .init( - forName: .NSMetadataQueryDidUpdate, - object: self.metadataQuery, - queue: nil) { notification in - Task { - await queryFinished(notification: notification) -// completable(.completed) - iterationComplete() - } - } - - self.metadataQuery.start() - return Disposables.create {} + func loadAllFromICloud(iterationComplete: (() -> Void)? = nil) -> Completable { + return Completable.create { [weak self] completable in + self?.setupObservers(completable: completable, iterationComplete: iterationComplete) + return Disposables.create() } } - - public func removeAllFromICloud() -> Completable { - return Completable.create { completable in - Task { - - guard self.containerURL != nil else { - completable(.error(SyncError.noUbiquityURL)) - return Disposables.create {} + + func setupObservers(completable: PrimitiveSequenceType.CompletableObserver, iterationComplete: (() -> Void)? = nil) { + guard containerURL != nil + else { + completable(.error(SyncError.noUbiquityURL)) + return + } + metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] + DLOG("directory: \(directory)") + metadataQuery.predicate = NSPredicate(format: "%K CONTAINS[c] %@", NSMetadataItemPathKey, "/Documents/\(directory)/") + //TODO: update to use Publishers.MergeMany + let _: NotificationObserver = .init( + forName: .NSMetadataQueryDidFinishGathering, + object: metadataQuery, + queue: nil) { [weak self] notification in + Task { + await self?.queryFinished(notification: notification) + iterationComplete?() } - return Disposables.create {} } - // metadataQuery = NSMetadataQuery() - self.metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] -// self.metadataQuery.predicate = self.metadataQueryPredicate - - let token: NSObjectProtocol? = NotificationCenter.default.addObserver( - forName: Notification.Name.NSMetadataQueryDidFinishGathering, - object: self.metadataQuery, - queue: nil) { notification in - self.removeQueryFinished(notification: notification) -// if let token = token { -// NotificationCenter.default.removeObserver(token) -// } - completable(.completed) - } - -// token = NotificationCenter.default.addObserver( -// forName: Notification.Name.NSMetadataQueryDidUpdate, -// object: self.metadataQuery, -// queue: nil) { notification in -// self.queryFinished(notification: notification) -// } - self.metadataQuery.start() - return Disposables.create { - if let token = token { - NotificationCenter.default.removeObserver(token) + //listen for deletions and new files. what about conflicts? + let _: NotificationObserver = .init( + forName: .NSMetadataQueryDidUpdate, + object: metadataQuery, + queue: nil) { [weak self] notification in + Task { + await self?.queryFinished(notification: notification) + iterationComplete?() } } - } - } - - func removeQueryFinished(notification: Notification) { - let mq = notification.object as! NSMetadataQuery - mq.disableUpdates() - mq.stop() - - for i in 0.. = [] - var newFiles: Set = [] +class SaveStateSyncer: iCloudContainerSyncer { var didFinishDownloadingAllFiles = false - init() { - NotificationCenter.default.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil, using: importNewSaves) + let notificationCenter: NotificationCenter + + init(notificationCenter: NotificationCenter) { + self.notificationCenter = notificationCenter + super.init(directory: "Save States") + notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in + self?.importNewSaves() + } } + deinit { - DLOG("dying") + notificationCenter.removeObserver(self) } - func setNewCloudFilesAvailable() { + override func setNewCloudFilesAvailable() { didFinishDownloadingAllFiles = pendingFilesToDownload.isEmpty && !newFiles.isEmpty } - var directory: String { - "Save States" - } - - func insertDownloadingFile(_ file: URL) { + override func insertDownloadingFile(_ file: URL) { pendingFilesToDownload.insert(file.absoluteString) } - func insertDownloadedFile(_ file: URL) { + override func insertDownloadedFile(_ file: URL) { guard let _ = pendingFilesToDownload.remove(file.absoluteString), "json".caseInsensitiveCompare(file.pathExtension) == .orderedSame else { @@ -443,16 +450,11 @@ class SaveStateSyncer: iCloudTypeSyncer { newFiles.insert(file) } - func deleteFromDatastore(_ file: URL) { + override func deleteFromDatastore(_ file: URL) { // PVSaveState //TODO: delete from database } - func importNewSaves(_ notification: Notification) { - NotificationCenter.default.removeObserver(self) - importNewSaves() - } - func importNewSaves() { guard RomDatabase.databaseInitialized else { @@ -539,40 +541,31 @@ class SaveStateSyncer: iCloudTypeSyncer { } } -class RomsSyncer: iCloudTypeSyncer { - var metadataQuery: NSMetadataQuery = .init() +class RomsSyncer: iCloudContainerSyncer { let gameImporter = GameImporter.shared var didFinishDownloadingAllFiles = false - var newFiles: Set = [] - var pendingFilesToDownload: Set = [] - - init() { - NotificationCenter.default.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil, using: handleNewRomFiles) + let notificationCenter: NotificationCenter + init(notificationCenter: NotificationCenter) { + self.notificationCenter = notificationCenter + super.init(directory: "ROMs") + notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in + self?.importNewRomFiles() + } } deinit { - DLOG("dying") + notificationCenter.removeObserver(self) } - var directory: String { - "ROMs" - } - - func setNewCloudFilesAvailable() { + override func setNewCloudFilesAvailable() { didFinishDownloadingAllFiles = pendingFilesToDownload.isEmpty && !newFiles.isEmpty } - /// sends a notification that rom files are ready to e - func handleNewRomFiles(_ notification: Notification) { - NotificationCenter.default.removeObserver(self) - handleNewRomFiles() - } - - func insertDownloadingFile(_ file: URL) { + override func insertDownloadingFile(_ file: URL) { pendingFilesToDownload.insert(file.absoluteString) } - func insertDownloadedFile(_ file: URL) { + override func insertDownloadedFile(_ file: URL) { guard let _ = pendingFilesToDownload.remove(file.absoluteString) else { return @@ -590,7 +583,7 @@ class RomsSyncer: iCloudTypeSyncer { newFiles.insert(file) } - func deleteFromDatastore(_ file: URL) { + override func deleteFromDatastore(_ file: URL) { guard let fileName = file.lastPathComponent.removingPercentEncoding, let parentDirectory = file.deletingLastPathComponent().lastPathComponent.removingPercentEncoding else { @@ -618,7 +611,7 @@ class RomsSyncer: iCloudTypeSyncer { } } - func handleNewRomFiles() { + func importNewRomFiles() { guard RomDatabase.databaseInitialized else { return @@ -639,38 +632,20 @@ class RomsSyncer: iCloudTypeSyncer { } } -class BiosSyncer: iCloudTypeSyncer { - var metadataQuery: NSMetadataQuery = .init() - - var directory: String { - "BIOS" - } - - deinit { - DLOG("dying") +class BiosSyncer: iCloudContainerSyncer { + convenience init() { + self.init(directory: "BIOS") } } -class BatterySavesSyncer: iCloudTypeSyncer { - var metadataQuery: NSMetadataQuery = .init() - - var directory: String { - "Battery Saves" - } - - deinit { - DLOG("dying") +class BatterySavesSyncer: iCloudContainerSyncer { + convenience init() { + self.init(directory: "Battery Saves") } } -class ScreenshotsSyncer: iCloudTypeSyncer { - var metadataQuery: NSMetadataQuery = .init() - //TODO: I think a base class that accepts the directory in the initializer may be better than this - var directory: String { - "Screenshots" - } - - deinit { - DLOG("dying") +class ScreenshotsSyncer: iCloudContainerSyncer { + convenience init() { + self.init(directory: "Screenshots") } } diff --git a/Provenance/Main UI/PVAppDelegate.swift b/Provenance/Main UI/PVAppDelegate.swift index 305a4d537d..ea94d5d2b1 100644 --- a/Provenance/Main UI/PVAppDelegate.swift +++ b/Provenance/Main UI/PVAppDelegate.swift @@ -342,14 +342,7 @@ final class PVAppDelegate: UIResponder, GameLaunchingAppDelegate, UIApplicationD func _initICloud() { PVEmulatorConfiguration.initICloud() - DispatchQueue.global(qos: .background).async { - let useiCloud = Defaults[.iCloudSync] && URL.supportsICloud - if useiCloud { - DispatchQueue.main.async { - iCloudSync.initICloudDocuments() - } - } - } + iCloudSync.initICloudDocuments() } var currentThemeObservation: Any? // AnyCancellable? From 96deb549806f48797812c76fe14df834d5109a91 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 26 Jan 2025 22:44:11 -0500 Subject: [PATCH 32/86] reverted --- .../PVSwiftUI/Components/GameContextMenu.swift | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/PVUI/Sources/PVSwiftUI/Components/GameContextMenu.swift b/PVUI/Sources/PVSwiftUI/Components/GameContextMenu.swift index 0d0c625ebf..a788c9c02c 100644 --- a/PVUI/Sources/PVSwiftUI/Components/GameContextMenu.swift +++ b/PVUI/Sources/PVSwiftUI/Components/GameContextMenu.swift @@ -38,13 +38,10 @@ struct GameContextMenu: View { self.rootDelegate = rootDelegate self.contextMenuDelegate = contextMenuDelegate - //when syncing the system may be nil - if let currentSystem = game.system { - // Initialize computed properties - _availableCores = State(initialValue: currentSystem.cores.filter { - !(AppState.shared.isAppStore && $0.appStoreDisabled) - }) - } + // Initialize computed properties + _availableCores = State(initialValue: game.system?.cores.filter { + !(AppState.shared.isAppStore && $0.appStoreDisabled) + } ?? []) _hasSaveStates = State(initialValue: !game.saveStates.isEmpty) } From 15ef275c7b8a7b795bc0fadbe6a485d4a397a163 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 26 Jan 2025 22:44:33 -0500 Subject: [PATCH 33/86] added TODO --- .../PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift index 158f5294d2..5735b05c19 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift @@ -50,6 +50,7 @@ public final class PVImageFile: PVFile { } private func calculateSizeData() { // async { + //TODO: path is wrong when switching to iCloud let path = url.path // let size = await Task { () -> CGSize in From fb8a981ea506170666677a1795973a2d58dba969 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 26 Jan 2025 22:46:52 -0500 Subject: [PATCH 34/86] added updating game importer's roms path that was causing issuing when turning icloud on and off --- .../Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 5fe22ada14..99341765c0 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -140,8 +140,8 @@ class iCloudContainerSyncer: iCloudTypeSyncer { iterationComplete?() } } - Task { @MainActor [weak self] in -// DispatchQueue.main.async { [weak self] in +// Task { @MainActor [weak self] in + DispatchQueue.main.async { [weak self] in self?.metadataQuery.start() } } @@ -310,6 +310,7 @@ public enum iCloudSync { static var biosSyncer: BiosSyncer! static var batterySavesSyncer: BatterySavesSyncer! static var screenshotsSyncer: ScreenshotsSyncer! + static var gameImporter = GameImporter.shared public static func initICloudDocuments() { Task { @@ -339,6 +340,8 @@ public enum iCloudSync { static func turnOn() { DLOG("turning on iCloud") + //reset ROMs path + gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) //TODO: move files from local to cloud container let fm = FileManager.default if let currentiCloudToken = fm.ubiquityIdentityToken { @@ -412,6 +415,8 @@ public enum iCloudSync { static func turnOff() { DLOG("turning off iCloud") disposeBag = nil + //reset ROMs path + gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) //TODO: remove iCloud downloads. do we also copy those files locally? } } From 6a8b45ef5606c04ff6d21fdf3c7334d04385d8dc Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 26 Jan 2025 23:28:40 -0500 Subject: [PATCH 35/86] added TODO for review --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 99341765c0..e4e8eb2217 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -140,6 +140,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { iterationComplete?() } } + //TODO: unsure if the Task doesn't work with NSMetadataQuery or if there's some other issue. // Task { @MainActor [weak self] in DispatchQueue.main.async { [weak self] in self?.metadataQuery.start() From 92e5400f57ceb1d485d8039875b05e1d6980d6e2 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Tue, 28 Jan 2025 08:43:47 -0500 Subject: [PATCH 36/86] added check for invalid path on PVImageFile --- .../PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift index 5735b05c19..c262472f8f 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift @@ -52,6 +52,10 @@ public final class PVImageFile: PVFile { private func calculateSizeData() { // async { //TODO: path is wrong when switching to iCloud let path = url.path + if path.contains("var/mobile/Containers/Data/Application") + && (path.contains("Mobile%20Documents") || path.contains("Mobile Documents")) { + ELOG("invalid path: \(path)") + } // let size = await Task { () -> CGSize in #if canImport(UIKit) From 0edab03742d20eb4c487f4b5e932531cc54e3b4f Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 2 Feb 2025 21:11:51 -0500 Subject: [PATCH 37/86] added todo --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index e4e8eb2217..caef3dac69 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -590,6 +590,7 @@ class RomsSyncer: iCloudContainerSyncer { } override func deleteFromDatastore(_ file: URL) { + //TODO: remove cloud download, but keep in iCloud. this way the ROM isn't deleted from all devices guard let fileName = file.lastPathComponent.removingPercentEncoding, let parentDirectory = file.deletingLastPathComponent().lastPathComponent.removingPercentEncoding else { From 1fe056de4265719274fad324414d7f605cd0b82b Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 2 Feb 2025 21:52:17 -0500 Subject: [PATCH 38/86] updated to use DLOG instead of print; added condition when root is iCloud and partial path is local path to fix the path --- .../RealmPlatform/Entities/Files/PVFile.swift | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift index f012c1993d..f3c289b469 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift @@ -79,7 +79,7 @@ public extension PVFile { var url: URL { get { let url2 = urlUpdate - print("url2=\(url2)\tpartialPath=\(partialPath)") + DLOG("url2=\(url2)\tpartialPath=\(partialPath)") if partialPath.contains("iCloud") || partialPath.contains("private") { var pathComponents = (partialPath as NSString).pathComponents pathComponents.removeFirst() @@ -89,37 +89,37 @@ public extension PVFile { if isDocumentsDir { let iCloudBase = URL.iCloudContainerDirectory let url = (iCloudBase ?? RelativeRoot.documentsDirectory).appendingPathComponent(path) - print("url:\(url)") + DLOG("url:\(url)") return url2 } else { if let iCloudBase = URL.iCloudDocumentsDirectory { let appendedICloudBase = iCloudBase.appendingPathComponent(path) - print("appendedICloudBase:\(appendedICloudBase))") + DLOG("appendedICloudBase:\(appendedICloudBase))") return url2 } else { let appendedRelativeRoot = RelativeRoot.documentsDirectory.appendingPathComponent(path) - print("appendedRelativeRoot:\(appendedRelativeRoot)") + DLOG("appendedRelativeRoot:\(appendedRelativeRoot)") return url2 } } } let root = relativeRoot let resolvedURL = root.appendingPath(partialPath) - print("resolvedURL:\(resolvedURL))") + DLOG("resolvedURL:\(resolvedURL))") return url2 } } var urlUpdate:URL { get { - print("relativeRoot=\(relativeRoot)\tpartialPath=\(partialPath)") + DLOG("relativeRoot=\(relativeRoot)\tpartialPath=\(partialPath)") //TODO: lazy load this so it's only done once let pathSuffix: String if let bundleIdentifier = Bundle.main.bundleIdentifier { - print("Bundle Identifier: \(bundleIdentifier)") + DLOG("Bundle Identifier: \(bundleIdentifier)") let bundleComponents = bundleIdentifier.split(separator: ".") - print("bundleComponents=\(bundleComponents)") + DLOG("bundleComponents=\(bundleComponents)") let joined = bundleComponents.joined(separator: "~") - print("joined=\(joined)") + DLOG("joined=\(joined)") pathSuffix = joined } else { pathSuffix = "org~provenance-emu~provenance" @@ -134,12 +134,12 @@ public extension PVFile { completePath = partialPath } if let urlPath = URL(string: completePath) { - print("urlPath=\(urlPath)") + DLOG("urlPath=\(urlPath)") return urlPath } var pathComponents = (partialPath as NSString).pathComponents - print("pathComponents=\(pathComponents)") + DLOG("pathComponents=\(pathComponents)") //["private", "var", "mobile", "Library", "Mobile Documents", "iCloud~\(pathSuffix)", "Documents"] let mobileDocumentsEncoded = "Mobile%20Documents" let mobileDocumentsDecoded = "Mobile Documents" @@ -155,7 +155,7 @@ public extension PVFile { } else { directoryPath = "\(privateDirectory)/var/mobile/Library/\(mobileDocumentsDecoded)/\(mobileDocumentsDecoded)/iCloud~\(pathSuffix)" } - print("directoryPath=\(directoryPath)") + DLOG("directoryPath=\(directoryPath)") var prefixes = directoryPath.split(separator: "/") let mobileDocumentsEncodedSub = mobileDocumentsEncoded.prefix(mobileDocumentsEncoded.count) //we also add an encoded one. @@ -167,7 +167,7 @@ public extension PVFile { if !prefixes.contains(mobileDocumentsDecodedSub) { prefixes.append(mobileDocumentsDecodedSub) } - print("prefixes=\(prefixes)") + DLOG("prefixes=\(prefixes)") while prefixes.contains(where: {String($0) == pathComponents.first}) { /* Action Button Pressed 1706495469592 Optional(1706495461875) @@ -181,30 +181,37 @@ public extension PVFile { url=file:///private/var/mobile/Library/Mobile%20Documents/iCloud~com~pqskapps~provenance/var/mobile/Library/Mobile%20Documents/iCloud~com~pqskapps~provenance/Documents/Save%20States/Gremlins%20(USA).a52/DC271E475B4766E80151F1DA5B764E52.728185244.058265.svs */ pathComponents.removeFirst() - print("pathComponentslremoveFirst()=\(pathComponents)") + DLOG("pathComponentslremoveFirst()=\(pathComponents)") } let path = pathComponents.joined(separator: "/") - print("path=\(path)") - print("PVEmulatorConfiguration.iCloudContainerDirectory=\(String(describing: URL.iCloudContainerDirectory))") - print("PVEmulatorConfiguration.iCloudDocumentsDirectory=\(String(describing: URL.iCloudDocumentsDirectory))") + DLOG("path=\(path)") + DLOG("PVEmulatorConfiguration.iCloudContainerDirectory=\(String(describing: URL.iCloudContainerDirectory))") + DLOG("PVEmulatorConfiguration.iCloudDocumentsDirectory=\(String(describing: URL.iCloudDocumentsDirectory))") let iCloudBase = path.contains("Documents") ? URL.iCloudContainerDirectory : URL.iCloudDocumentsDirectory - print("iCloudBase=\(String(describing: iCloudBase))") + DLOG("iCloudBase=\(String(describing: iCloudBase))") let url = (iCloudBase ?? RelativeRoot.documentsDirectory).appendingPathComponent(path) - print("url=\(url)") + DLOG("url=\(url)") return url } let root = relativeRoot - print("root=\(root)") + DLOG("root=\(root)") var actualPartialPath = partialPath - print("actualPartialPath=\(actualPartialPath)") + if root == .iCloud && partialPath.starts(with: "var/mobile/Containers/Data/Application/") { + DLOG("iCloud path, but partialPath does NOT contain iCloud path") + var partialPathComponents = partialPath.components(separatedBy: "/") + let directoriesToRemove = partialPathComponents.count >= 7 ? 7 : partialPathComponents.count + partialPathComponents.removeFirst(directoriesToRemove) + actualPartialPath = partialPathComponents.joined(separator: "/") + } + DLOG("actualPartialPath=\(actualPartialPath)") if partialPath.hasPrefix(privateDirectory) { var tmp = partialPath.split(separator: "/") tmp.removeFirst() actualPartialPath = tmp.joined(separator: "/") } - print("actualPartialPath=\(actualPartialPath)") + DLOG("actualPartialPath=\(actualPartialPath)") let resolvedURL = root.appendingPath(actualPartialPath) - print("resolvedURL=\(resolvedURL)") + DLOG("resolvedURL=\(resolvedURL)") return resolvedURL /* relativeRoot=iCloud partialPath=var/mobile/Containers/Data/Application/B8153B85-9BB5-44B6-A189-FDE9D8ABC29C/Documents/PVCache/F62D5AA941BB70E1913B787A65CD7EFC From b5d647fabbe710fb3750af36ebd52eccb07796d3 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 2 Feb 2025 22:04:42 -0500 Subject: [PATCH 39/86] updated cache to save locally; updated debug log print statement --- .../Services/GameImporter/GameImporterDatabaseService.swift | 2 +- .../Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index de4c0732e0..00e63122c6 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -237,7 +237,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { if let artwork = UIImage(data: data) { do { let localURL = try PVMediaCache.writeImage(toDisk: artwork, withKey: url) - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) + let file = PVImageFile(withURL: localURL, relativeRoot: .documents) game.originalArtworkFile = file } catch { ELOG("\(error.localizedDescription)") } } diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift index f3c289b469..46a896c554 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift @@ -197,7 +197,7 @@ public extension PVFile { DLOG("root=\(root)") var actualPartialPath = partialPath if root == .iCloud && partialPath.starts(with: "var/mobile/Containers/Data/Application/") { - DLOG("iCloud path, but partialPath does NOT contain iCloud path") + DLOG("iCloud path, but partialPath does NOT contain iCloud path, but instead local path") var partialPathComponents = partialPath.components(separatedBy: "/") let directoriesToRemove = partialPathComponents.count >= 7 ? 7 : partialPathComponents.count partialPathComponents.removeFirst(directoriesToRemove) From 1757a7feaf7e1479d33532672ee420d75dfc9373 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 2 Feb 2025 22:06:52 -0500 Subject: [PATCH 40/86] added todo --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index caef3dac69..1ce0ad3fb3 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -474,6 +474,7 @@ class SaveStateSyncer: iCloudContainerSyncer { else { return } + //TODO: initially when importing icloud files, we should wait for ROMs to finish importing. when that completes, then we no longer have to wait for that, ie when syncing single changes say 2 devices are open at the same time and 1 save file is added from another device didFinishDownloadingAllFiles = false let jsonFiles = newFiles newFiles.removeAll() From 254d5babfedecf3700286a6c89d02dcc0b3ae7c5 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 3 Feb 2025 20:59:52 -0500 Subject: [PATCH 41/86] updated save path --- .../Conversion/Extensions/SaveState+PVSaveState.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift index 83aa74e8eb..16b03c11b5 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Conversion/Extensions/SaveState+PVSaveState.swift @@ -63,8 +63,8 @@ extension SaveState: RealmRepresentable { } else { object.core = core.asRealm() } - - let path = game.file.fileName.saveStatePath.appendingPathComponent(file.fileName) + //we remove the extension in order to get the correct path + let path = game.file.fileName.saveStatePath.deletingPathExtension().appendingPathComponent(file.fileName) object.file = PVFile(withURL: path) DLOG("file path: \(path)") From 0de1a038150c2dac23f1bfe3920beff7c5860ce4 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 3 Feb 2025 21:33:36 -0500 Subject: [PATCH 42/86] removed unused code; updated try to not force on Realm in case of db error --- .../Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 1ce0ad3fb3..2f02f56ed7 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -333,10 +333,7 @@ public enum iCloudSync { DLOG("attempted to turn on iCloud, but iCloud is NOT setup on the device") return } -// Task { @MainActor in -// DispatchQueue.main.async { - turnOn() -// } + turnOn() } static func turnOn() { @@ -484,9 +481,8 @@ class SaveStateSyncer: iCloudContainerSyncer { Task.detached { // @MainActor in await jsonFiles.concurrentForEach { @MainActor json in - let realm = try! await Realm() do { - + let realm = try await Realm() let secureDoc = json.startAccessingSecurityScopedResource() defer { From cbe334ec09fc8c202fb80b777ed8dcf146aac697 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 3 Feb 2025 22:04:02 -0500 Subject: [PATCH 43/86] removed unused closure onCompleted --- .../Importer/iCloud/iCloudSync.swift | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 2f02f56ed7..4ca88dcc7f 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -358,16 +358,15 @@ public enum iCloudSync { saveStateSyncer = .init(notificationCenter: .default) saveStateSyncer.loadAllFromICloud() { - saveStateSyncer.importNewSaves() - }.observe(on: MainScheduler.instance) - .subscribe(onCompleted: { - }, onError: { error in - ELOG(error.localizedDescription) - }) { - DLOG("disposing saveStateSyncer") - saveStateSyncer = nil - - }.disposed(by: disposeBag) + saveStateSyncer.importNewSaves() + }.observe(on: MainScheduler.instance) + .subscribe(onError: { error in + ELOG(error.localizedDescription) + }) { + DLOG("disposing saveStateSyncer") + saveStateSyncer = nil + + }.disposed(by: disposeBag) var romsSyncer: RomsSyncer! = .init(notificationCenter: .default) romsSyncer.loadAllFromICloud() { From bfd7b14dd9679d7907767f30ae2c33247dbff26b Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 3 Feb 2025 22:05:46 -0500 Subject: [PATCH 44/86] removed commented out code --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 4ca88dcc7f..f7bba4df43 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -365,7 +365,6 @@ public enum iCloudSync { }) { DLOG("disposing saveStateSyncer") saveStateSyncer = nil - }.disposed(by: disposeBag) var romsSyncer: RomsSyncer! = .init(notificationCenter: .default) @@ -376,7 +375,6 @@ public enum iCloudSync { ELOG(error.localizedDescription) }) { DLOG("disposing romsSyncer") -// romsSyncer.removeObservers() romsSyncer = nil }.disposed(by: disposeBag) //TODO: set the following to merge onto a single class that just does a query for icloud for all of those directories. From 561be703bf421a82a2f36e7dda6936bfe1c3d55a Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Wed, 5 Feb 2025 23:06:02 -0500 Subject: [PATCH 45/86] added recursive code to move files from local to icloud container --- .../Importer/iCloud/iCloudSync.swift | 97 ++++++++++++++----- 1 file changed, 71 insertions(+), 26 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index f7bba4df43..bb44127edf 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -37,10 +37,11 @@ extension Container { } public protocol SyncFileToiCloud: Container { + var fileManager: FileManager { get } var metadataQuery: NSMetadataQuery { get } - func syncToiCloud(completionHandler: @escaping (SyncResult) -> Void) // -> Single + func syncToiCloud(completionHandler: @escaping (SyncResult) -> Void) async// -> Single func queryFile(completionHandler: @escaping (URL?) -> Void) // -> Single - func downloadingFile(completionHandler: @escaping (SyncResult) -> Void) // -> Single + func downloadingFile(completionHandler: @escaping (SyncResult) -> Void) async // -> Single } public protocol iCloudTypeSyncer: Container { @@ -66,7 +67,7 @@ final class NotificationObserver { observer = center.addObserver(forName: name, object: object, queue: queue, using: block) } - deinit {//because this was created inline, deinit gets called right away. does this ever need to be removed? shouldn't this be in the lifetime of the application? + deinit {//TODO: because this was created inline, deinit gets called right away. does this ever need to be removed? shouldn't this be in the lifetime of the application? //center.removeObserver(observer, name: name, object: object) } } @@ -86,7 +87,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { DLOG("dying") } - var metadataQuery: NSMetadataQuery = .init() + let metadataQuery: NSMetadataQuery = .init() func insertDownloadingFile(_ file: URL) { //no-op @@ -208,35 +209,65 @@ extension SyncFileToiCloud where Self: LocalFileInfoProvider { return containerURL.appendingPathComponent(url.relativePath) }.value }} - - func syncToiCloud() async -> SyncResult { + + //TODO: refactor this on the syncer + func syncToiCloud(completionHandler: @escaping (SyncResult) -> Void) async { await Task { - guard let destinationURL = await self.destinationURL else { - return SyncResult.denied - } - let url = self.url - - self.metadataQuery.disableUpdates() - defer { - self.metadataQuery.enableUpdates() + DLOG("url: \(url)") + guard fileManager.fileExists(atPath: url.path), + let actualContainerUrl = containerURL + else { + completionHandler(.fileNotExist) + return } - - let fm = FileManager.default - if fm.fileExists(atPath: url.path) { - try! await fm.removeItem(at: url) + guard let destinationURL = await self.destinationURL else { + return completionHandler(.denied) } +// let url = self.url +// +// self.metadataQuery.disableUpdates() +// defer { +// self.metadataQuery.enableUpdates() +// } - do { - ILOG("Trying to set Ubiquitious from local (\(url.path)) to ICloud (\(destinationURL.path))") - try fm.setUbiquitous(true, itemAt: url, destinationURL: destinationURL) - return .success - } catch { - ELOG("iCloud failed to set Ubiquitous: \(error.localizedDescription)") - return .saveFailure - } + completionHandler(await moveFiles(at: url, container: actualContainerUrl)) }.value } + + func moveFiles(at current: URL, container: URL) async -> SyncResult { + do { + let subdirectories = try fileManager.subpathsOfDirectory(atPath: current.path) + DLOG("subdirectories of \(current): \(subdirectories)") + let directoryContents = try try fileManager.contentsOfDirectory(at: current, includingPropertiesForKeys: []) + DLOG("directoryContents of \(current): \(directoryContents)") + for currentItem in directoryContents { + var isDirectory: ObjCBool = false + let exists = fileManager.fileExists(atPath: currentItem.path, isDirectory: &isDirectory) + if exists && isDirectory.boolValue { + //TODO: should we just ignore and try to move as many as we can? this could be if storage is low + let resultSub = await moveFiles(at: currentItem, container: container) + guard resultSub == .success + else { + return resultSub + } + } + do { + let destination = container.appendingPathComponent(currentItem.relativePath) + ILOG("Trying to set Ubiquitious from local (\(current.path)) to ICloud (\(destination.path))") + try fileManager.setUbiquitous(true, itemAt: currentItem, destinationURL: destination) + try await fileManager.removeItem(at: current) + } catch { + //this could indicate no more space is left + ELOG("iCloud failed to set Ubiquitous: \(error.localizedDescription)") + } + } + return .success + } catch { + ELOG("failed to get directory contents: \(error.localizedDescription)") + return .saveFailure + } + } /// - Parameter completionHandler: Non-main func queryFile(completionHandler: @escaping (URL?) -> Void) { @@ -307,11 +338,14 @@ enum iCloudError: Error { public enum iCloudSync { static var disposeBag: DisposeBag! + //syncers static var saveStateSyncer: SaveStateSyncer! static var biosSyncer: BiosSyncer! static var batterySavesSyncer: BatterySavesSyncer! static var screenshotsSyncer: ScreenshotsSyncer! static var gameImporter = GameImporter.shared + //initial uploaders +// static var /*saveStateUploader: SyncFileToiCloud = */ public static func initICloudDocuments() { Task { @@ -415,6 +449,9 @@ public enum iCloudSync { //TODO: remove iCloud downloads. do we also copy those files locally? } } + +//MARK: - iCloud syncers + //TODO: perhaps 1 generic class since a lot of this code is similar and move the extension onto generic class. we could just add a protocol delegate dependency for ROMs and SaveState classes that does specific code class SaveStateSyncer: iCloudContainerSyncer { var didFinishDownloadingAllFiles = false @@ -650,3 +687,11 @@ class ScreenshotsSyncer: iCloudContainerSyncer { self.init(directory: "Screenshots") } } + +//MARK: - iCloud initial container uploaders + +class SaveStateUploader: SyncFileToiCloud, LocalFileInfoProvider { + let fileManager = FileManager.default + let metadataQuery: NSMetadataQuery = .init() + let url: URL = URL.documentsDirectory.appendingPathComponent("Save States") +} From 7bbf493c099ef301e55801e4f0fdff8999119e59 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Thu, 6 Feb 2025 22:10:33 -0500 Subject: [PATCH 46/86] updated to get correct files and tree and to identify directories that need to be created and new location of cloud file --- .../Importer/iCloud/iCloudSync.swift | 58 +++++++++++++------ 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index bb44127edf..e4251dc054 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -221,9 +221,9 @@ extension SyncFileToiCloud where Self: LocalFileInfoProvider { completionHandler(.fileNotExist) return } - guard let destinationURL = await self.destinationURL else { - return completionHandler(.denied) - } +// guard let destinationURL = await self.destinationURL else { +// return completionHandler(.denied) +// } // let url = self.url // // self.metadataQuery.disableUpdates() @@ -231,7 +231,7 @@ extension SyncFileToiCloud where Self: LocalFileInfoProvider { // self.metadataQuery.enableUpdates() // } - completionHandler(await moveFiles(at: url, container: actualContainerUrl)) + completionHandler(await moveFiles(at: url, container: actualContainerUrl.appendingPathComponent("Documents"))) }.value } @@ -241,22 +241,34 @@ extension SyncFileToiCloud where Self: LocalFileInfoProvider { DLOG("subdirectories of \(current): \(subdirectories)") let directoryContents = try try fileManager.contentsOfDirectory(at: current, includingPropertiesForKeys: []) DLOG("directoryContents of \(current): \(directoryContents)") - for currentItem in directoryContents { - var isDirectory: ObjCBool = false - let exists = fileManager.fileExists(atPath: currentItem.path, isDirectory: &isDirectory) - if exists && isDirectory.boolValue { - //TODO: should we just ignore and try to move as many as we can? this could be if storage is low - let resultSub = await moveFiles(at: currentItem, container: container) - guard resultSub == .success - else { - return resultSub - } + for currentChild in subdirectories { + let currentItem = current.appendingPathComponent(currentChild) +// var isDirectory: ObjCBool = false +// let exists = fileManager.fileExists(atPath: currentItem.path, isDirectory: &isDirectory) + let isDirectory = currentItem.pathExtension.allSatisfy({$0.isWhitespace}) +// DLOG("\(currentItem): isDirectory?\(isDirectory), exists?\(exists)") + DLOG("\(currentItem) isDirectory?\(isDirectory)") + let iCloudDestination = container.appendingPathComponent(currentChild) + DLOG("new iCloud directory: \(iCloudDestination)") + if isDirectory && !fileManager.fileExists(atPath: iCloudDestination.path) { + DLOG("\(iCloudDestination) does NOT exist") +// try fileManager.createDirectory(atPath: iCloudDirectory.path, withIntermediateDirectories: false) +// if !exists || !isDirectory.boolValue { +// //TODO: should we just ignore and try to move as many as we can? this could be if storage is low +// let resultSub = await moveFiles(at: currentItem, container: container) +// guard resultSub == .success +// else { +// return resultSub +// } + } + if isDirectory { + continue } do { - let destination = container.appendingPathComponent(currentItem.relativePath) - ILOG("Trying to set Ubiquitious from local (\(current.path)) to ICloud (\(destination.path))") - try fileManager.setUbiquitous(true, itemAt: currentItem, destinationURL: destination) - try await fileManager.removeItem(at: current) + ILOG("Trying to set Ubiquitious from local (\(currentItem.path)) to ICloud (\(iCloudDestination.path))") + //TODO: uncomment when ready +// try fileManager.setUbiquitous(true, itemAt: currentItem, destinationURL: destination) +// try await fileManager.removeItem(at: currentItem) } catch { //this could indicate no more space is left ELOG("iCloud failed to set Ubiquitous: \(error.localizedDescription)") @@ -345,7 +357,7 @@ public enum iCloudSync { static var screenshotsSyncer: ScreenshotsSyncer! static var gameImporter = GameImporter.shared //initial uploaders -// static var /*saveStateUploader: SyncFileToiCloud = */ + static var saveStateUploader: SyncFileToiCloud = SaveStateUploader() public static func initICloudDocuments() { Task { @@ -375,6 +387,14 @@ public enum iCloudSync { //reset ROMs path gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) //TODO: move files from local to cloud container + Task { + await saveStateUploader.syncToiCloud { completion in + DLOG("syncToiCloud result: \(completion)") + } + } + if 1==1 { + return + } let fm = FileManager.default if let currentiCloudToken = fm.ubiquityIdentityToken { do { From 957fc53cd13e1f0c65283be12856957685b6da3a Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Thu, 6 Feb 2025 22:34:09 -0500 Subject: [PATCH 47/86] Finished main code to move files to cloud container --- .../Importer/iCloud/iCloudSync.swift | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index e4251dc054..2c991ba0ee 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -231,7 +231,7 @@ extension SyncFileToiCloud where Self: LocalFileInfoProvider { // self.metadataQuery.enableUpdates() // } - completionHandler(await moveFiles(at: url, container: actualContainerUrl.appendingPathComponent("Documents"))) + completionHandler(await moveFiles(at: url, container: actualContainerUrl.appendingPathComponent("Documents").appendingPathComponent(url.lastPathComponent))) }.value } @@ -239,36 +239,22 @@ extension SyncFileToiCloud where Self: LocalFileInfoProvider { do { let subdirectories = try fileManager.subpathsOfDirectory(atPath: current.path) DLOG("subdirectories of \(current): \(subdirectories)") - let directoryContents = try try fileManager.contentsOfDirectory(at: current, includingPropertiesForKeys: []) - DLOG("directoryContents of \(current): \(directoryContents)") for currentChild in subdirectories { let currentItem = current.appendingPathComponent(currentChild) -// var isDirectory: ObjCBool = false -// let exists = fileManager.fileExists(atPath: currentItem.path, isDirectory: &isDirectory) let isDirectory = currentItem.pathExtension.allSatisfy({$0.isWhitespace}) -// DLOG("\(currentItem): isDirectory?\(isDirectory), exists?\(exists)") DLOG("\(currentItem) isDirectory?\(isDirectory)") let iCloudDestination = container.appendingPathComponent(currentChild) DLOG("new iCloud directory: \(iCloudDestination)") if isDirectory && !fileManager.fileExists(atPath: iCloudDestination.path) { DLOG("\(iCloudDestination) does NOT exist") -// try fileManager.createDirectory(atPath: iCloudDirectory.path, withIntermediateDirectories: false) -// if !exists || !isDirectory.boolValue { -// //TODO: should we just ignore and try to move as many as we can? this could be if storage is low -// let resultSub = await moveFiles(at: currentItem, container: container) -// guard resultSub == .success -// else { -// return resultSub -// } + try fileManager.createDirectory(atPath: iCloudDestination.path, withIntermediateDirectories: false) } if isDirectory { continue } do { ILOG("Trying to set Ubiquitious from local (\(currentItem.path)) to ICloud (\(iCloudDestination.path))") - //TODO: uncomment when ready -// try fileManager.setUbiquitous(true, itemAt: currentItem, destinationURL: destination) -// try await fileManager.removeItem(at: currentItem) + try fileManager.setUbiquitous(true, itemAt: currentItem, destinationURL: iCloudDestination) } catch { //this could indicate no more space is left ELOG("iCloud failed to set Ubiquitous: \(error.localizedDescription)") From 1c0529df94d9b4836e505d5900da75d731c9937e Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Thu, 6 Feb 2025 22:44:40 -0500 Subject: [PATCH 48/86] added check if file exists in cloud and deleting local version --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 2c991ba0ee..4dbc75b7b4 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -252,6 +252,10 @@ extension SyncFileToiCloud where Self: LocalFileInfoProvider { if isDirectory { continue } + if fileManager.fileExists(atPath: iCloudDestination.path) { + try fileManager.removeItem(atPath: currentItem.path) + continue + } do { ILOG("Trying to set Ubiquitious from local (\(currentItem.path)) to ICloud (\(iCloudDestination.path))") try fileManager.setUbiquitous(true, itemAt: currentItem, destinationURL: iCloudDestination) From ed1eff429d602f528032c1ea6eaede9a1b17ed9c Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Thu, 6 Feb 2025 23:08:58 -0500 Subject: [PATCH 49/86] fixed check for directory, added uploader for ROMs --- .../Importer/iCloud/iCloudSync.swift | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 4dbc75b7b4..8848108528 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -241,15 +241,17 @@ extension SyncFileToiCloud where Self: LocalFileInfoProvider { DLOG("subdirectories of \(current): \(subdirectories)") for currentChild in subdirectories { let currentItem = current.appendingPathComponent(currentChild) - let isDirectory = currentItem.pathExtension.allSatisfy({$0.isWhitespace}) - DLOG("\(currentItem) isDirectory?\(isDirectory)") + + var isDirectory: ObjCBool = false + let exists = fileManager.fileExists(atPath: currentItem.path, isDirectory: &isDirectory) + DLOG("\(currentItem) isDirectory?\(isDirectory) exists?\(exists)") let iCloudDestination = container.appendingPathComponent(currentChild) DLOG("new iCloud directory: \(iCloudDestination)") - if isDirectory && !fileManager.fileExists(atPath: iCloudDestination.path) { + if isDirectory.boolValue && !fileManager.fileExists(atPath: iCloudDestination.path) { DLOG("\(iCloudDestination) does NOT exist") try fileManager.createDirectory(atPath: iCloudDestination.path, withIntermediateDirectories: false) } - if isDirectory { + if isDirectory.boolValue { continue } if fileManager.fileExists(atPath: iCloudDestination.path) { @@ -347,7 +349,8 @@ public enum iCloudSync { static var screenshotsSyncer: ScreenshotsSyncer! static var gameImporter = GameImporter.shared //initial uploaders - static var saveStateUploader: SyncFileToiCloud = SaveStateUploader() + static var saveStateUploader = SaveStateUploader() + static var romsUploader = RomsUploader() public static func initICloudDocuments() { Task { @@ -379,7 +382,10 @@ public enum iCloudSync { //TODO: move files from local to cloud container Task { await saveStateUploader.syncToiCloud { completion in - DLOG("syncToiCloud result: \(completion)") + DLOG("saveStateUploader syncToiCloud result: \(completion)") + } + await romsUploader.syncToiCloud { completion in + DLOG("romsUploader syncToiCloud result: \(completion)") } } if 1==1 { @@ -705,3 +711,9 @@ class SaveStateUploader: SyncFileToiCloud, LocalFileInfoProvider { let metadataQuery: NSMetadataQuery = .init() let url: URL = URL.documentsDirectory.appendingPathComponent("Save States") } + +class RomsUploader: SyncFileToiCloud, LocalFileInfoProvider { + let fileManager = FileManager.default + let metadataQuery: NSMetadataQuery = .init() + let url: URL = URL.documentsDirectory.appendingPathComponent("ROMs") +} From c70e781293e669e42d8e6a0c1eb3b287f3084c2d Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Thu, 6 Feb 2025 23:10:45 -0500 Subject: [PATCH 50/86] added TODO --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 8848108528..9f5f9486a9 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -388,6 +388,7 @@ public enum iCloudSync { DLOG("romsUploader syncToiCloud result: \(completion)") } } + //TODO: first we upload anything pending, then we download anything that already exists and import those. we will have to update the locations of existing saves and ROMs in the db if 1==1 { return } From 0827eb927c04ece1e29c000bfad22cf1713583bf Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 8 Feb 2025 19:57:00 -0500 Subject: [PATCH 51/86] updated relativeRoot and added todos to check invalid paths --- .../Realm Database/RomDatabase+Saves.swift | 4 +- .../GameImporterDatabaseService.swift | 6 +-- .../RealmPlatform/Entities/Files/PVFile.swift | 37 +++++++++++++++++-- .../Entities/Files/PVImageFile.swift | 4 ++ 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase+Saves.swift b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase+Saves.swift index ac80f05044..752c0b587f 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase+Saves.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase+Saves.swift @@ -198,8 +198,8 @@ public extension RomDatabase { guard let core = realm.object(ofType: PVCore.self, forPrimaryKey: core.coreIdentifier) else { throw SaveStateError.noCoreFound(core.coreIdentifier ?? "null") } - let imgFile = PVImageFile(withURL: URL(fileURLWithPath: url.path.replacingOccurrences(of: "svs", with: "jpg"))) - let saveFile = PVFile(withURL: url) + let imgFile = PVImageFile(withURL: URL(fileURLWithPath: url.path.replacingOccurrences(of: "svs", with: "jpg")), relativeRoot: .iCloud) + let saveFile = PVFile(withURL: url, relativeRoot: .iCloud) let newState = PVSaveState(withGame: game, core: core, file: saveFile, image: imgFile, isAutosave: false) try realm.write { realm.add(newState) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift index 00e63122c6..330dc31665 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterDatabaseService.swift @@ -135,7 +135,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { throw GameImporterError.noSystemMatched } - let file = PVFile(withURL: queueItem.destinationUrl!) + let file = PVFile(withURL: queueItem.destinationUrl!, relativeRoot: .iCloud) let game = PVGame(withFile: file, system: system) game.romPath = partialPath game.title = title @@ -205,7 +205,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { } if PVMediaCache.fileExists(forKey: url) { if let localURL = PVMediaCache.filePath(forKey: url) { - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) + let file = PVImageFile(withURL: localURL, relativeRoot: .documents) game.originalArtworkFile = file return game } @@ -229,7 +229,7 @@ class GameImporterDatabaseService : GameImporterDatabaseServicing { if let artwork = NSImage(data: data) { do { let localURL = try PVMediaCache.writeImage(toDisk: artwork, withKey: url) - let file = PVImageFile(withURL: localURL, relativeRoot: .iCloud) + let file = PVImageFile(withURL: localURL, relativeRoot: .documents) game.originalArtworkFile = file } catch { ELOG("\(error.localizedDescription)") } } diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift index 46a896c554..b8b87323d4 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift @@ -52,7 +52,12 @@ public class PVFile: Object, LocalFileProvider, Codable, DomainConvertibleType { public convenience init(withURL url: URL, relativeRoot: RelativeRoot = RelativeRoot.platformDefault, size: Int = 0, md5: String? = nil) { self.init() self.relativeRoot = relativeRoot + //TODO: this isn't working to get the partial path in all cases partialPath = relativeRoot.createRelativePath(fromURL: url) + //TODO: remove + if doesPathContainParent(partialPath) { + DLOG("partialPath: \(partialPath)") + } self.md5Cache = md5 if size > 0 { self.sizeCache = size @@ -78,6 +83,7 @@ public extension PVFile { var url: URL { get { + //TODO: if relativeRoot == .iCloud, AND partialPath is NOT a partial path, then remove the prefix (cloudContainer/Documents), this will be older db entries let url2 = urlUpdate DLOG("url2=\(url2)\tpartialPath=\(partialPath)") if partialPath.contains("iCloud") || partialPath.contains("private") { @@ -90,23 +96,36 @@ public extension PVFile { let iCloudBase = URL.iCloudContainerDirectory let url = (iCloudBase ?? RelativeRoot.documentsDirectory).appendingPathComponent(path) DLOG("url:\(url)") - return url2 + //TODO: new return url2 + if doesPathContainParent(url.path) { + DLOG("invalid url:\(url)") + } + return url } else { if let iCloudBase = URL.iCloudDocumentsDirectory { let appendedICloudBase = iCloudBase.appendingPathComponent(path) DLOG("appendedICloudBase:\(appendedICloudBase))") - return url2 + //TODO: new return url2 + if doesPathContainParent(appendedICloudBase.path) { + DLOG("invalid url:\(appendedICloudBase)") + } + return appendedICloudBase } else { let appendedRelativeRoot = RelativeRoot.documentsDirectory.appendingPathComponent(path) DLOG("appendedRelativeRoot:\(appendedRelativeRoot)") - return url2 + //TODO: new return url2 + if doesPathContainParent(appendedRelativeRoot.path) { + DLOG("invalid url:\(appendedRelativeRoot)") + } + return appendedRelativeRoot } } } let root = relativeRoot let resolvedURL = root.appendingPath(partialPath) DLOG("resolvedURL:\(resolvedURL))") - return url2 + //TODO: new return url2 + return resolvedURL } } var urlUpdate:URL { @@ -195,6 +214,9 @@ public extension PVFile { } let root = relativeRoot DLOG("root=\(root)") + if doesPathContainParent(partialPath) { + DLOG("invalid path: \(partialPath)") + } var actualPartialPath = partialPath if root == .iCloud && partialPath.starts(with: "var/mobile/Containers/Data/Application/") { DLOG("iCloud path, but partialPath does NOT contain iCloud path, but instead local path") @@ -222,6 +244,13 @@ public extension PVFile { */ } } + + func doesPathContainParent(_ path: String) -> Bool { + return path.starts(with: "private/var/mobile/Library/Mobile") + || path.starts(with: "var/mobile/Containers/Data/Application") + || path.starts(with: "var/mobile/Containers/") + || path.starts(with: "private/var/mobile") + } private func setURL(_ url: URL) { do { diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift index c262472f8f..66a57ea1bc 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift @@ -41,6 +41,10 @@ public final class PVImageFile: PVFile { self.init() self.relativeRoot = relativeRoot let partialPath = relativeRoot.createRelativePath(fromURL: url) + //TODO: remove + if doesPathContainParent(partialPath) { + DLOG("invalid path: \(partialPath)") + } self.partialPath = partialPath calculateSizeData() From fe25a2203ed76cadbefbd23edef5c49e0417c9f1 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 8 Feb 2025 20:04:00 -0500 Subject: [PATCH 52/86] added debug block --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 9f5f9486a9..d14340a4b5 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -389,9 +389,11 @@ public enum iCloudSync { } } //TODO: first we upload anything pending, then we download anything that already exists and import those. we will have to update the locations of existing saves and ROMs in the db +#if DEBUG if 1==1 { return } +#endif let fm = FileManager.default if let currentiCloudToken = fm.ubiquityIdentityToken { do { From aac5da6fb1f2b1c600bb9cdd74dc5790bd853454 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 8 Feb 2025 22:27:25 -0500 Subject: [PATCH 53/86] reverted returns; updated TODO text --- .../RealmPlatform/Entities/Files/PVFile.swift | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift index b8b87323d4..3f8b9d7db2 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift @@ -83,7 +83,7 @@ public extension PVFile { var url: URL { get { - //TODO: if relativeRoot == .iCloud, AND partialPath is NOT a partial path, then remove the prefix (cloudContainer/Documents), this will be older db entries + //TODO: if partialPath is NOT a partial path, ie it has the prefix OR contains the app sandbox container OR contains or has the prefix (cloudContainer/Documents), then remove either of those, this will be older db entries from older versions that erronously don't remove the prefix let url2 = urlUpdate DLOG("url2=\(url2)\tpartialPath=\(partialPath)") if partialPath.contains("iCloud") || partialPath.contains("private") { @@ -96,11 +96,11 @@ public extension PVFile { let iCloudBase = URL.iCloudContainerDirectory let url = (iCloudBase ?? RelativeRoot.documentsDirectory).appendingPathComponent(path) DLOG("url:\(url)") - //TODO: new return url2 if doesPathContainParent(url.path) { DLOG("invalid url:\(url)") } - return url +// return url + return url2 } else { if let iCloudBase = URL.iCloudDocumentsDirectory { let appendedICloudBase = iCloudBase.appendingPathComponent(path) @@ -113,19 +113,19 @@ public extension PVFile { } else { let appendedRelativeRoot = RelativeRoot.documentsDirectory.appendingPathComponent(path) DLOG("appendedRelativeRoot:\(appendedRelativeRoot)") - //TODO: new return url2 if doesPathContainParent(appendedRelativeRoot.path) { DLOG("invalid url:\(appendedRelativeRoot)") } - return appendedRelativeRoot +// return appendedRelativeRoot + return url2 } } } let root = relativeRoot let resolvedURL = root.appendingPath(partialPath) DLOG("resolvedURL:\(resolvedURL))") - //TODO: new return url2 - return resolvedURL +// return resolvedURL + return url2 } } var urlUpdate:URL { From ed8664743f2bccf1414332490a50c4b30540d97b Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 00:29:20 -0500 Subject: [PATCH 54/86] moved upload icloud to syncer class; added enumeration status icloud upload; added importing already downloaded files when switching icloud on; added deleting saves from db when syncing from icloud; added check to see if a ROM exists in db in same path to not re-add the same ROM --- .../Importer/iCloud/iCloudSync.swift | 236 ++++++++---------- 1 file changed, 98 insertions(+), 138 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index d14340a4b5..523e1c7038 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -36,14 +36,6 @@ extension Container { var documentsURL: URL? { get { return URL.iCloudDocumentsDirectory }} } -public protocol SyncFileToiCloud: Container { - var fileManager: FileManager { get } - var metadataQuery: NSMetadataQuery { get } - func syncToiCloud(completionHandler: @escaping (SyncResult) -> Void) async// -> Single - func queryFile(completionHandler: @escaping (URL?) -> Void) // -> Single - func downloadingFile(completionHandler: @escaping (SyncResult) -> Void) async // -> Single -} - public protocol iCloudTypeSyncer: Container { var directory: String { get } var metadataQuery: NSMetadataQuery { get } @@ -51,6 +43,7 @@ public protocol iCloudTypeSyncer: Container { func loadAllFromICloud(iterationComplete: (() -> Void)?) -> Completable func insertDownloadingFile(_ file: URL) func insertDownloadedFile(_ file: URL) + func insertUploadedFile(_ file: URL) func deleteFromDatastore(_ file: URL) func setNewCloudFilesAvailable() } @@ -72,10 +65,18 @@ final class NotificationObserver { } } +enum iCloudSyncStatus { + case initialUpload + case filesAlreadyMoved +} + class iCloudContainerSyncer: iCloudTypeSyncer { lazy var pendingFilesToDownload: Set = [] lazy var newFiles: Set = [] + lazy var uploadedFiles: Set = [] let directory: String + let fileManager = FileManager.default + var status: iCloudSyncStatus = .initialUpload init(directory: String) { self.directory = directory @@ -97,6 +98,10 @@ class iCloudContainerSyncer: iCloudTypeSyncer { //no-op } + func insertUploadedFile(_ file: URL) { + //no-op + } + func setNewCloudFilesAvailable() { //no-op } @@ -118,6 +123,16 @@ class iCloudContainerSyncer: iCloudTypeSyncer { completable(.error(SyncError.noUbiquityURL)) return } + + let completion = syncToiCloud() + DLOG("saveStateUploader syncToiCloud result: \(completion)") + guard completion != .saveFailure, + completion != .denied + else { + ELOG("error moving files to iCloud container") + return + } + metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] DLOG("directory: \(directory)") metadataQuery.predicate = NSPredicate(format: "%K CONTAINS[c] %@", NSMetadataItemPathKey, "/Documents/\(directory)/") @@ -188,6 +203,11 @@ class iCloudContainerSyncer: iCloudTypeSyncer { self?.deleteFromDatastore(file) } else { queue.sync { + //in the case when we are initially turning on iCloud, we try to import any files already downloaded + //TODO: this should only happen one time per turning on the iCloud switch to avoid doing this every time the app opens, comes back to the foreground + if self?.status == .initialUpload { + self?.insertDownloadingFile(file) + } filesDownloaded.insert(file) self?.insertDownloadedFile(file) } @@ -200,42 +220,20 @@ class iCloudContainerSyncer: iCloudTypeSyncer { setNewCloudFilesAvailable() DLOG("\(directory): current iteration: files pending to be downloaded: \(files.count), files downloaded : \(filesDownloaded.count)") } -} - -extension SyncFileToiCloud where Self: LocalFileInfoProvider { - private var destinationURL: URL? { get async { - await Task { - guard let containerURL = containerURL else { return nil } - return containerURL.appendingPathComponent(url.relativePath) - }.value - }} - - //TODO: refactor this on the syncer - func syncToiCloud(completionHandler: @escaping (SyncResult) -> Void) async { - await Task { - - DLOG("url: \(url)") - guard fileManager.fileExists(atPath: url.path), - let actualContainerUrl = containerURL - else { - completionHandler(.fileNotExist) - return - } -// guard let destinationURL = await self.destinationURL else { -// return completionHandler(.denied) -// } -// let url = self.url -// -// self.metadataQuery.disableUpdates() -// defer { -// self.metadataQuery.enableUpdates() -// } + + func syncToiCloud() -> SyncResult { + let url = URL.documentsDirectory.appendingPathComponent(directory) + DLOG("url: \(url)") + guard fileManager.fileExists(atPath: url.path), + let actualContainerUrl = containerURL + else { + return .fileNotExist + } - completionHandler(await moveFiles(at: url, container: actualContainerUrl.appendingPathComponent("Documents").appendingPathComponent(url.lastPathComponent))) - }.value + return moveFiles(at: url, container: actualContainerUrl.appendingPathComponent("Documents").appendingPathComponent(url.lastPathComponent)) } - func moveFiles(at current: URL, container: URL) async -> SyncResult { + func moveFiles(at current: URL, container: URL) -> SyncResult { do { let subdirectories = try fileManager.subpathsOfDirectory(atPath: current.path) DLOG("subdirectories of \(current): \(subdirectories)") @@ -261,6 +259,7 @@ extension SyncFileToiCloud where Self: LocalFileInfoProvider { do { ILOG("Trying to set Ubiquitious from local (\(currentItem.path)) to ICloud (\(iCloudDestination.path))") try fileManager.setUbiquitous(true, itemAt: currentItem, destinationURL: iCloudDestination) + insertUploadedFile(iCloudDestination) } catch { //this could indicate no more space is left ELOG("iCloud failed to set Ubiquitous: \(error.localizedDescription)") @@ -272,68 +271,6 @@ extension SyncFileToiCloud where Self: LocalFileInfoProvider { return .saveFailure } } - - /// - Parameter completionHandler: Non-main - func queryFile(completionHandler: @escaping (URL?) -> Void) { - metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] - - let center = NotificationCenter.default - - center.addObserver(forName: .NSMetadataQueryDidFinishGathering, object: metadataQuery, queue: nil) { _ in - // guard let `self` = self else {return} - - self.metadataQuery.disableUpdates() - defer { - self.metadataQuery.enableUpdates() - } - - guard self.metadataQuery.resultCount >= 1, - let item = self.metadataQuery.results.first as? NSMetadataItem, - let fileURL = item.value(forAttribute: NSMetadataItemURLKey) as? URL - else { - self.metadataQuery.enableUpdates() - return completionHandler(nil) - } - - completionHandler(fileURL) - self.metadataQuery.enableUpdates() - } - - metadataQuery.start() - } - - /// - Parameters: - /// - completionHandler: Non-main - func downloadingFile(completionHandler: @escaping (SyncResult) -> Void) async { - guard let destinationURL = await destinationURL else { - completionHandler(.denied) - return - } - - DispatchQueue.global(qos: .utility).async { - if !FileManager.default.isUbiquitousItem(at: destinationURL) { - completionHandler(.fileNotExist) - return - } - - self.metadataQuery.disableUpdates() - defer { - self.metadataQuery.enableUpdates() - } - - let fm = FileManager.default - - do { - // TODO: Should really wait and listen for it to finish downloading, this call is async - try fm.startDownloadingUbiquitousItem(at: destinationURL) - completionHandler(.success) - } catch { - ELOG("iCloud Download error: \(error.localizedDescription)") - completionHandler(.saveFailure) - return - } - } - } } enum iCloudError: Error { @@ -348,9 +285,6 @@ public enum iCloudSync { static var batterySavesSyncer: BatterySavesSyncer! static var screenshotsSyncer: ScreenshotsSyncer! static var gameImporter = GameImporter.shared - //initial uploaders - static var saveStateUploader = SaveStateUploader() - static var romsUploader = RomsUploader() public static func initICloudDocuments() { Task { @@ -379,21 +313,7 @@ public enum iCloudSync { DLOG("turning on iCloud") //reset ROMs path gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) - //TODO: move files from local to cloud container - Task { - await saveStateUploader.syncToiCloud { completion in - DLOG("saveStateUploader syncToiCloud result: \(completion)") - } - await romsUploader.syncToiCloud { completion in - DLOG("romsUploader syncToiCloud result: \(completion)") - } - } - //TODO: first we upload anything pending, then we download anything that already exists and import those. we will have to update the locations of existing saves and ROMs in the db -#if DEBUG - if 1==1 { - return - } -#endif + let fm = FileManager.default if let currentiCloudToken = fm.ubiquityIdentityToken { do { @@ -493,6 +413,10 @@ class SaveStateSyncer: iCloudContainerSyncer { } override func insertDownloadingFile(_ file: URL) { + guard !uploadedFiles.contains(file) + else { + return + } pendingFilesToDownload.insert(file.absoluteString) } @@ -506,9 +430,32 @@ class SaveStateSyncer: iCloudContainerSyncer { newFiles.insert(file) } + override func insertUploadedFile(_ file: URL) { + uploadedFiles.insert(file) + } + override func deleteFromDatastore(_ file: URL) { -// PVSaveState - //TODO: delete from database + guard "jpg'".caseInsensitiveCompare(file.pathExtension) == .orderedSame + else { + return + } + do { + let realm = try Realm() + DLOG("attempting to query PVSaveState by file: \(file)") + let imageField = NSExpression(forKeyPath: \PVSaveState.image.self).keyPath + let urlField = NSExpression(forKeyPath: \PVImageFile.url.self).keyPath + let absoluteStringField = NSExpression(forKeyPath: \URL.absoluteString.self).keyPath + let results = realm.objects(PVSaveState.self).filter(NSPredicate(format: "\(imageField).\(urlField).\(absoluteStringField) == %@", file.absoluteString)) + guard let save: PVSaveState = results.first + else { + return + } + try realm.write { + realm.delete(save) + } + } catch { + ELOG(error.localizedDescription) + } } func importNewSaves() { @@ -526,8 +473,10 @@ class SaveStateSyncer: iCloudContainerSyncer { } //TODO: initially when importing icloud files, we should wait for ROMs to finish importing. when that completes, then we no longer have to wait for that, ie when syncing single changes say 2 devices are open at the same time and 1 save file is added from another device didFinishDownloadingAllFiles = false + status = .filesAlreadyMoved let jsonFiles = newFiles newFiles.removeAll() + uploadedFiles.removeAll() Task { let jsonDecorder = JSONDecoder() jsonDecorder.dataDecodingStrategy = .deferredToData @@ -618,6 +567,11 @@ class RomsSyncer: iCloudContainerSyncer { } override func insertDownloadingFile(_ file: URL) { + //we ensure we don't re-add any files we just moved over + guard !uploadedFiles.contains(file) + else { + return + } pendingFilesToDownload.insert(file.absoluteString) } @@ -631,14 +585,32 @@ class RomsSyncer: iCloudContainerSyncer { DLOG("adding file to game import queue: \(file), parent directory: \(parentDirectory)") //we should only add to the import queue files that are actual ROMs, anything else can be ignored. guard parentDirectory.range(of: "com.provenance.", - options: [.caseInsensitive, .anchored]) != nil + options: [.caseInsensitive, .anchored]) != nil, + let fileName = file.lastPathComponent.removingPercentEncoding else { return } + do { + let realm = try Realm() + let romPath = "\(parentDirectory)/\(fileName)" + DLOG("attempting to query PVGame by romPath: \(romPath)") + let results = realm.objects(PVGame.self).filter(NSPredicate(format: "\(NSExpression(forKeyPath: \PVGame.romPath.self).keyPath) == %@", romPath)) + guard results.first == nil + else { + return + } + } catch { + ELOG(error.localizedDescription) + } + newFiles.insert(file) } + override func insertUploadedFile(_ file: URL) { + uploadedFiles.insert(file) + } + override func deleteFromDatastore(_ file: URL) { //TODO: remove cloud download, but keep in iCloud. this way the ROM isn't deleted from all devices guard let fileName = file.lastPathComponent.removingPercentEncoding, @@ -682,8 +654,10 @@ class RomsSyncer: iCloudContainerSyncer { return } didFinishDownloadingAllFiles = false + status = .filesAlreadyMoved let importPaths = [URL](newFiles) newFiles.removeAll() + uploadedFiles.removeAll() gameImporter.addImports(forPaths: importPaths) gameImporter.startProcessing() } @@ -706,17 +680,3 @@ class ScreenshotsSyncer: iCloudContainerSyncer { self.init(directory: "Screenshots") } } - -//MARK: - iCloud initial container uploaders - -class SaveStateUploader: SyncFileToiCloud, LocalFileInfoProvider { - let fileManager = FileManager.default - let metadataQuery: NSMetadataQuery = .init() - let url: URL = URL.documentsDirectory.appendingPathComponent("Save States") -} - -class RomsUploader: SyncFileToiCloud, LocalFileInfoProvider { - let fileManager = FileManager.default - let metadataQuery: NSMetadataQuery = .init() - let url: URL = URL.documentsDirectory.appendingPathComponent("ROMs") -} From e83acf7ff672e86c61b32cc8cafbb1fc767ae1b3 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 11:11:02 -0500 Subject: [PATCH 55/86] updated to use notificationCenter directly and removed unused NotificationObserver --- .../Importer/iCloud/iCloudSync.swift | 53 +++++++------------ 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 523e1c7038..407a39d89c 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -48,23 +48,6 @@ public protocol iCloudTypeSyncer: Container { func setNewCloudFilesAvailable() } -final class NotificationObserver { - - var name: Notification.Name - var observer: NSObjectProtocol - var center = NotificationCenter.default - var object: Any? - - init(forName name: Notification.Name, object: Any? = nil, queue: OperationQueue? = nil, block: @escaping (Notification) -> Void) { - self.name = name - observer = center.addObserver(forName: name, object: object, queue: queue, using: block) - } - - deinit {//TODO: because this was created inline, deinit gets called right away. does this ever need to be removed? shouldn't this be in the lifetime of the application? - //center.removeObserver(observer, name: name, object: object) - } -} - enum iCloudSyncStatus { case initialUpload case filesAlreadyMoved @@ -77,14 +60,18 @@ class iCloudContainerSyncer: iCloudTypeSyncer { let directory: String let fileManager = FileManager.default var status: iCloudSyncStatus = .initialUpload + let notificationCenter: NotificationCenter - init(directory: String) { + init(directory: String, notificationCenter: NotificationCenter) { + self.notificationCenter = notificationCenter self.directory = directory } deinit { metadataQuery.disableUpdates() metadataQuery.stop() + notificationCenter.removeObserver(self, name: .NSMetadataQueryDidFinishGathering, object: metadataQuery) + notificationCenter.removeObserver(self, name: .NSMetadataQueryDidUpdate, object: metadataQuery) DLOG("dying") } @@ -137,7 +124,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { DLOG("directory: \(directory)") metadataQuery.predicate = NSPredicate(format: "%K CONTAINS[c] %@", NSMetadataItemPathKey, "/Documents/\(directory)/") //TODO: update to use Publishers.MergeMany - let _: NotificationObserver = .init( + notificationCenter.addObserver( forName: .NSMetadataQueryDidFinishGathering, object: metadataQuery, queue: nil) { [weak self] notification in @@ -147,7 +134,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } } //listen for deletions and new files. what about conflicts? - let _: NotificationObserver = .init( + notificationCenter.addObserver( forName: .NSMetadataQueryDidUpdate, object: metadataQuery, queue: nil) { [weak self] notification in @@ -351,7 +338,7 @@ public enum iCloudSync { romsSyncer = nil }.disposed(by: disposeBag) //TODO: set the following to merge onto a single class that just does a query for icloud for all of those directories. - biosSyncer = .init() + biosSyncer = .init(notificationCenter: .default) biosSyncer.loadAllFromICloud() .observe(on: MainScheduler.instance) .subscribe(onError: { error in @@ -360,7 +347,7 @@ public enum iCloudSync { DLOG("disposing BiosSyncer") biosSyncer = nil }.disposed(by: disposeBag) - batterySavesSyncer = .init() + batterySavesSyncer = .init(notificationCenter: .default) batterySavesSyncer.loadAllFromICloud() .observe(on: MainScheduler.instance) .subscribe(onError: { error in @@ -369,7 +356,7 @@ public enum iCloudSync { DLOG("disposing BatterySavesSyncer") batterySavesSyncer = nil }.disposed(by: disposeBag) - screenshotsSyncer = .init() + screenshotsSyncer = .init(notificationCenter: .default) screenshotsSyncer.loadAllFromICloud() .observe(on: MainScheduler.instance) .subscribe(onError: { error in @@ -394,11 +381,9 @@ public enum iCloudSync { //TODO: perhaps 1 generic class since a lot of this code is similar and move the extension onto generic class. we could just add a protocol delegate dependency for ROMs and SaveState classes that does specific code class SaveStateSyncer: iCloudContainerSyncer { var didFinishDownloadingAllFiles = false - let notificationCenter: NotificationCenter init(notificationCenter: NotificationCenter) { - self.notificationCenter = notificationCenter - super.init(directory: "Save States") + super.init(directory: "Save States", notificationCenter: notificationCenter) notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in self?.importNewSaves() } @@ -549,10 +534,8 @@ class SaveStateSyncer: iCloudContainerSyncer { class RomsSyncer: iCloudContainerSyncer { let gameImporter = GameImporter.shared var didFinishDownloadingAllFiles = false - let notificationCenter: NotificationCenter init(notificationCenter: NotificationCenter) { - self.notificationCenter = notificationCenter - super.init(directory: "ROMs") + super.init(directory: "ROMs", notificationCenter: notificationCenter) notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in self?.importNewRomFiles() } @@ -664,19 +647,19 @@ class RomsSyncer: iCloudContainerSyncer { } class BiosSyncer: iCloudContainerSyncer { - convenience init() { - self.init(directory: "BIOS") + convenience init(notificationCenter: NotificationCenter) { + self.init(directory: "BIOS", notificationCenter: notificationCenter) } } class BatterySavesSyncer: iCloudContainerSyncer { - convenience init() { - self.init(directory: "Battery Saves") + convenience init(notificationCenter: NotificationCenter) { + self.init(directory: "Battery Saves", notificationCenter: notificationCenter) } } class ScreenshotsSyncer: iCloudContainerSyncer { - convenience init() { - self.init(directory: "Screenshots") + convenience init(notificationCenter: NotificationCenter) { + self.init(directory: "Screenshots", notificationCenter: notificationCenter) } } From e1e599a534a8d562484a6044bd0bed2a35b7b513 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 11:34:49 -0500 Subject: [PATCH 56/86] added userdefaults to know when to initially import files from cloud --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 3 ++- .../Sources/PVSettings/Settings/Model/PVSettingsModel.swift | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 407a39d89c..8554517eda 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -192,7 +192,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { queue.sync { //in the case when we are initially turning on iCloud, we try to import any files already downloaded //TODO: this should only happen one time per turning on the iCloud switch to avoid doing this every time the app opens, comes back to the foreground - if self?.status == .initialUpload { + if !Defaults[.iCloudInitialSetupComplete] && self?.status == .initialUpload { self?.insertDownloadingFile(file) } filesDownloaded.insert(file) @@ -638,6 +638,7 @@ class RomsSyncer: iCloudContainerSyncer { } didFinishDownloadingAllFiles = false status = .filesAlreadyMoved + Defaults[.iCloudInitialSetupComplete] = true let importPaths = [URL](newFiles) newFiles.removeAll() uploadedFiles.removeAll() diff --git a/PVSettings/Sources/PVSettings/Settings/Model/PVSettingsModel.swift b/PVSettings/Sources/PVSettings/Settings/Model/PVSettingsModel.swift index cd221561cb..214ce18e95 100644 --- a/PVSettings/Sources/PVSettings/Settings/Model/PVSettingsModel.swift +++ b/PVSettings/Sources/PVSettings/Settings/Model/PVSettingsModel.swift @@ -217,6 +217,7 @@ public extension Defaults.Keys { static let useUIKit = Key("useUIKit", default:false) #endif static let iCloudSync = Key("iCloudSync", default: false) + static let iCloudInitialSetupComplete = Key("iCloudInitialSetupComplete", default: false) static let unsupportedCores = Key("unsupportedCores", default: false) #if os(tvOS) static let tvOSThemes = Key("tvOSThemes", default: false) From 7f408435f9c0c49d1a4de48100b0a6a32efa1714 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 14:11:02 -0500 Subject: [PATCH 57/86] added removal of icloud files when flag is turned off; added todo on working with existing files in icloud container --- .../Importer/iCloud/iCloudSync.swift | 88 +++++++++++++------ 1 file changed, 61 insertions(+), 27 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 8554517eda..7b36f3a6f3 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -59,7 +59,6 @@ class iCloudContainerSyncer: iCloudTypeSyncer { lazy var uploadedFiles: Set = [] let directory: String let fileManager = FileManager.default - var status: iCloudSyncStatus = .initialUpload let notificationCenter: NotificationCenter init(directory: String, notificationCenter: NotificationCenter) { @@ -72,9 +71,20 @@ class iCloudContainerSyncer: iCloudTypeSyncer { metadataQuery.stop() notificationCenter.removeObserver(self, name: .NSMetadataQueryDidFinishGathering, object: metadataQuery) notificationCenter.removeObserver(self, name: .NSMetadataQueryDidUpdate, object: metadataQuery) + let removed = removeFromiCloud() + DLOG("removed: \(removed)") DLOG("dying") } + var iCloudDocumentsDirectory: URL? { + containerURL?.appendingPathComponent("Documents") + .appendingPathComponent(directory) + } + + var localDirectory: URL { + URL.documentsDirectory.appendingPathComponent(directory) + } + let metadataQuery: NSMetadataQuery = .init() func insertDownloadingFile(_ file: URL) { @@ -192,7 +202,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { queue.sync { //in the case when we are initially turning on iCloud, we try to import any files already downloaded //TODO: this should only happen one time per turning on the iCloud switch to avoid doing this every time the app opens, comes back to the foreground - if !Defaults[.iCloudInitialSetupComplete] && self?.status == .initialUpload { + if !Defaults[.iCloudInitialSetupComplete] { self?.insertDownloadingFile(file) } filesDownloaded.insert(file) @@ -209,47 +219,72 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } func syncToiCloud() -> SyncResult { - let url = URL.documentsDirectory.appendingPathComponent(directory) - DLOG("url: \(url)") - guard fileManager.fileExists(atPath: url.path), - let actualContainerUrl = containerURL + guard let destination = iCloudDocumentsDirectory else { - return .fileNotExist + return .denied + } + return moveFiles(at: localDirectory, + containerDestination: destination, + existingClosure: { existing in + try fileManager.removeItem(atPath: existing.path) + }) { currentSource, currentDestination in + try fileManager.setUbiquitous(true, itemAt: currentSource, destinationURL: currentDestination) } - - return moveFiles(at: url, container: actualContainerUrl.appendingPathComponent("Documents").appendingPathComponent(url.lastPathComponent)) } - func moveFiles(at current: URL, container: URL) -> SyncResult { + func removeFromiCloud() -> SyncResult { + guard let source = iCloudDocumentsDirectory + else { + return .denied + } + return moveFiles(at: source, + containerDestination: localDirectory, + existingClosure: { existing in + try fileManager.evictUbiquitousItem(at: existing) + }) { currentSource, currentDestination in + try fileManager.copyItem(at: currentSource, to: currentDestination) + try fileManager.evictUbiquitousItem(at: currentSource) + } + } + + func moveFiles(at source: URL, + containerDestination: URL, + existingClosure: ((URL) throws -> Void), + moveClosure: (URL, URL) throws -> Void) -> SyncResult { + DLOG("source: \(source)") + guard fileManager.fileExists(atPath: source.path) + else { + return .fileNotExist + } do { - let subdirectories = try fileManager.subpathsOfDirectory(atPath: current.path) - DLOG("subdirectories of \(current): \(subdirectories)") + let subdirectories = try fileManager.subpathsOfDirectory(atPath: source.path) + DLOG("subdirectories of \(source): \(subdirectories)") for currentChild in subdirectories { - let currentItem = current.appendingPathComponent(currentChild) + let currentItem = source.appendingPathComponent(currentChild) var isDirectory: ObjCBool = false let exists = fileManager.fileExists(atPath: currentItem.path, isDirectory: &isDirectory) DLOG("\(currentItem) isDirectory?\(isDirectory) exists?\(exists)") - let iCloudDestination = container.appendingPathComponent(currentChild) - DLOG("new iCloud directory: \(iCloudDestination)") - if isDirectory.boolValue && !fileManager.fileExists(atPath: iCloudDestination.path) { - DLOG("\(iCloudDestination) does NOT exist") - try fileManager.createDirectory(atPath: iCloudDestination.path, withIntermediateDirectories: false) + let destination = containerDestination.appendingPathComponent(currentChild) + DLOG("new destination: \(destination)") + if isDirectory.boolValue && !fileManager.fileExists(atPath: destination.path) { + DLOG("\(destination) does NOT exist") + try fileManager.createDirectory(atPath: destination.path, withIntermediateDirectories: false) } if isDirectory.boolValue { continue } - if fileManager.fileExists(atPath: iCloudDestination.path) { - try fileManager.removeItem(atPath: currentItem.path) + if fileManager.fileExists(atPath: destination.path) { + try existingClosure(currentItem) continue } do { - ILOG("Trying to set Ubiquitious from local (\(currentItem.path)) to ICloud (\(iCloudDestination.path))") - try fileManager.setUbiquitous(true, itemAt: currentItem, destinationURL: iCloudDestination) - insertUploadedFile(iCloudDestination) + ILOG("Trying to move (\(currentItem.path)) to (\(destination.path))") + try moveClosure(currentItem, destination) + insertUploadedFile(destination) } catch { - //this could indicate no more space is left - ELOG("iCloud failed to set Ubiquitous: \(error.localizedDescription)") + //this could indicate no more space is left when moving to iCloud + ELOG("failed to move: \(error.localizedDescription)") } } return .success @@ -300,6 +335,7 @@ public enum iCloudSync { DLOG("turning on iCloud") //reset ROMs path gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) + Defaults[.iCloudInitialSetupComplete] = false//TODO: when the app first opens, this shouldn't be done, only when the user taps the flag on let fm = FileManager.default if let currentiCloudToken = fm.ubiquityIdentityToken { @@ -458,7 +494,6 @@ class SaveStateSyncer: iCloudContainerSyncer { } //TODO: initially when importing icloud files, we should wait for ROMs to finish importing. when that completes, then we no longer have to wait for that, ie when syncing single changes say 2 devices are open at the same time and 1 save file is added from another device didFinishDownloadingAllFiles = false - status = .filesAlreadyMoved let jsonFiles = newFiles newFiles.removeAll() uploadedFiles.removeAll() @@ -637,7 +672,6 @@ class RomsSyncer: iCloudContainerSyncer { return } didFinishDownloadingAllFiles = false - status = .filesAlreadyMoved Defaults[.iCloudInitialSetupComplete] = true let importPaths = [URL](newFiles) newFiles.removeAll() From 4803b01ffa1780a589b9b9a4896d4e5d80c0ad27 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 15:44:27 -0500 Subject: [PATCH 58/86] updated to use shared function --- .../PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift index 66a57ea1bc..b11a3d2152 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVImageFile.swift @@ -56,8 +56,7 @@ public final class PVImageFile: PVFile { private func calculateSizeData() { // async { //TODO: path is wrong when switching to iCloud let path = url.path - if path.contains("var/mobile/Containers/Data/Application") - && (path.contains("Mobile%20Documents") || path.contains("Mobile Documents")) { + if doesPathContainParent(path) { ELOG("invalid path: \(path)") } From 2c7cd8577d572dcaa48461db8daae6a66732e0c0 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 15:44:46 -0500 Subject: [PATCH 59/86] removed finished todo --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 7b36f3a6f3..4a6e4dc369 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -408,7 +408,6 @@ public enum iCloudSync { disposeBag = nil //reset ROMs path gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) - //TODO: remove iCloud downloads. do we also copy those files locally? } } From 8f68d4f55ee65b1a04e53238e35827ffbe3722e0 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 15:47:15 -0500 Subject: [PATCH 60/86] updated initializer --- .../Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 4a6e4dc369..4f110824df 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -417,8 +417,8 @@ public enum iCloudSync { class SaveStateSyncer: iCloudContainerSyncer { var didFinishDownloadingAllFiles = false - init(notificationCenter: NotificationCenter) { - super.init(directory: "Save States", notificationCenter: notificationCenter) + convenience init(notificationCenter: NotificationCenter) { + self.init(directory: "Save States", notificationCenter: notificationCenter) notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in self?.importNewSaves() } @@ -568,8 +568,8 @@ class SaveStateSyncer: iCloudContainerSyncer { class RomsSyncer: iCloudContainerSyncer { let gameImporter = GameImporter.shared var didFinishDownloadingAllFiles = false - init(notificationCenter: NotificationCenter) { - super.init(directory: "ROMs", notificationCenter: notificationCenter) + convenience init(notificationCenter: NotificationCenter) { + self.init(directory: "ROMs", notificationCenter: notificationCenter) notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in self?.importNewRomFiles() } From 46465be7546bbc1b2de5636d330d249fb5d3b45c Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 18:12:08 -0500 Subject: [PATCH 61/86] refactored to merge files that do NOT have db entry into a single class; fixed files not storing correctly when turning off icloud --- .../Importer/iCloud/iCloudSync.swift | 230 +++++++++--------- 1 file changed, 111 insertions(+), 119 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 4f110824df..e3ccb57f7d 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -37,7 +37,7 @@ extension Container { } public protocol iCloudTypeSyncer: Container { - var directory: String { get } + var directories: Set { get } var metadataQuery: NSMetadataQuery { get } func loadAllFromICloud(iterationComplete: (() -> Void)?) -> Completable @@ -57,13 +57,13 @@ class iCloudContainerSyncer: iCloudTypeSyncer { lazy var pendingFilesToDownload: Set = [] lazy var newFiles: Set = [] lazy var uploadedFiles: Set = [] - let directory: String + let directories: Set let fileManager = FileManager.default let notificationCenter: NotificationCenter - init(directory: String, notificationCenter: NotificationCenter) { + init(directories: Set, notificationCenter: NotificationCenter) { self.notificationCenter = notificationCenter - self.directory = directory + self.directories = directories } deinit { @@ -76,13 +76,17 @@ class iCloudContainerSyncer: iCloudTypeSyncer { DLOG("dying") } - var iCloudDocumentsDirectory: URL? { - containerURL?.appendingPathComponent("Documents") - .appendingPathComponent(directory) - } - - var localDirectory: URL { - URL.documentsDirectory.appendingPathComponent(directory) + var localAndCloudDirectories: [URL: URL] { + var alliCloudDirectories = [URL: URL]() + guard let actualContainrUrl = containerURL + else { + return alliCloudDirectories + } + let parentContainer = actualContainrUrl.appendingPathComponent("Documents") + directories.forEach { directory in + alliCloudDirectories[URL.documentsDirectory.appendingPathComponent(directory)] = parentContainer.appendingPathComponent(directory) + } + return alliCloudDirectories } let metadataQuery: NSMetadataQuery = .init() @@ -115,7 +119,8 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } func setupObservers(completable: PrimitiveSequenceType.CompletableObserver, iterationComplete: (() -> Void)? = nil) { - guard containerURL != nil + guard containerURL != nil, + directories.count > 0 else { completable(.error(SyncError.noUbiquityURL)) return @@ -131,8 +136,18 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] - DLOG("directory: \(directory)") - metadataQuery.predicate = NSPredicate(format: "%K CONTAINS[c] %@", NSMetadataItemPathKey, "/Documents/\(directory)/") + DLOG("directory: \(directories)") + var predicateFormat = "" + var predicateArgs = [CVarArg]() + directories.forEach { directory in + if !predicateFormat.isEmpty { + predicateFormat += " OR " + } + predicateFormat += "%K CONTAINS[c] %@" + predicateArgs.append(NSMetadataItemPathKey) + predicateArgs.append("/Documents/\(directory)/") + } + metadataQuery.predicate = NSPredicate(format: predicateFormat, argumentArray: predicateArgs) //TODO: update to use Publishers.MergeMany notificationCenter.addObserver( forName: .NSMetadataQueryDidFinishGathering, @@ -161,7 +176,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } func queryFinished(notification: Notification) async { - DLOG("directory: \(directory)") + DLOG("directory: \(directories)") guard (notification.object as? NSMetadataQuery) == metadataQuery else { return @@ -169,9 +184,9 @@ class iCloudContainerSyncer: iCloudTypeSyncer { let fileManager = FileManager.default var files: Set = [] var filesDownloaded: Set = [] - let queue = DispatchQueue(label: "org.provenance-emu.provenance.newFiles") + let queue = DispatchQueue(label: "com.provenance.newFiles") let removedObjects = notification.userInfo?[NSMetadataQueryUpdateRemovedItemsKey] - DLOG("\(directory): removedObjects: \(removedObjects)") + DLOG("\(directories): removedObjects: \(removedObjects)") //accessing results automatically pauses updates and resumes after deallocated await metadataQuery.results.concurrentForEach { [weak self] item in @@ -215,36 +230,53 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } //TODO: for ROMs and saves, perhaps we need to store the downloaded files that need to be process in the case of a crash or the user puts the app in the background. setNewCloudFilesAvailable() - DLOG("\(directory): current iteration: files pending to be downloaded: \(files.count), files downloaded : \(filesDownloaded.count)") + DLOG("\(directories): current iteration: files pending to be downloaded: \(files.count), files downloaded : \(filesDownloaded.count)") } func syncToiCloud() -> SyncResult { - guard let destination = iCloudDocumentsDirectory + let allDirectories = localAndCloudDirectories + guard allDirectories.count > 0 else { return .denied } - return moveFiles(at: localDirectory, - containerDestination: destination, - existingClosure: { existing in - try fileManager.removeItem(atPath: existing.path) - }) { currentSource, currentDestination in - try fileManager.setUbiquitous(true, itemAt: currentSource, destinationURL: currentDestination) + var moveResult: SyncResult? = nil + allDirectories.forEach { (localDirectory: URL, iCloudDirectory: URL) in + + let moved = moveFiles(at: localDirectory, + containerDestination: iCloudDirectory, + existingClosure: { existing in + try fileManager.removeItem(atPath: existing.path) + }) { currentSource, currentDestination in + try fileManager.setUbiquitous(true, itemAt: currentSource, destinationURL: currentDestination) + } + if moved == .saveFailure { + moveResult = .saveFailure + } } + return moveResult ?? .success } func removeFromiCloud() -> SyncResult { - guard let source = iCloudDocumentsDirectory + let allDirectories = localAndCloudDirectories + guard allDirectories.count > 0 else { return .denied } - return moveFiles(at: source, - containerDestination: localDirectory, - existingClosure: { existing in - try fileManager.evictUbiquitousItem(at: existing) - }) { currentSource, currentDestination in - try fileManager.copyItem(at: currentSource, to: currentDestination) - try fileManager.evictUbiquitousItem(at: currentSource) + var moveResult: SyncResult? + allDirectories.forEach { (localDirectory: URL, iCloudDirectory: URL) in + let moved = moveFiles(at: iCloudDirectory, + containerDestination: localDirectory, + existingClosure: { existing in + try fileManager.evictUbiquitousItem(at: existing) + }) { currentSource, currentDestination in + try fileManager.copyItem(at: currentSource, to: currentDestination) + try fileManager.evictUbiquitousItem(at: currentSource) + } + if moved == .saveFailure { + moveResult = .saveFailure + } } + return moveResult ?? .success } func moveFiles(at source: URL, @@ -269,7 +301,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { DLOG("new destination: \(destination)") if isDirectory.boolValue && !fileManager.fileExists(atPath: destination.path) { DLOG("\(destination) does NOT exist") - try fileManager.createDirectory(atPath: destination.path, withIntermediateDirectories: false) + try fileManager.createDirectory(atPath: destination.path, withIntermediateDirectories: true) } if isDirectory.boolValue { continue @@ -301,11 +333,6 @@ enum iCloudError: Error { public enum iCloudSync { static var disposeBag: DisposeBag! - //syncers - static var saveStateSyncer: SaveStateSyncer! - static var biosSyncer: BiosSyncer! - static var batterySavesSyncer: BatterySavesSyncer! - static var screenshotsSyncer: ScreenshotsSyncer! static var gameImporter = GameImporter.shared public static func initICloudDocuments() { @@ -351,8 +378,16 @@ public enum iCloudSync { //TODO: should we pause when a game starts so we don't interfere with the game and continue listening when no game is running? disposeBag = DisposeBag() - - saveStateSyncer = .init(notificationCenter: .default) + var nonDatabaseFileSyncer: iCloudContainerSyncer! = .init(directories: ["BIOS", "Battery Saves", "Screenshots"], notificationCenter: .default) + nonDatabaseFileSyncer.loadAllFromICloud() + .observe(on: MainScheduler.instance) + .subscribe(onError: { error in + ELOG(error.localizedDescription) + }) { + DLOG("disposing nonDatabaseFileSyncer") + nonDatabaseFileSyncer = nil + }.disposed(by: disposeBag) + var saveStateSyncer: SaveStateSyncer! = .init(notificationCenter: .default) saveStateSyncer.loadAllFromICloud() { saveStateSyncer.importNewSaves() }.observe(on: MainScheduler.instance) @@ -373,34 +408,6 @@ public enum iCloudSync { DLOG("disposing romsSyncer") romsSyncer = nil }.disposed(by: disposeBag) - //TODO: set the following to merge onto a single class that just does a query for icloud for all of those directories. - biosSyncer = .init(notificationCenter: .default) - biosSyncer.loadAllFromICloud() - .observe(on: MainScheduler.instance) - .subscribe(onError: { error in - ELOG(error.localizedDescription) - }) { - DLOG("disposing BiosSyncer") - biosSyncer = nil - }.disposed(by: disposeBag) - batterySavesSyncer = .init(notificationCenter: .default) - batterySavesSyncer.loadAllFromICloud() - .observe(on: MainScheduler.instance) - .subscribe(onError: { error in - ELOG(error.localizedDescription) - }) { - DLOG("disposing BatterySavesSyncer") - batterySavesSyncer = nil - }.disposed(by: disposeBag) - screenshotsSyncer = .init(notificationCenter: .default) - screenshotsSyncer.loadAllFromICloud() - .observe(on: MainScheduler.instance) - .subscribe(onError: { error in - ELOG(error.localizedDescription) - }) { - DLOG("disposing ScreenshotsSyncer") - screenshotsSyncer = nil - }.disposed(by: disposeBag) } static func turnOff() { @@ -416,9 +423,10 @@ public enum iCloudSync { //TODO: perhaps 1 generic class since a lot of this code is similar and move the extension onto generic class. we could just add a protocol delegate dependency for ROMs and SaveState classes that does specific code class SaveStateSyncer: iCloudContainerSyncer { var didFinishDownloadingAllFiles = false - + let jsonDecorder = JSONDecoder() convenience init(notificationCenter: NotificationCenter) { - self.init(directory: "Save States", notificationCenter: notificationCenter) + self.init(directories: ["Save States"], notificationCenter: notificationCenter) + jsonDecorder.dataDecodingStrategy = .deferredToData notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in self?.importNewSaves() } @@ -459,25 +467,45 @@ class SaveStateSyncer: iCloudContainerSyncer { else { return } - do { + do {//TODO: querying via the id will be better let realm = try Realm() DLOG("attempting to query PVSaveState by file: \(file)") - let imageField = NSExpression(forKeyPath: \PVSaveState.image.self).keyPath - let urlField = NSExpression(forKeyPath: \PVImageFile.url.self).keyPath - let absoluteStringField = NSExpression(forKeyPath: \URL.absoluteString.self).keyPath - let results = realm.objects(PVSaveState.self).filter(NSPredicate(format: "\(imageField).\(urlField).\(absoluteStringField) == %@", file.absoluteString)) - guard let save: PVSaveState = results.first + let save = try getSaveFrom(file) + guard let existingSave = realm.object(ofType: PVSaveState.self, forPrimaryKey: save.id) else { return } try realm.write { - realm.delete(save) + realm.delete(existingSave) } } catch { ELOG(error.localizedDescription) } } + func getSaveFrom(_ json: URL) throws -> SaveState { + let secureDoc = json.startAccessingSecurityScopedResource() + + defer { + if secureDoc { + json.stopAccessingSecurityScopedResource() + } + } + + var dataMaybe = fileManager.contents(atPath: json.path) + if dataMaybe == nil { + dataMaybe = try Data(contentsOf: json, options: [.uncached]) + } + guard let data = dataMaybe else { + throw iCloudError.dataReadFail + } + + DLOG("Data read \(String(data: data, encoding: .utf8) ?? "Nil")") + let save = try jsonDecorder.decode(SaveState.self, from: data) + DLOG("Read JSON data at (\(json.absoluteString)") + return save + } + func importNewSaves() { guard RomDatabase.databaseInitialized else { @@ -497,33 +525,15 @@ class SaveStateSyncer: iCloudContainerSyncer { newFiles.removeAll() uploadedFiles.removeAll() Task { - let jsonDecorder = JSONDecoder() - jsonDecorder.dataDecodingStrategy = .deferredToData - Task.detached { // @MainActor in - await jsonFiles.concurrentForEach { @MainActor json in + await jsonFiles.concurrentForEach { @MainActor [weak self] json in do { let realm = try await Realm() - let secureDoc = json.startAccessingSecurityScopedResource() - - defer { - if secureDoc { - json.stopAccessingSecurityScopedResource() - } + guard let save = try self?.getSaveFrom(json) + else { + return } - var dataMaybe = FileManager.default.contents(atPath: json.path) - if dataMaybe == nil { - dataMaybe = try Data(contentsOf: json, options: [.uncached]) - } - guard let data = dataMaybe else { - throw iCloudError.dataReadFail - } - - DLOG("Data read \(String(data: data, encoding: .utf8) ?? "Nil")") - let save = try jsonDecorder.decode(SaveState.self, from: data) - DLOG("Read JSON data at (\(json.absoluteString)") - let existing = realm.object(ofType: PVSaveState.self, forPrimaryKey: save.id) if let existing = existing { // Skip if Save already exists @@ -569,7 +579,7 @@ class RomsSyncer: iCloudContainerSyncer { let gameImporter = GameImporter.shared var didFinishDownloadingAllFiles = false convenience init(notificationCenter: NotificationCenter) { - self.init(directory: "ROMs", notificationCenter: notificationCenter) + self.init(directories: ["ROMs"], notificationCenter: notificationCenter) notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in self?.importNewRomFiles() } @@ -679,21 +689,3 @@ class RomsSyncer: iCloudContainerSyncer { gameImporter.startProcessing() } } - -class BiosSyncer: iCloudContainerSyncer { - convenience init(notificationCenter: NotificationCenter) { - self.init(directory: "BIOS", notificationCenter: notificationCenter) - } -} - -class BatterySavesSyncer: iCloudContainerSyncer { - convenience init(notificationCenter: NotificationCenter) { - self.init(directory: "Battery Saves", notificationCenter: notificationCenter) - } -} - -class ScreenshotsSyncer: iCloudContainerSyncer { - convenience init(notificationCenter: NotificationCenter) { - self.init(directory: "Screenshots", notificationCenter: notificationCenter) - } -} From 390ef40c1dbebacc1904113ceb062776ac2acdcd Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 18:46:17 -0500 Subject: [PATCH 62/86] fixed battery states directory name --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index e3ccb57f7d..85fe169fb7 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -314,7 +314,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { ILOG("Trying to move (\(currentItem.path)) to (\(destination.path))") try moveClosure(currentItem, destination) insertUploadedFile(destination) - } catch { + } catch {//TODO: failed to move: The file couldn’t be locked //this could indicate no more space is left when moving to iCloud ELOG("failed to move: \(error.localizedDescription)") } @@ -378,7 +378,7 @@ public enum iCloudSync { //TODO: should we pause when a game starts so we don't interfere with the game and continue listening when no game is running? disposeBag = DisposeBag() - var nonDatabaseFileSyncer: iCloudContainerSyncer! = .init(directories: ["BIOS", "Battery Saves", "Screenshots"], notificationCenter: .default) + var nonDatabaseFileSyncer: iCloudContainerSyncer! = .init(directories: ["BIOS", "Battery States", "Screenshots"], notificationCenter: .default) nonDatabaseFileSyncer.loadAllFromICloud() .observe(on: MainScheduler.instance) .subscribe(onError: { error in From 441831f8a5cfd290fea103fc233df812ff27c229 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 20:09:53 -0500 Subject: [PATCH 63/86] updated to add full error; added check for storage available --- .../Importer/iCloud/iCloudSync.swift | 94 +++++++++++++++---- 1 file changed, 77 insertions(+), 17 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 85fe169fb7..21e6d4bc4e 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -245,7 +245,11 @@ class iCloudContainerSyncer: iCloudTypeSyncer { let moved = moveFiles(at: localDirectory, containerDestination: iCloudDirectory, existingClosure: { existing in - try fileManager.removeItem(atPath: existing.path) + do { + try fileManager.removeItem(atPath: existing.path) + } catch { + ELOG("error deleting existing file that already exists in iCloud: \(existing), \(error)") + } }) { currentSource, currentDestination in try fileManager.setUbiquitous(true, itemAt: currentSource, destinationURL: currentDestination) } @@ -267,10 +271,18 @@ class iCloudContainerSyncer: iCloudTypeSyncer { let moved = moveFiles(at: iCloudDirectory, containerDestination: localDirectory, existingClosure: { existing in - try fileManager.evictUbiquitousItem(at: existing) + do { + try fileManager.evictUbiquitousItem(at: existing) + } catch {//this happens when a file is being presented on the UI (saved states image) and thus we can't remove the icloud download + ELOG("error evicting iCloud file: \(existing), \(error)") + } }) { currentSource, currentDestination in try fileManager.copyItem(at: currentSource, to: currentDestination) - try fileManager.evictUbiquitousItem(at: currentSource) + do { + try fileManager.evictUbiquitousItem(at: currentSource) + } catch {//this happens when a file is being presented on the UI (saved states image) and thus we can't remove the icloud download + ELOG("error evicting iCloud file: \(currentSource), \(error)") + } } if moved == .saveFailure { moveResult = .saveFailure @@ -281,8 +293,10 @@ class iCloudContainerSyncer: iCloudTypeSyncer { func moveFiles(at source: URL, containerDestination: URL, - existingClosure: ((URL) throws -> Void), + existingClosure: ((URL) -> Void), moveClosure: (URL, URL) throws -> Void) -> SyncResult { + getDeviceAvailableStorage() + getICloudAvailableStorage() DLOG("source: \(source)") guard fileManager.fileExists(atPath: source.path) else { @@ -301,30 +315,76 @@ class iCloudContainerSyncer: iCloudTypeSyncer { DLOG("new destination: \(destination)") if isDirectory.boolValue && !fileManager.fileExists(atPath: destination.path) { DLOG("\(destination) does NOT exist") - try fileManager.createDirectory(atPath: destination.path, withIntermediateDirectories: true) + do { + try fileManager.createDirectory(atPath: destination.path, withIntermediateDirectories: true) + } catch { + DLOG("error creating directory: \(destination.path), \(error)") + } } if isDirectory.boolValue { continue } if fileManager.fileExists(atPath: destination.path) { - try existingClosure(currentItem) + existingClosure(currentItem) continue } do { - ILOG("Trying to move (\(currentItem.path)) to (\(destination.path))") + ILOG("Trying to move \(currentItem.path) to \(destination.path)") try moveClosure(currentItem, destination) insertUploadedFile(destination) - } catch {//TODO: failed to move: The file couldn’t be locked + } catch { //this could indicate no more space is left when moving to iCloud - ELOG("failed to move: \(error.localizedDescription)") + ELOG("failed to move \(currentItem.path) to \(destination.path): \(error)") } } return .success } catch { - ELOG("failed to get directory contents: \(error.localizedDescription)") + ELOG("failed to get directory contents: \(error)") return .saveFailure } } + + func getICloudAvailableStorage() { + guard let iCloudContainer = containerURL + else { + ELOG("iCloud is not enabled.") + return + } + do { + let values = try iCloudContainer.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + if let availableSpace = values.volumeAvailableCapacityForImportantUsage { + ILOG("Available iCloud storage: \(availableSpace.toGb)") + } else { + ELOG("Could not retrieve available iCloud storage.") + } + } catch { + ELOG("Error retrieving iCloud storage info: \(error)") + } + } + + func getDeviceAvailableStorage() { + guard let systemURL = fileManager.urls(for: .documentDirectory, in: .userDomainMask).first + else { + ELOG("unable to determine available storage on device") + return + } + do { + let values = try systemURL.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + if let availableSpace = values.volumeAvailableCapacityForImportantUsage { + ILOG("Available device storage: \(availableSpace.toGb)") + } else { + ELOG("Could not retrieve available storage.") + } + } catch { + ELOG("Error retrieving storage info: \(error)") + } + } +} + +extension Int64 { + var toGb: String { + String(format: "%.2f GBs", Double(self / (1024 * 1024 * 1024))) + } } enum iCloudError: Error { @@ -370,7 +430,7 @@ public enum iCloudSync { let newTokenData = try NSKeyedArchiver.archivedData(withRootObject: currentiCloudToken, requiringSecureCoding: false) UserDefaults.standard.set(newTokenData, forKey: UbiquityIdentityTokenKey) } catch { - ELOG("\(error.localizedDescription)") + ELOG("error serializing iCloud token: \(error)") } } else { UserDefaults.standard.removeObject(forKey: UbiquityIdentityTokenKey) @@ -479,7 +539,7 @@ class SaveStateSyncer: iCloudContainerSyncer { realm.delete(existingSave) } } catch { - ELOG(error.localizedDescription) + ELOG("error delating from database: \(error)") } } @@ -545,7 +605,7 @@ class SaveStateSyncer: iCloudContainerSyncer { existing.game = game } } catch { - ELOG("Failed to update game: \(error.localizedDescription)") + ELOG("Failed to update game: \(error)") } } // TODO: Maybe any other missing data updates or update values in general? @@ -559,14 +619,14 @@ class SaveStateSyncer: iCloudContainerSyncer { realm.add(newSave, update: .all) } } catch { - ELOG(error.localizedDescription) + ELOG("error adding new save: \(error)") } } else { realm.add(newSave, update: .all) } ILOG("Added new save \(newSave.debugDescription)") } catch { - ELOG("Decode error: " + error.localizedDescription) + ELOG("Decode error: \(error)") return } } @@ -628,7 +688,7 @@ class RomsSyncer: iCloudContainerSyncer { return } } catch { - ELOG(error.localizedDescription) + ELOG("error searching existing ROM: \(error)") } newFiles.insert(file) @@ -663,7 +723,7 @@ class RomsSyncer: iCloudContainerSyncer { realm.delete(game) } } catch { - ELOG(error.localizedDescription) + ELOG("error deleting ROM from database: \(error)") } } From 70272018c6d561cdb1333963c67c2a3d87ee62f5 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 21:41:12 -0500 Subject: [PATCH 64/86] removed unused field --- .../Sources/PVSettings/Settings/Model/PVSettingsModel.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/PVSettings/Sources/PVSettings/Settings/Model/PVSettingsModel.swift b/PVSettings/Sources/PVSettings/Settings/Model/PVSettingsModel.swift index 214ce18e95..cd221561cb 100644 --- a/PVSettings/Sources/PVSettings/Settings/Model/PVSettingsModel.swift +++ b/PVSettings/Sources/PVSettings/Settings/Model/PVSettingsModel.swift @@ -217,7 +217,6 @@ public extension Defaults.Keys { static let useUIKit = Key("useUIKit", default:false) #endif static let iCloudSync = Key("iCloudSync", default: false) - static let iCloudInitialSetupComplete = Key("iCloudInitialSetupComplete", default: false) static let unsupportedCores = Key("unsupportedCores", default: false) #if os(tvOS) static let tvOSThemes = Key("tvOSThemes", default: false) From 8b39cab5f266de843cacea76b65849f1f7e2b8af Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 9 Feb 2025 23:03:46 -0500 Subject: [PATCH 65/86] refactored insert code into base class; updated logs; remove unnecessary use of user defaults flag; updated logic so initial opening of app or switching icloud on tries to import anything already downloaded --- .../Importer/iCloud/iCloudSync.swift | 139 ++++++------------ 1 file changed, 46 insertions(+), 93 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 21e6d4bc4e..8cf5d4d9c9 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -60,6 +60,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { let directories: Set let fileManager = FileManager.default let notificationCenter: NotificationCenter + var status: iCloudSyncStatus = .initialUpload init(directories: Set, notificationCenter: NotificationCenter) { self.notificationCenter = notificationCenter @@ -92,19 +93,27 @@ class iCloudContainerSyncer: iCloudTypeSyncer { let metadataQuery: NSMetadataQuery = .init() func insertDownloadingFile(_ file: URL) { - //no-op + guard !uploadedFiles.contains(file) + else { + return + } + pendingFilesToDownload.insert(file.absoluteString) } func insertDownloadedFile(_ file: URL) { - //no-op + pendingFilesToDownload.remove(file.absoluteString) } func insertUploadedFile(_ file: URL) { - //no-op + uploadedFiles.insert(file) } func setNewCloudFilesAvailable() { - //no-op + if pendingFilesToDownload.isEmpty { + status = .filesAlreadyMoved + DLOG("set status to \(status) and removing all uploaded files in \(directories)") + uploadedFiles.removeAll() + } } func deleteFromDatastore(_ file: URL) { @@ -136,7 +145,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } metadataQuery.searchScopes = [NSMetadataQueryUbiquitousDocumentsScope] - DLOG("directory: \(directories)") + DLOG("directories: \(directories)") var predicateFormat = "" var predicateArgs = [CVarArg]() directories.forEach { directory in @@ -176,7 +185,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } func queryFinished(notification: Notification) async { - DLOG("directory: \(directories)") + DLOG("directories: \(directories)") guard (notification.object as? NSMetadataQuery) == metadataQuery else { return @@ -204,7 +213,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { files.insert(file) self?.insertDownloadingFile(file) } - DLOG("Download started for: \(file.lastPathComponent)") + DLOG("Download started for: \(file)") } catch { DLOG("Failed to start download: \(error)") } @@ -215,16 +224,15 @@ class iCloudContainerSyncer: iCloudTypeSyncer { self?.deleteFromDatastore(file) } else { queue.sync { - //in the case when we are initially turning on iCloud, we try to import any files already downloaded - //TODO: this should only happen one time per turning on the iCloud switch to avoid doing this every time the app opens, comes back to the foreground - if !Defaults[.iCloudInitialSetupComplete] { + //in the case when we are initially turning on iCloud or the app is opened and coming into the foreground for the first time, we try to import any files already downloaded + if self?.status == .initialUpload { self?.insertDownloadingFile(file) } filesDownloaded.insert(file) self?.insertDownloadedFile(file) } } - default: DLOG("\(file.lastPathComponent): download status: \(downloadStatus)") + default: DLOG("\(file): download status: \(downloadStatus)") } } } @@ -295,8 +303,6 @@ class iCloudContainerSyncer: iCloudTypeSyncer { containerDestination: URL, existingClosure: ((URL) -> Void), moveClosure: (URL, URL) throws -> Void) -> SyncResult { - getDeviceAvailableStorage() - getICloudAvailableStorage() DLOG("source: \(source)") guard fileManager.fileExists(atPath: source.path) else { @@ -343,42 +349,6 @@ class iCloudContainerSyncer: iCloudTypeSyncer { return .saveFailure } } - - func getICloudAvailableStorage() { - guard let iCloudContainer = containerURL - else { - ELOG("iCloud is not enabled.") - return - } - do { - let values = try iCloudContainer.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) - if let availableSpace = values.volumeAvailableCapacityForImportantUsage { - ILOG("Available iCloud storage: \(availableSpace.toGb)") - } else { - ELOG("Could not retrieve available iCloud storage.") - } - } catch { - ELOG("Error retrieving iCloud storage info: \(error)") - } - } - - func getDeviceAvailableStorage() { - guard let systemURL = fileManager.urls(for: .documentDirectory, in: .userDomainMask).first - else { - ELOG("unable to determine available storage on device") - return - } - do { - let values = try systemURL.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) - if let availableSpace = values.volumeAvailableCapacityForImportantUsage { - ILOG("Available device storage: \(availableSpace.toGb)") - } else { - ELOG("Could not retrieve available storage.") - } - } catch { - ELOG("Error retrieving storage info: \(error)") - } - } } extension Int64 { @@ -392,11 +362,16 @@ enum iCloudError: Error { } public enum iCloudSync { + case initialAppLoad + case appLoaded + static var disposeBag: DisposeBag! static var gameImporter = GameImporter.shared + static var state: iCloudSync = .initialAppLoad public static func initICloudDocuments() { Task { + printDeviceAvailableStorage() for await value in Defaults.updates(.iCloudSync) { iCloudSyncChanged(value) } @@ -422,8 +397,6 @@ public enum iCloudSync { DLOG("turning on iCloud") //reset ROMs path gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) - Defaults[.iCloudInitialSetupComplete] = false//TODO: when the app first opens, this shouldn't be done, only when the user taps the flag on - let fm = FileManager.default if let currentiCloudToken = fm.ubiquityIdentityToken { do { @@ -476,13 +449,30 @@ public enum iCloudSync { //reset ROMs path gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) } + + static func printDeviceAvailableStorage() { + guard let systemURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first + else { + ELOG("unable to determine available storage on device") + return + } + do { + let values = try systemURL.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + if let availableSpace = values.volumeAvailableCapacityForImportantUsage { + ILOG("available device storage: \(availableSpace.toGb)") + } else { + ELOG("could not retrieve available storage.") + } + } catch { + ELOG("error retrieving storage available: \(error)") + } + } } //MARK: - iCloud syncers //TODO: perhaps 1 generic class since a lot of this code is similar and move the extension onto generic class. we could just add a protocol delegate dependency for ROMs and SaveState classes that does specific code class SaveStateSyncer: iCloudContainerSyncer { - var didFinishDownloadingAllFiles = false let jsonDecorder = JSONDecoder() convenience init(notificationCenter: NotificationCenter) { self.init(directories: ["Save States"], notificationCenter: notificationCenter) @@ -496,32 +486,16 @@ class SaveStateSyncer: iCloudContainerSyncer { notificationCenter.removeObserver(self) } - override func setNewCloudFilesAvailable() { - didFinishDownloadingAllFiles = pendingFilesToDownload.isEmpty && !newFiles.isEmpty - } - - override func insertDownloadingFile(_ file: URL) { - guard !uploadedFiles.contains(file) - else { - return - } - pendingFilesToDownload.insert(file.absoluteString) - } - override func insertDownloadedFile(_ file: URL) { guard let _ = pendingFilesToDownload.remove(file.absoluteString), "json".caseInsensitiveCompare(file.pathExtension) == .orderedSame else { return } - DLOG("downloaded save file: \(file.lastPathComponent)") + DLOG("downloaded save file: \(file)") newFiles.insert(file) } - override func insertUploadedFile(_ file: URL) { - uploadedFiles.insert(file) - } - override func deleteFromDatastore(_ file: URL) { guard "jpg'".caseInsensitiveCompare(file.pathExtension) == .orderedSame else { @@ -575,12 +549,10 @@ class SaveStateSyncer: iCloudContainerSyncer { else { return } - guard didFinishDownloadingAllFiles + guard pendingFilesToDownload.isEmpty else { return } - //TODO: initially when importing icloud files, we should wait for ROMs to finish importing. when that completes, then we no longer have to wait for that, ie when syncing single changes say 2 devices are open at the same time and 1 save file is added from another device - didFinishDownloadingAllFiles = false let jsonFiles = newFiles newFiles.removeAll() uploadedFiles.removeAll() @@ -637,7 +609,7 @@ class SaveStateSyncer: iCloudContainerSyncer { class RomsSyncer: iCloudContainerSyncer { let gameImporter = GameImporter.shared - var didFinishDownloadingAllFiles = false + convenience init(notificationCenter: NotificationCenter) { self.init(directories: ["ROMs"], notificationCenter: notificationCenter) notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in @@ -649,19 +621,6 @@ class RomsSyncer: iCloudContainerSyncer { notificationCenter.removeObserver(self) } - override func setNewCloudFilesAvailable() { - didFinishDownloadingAllFiles = pendingFilesToDownload.isEmpty && !newFiles.isEmpty - } - - override func insertDownloadingFile(_ file: URL) { - //we ensure we don't re-add any files we just moved over - guard !uploadedFiles.contains(file) - else { - return - } - pendingFilesToDownload.insert(file.absoluteString) - } - override func insertDownloadedFile(_ file: URL) { guard let _ = pendingFilesToDownload.remove(file.absoluteString) else { @@ -694,10 +653,6 @@ class RomsSyncer: iCloudContainerSyncer { newFiles.insert(file) } - override func insertUploadedFile(_ file: URL) { - uploadedFiles.insert(file) - } - override func deleteFromDatastore(_ file: URL) { //TODO: remove cloud download, but keep in iCloud. this way the ROM isn't deleted from all devices guard let fileName = file.lastPathComponent.removingPercentEncoding, @@ -736,12 +691,10 @@ class RomsSyncer: iCloudContainerSyncer { else { return } - guard didFinishDownloadingAllFiles + guard pendingFilesToDownload.isEmpty else { return } - didFinishDownloadingAllFiles = false - Defaults[.iCloudInitialSetupComplete] = true let importPaths = [URL](newFiles) newFiles.removeAll() uploadedFiles.removeAll() From c2f97f7e5dca1affd82f1da477c6c139711f2588 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 15 Feb 2025 16:46:08 -0500 Subject: [PATCH 66/86] added error queue to store errors when syncing with iCloud --- .../Importer/iCloud/iCloudSync.swift | 147 +++++++++++++++--- 1 file changed, 127 insertions(+), 20 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 8cf5d4d9c9..814b3004f9 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -61,10 +61,14 @@ class iCloudContainerSyncer: iCloudTypeSyncer { let fileManager = FileManager.default let notificationCenter: NotificationCenter var status: iCloudSyncStatus = .initialUpload + let errorHandler: ErrorHandler - init(directories: Set, notificationCenter: NotificationCenter) { + init(directories: Set, + notificationCenter: NotificationCenter, + errorHandler: ErrorHandler) { self.notificationCenter = notificationCenter self.directories = directories + self.errorHandler = errorHandler } deinit { @@ -215,11 +219,12 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } DLOG("Download started for: \(file)") } catch { + self?.errorHandler.handleError(error, file: file) DLOG("Failed to start download: \(error)") } case NSMetadataUbiquitousItemDownloadingStatusCurrent: DLOG("item up to date: \(file)") - if !fileManager.fileExists(atPath: file.path) { + if !fileManager.fileExists(atPath: file.pathDecoded) { DLOG("file DELETED from iCloud: \(file)") self?.deleteFromDatastore(file) } else { @@ -254,8 +259,9 @@ class iCloudContainerSyncer: iCloudTypeSyncer { containerDestination: iCloudDirectory, existingClosure: { existing in do { - try fileManager.removeItem(atPath: existing.path) + try fileManager.removeItem(atPath: existing.pathDecoded) } catch { + errorHandler.handleError(error, file: existing) ELOG("error deleting existing file that already exists in iCloud: \(existing), \(error)") } }) { currentSource, currentDestination in @@ -282,6 +288,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { do { try fileManager.evictUbiquitousItem(at: existing) } catch {//this happens when a file is being presented on the UI (saved states image) and thus we can't remove the icloud download + errorHandler.handleError(error, file: existing) ELOG("error evicting iCloud file: \(existing), \(error)") } }) { currentSource, currentDestination in @@ -289,6 +296,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { do { try fileManager.evictUbiquitousItem(at: currentSource) } catch {//this happens when a file is being presented on the UI (saved states image) and thus we can't remove the icloud download + errorHandler.handleError(error, file: currentSource) ELOG("error evicting iCloud file: \(currentSource), \(error)") } } @@ -304,47 +312,50 @@ class iCloudContainerSyncer: iCloudTypeSyncer { existingClosure: ((URL) -> Void), moveClosure: (URL, URL) throws -> Void) -> SyncResult { DLOG("source: \(source)") - guard fileManager.fileExists(atPath: source.path) + guard fileManager.fileExists(atPath: source.pathDecoded) else { return .fileNotExist } do { - let subdirectories = try fileManager.subpathsOfDirectory(atPath: source.path) + let subdirectories = try fileManager.subpathsOfDirectory(atPath: source.pathDecoded) DLOG("subdirectories of \(source): \(subdirectories)") for currentChild in subdirectories { let currentItem = source.appendingPathComponent(currentChild) var isDirectory: ObjCBool = false - let exists = fileManager.fileExists(atPath: currentItem.path, isDirectory: &isDirectory) + let exists = fileManager.fileExists(atPath: currentItem.pathDecoded, isDirectory: &isDirectory) DLOG("\(currentItem) isDirectory?\(isDirectory) exists?\(exists)") let destination = containerDestination.appendingPathComponent(currentChild) DLOG("new destination: \(destination)") - if isDirectory.boolValue && !fileManager.fileExists(atPath: destination.path) { + if isDirectory.boolValue && !fileManager.fileExists(atPath: destination.pathDecoded) { DLOG("\(destination) does NOT exist") do { - try fileManager.createDirectory(atPath: destination.path, withIntermediateDirectories: true) + try fileManager.createDirectory(atPath: destination.pathDecoded, withIntermediateDirectories: true) } catch { - DLOG("error creating directory: \(destination.path), \(error)") + errorHandler.handleError(error, file: destination) + DLOG("error creating directory: \(destination.pathDecoded), \(error)") } } if isDirectory.boolValue { continue } - if fileManager.fileExists(atPath: destination.path) { + if fileManager.fileExists(atPath: destination.pathDecoded) { existingClosure(currentItem) continue } do { - ILOG("Trying to move \(currentItem.path) to \(destination.path)") + ILOG("Trying to move \(currentItem.pathDecoded) to \(destination.pathDecoded)") try moveClosure(currentItem, destination) insertUploadedFile(destination) } catch { + errorHandler.handleError(error, file: currentItem) //this could indicate no more space is left when moving to iCloud - ELOG("failed to move \(currentItem.path) to \(destination.path): \(error)") + ELOG("failed to move \(currentItem.pathDecoded) to \(destination.pathDecoded): \(error)") } } return .success } catch { + errorHandler.handleError(error, file: source) ELOG("failed to get directory contents: \(error)") return .saveFailure } @@ -357,6 +368,13 @@ extension Int64 { } } +extension URL { + /// calls URL.path(percentEncoded: false) which is the same as the upcoming deprecation of URL.path + var pathDecoded: String { + path(percentEncoded: false) + } +} + enum iCloudError: Error { case dataReadFail } @@ -368,6 +386,7 @@ public enum iCloudSync { static var disposeBag: DisposeBag! static var gameImporter = GameImporter.shared static var state: iCloudSync = .initialAppLoad + static let errorHandler: ErrorHandler = iCloudErrorHandler.shared public static func initICloudDocuments() { Task { @@ -403,6 +422,7 @@ public enum iCloudSync { let newTokenData = try NSKeyedArchiver.archivedData(withRootObject: currentiCloudToken, requiringSecureCoding: false) UserDefaults.standard.set(newTokenData, forKey: UbiquityIdentityTokenKey) } catch { + errorHandler.handleError(error, file: nil) ELOG("error serializing iCloud token: \(error)") } } else { @@ -411,7 +431,9 @@ public enum iCloudSync { //TODO: should we pause when a game starts so we don't interfere with the game and continue listening when no game is running? disposeBag = DisposeBag() - var nonDatabaseFileSyncer: iCloudContainerSyncer! = .init(directories: ["BIOS", "Battery States", "Screenshots"], notificationCenter: .default) + var nonDatabaseFileSyncer: iCloudContainerSyncer! = .init(directories: ["BIOS", "Battery States", "Screenshots"], + notificationCenter: .default, + errorHandler: iCloudErrorHandler.shared) nonDatabaseFileSyncer.loadAllFromICloud() .observe(on: MainScheduler.instance) .subscribe(onError: { error in @@ -420,7 +442,7 @@ public enum iCloudSync { DLOG("disposing nonDatabaseFileSyncer") nonDatabaseFileSyncer = nil }.disposed(by: disposeBag) - var saveStateSyncer: SaveStateSyncer! = .init(notificationCenter: .default) + var saveStateSyncer: SaveStateSyncer! = .init(notificationCenter: .default, errorHandler: iCloudErrorHandler.shared) saveStateSyncer.loadAllFromICloud() { saveStateSyncer.importNewSaves() }.observe(on: MainScheduler.instance) @@ -431,7 +453,7 @@ public enum iCloudSync { saveStateSyncer = nil }.disposed(by: disposeBag) - var romsSyncer: RomsSyncer! = .init(notificationCenter: .default) + var romsSyncer: RomsSyncer! = .init(notificationCenter: .default, errorHandler: iCloudErrorHandler.shared) romsSyncer.loadAllFromICloud() { romsSyncer.importNewRomFiles() }.observe(on: MainScheduler.instance) @@ -474,8 +496,8 @@ public enum iCloudSync { //TODO: perhaps 1 generic class since a lot of this code is similar and move the extension onto generic class. we could just add a protocol delegate dependency for ROMs and SaveState classes that does specific code class SaveStateSyncer: iCloudContainerSyncer { let jsonDecorder = JSONDecoder() - convenience init(notificationCenter: NotificationCenter) { - self.init(directories: ["Save States"], notificationCenter: notificationCenter) + convenience init(notificationCenter: NotificationCenter, errorHandler: ErrorHandler) { + self.init(directories: ["Save States"], notificationCenter: notificationCenter, errorHandler: errorHandler) jsonDecorder.dataDecodingStrategy = .deferredToData notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in self?.importNewSaves() @@ -513,6 +535,7 @@ class SaveStateSyncer: iCloudContainerSyncer { realm.delete(existingSave) } } catch { + errorHandler.handleError(error, file: file) ELOG("error delating from database: \(error)") } } @@ -526,7 +549,7 @@ class SaveStateSyncer: iCloudContainerSyncer { } } - var dataMaybe = fileManager.contents(atPath: json.path) + var dataMaybe = fileManager.contents(atPath: json.pathDecoded) if dataMaybe == nil { dataMaybe = try Data(contentsOf: json, options: [.uncached]) } @@ -577,6 +600,7 @@ class SaveStateSyncer: iCloudContainerSyncer { existing.game = game } } catch { + self?.errorHandler.handleError(error, file: json) ELOG("Failed to update game: \(error)") } } @@ -591,6 +615,7 @@ class SaveStateSyncer: iCloudContainerSyncer { realm.add(newSave, update: .all) } } catch { + self?.errorHandler.handleError(error, file: json) ELOG("error adding new save: \(error)") } } else { @@ -598,6 +623,7 @@ class SaveStateSyncer: iCloudContainerSyncer { } ILOG("Added new save \(newSave.debugDescription)") } catch { + self?.errorHandler.handleError(error, file: json) ELOG("Decode error: \(error)") return } @@ -610,8 +636,8 @@ class SaveStateSyncer: iCloudContainerSyncer { class RomsSyncer: iCloudContainerSyncer { let gameImporter = GameImporter.shared - convenience init(notificationCenter: NotificationCenter) { - self.init(directories: ["ROMs"], notificationCenter: notificationCenter) + convenience init(notificationCenter: NotificationCenter, errorHandler: ErrorHandler) { + self.init(directories: ["ROMs"], notificationCenter: notificationCenter, errorHandler: errorHandler) notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in self?.importNewRomFiles() } @@ -647,6 +673,7 @@ class RomsSyncer: iCloudContainerSyncer { return } } catch { + errorHandler.handleError(error, file: file) ELOG("error searching existing ROM: \(error)") } @@ -678,6 +705,7 @@ class RomsSyncer: iCloudContainerSyncer { realm.delete(game) } } catch { + errorHandler.handleError(error, file: file) ELOG("error deleting ROM from database: \(error)") } } @@ -702,3 +730,82 @@ class RomsSyncer: iCloudContainerSyncer { gameImporter.startProcessing() } } + +struct iCloudSyncError { + let file: String? + var summary: String { + error.localizedDescription + } + let error: Error +} + +protocol Queue { + associatedtype Entry + var count: Int { get } + mutating func enqueue(entry: Entry) + mutating func dequeue() -> Entry? + func peek() -> Entry? + mutating func clear() +} + +struct iCloudErrorsQueue: Queue { + var errors = [iCloudSyncError]() + + var count: Int { + errors.count + } + + mutating func enqueue(entry: iCloudSyncError) { + errors.insert(entry, at: 0) + } + + mutating func dequeue() -> iCloudSyncError? { + guard !errors.isEmpty + else { + return nil + } + return errors.removeFirst() + } + + func peek() -> iCloudSyncError? { + errors.first + } + + mutating func clear() { + errors.removeAll() + } +} + +protocol ErrorHandler { + var allErrorSummaries: [String] { get } + var allFullErrors: [String] { get } + var allErrors: [iCloudSyncError] { get } + func handleError(_ error: Error, file: URL?) + func clear() +} + +class iCloudErrorHandler: ErrorHandler { + static let shared = iCloudErrorHandler() + var queue = iCloudErrorsQueue() + + var allErrorSummaries: [String] { + queue.errors.map { $0.summary } + } + + var allFullErrors: [String] { + queue.errors.map { "\($0.error)" } + } + + var allErrors: [iCloudSyncError] { + queue.errors + } + + func handleError(_ error: any Error, file: URL?) { + let syncError = iCloudSyncError(file: file?.path(percentEncoded: false), error: error) + queue.enqueue(entry: syncError) + } + + func clear() { + queue.clear() + } +} From 99b490a9e1a46dbcfb8ae19ccdce4d7fe993852f Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 15 Feb 2025 17:04:33 -0500 Subject: [PATCH 67/86] refactored move files to reduce nesting; updated to handle the case when importer is curently importing files and to do the importing when it finishes --- .../Importer/iCloud/iCloudSync.swift | 101 +++++++++++------- 1 file changed, 61 insertions(+), 40 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 814b3004f9..ff7644941f 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -316,49 +316,50 @@ class iCloudContainerSyncer: iCloudTypeSyncer { else { return .fileNotExist } + let subdirectories: [String] do { - let subdirectories = try fileManager.subpathsOfDirectory(atPath: source.pathDecoded) - DLOG("subdirectories of \(source): \(subdirectories)") - for currentChild in subdirectories { - let currentItem = source.appendingPathComponent(currentChild) - - var isDirectory: ObjCBool = false - let exists = fileManager.fileExists(atPath: currentItem.pathDecoded, isDirectory: &isDirectory) - DLOG("\(currentItem) isDirectory?\(isDirectory) exists?\(exists)") - let destination = containerDestination.appendingPathComponent(currentChild) - DLOG("new destination: \(destination)") - if isDirectory.boolValue && !fileManager.fileExists(atPath: destination.pathDecoded) { - DLOG("\(destination) does NOT exist") - do { - try fileManager.createDirectory(atPath: destination.pathDecoded, withIntermediateDirectories: true) - } catch { - errorHandler.handleError(error, file: destination) - DLOG("error creating directory: \(destination.pathDecoded), \(error)") - } - } - if isDirectory.boolValue { - continue - } - if fileManager.fileExists(atPath: destination.pathDecoded) { - existingClosure(currentItem) - continue - } - do { - ILOG("Trying to move \(currentItem.pathDecoded) to \(destination.pathDecoded)") - try moveClosure(currentItem, destination) - insertUploadedFile(destination) - } catch { - errorHandler.handleError(error, file: currentItem) - //this could indicate no more space is left when moving to iCloud - ELOG("failed to move \(currentItem.pathDecoded) to \(destination.pathDecoded): \(error)") - } - } - return .success + subdirectories = try fileManager.subpathsOfDirectory(atPath: source.pathDecoded) } catch { errorHandler.handleError(error, file: source) ELOG("failed to get directory contents: \(error)") return .saveFailure } + DLOG("subdirectories of \(source): \(subdirectories)") + for currentChild in subdirectories { + let currentItem = source.appendingPathComponent(currentChild) + + var isDirectory: ObjCBool = false + let exists = fileManager.fileExists(atPath: currentItem.pathDecoded, isDirectory: &isDirectory) + DLOG("\(currentItem) isDirectory?\(isDirectory) exists?\(exists)") + let destination = containerDestination.appendingPathComponent(currentChild) + DLOG("new destination: \(destination)") + if isDirectory.boolValue && !fileManager.fileExists(atPath: destination.pathDecoded) { + DLOG("\(destination) does NOT exist") + do { + try fileManager.createDirectory(atPath: destination.pathDecoded, withIntermediateDirectories: true) + } catch { + errorHandler.handleError(error, file: destination) + DLOG("error creating directory: \(destination.pathDecoded), \(error)") + } + } + if isDirectory.boolValue { + continue + } + if fileManager.fileExists(atPath: destination.pathDecoded) { + existingClosure(currentItem) + continue + } + do { + ILOG("Trying to move \(currentItem.pathDecoded) to \(destination.pathDecoded)") + try moveClosure(currentItem, destination) + insertUploadedFile(destination) + } catch { + errorHandler.handleError(error, file: currentItem) + //this could indicate no more space is left when moving to iCloud + ELOG("failed to move \(currentItem.pathDecoded) to \(destination.pathDecoded): \(error)") + } + } + return .success } } @@ -455,7 +456,7 @@ public enum iCloudSync { var romsSyncer: RomsSyncer! = .init(notificationCenter: .default, errorHandler: iCloudErrorHandler.shared) romsSyncer.loadAllFromICloud() { - romsSyncer.importNewRomFiles() + romsSyncer.handleImportNewRomFiles() }.observe(on: MainScheduler.instance) .subscribe(onError: { error in ELOG(error.localizedDescription) @@ -639,7 +640,10 @@ class RomsSyncer: iCloudContainerSyncer { convenience init(notificationCenter: NotificationCenter, errorHandler: ErrorHandler) { self.init(directories: ["ROMs"], notificationCenter: notificationCenter, errorHandler: errorHandler) notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in - self?.importNewRomFiles() + self?.handleImportNewRomFiles() + } + notificationCenter.addObserver(forName: .RomsFinishedImporting, object: nil, queue: nil) { [weak self] _ in + self?.handleImportNewRomFiles() } } @@ -710,7 +714,7 @@ class RomsSyncer: iCloudContainerSyncer { } } - func importNewRomFiles() { + func handleImportNewRomFiles() { guard RomDatabase.databaseInitialized else { return @@ -723,6 +727,23 @@ class RomsSyncer: iCloudContainerSyncer { else { return } + Task { @MainActor in + tryToImportNewRomFiles() + } + } + + func tryToImportNewRomFiles() { + //if the importer is currently importing files, we have to wait + guard gameImporter.processingState == .idle + else { + return + } + Task { + importNewRomFiles() + } + } + + func importNewRomFiles() { let importPaths = [URL](newFiles) newFiles.removeAll() uploadedFiles.removeAll() From 281b0bf8ce9f8ab0cc56da2667da052127ac6697 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 15 Feb 2025 21:23:20 -0500 Subject: [PATCH 68/86] added deleting of json file when deleting save states; added trying to get system from icloud parent directory --- .../PVLibrary/Database/Realm Database/RomDatabase.swift | 9 +++++++++ .../GameImporter/GameImporterSystemsService.swift | 7 ++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift index fcce9daba4..fbc0e307ce 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift @@ -782,6 +782,15 @@ public extension RomDatabase { throw RomDeletionError.fileManagerDeletionError(error) } } + let jsonFile = actualSavePath.pathDecoded.appending(".json") + if FileManager.default.fileExists(atPath: jsonFile) { + do { + try FileManager.default.removeItem(atPath: jsonFile) + } catch { + ELOG("Unable to delete json at path: \(jsonFile) because: \(error.localizedDescription)") + throw RomDeletionError.fileManagerDeletionError(error) + } + } } func deleteRelatedFilesGame(_ game: PVGame) { diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift index 74860f2787..6882a7fa86 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift @@ -42,7 +42,12 @@ class GameImporterSystemsService: GameImporterSystemsServicing { } func determineSystems(for item: ImportQueueItem) async throws -> [SystemIdentifier] { - // First try MD5 lookup + // if syncing from icloud, we have the system, so try to get the system this way + if let system = SystemIdentifier(rawValue: item.url.deletingLastPathComponent().lastPathComponent.lowercased()) { + DLOG("found system: \(system)") + return [system] + } + // next try MD5 lookup if let md5 = item.md5 { if let systemID = try await lookup.systemIdentifier(forRomMD5: md5, or: item.url.lastPathComponent) { return [systemID] From 387b539f245acf5ebd7bcbef64f71f5c01b8742e Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 16 Feb 2025 01:07:44 -0500 Subject: [PATCH 69/86] added confirmation when deleting a ROM --- .../PVRootViewController+DelegateMethods.swift | 12 +++++++----- .../PVUIBase/UIViewController+Alerts.swift | 17 +++++++++++++++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/PVUI/Sources/PVSwiftUI/RootView/PVRootViewController+DelegateMethods.swift b/PVUI/Sources/PVSwiftUI/RootView/PVRootViewController+DelegateMethods.swift index d21486216f..629de4e96d 100644 --- a/PVUI/Sources/PVSwiftUI/RootView/PVRootViewController+DelegateMethods.swift +++ b/PVUI/Sources/PVSwiftUI/RootView/PVRootViewController+DelegateMethods.swift @@ -48,11 +48,13 @@ extension PVRootViewController: PVRootDelegate { self.presentCoreSelection(forGame: game.warmUp(), sender: sender) } - public func attemptToDelete(game: PVGame, deleteSaves: Bool) { - do { - try self.delete(game: game, deleteSaves: deleteSaves) - } catch { - self.presentError(error.localizedDescription, source: self.view) + public func attemptToDelete(game: PVGame, deleteSaves: Bool) {//TODO: add localization string + presentCancellableMessage("Are you sure you want to delete \"\(game.title)\"?", title: "Delete ROM?", source: view) { + do { + try self.delete(game: game, deleteSaves: deleteSaves) + } catch { + self.presentError(error.localizedDescription, source: self.view) + } } } diff --git a/PVUI/Sources/PVUIBase/UIViewController+Alerts.swift b/PVUI/Sources/PVUIBase/UIViewController+Alerts.swift index 1ea00015c6..f69b315ee2 100644 --- a/PVUI/Sources/PVUIBase/UIViewController+Alerts.swift +++ b/PVUI/Sources/PVUIBase/UIViewController+Alerts.swift @@ -13,7 +13,15 @@ import PVLogging public extension UIViewController { - func presentMessage(_ message: String, title: String, source: UIView, completion _: (() -> Swift.Void)? = nil) { + func presentMessage(_ message: String, title: String, source: UIView, completion: (() -> Swift.Void)? = nil) { + presentMessage(message, title: title, addCancel: false, source: source, completion: completion) + } + + func presentCancellableMessage(_ message: String, title: String, source: UIView, completion: (() -> Swift.Void)? = nil) { + presentMessage(message, title: title, addCancel: true, source: source, completion: completion) + } + + func presentMessage(_ message: String, title: String, addCancel: Bool, source: UIView, completion: (() -> Swift.Void)? = nil) { NSLog("Title: %@ Message: %@", title, message); let alert = UIAlertController(title: title, message: message, preferredStyle: .alert) alert.preferredContentSize = CGSize(width: 300, height: 300) @@ -22,7 +30,12 @@ extension UIViewController { alert.popoverPresentationController?.sourceView = source alert.popoverPresentationController?.sourceRect = UIScreen.main.bounds - alert.addAction(UIAlertAction(title: "OK", style: .default, handler: nil)) + alert.addAction(UIAlertAction(title: "OK", style: .default) { _ in + completion?() + }) + if addCancel { + alert.addAction(UIAlertAction(title: "Cancel", style: .cancel)) + } let presentingVC = presentedViewController ?? self From 6029c614c6aec30db4b01c25a1aa801db9abec75 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 16 Feb 2025 01:10:29 -0500 Subject: [PATCH 70/86] added deletion of db items when deleted from icloud; fixed extension check; fixed predicate; updated to check if file exists first --- .../Importer/iCloud/iCloudSync.swift | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index ff7644941f..045529a712 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -199,7 +199,15 @@ class iCloudContainerSyncer: iCloudTypeSyncer { var filesDownloaded: Set = [] let queue = DispatchQueue(label: "com.provenance.newFiles") let removedObjects = notification.userInfo?[NSMetadataQueryUpdateRemovedItemsKey] - DLOG("\(directories): removedObjects: \(removedObjects)") + if let actualRemovedObjects = removedObjects as? [NSMetadataItem] { + DLOG("\(directories): actualRemovedObjects: (\(actualRemovedObjects.count)) \(actualRemovedObjects)") + await actualRemovedObjects.concurrentForEach { [weak self] item in + if let file = item.value(forAttribute: NSMetadataItemURLKey) as? URL { + DLOG("file DELETED from iCloud: \(file)") + self?.deleteFromDatastore(file) + } + } + } //accessing results automatically pauses updates and resumes after deallocated await metadataQuery.results.concurrentForEach { [weak self] item in @@ -224,7 +232,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } case NSMetadataUbiquitousItemDownloadingStatusCurrent: DLOG("item up to date: \(file)") - if !fileManager.fileExists(atPath: file.pathDecoded) { + if !fileManager.fileExists(atPath: file.pathDecoded) { DLOG("file DELETED from iCloud: \(file)") self?.deleteFromDatastore(file) } else { @@ -520,20 +528,26 @@ class SaveStateSyncer: iCloudContainerSyncer { } override func deleteFromDatastore(_ file: URL) { - guard "jpg'".caseInsensitiveCompare(file.pathExtension) == .orderedSame + guard "jpg".caseInsensitiveCompare(file.pathExtension) == .orderedSame else { return } - do {//TODO: querying via the id will be better + do { let realm = try Realm() DLOG("attempting to query PVSaveState by file: \(file)") - let save = try getSaveFrom(file) - guard let existingSave = realm.object(ofType: PVSaveState.self, forPrimaryKey: save.id) + let gameDirectory = file.deletingLastPathComponent().lastPathComponent + let savesDirectory = file.deletingLastPathComponent().deletingLastPathComponent().lastPathComponent + let partialPath = "\(savesDirectory)/\(gameDirectory)/\(file.lastPathComponent)" + let imageField = NSExpression(forKeyPath: \PVSaveState.image.self).keyPath + let partialPathField = NSExpression(forKeyPath: \PVImageFile.partialPath.self).keyPath + let results = realm.objects(PVSaveState.self).filter(NSPredicate(format: "\(imageField).\(partialPathField) CONTAINS[c] %@", partialPath)) + DLOG("saves found: \(results.count)") + guard let save: PVSaveState = results.first else { return } try realm.write { - realm.delete(existingSave) + realm.delete(save) } } catch { errorHandler.handleError(error, file: file) @@ -541,7 +555,11 @@ class SaveStateSyncer: iCloudContainerSyncer { } } - func getSaveFrom(_ json: URL) throws -> SaveState { + func getSaveFrom(_ json: URL) throws -> SaveState? { + guard fileManager.fileExists(atPath: json.pathDecoded) + else { + return nil + } let secureDoc = json.startAccessingSecurityScopedResource() defer { @@ -745,6 +763,7 @@ class RomsSyncer: iCloudContainerSyncer { func importNewRomFiles() { let importPaths = [URL](newFiles) + //"m3u", "cue" newFiles.removeAll() uploadedFiles.removeAll() gameImporter.addImports(forPaths: importPaths) From 21d31003e794e4ec316733ab7a12b861145c83da Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 16 Feb 2025 14:44:20 -0500 Subject: [PATCH 71/86] updated alerts --- .../Resources/en.lproj/Localizable.strings | 7 +++++ ...PVRootViewController+DelegateMethods.swift | 6 ++-- .../PVUIBase/UIViewController+Alerts.swift | 30 +++++++++++++++---- 3 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 PVUI/Sources/PVSwiftUI/Resources/en.lproj/Localizable.strings diff --git a/PVUI/Sources/PVSwiftUI/Resources/en.lproj/Localizable.strings b/PVUI/Sources/PVSwiftUI/Resources/en.lproj/Localizable.strings new file mode 100644 index 0000000000..d249105ac4 --- /dev/null +++ b/PVUI/Sources/PVSwiftUI/Resources/en.lproj/Localizable.strings @@ -0,0 +1,7 @@ +// +// Untitled.swift +// PVUI +// +// Created by Pablo Arista on 2/16/25. +// + diff --git a/PVUI/Sources/PVSwiftUI/RootView/PVRootViewController+DelegateMethods.swift b/PVUI/Sources/PVSwiftUI/RootView/PVRootViewController+DelegateMethods.swift index 629de4e96d..2d7b753efc 100644 --- a/PVUI/Sources/PVSwiftUI/RootView/PVRootViewController+DelegateMethods.swift +++ b/PVUI/Sources/PVSwiftUI/RootView/PVRootViewController+DelegateMethods.swift @@ -48,8 +48,10 @@ extension PVRootViewController: PVRootDelegate { self.presentCoreSelection(forGame: game.warmUp(), sender: sender) } - public func attemptToDelete(game: PVGame, deleteSaves: Bool) {//TODO: add localization string - presentCancellableMessage("Are you sure you want to delete \"\(game.title)\"?", title: "Delete ROM?", source: view) { + public func attemptToDelete(game: PVGame, deleteSaves: Bool) { + //String(format: NSLocalizedString("DeleteGameBody", bundle: Bundle.module, comment: ""), game.title) + //NSLocalizedString("DeleteGameTitle", comment: "") + presentCancellableMessage("Are you sure you want to delete game \"\(game.title)\"?", title: "Delete Game?", source: view) { do { try self.delete(game: game, deleteSaves: deleteSaves) } catch { diff --git a/PVUI/Sources/PVUIBase/UIViewController+Alerts.swift b/PVUI/Sources/PVUIBase/UIViewController+Alerts.swift index f69b315ee2..f5829331db 100644 --- a/PVUI/Sources/PVUIBase/UIViewController+Alerts.swift +++ b/PVUI/Sources/PVUIBase/UIViewController+Alerts.swift @@ -14,14 +14,32 @@ import PVLogging public extension UIViewController { func presentMessage(_ message: String, title: String, source: UIView, completion: (() -> Swift.Void)? = nil) { - presentMessage(message, title: title, addCancel: false, source: source, completion: completion) + presentMessage(message, + title: title, + source: source, + completion: completion) } func presentCancellableMessage(_ message: String, title: String, source: UIView, completion: (() -> Swift.Void)? = nil) { - presentMessage(message, title: title, addCancel: true, source: source, completion: completion) + presentMessage(message, title: title, + source: source, + secondaryActionTitle: NSLocalizedString("Cancel", comment: ""), + secondaryActionStyle: .cancel, + secondaryCompletion: nil, + defaultActionTitle: NSLocalizedString("Delete", comment: ""), + defaultActionStyle: .destructive, + completion: completion) } - func presentMessage(_ message: String, title: String, addCancel: Bool, source: UIView, completion: (() -> Swift.Void)? = nil) { + func presentMessage(_ message: String, + title: String, + source: UIView, + secondaryActionTitle: String? = nil, + secondaryActionStyle: UIAlertAction.Style = .cancel, + secondaryCompletion: ((UIAlertAction) -> Void)? = nil, + defaultActionTitle: String = NSLocalizedString("OK", comment: ""), + defaultActionStyle: UIAlertAction.Style = .default, + completion: (() -> Swift.Void)? = nil) { NSLog("Title: %@ Message: %@", title, message); let alert = UIAlertController(title: title, message: message, preferredStyle: .alert) alert.preferredContentSize = CGSize(width: 300, height: 300) @@ -30,11 +48,11 @@ extension UIViewController { alert.popoverPresentationController?.sourceView = source alert.popoverPresentationController?.sourceRect = UIScreen.main.bounds - alert.addAction(UIAlertAction(title: "OK", style: .default) { _ in + alert.addAction(UIAlertAction(title: defaultActionTitle, style: defaultActionStyle) { _ in completion?() }) - if addCancel { - alert.addAction(UIAlertAction(title: "Cancel", style: .cancel)) + if let actualSecondaryActionTitle = secondaryActionTitle { + alert.addAction(UIAlertAction(title: actualSecondaryActionTitle, style: secondaryActionStyle, handler: secondaryCompletion)) } let presentingVC = presentedViewController ?? self From ed042061670e8e2b0421328bfae021dad7b3b6e4 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 16 Feb 2025 17:09:02 -0500 Subject: [PATCH 72/86] fixed bug when deleting file and then re-adding the same file by removing game from cache after deletion either from manual deletion or icloud sync deletion --- .../Database/Realm Database/RomDatabase.swift | 5 +++- .../Services/GameImporter/GameImporter.swift | 26 +++++++++++++++++-- .../Importer/iCloud/iCloudSync.swift | 9 ++++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift index fbc0e307ce..cb7f84c0d9 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift @@ -727,9 +727,12 @@ public extension RomDatabase { game.recentPlays.forEach { try? $0.delete() } game.screenShots.forEach { try? $0.delete() } try game.delete() + RomDatabase.reloadGamesCache() } catch { // Delete the DB entry anyway if any of the above files couldn't be removed - do { try game.delete() } catch { + do { try game.delete() + RomDatabase.reloadGamesCache() + } catch { ELOG("\(error.localizedDescription)") } ELOG("\(error.localizedDescription)") diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 305c1dafb4..bc96e6fc26 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -416,6 +416,27 @@ public final class GameImporter: GameImporting, ObservableObject { importQueue.remove(atOffsets: offsets) } + + /// Searches for successful imports filtered by files and removes from importQueue and filies. This is so that only files imported by iCloud can be removed + /// - Parameter files: set of files to check + public func removeSuccessfulImports(from files: inout Set) { + guard !files.isEmpty + else { + return + } + importQueueLock.lock() + defer { + importQueueLock.unlock() + } + var removed = [URL]() + importQueue.enumerated().forEach { index, item in + if item.status == .success && files.contains(item.url) { + files.remove(item.url) + importQueue.remove(at: index) + + } + } + } // Public method to manually start processing if needed public func startProcessing() { @@ -651,6 +672,9 @@ public final class GameImporter: GameImporting, ObservableObject { // Processes each ImportItem in the queue sequentially @MainActor private func processQueue() async { + defer { + NotificationCenter.default.post(name: .RomsFinishedImporting, object: nil) + } // Check for items that are either queued or have a user-chosen system let itemsToProcess = importQueue.filter { $0.status == .queued || $0.userChosenSystem != nil @@ -680,8 +704,6 @@ public final class GameImporter: GameImporting, ObservableObject { self.processingState = .idle } ILOG("GameImportQueue - processQueue complete Import Processing") - //TODO: this doesn't appear to be needed anymore. - NotificationCenter.default.post(name: .RomsFinishedImporting, object: nil) } // Process a single ImportItem and update its status diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 045529a712..1ed51cdf9b 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -654,6 +654,7 @@ class SaveStateSyncer: iCloudContainerSyncer { class RomsSyncer: iCloudContainerSyncer { let gameImporter = GameImporter.shared + var processingFiles = Set() convenience init(notificationCenter: NotificationCenter, errorHandler: ErrorHandler) { self.init(directories: ["ROMs"], notificationCenter: notificationCenter, errorHandler: errorHandler) @@ -725,6 +726,7 @@ class RomsSyncer: iCloudContainerSyncer { game.recentPlays.forEach { try? $0.delete() } game.screenShots.forEach { try? $0.delete() } realm.delete(game) + RomDatabase.reloadGamesCache() } } catch { errorHandler.handleError(error, file: file) @@ -737,6 +739,7 @@ class RomsSyncer: iCloudContainerSyncer { else { return } + clearProcessedFiles() guard !newFiles.isEmpty else { return @@ -761,9 +764,13 @@ class RomsSyncer: iCloudContainerSyncer { } } + func clearProcessedFiles() { + gameImporter.removeSuccessfulImports(from: &processingFiles) + } + func importNewRomFiles() { + processingFiles = newFiles let importPaths = [URL](newFiles) - //"m3u", "cue" newFiles.removeAll() uploadedFiles.removeAll() gameImporter.addImports(forPaths: importPaths) From 5e2c599a9a80bb5f360ee742da78becf71f5f910 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 16 Feb 2025 19:23:04 -0500 Subject: [PATCH 73/86] fixed deletion of ROM and related files --- .../Database/Realm Database/RomDatabase.swift | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift index cb7f84c0d9..f5ff0ba6cf 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift @@ -811,6 +811,40 @@ public extension RomDatabase { NSLog(error.localizedDescription) } } + //attempt to delete files with the same name. There's an issue when importing that the files do NOT get associated, so if we assume the user imported properly, the name should just be the same with different extensions. + let parentDirectory = game.file.url.deletingLastPathComponent() + let fileManager: FileManager = .default + guard fileManager.fileExists(atPath: parentDirectory.pathDecoded) + else { + return + } + let children: [String] + do { + children = try fileManager.subpathsOfDirectory(atPath: parentDirectory.pathDecoded) + } catch { + ELOG("error retrieving files at directory: \(parentDirectory), \(error)") + return + } + guard !children.isEmpty + else { + return + } + DLOG("children: \(children)") + let fileName = game.file.url.deletingPathExtension().lastPathComponent + DLOG("fileName without extension: \(fileName)") + children.forEach { child in + let currentChildUrl = parentDirectory.appendingPathComponent(child) + let currentExtension = currentChildUrl.pathExtension + let currentChildFileName = currentChildUrl.deletingPathExtension().lastPathComponent + DLOG("current extension: \(currentExtension), current file name: \(currentChildFileName), current url: \(currentChildUrl)") + if !currentExtension.allSatisfy({$0.isWhitespace}) && currentChildFileName == fileName { + do { + try fileManager.removeItem(at: currentChildUrl) + } catch { + ELOG("error deleting file: \(currentChildUrl)") + } + } + } } } From d1106326a1fd0dfd91b4fc9f5b152ab1dee56b20 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 16 Feb 2025 19:57:53 -0500 Subject: [PATCH 74/86] removed unused storage code --- .../Importer/iCloud/iCloudSync.swift | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 1ed51cdf9b..7a6ceafd3e 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -399,7 +399,6 @@ public enum iCloudSync { public static func initICloudDocuments() { Task { - printDeviceAvailableStorage() for await value in Defaults.updates(.iCloudSync) { iCloudSyncChanged(value) } @@ -480,24 +479,6 @@ public enum iCloudSync { //reset ROMs path gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) } - - static func printDeviceAvailableStorage() { - guard let systemURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first - else { - ELOG("unable to determine available storage on device") - return - } - do { - let values = try systemURL.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) - if let availableSpace = values.volumeAvailableCapacityForImportantUsage { - ILOG("available device storage: \(availableSpace.toGb)") - } else { - ELOG("could not retrieve available storage.") - } - } catch { - ELOG("error retrieving storage available: \(error)") - } - } } //MARK: - iCloud syncers From 43e47c4665853edefe0ea6a8f556926795e59ce5 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 16 Feb 2025 22:22:24 -0500 Subject: [PATCH 75/86] refactored code to get indexset to fix index out of bounds when multiple items in queue are removed; added deletion of games when the application is closed and the cloud container ROMs are removed --- .../Services/GameImporter/GameImporter.swift | 9 +- .../Importer/iCloud/iCloudSync.swift | 89 ++++++++++++++++--- 2 files changed, 81 insertions(+), 17 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index bc96e6fc26..4ac53328d9 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -429,13 +429,14 @@ public final class GameImporter: GameImporting, ObservableObject { importQueueLock.unlock() } var removed = [URL]() - importQueue.enumerated().forEach { index, item in + let offsets = IndexSet(importQueue.enumerated().compactMap { index, item in if item.status == .success && files.contains(item.url) { files.remove(item.url) - importQueue.remove(at: index) - + return index } - } + return nil + }) + importQueue.remove(atOffsets: offsets) } // Public method to manually start processing if needed diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 7a6ceafd3e..1d499fe4ba 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -25,6 +25,7 @@ public enum SyncResult { case saveFailure case fileNotExist case success + case indeterminate } public protocol Container { @@ -53,6 +54,11 @@ enum iCloudSyncStatus { case filesAlreadyMoved } +enum GamePurgeStatus { + case incomplete + case complete +} + class iCloudContainerSyncer: iCloudTypeSyncer { lazy var pendingFilesToDownload: Set = [] lazy var newFiles: Set = [] @@ -62,6 +68,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { let notificationCenter: NotificationCenter var status: iCloudSyncStatus = .initialUpload let errorHandler: ErrorHandler + var initialSyncResult: SyncResult = .indeterminate init(directories: Set, notificationCenter: NotificationCenter, @@ -87,7 +94,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { else { return alliCloudDirectories } - let parentContainer = actualContainrUrl.appendingPathComponent("Documents") + let parentContainer = actualContainrUrl.appendDocumentsDirectory directories.forEach { directory in alliCloudDirectories[URL.documentsDirectory.appendingPathComponent(directory)] = parentContainer.appendingPathComponent(directory) } @@ -139,10 +146,10 @@ class iCloudContainerSyncer: iCloudTypeSyncer { return } - let completion = syncToiCloud() - DLOG("saveStateUploader syncToiCloud result: \(completion)") - guard completion != .saveFailure, - completion != .denied + initialSyncResult = syncToiCloud() + DLOG("syncToiCloud result: \(initialSyncResult)") + guard initialSyncResult != .saveFailure, + initialSyncResult != .denied else { ELOG("error moving files to iCloud container") return @@ -382,6 +389,22 @@ extension URL { var pathDecoded: String { path(percentEncoded: false) } + var appendDocumentsDirectory: URL { + appendingPathComponent("Documents") + } +} + +extension Realm { + func deleteGame(_ game: PVGame) throws { + try write { + game.saveStates.forEach { try? $0.delete() } + game.cheats.forEach { try? $0.delete() } + game.recentPlays.forEach { try? $0.delete() } + game.screenShots.forEach { try? $0.delete() } + delete(game) + RomDatabase.reloadGamesCache() + } + } } enum iCloudError: Error { @@ -424,6 +447,7 @@ public enum iCloudSync { DLOG("turning on iCloud") //reset ROMs path gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) + errorHandler.clear() let fm = FileManager.default if let currentiCloudToken = fm.ubiquityIdentityToken { do { @@ -475,6 +499,7 @@ public enum iCloudSync { static func turnOff() { DLOG("turning off iCloud") + errorHandler.clear() disposeBag = nil //reset ROMs path gameImporter.gameImporterDatabaseService.setRomsPath(url: gameImporter.romsPath) @@ -636,10 +661,12 @@ class SaveStateSyncer: iCloudContainerSyncer { class RomsSyncer: iCloudContainerSyncer { let gameImporter = GameImporter.shared var processingFiles = Set() + var purgeStatus: GamePurgeStatus = .incomplete convenience init(notificationCenter: NotificationCenter, errorHandler: ErrorHandler) { self.init(directories: ["ROMs"], notificationCenter: notificationCenter, errorHandler: errorHandler) notificationCenter.addObserver(forName: .RomDatabaseInitialized, object: nil, queue: nil) { [weak self] _ in + self?.removeGamesDeletedWhileApplicationClosed() self?.handleImportNewRomFiles() } notificationCenter.addObserver(forName: .RomsFinishedImporting, object: nil, queue: nil) { [weak self] _ in @@ -651,6 +678,44 @@ class RomsSyncer: iCloudContainerSyncer { notificationCenter.removeObserver(self) } + func removeGamesDeletedWhileApplicationClosed() { + guard purgeStatus == .incomplete, + initialSyncResult == .success, + errorHandler.numberOfErrors == 0 + else { + return + } + + defer { + purgeStatus = .complete + } + guard let actualContainrUrl = containerURL, + let romsDirectoryName = directories.first + else { + return + } + let romsPath = actualContainrUrl.appendDocumentsDirectory.appendingPathComponent(romsDirectoryName) + DLOG("romsPath: \(romsPath)") + let realm: Realm + do { + realm = try Realm() + } catch { + ELOG("error removing game entries that do NOT exist in the cloud container") + return + } + var games = realm.objects(PVGame.self) + games.forEach { game in + let gameUrl = romsPath.appendingPathComponent(game.romPath) + if !fileManager.fileExists(atPath: gameUrl.pathDecoded) { + do { + try realm.deleteGame(game) + } catch { + ELOG("error deleting \(gameUrl), \(error)") + } + } + } + } + override func insertDownloadedFile(_ file: URL) { guard let _ = pendingFilesToDownload.remove(file.absoluteString) else { @@ -701,14 +766,7 @@ class RomsSyncer: iCloudContainerSyncer { return } - try realm.write { - game.saveStates.forEach { try? $0.delete() } - game.cheats.forEach { try? $0.delete() } - game.recentPlays.forEach { try? $0.delete() } - game.screenShots.forEach { try? $0.delete() } - realm.delete(game) - RomDatabase.reloadGamesCache() - } + try realm.deleteGame(game) } catch { errorHandler.handleError(error, file: file) ELOG("error deleting ROM from database: \(error)") @@ -808,6 +866,7 @@ protocol ErrorHandler { var allErrorSummaries: [String] { get } var allFullErrors: [String] { get } var allErrors: [iCloudSyncError] { get } + var numberOfErrors: Int { get } func handleError(_ error: Error, file: URL?) func clear() } @@ -828,6 +887,10 @@ class iCloudErrorHandler: ErrorHandler { queue.errors } + var numberOfErrors: Int { + queue.count + } + func handleError(_ error: any Error, file: URL?) { let syncError = iCloudSyncError(file: file?.path(percentEncoded: false), error: error) queue.enqueue(entry: syncError) From 06d6612d2082117d88e54f0e1bf8128593471a0e Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 17 Feb 2025 10:07:15 -0500 Subject: [PATCH 76/86] refactored to defer instead of duplicating code; updated to check for different kinds of multiple disc ROM files; fixed comment --- .../Database/Realm Database/RomDatabase.swift | 12 ++++++++---- .../Services/GameImporter/GameImporter.swift | 2 +- .../GameImporter/GameImporterSystemsService.swift | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift index f5ff0ba6cf..bfb193d753 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift @@ -329,7 +329,6 @@ public final class RomDatabase { ILOG("Database initialization completed") databaseInitialized = true NotificationCenter.default.post(name: .RomDatabaseInitialized, object: nil, userInfo: nil) - } else { ILOG("Database already initialized") } @@ -720,6 +719,9 @@ public extension RomDatabase { #if os(iOS) deleteFromSpotlight(game: game) #endif + defer { + RomDatabase.reloadGamesCache() + } do { deleteRelatedFilesGame(game) game.saveStates.forEach { try? $0.delete() } @@ -727,11 +729,9 @@ public extension RomDatabase { game.recentPlays.forEach { try? $0.delete() } game.screenShots.forEach { try? $0.delete() } try game.delete() - RomDatabase.reloadGamesCache() } catch { // Delete the DB entry anyway if any of the above files couldn't be removed do { try game.delete() - RomDatabase.reloadGamesCache() } catch { ELOG("\(error.localizedDescription)") } @@ -837,7 +837,11 @@ public extension RomDatabase { let currentExtension = currentChildUrl.pathExtension let currentChildFileName = currentChildUrl.deletingPathExtension().lastPathComponent DLOG("current extension: \(currentExtension), current file name: \(currentChildFileName), current url: \(currentChildUrl)") - if !currentExtension.allSatisfy({$0.isWhitespace}) && currentChildFileName == fileName { + if !currentExtension.isEmpty + && (currentChildFileName == fileName + || currentChildFileName.starts(with: "\(fileName) (Track ") + || currentChildFileName.starts(with: "\(fileName) (Disc ") + ) { do { try fileManager.removeItem(at: currentChildUrl) } catch { diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 4ac53328d9..b33a4a766c 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -417,7 +417,7 @@ public final class GameImporter: GameImporting, ObservableObject { importQueue.remove(atOffsets: offsets) } - /// Searches for successful imports filtered by files and removes from importQueue and filies. This is so that only files imported by iCloud can be removed + /// Searches for successful imports filtered by files and removes from importQueue and files. This is so that only files imported by iCloud can be removed /// - Parameter files: set of files to check public func removeSuccessfulImports(from files: inout Set) { guard !files.isEmpty diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift index 6882a7fa86..f2af6ddecb 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporterSystemsService.swift @@ -43,7 +43,7 @@ class GameImporterSystemsService: GameImporterSystemsServicing { func determineSystems(for item: ImportQueueItem) async throws -> [SystemIdentifier] { // if syncing from icloud, we have the system, so try to get the system this way - if let system = SystemIdentifier(rawValue: item.url.deletingLastPathComponent().lastPathComponent.lowercased()) { + if let system = SystemIdentifier(rawValue: item.url.deletingLastPathComponent().lastPathComponent) { DLOG("found system: \(system)") return [system] } From edae4e7940aae101b8ceca22810c17836d0e349f Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 17 Feb 2025 10:07:31 -0500 Subject: [PATCH 77/86] updated localization --- .../PVSwiftUI/Resources/en.lproj/Localizable.strings | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/PVUI/Sources/PVSwiftUI/Resources/en.lproj/Localizable.strings b/PVUI/Sources/PVSwiftUI/Resources/en.lproj/Localizable.strings index d249105ac4..7bb679d943 100644 --- a/PVUI/Sources/PVSwiftUI/Resources/en.lproj/Localizable.strings +++ b/PVUI/Sources/PVSwiftUI/Resources/en.lproj/Localizable.strings @@ -1,7 +1,4 @@ -// -// Untitled.swift -// PVUI -// -// Created by Pablo Arista on 2/16/25. -// - +"DeleteGameTitle" = "Delete Game?"; +"DeleteGameBody" = "Are you sure you want to delete game \"%@\"?"; +"OK" = "OK"; +"Cancel" = "Cancel"; From 7909e06748e61a6273508eed74fdb4cc0b0fdfe5 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Mon, 17 Feb 2025 23:32:45 -0500 Subject: [PATCH 78/86] removed unused variable --- .../PVLibrary/Importer/Services/GameImporter/GameImporter.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 369a71eddb..80f47e97ce 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -429,7 +429,6 @@ public final class GameImporter: GameImporting, ObservableObject { defer { importQueueLock.unlock() } - var removed = [URL]() let offsets = IndexSet(importQueue.enumerated().compactMap { index, item in if item.status == .success && files.contains(item.url) { files.remove(item.url) From 3d3e2e123e178eebdf607a60843852faf4d1589f Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Wed, 19 Feb 2025 00:05:53 -0500 Subject: [PATCH 79/86] updated to process 10 ROMs at a time. --- .../PVLibrary/Importer/iCloud/iCloudSync.swift | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 1d499fe4ba..7d78afa9e9 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -69,6 +69,8 @@ class iCloudContainerSyncer: iCloudTypeSyncer { var status: iCloudSyncStatus = .initialUpload let errorHandler: ErrorHandler var initialSyncResult: SyncResult = .indeterminate + let queue = DispatchQueue(label: "com.provenance.newFiles") + let fileImportQueueMaxCount = 10 init(directories: Set, notificationCenter: NotificationCenter, @@ -204,7 +206,6 @@ class iCloudContainerSyncer: iCloudTypeSyncer { let fileManager = FileManager.default var files: Set = [] var filesDownloaded: Set = [] - let queue = DispatchQueue(label: "com.provenance.newFiles") let removedObjects = notification.userInfo?[NSMetadataQueryUpdateRemovedItemsKey] if let actualRemovedObjects = removedObjects as? [NSMetadataItem] { DLOG("\(directories): actualRemovedObjects: (\(actualRemovedObjects.count)) \(actualRemovedObjects)") @@ -228,7 +229,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { case NSMetadataUbiquitousItemDownloadingStatusNotDownloaded: do { try fileManager.startDownloadingUbiquitousItem(at: file) - queue.sync { + self?.queue.async(flags: .barrier) { files.insert(file) self?.insertDownloadingFile(file) } @@ -243,7 +244,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { DLOG("file DELETED from iCloud: \(file)") self?.deleteFromDatastore(file) } else { - queue.sync { + self?.queue.async(flags: .barrier) { //in the case when we are initially turning on iCloud or the app is opened and coming into the foreground for the first time, we try to import any files already downloaded if self?.status == .initialUpload { self?.insertDownloadingFile(file) @@ -808,10 +809,13 @@ class RomsSyncer: iCloudContainerSyncer { } func importNewRomFiles() { - processingFiles = newFiles - let importPaths = [URL](newFiles) - newFiles.removeAll() - uploadedFiles.removeAll() + let nextFilesToProcess = newFiles.prefix(fileImportQueueMaxCount) + newFiles.subtract(nextFilesToProcess) + processingFiles.formUnion(nextFilesToProcess) + let importPaths = [URL](nextFilesToProcess) + if newFiles.isEmpty {//TODO: does this make sense? + uploadedFiles.removeAll() + } gameImporter.addImports(forPaths: importPaths) gameImporter.startProcessing() } From 3459b90d58094d09a9cdfd3aaf263d751967bb38 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Wed, 19 Feb 2025 00:06:55 -0500 Subject: [PATCH 80/86] added todo --- PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 7d78afa9e9..d4904a3d91 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -603,6 +603,7 @@ class SaveStateSyncer: iCloudContainerSyncer { return } let jsonFiles = newFiles + //TODO: process 10 at a time, do it like the RomsSyncer newFiles.removeAll() uploadedFiles.removeAll() Task { From d12cbc8dd84fe5c62d90606c1c7030461014198d Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 22 Feb 2025 18:29:57 -0500 Subject: [PATCH 81/86] fixed compilation issues --- .../Database/Realm Database/RomDatabase.swift | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift index 31bf9e535c..fcd90394e2 100644 --- a/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift +++ b/PVLibrary/Sources/PVLibrary/Database/Realm Database/RomDatabase.swift @@ -829,8 +829,8 @@ public extension RomDatabase { throw RomDeletionError.fileManagerDeletionError(error) } } - let jsonFile = actualSavePath.pathDecoded.appending(".json") - if FileManager.default.fileExists(atPath: jsonFile) { + if let jsonFile = actualSavePath?.pathDecoded.appending(".json"), + FileManager.default.fileExists(atPath: jsonFile) { do { try FileManager.default.removeItem(atPath: jsonFile) } catch { @@ -856,8 +856,12 @@ public extension RomDatabase { } } //attempt to delete files with the same name. There's an issue when importing that the files do NOT get associated, so if we assume the user imported properly, the name should just be the same with different extensions. - let parentDirectory = game.file.url.deletingLastPathComponent() let fileManager: FileManager = .default + guard let gameFileUrl = game.file?.url + else { + return + } + let parentDirectory = gameFileUrl.deletingLastPathComponent() guard fileManager.fileExists(atPath: parentDirectory.pathDecoded) else { return @@ -874,7 +878,7 @@ public extension RomDatabase { return } DLOG("children: \(children)") - let fileName = game.file.url.deletingPathExtension().lastPathComponent + let fileName = gameFileUrl.deletingPathExtension().lastPathComponent DLOG("fileName without extension: \(fileName)") children.forEach { child in let currentChildUrl = parentDirectory.appendingPathComponent(child) From b12269d8f0fe64452de3c7079c725170762c832b Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 22 Feb 2025 21:51:48 -0500 Subject: [PATCH 82/86] added sending signal on the main queue --- .../Importer/Services/GameImporter/GameImporter.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift index 5ee157b331..3dfccbcac0 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/Services/GameImporter/GameImporter.swift @@ -680,7 +680,9 @@ public final class GameImporter: GameImporting, ObservableObject { // Processes each ImportItem in the queue sequentially private func processQueue() async { defer { - NotificationCenter.default.post(name: .RomsFinishedImporting, object: nil) + DispatchQueue.main.async { + NotificationCenter.default.post(name: .RomsFinishedImporting, object: nil) + } } // Check for items that are either queued or have a user-chosen system let itemsToProcess = importQueue.filter { From 3f7911b40e6cec1c10404a17a4e78af66eff7e6c Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 22 Feb 2025 21:52:10 -0500 Subject: [PATCH 83/86] removed extra parenthesis --- .../Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift index 029afd894b..71f9fbed28 100644 --- a/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift +++ b/PVLibrary/Sources/PVRealm/RealmPlatform/Entities/Files/PVFile.swift @@ -123,7 +123,7 @@ public extension PVFile { } let root = relativeRoot let resolvedURL = root.appendingPath(partialPath) - DLOG("resolvedURL:\(resolvedURL))") + DLOG("resolvedURL:\(resolvedURL)") // return resolvedURL return url2 } From ce2594bf1632eb41e62bdcd136cbe274b7a17328 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 22 Feb 2025 21:53:33 -0500 Subject: [PATCH 84/86] added more logs, change foreach to synchronous to handle the case when the app first loads or the user turns on icloud and icloud files are already downloaded, but not imported --- .../Importer/iCloud/iCloudSync.swift | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index d4904a3d91..dfd2b3561d 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -218,7 +218,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { } //accessing results automatically pauses updates and resumes after deallocated - await metadataQuery.results.concurrentForEach { [weak self] item in + /*await*/ metadataQuery.results.forEach/*concurrentForEach*/ { [weak self] item in if let fileItem = item as? NSMetadataItem, let file = fileItem.value(forAttribute: NSMetadataItemURLKey) as? URL, let isDirectory = try? file.resourceValues(forKeys: [.isDirectoryKey]).isDirectory, @@ -229,10 +229,10 @@ class iCloudContainerSyncer: iCloudTypeSyncer { case NSMetadataUbiquitousItemDownloadingStatusNotDownloaded: do { try fileManager.startDownloadingUbiquitousItem(at: file) - self?.queue.async(flags: .barrier) { + //self?.queue.async(flags: .barrier) { files.insert(file) self?.insertDownloadingFile(file) - } + //} DLOG("Download started for: \(file)") } catch { self?.errorHandler.handleError(error, file: file) @@ -244,14 +244,14 @@ class iCloudContainerSyncer: iCloudTypeSyncer { DLOG("file DELETED from iCloud: \(file)") self?.deleteFromDatastore(file) } else { - self?.queue.async(flags: .barrier) { + //self?.queue.async(flags: .barrier) { //in the case when we are initially turning on iCloud or the app is opened and coming into the foreground for the first time, we try to import any files already downloaded if self?.status == .initialUpload { self?.insertDownloadingFile(file) } filesDownloaded.insert(file) self?.insertDownloadedFile(file) - } + //} } default: DLOG("\(file): download status: \(downloadStatus)") } @@ -606,6 +606,7 @@ class SaveStateSyncer: iCloudContainerSyncer { //TODO: process 10 at a time, do it like the RomsSyncer newFiles.removeAll() uploadedFiles.removeAll() + //TODO: try to change this to a single task and can we do this on a background thread instead of the main? Task { Task.detached { // @MainActor in await jsonFiles.concurrentForEach { @MainActor [weak self] json in @@ -725,7 +726,7 @@ class RomsSyncer: iCloudContainerSyncer { } let parentDirectory = file.deletingLastPathComponent().lastPathComponent - DLOG("adding file to game import queue: \(file), parent directory: \(parentDirectory)") + DLOG("attempting to add file to game import queue: \(file), parent directory: \(parentDirectory)") //we should only add to the import queue files that are actual ROMs, anything else can be ignored. guard parentDirectory.range(of: "com.provenance.", options: [.caseInsensitive, .anchored]) != nil, @@ -741,13 +742,14 @@ class RomsSyncer: iCloudContainerSyncer { let results = realm.objects(PVGame.self).filter(NSPredicate(format: "\(NSExpression(forKeyPath: \PVGame.romPath.self).keyPath) == %@", romPath)) guard results.first == nil else { + DLOG("\(file) already exists in database") return } } catch { errorHandler.handleError(error, file: file) ELOG("error searching existing ROM: \(error)") } - + DLOG("\(file) does NOT exist in database, adding to import set") newFiles.insert(file) } @@ -785,10 +787,10 @@ class RomsSyncer: iCloudContainerSyncer { else { return } - guard pendingFilesToDownload.isEmpty - else { - return - } +// guard pendingFilesToDownload.isEmpty +// else { +// return +// } Task { @MainActor in tryToImportNewRomFiles() } @@ -810,9 +812,13 @@ class RomsSyncer: iCloudContainerSyncer { } func importNewRomFiles() { + DLOG("newFiles: (\(newFiles.count)): \(newFiles)") let nextFilesToProcess = newFiles.prefix(fileImportQueueMaxCount) newFiles.subtract(nextFilesToProcess) + DLOG("newFiles minus processing files: (\(newFiles.count)): \(newFiles)") + DLOG("processingFiles: (\(processingFiles.count)): \(processingFiles)") processingFiles.formUnion(nextFilesToProcess) + DLOG("processingFiles plus new files: (\(processingFiles.count)): \(processingFiles)") let importPaths = [URL](nextFilesToProcess) if newFiles.isEmpty {//TODO: does this make sense? uploadedFiles.removeAll() From 71c5e3314c28b9ba8e09054362af557327711308 Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sat, 22 Feb 2025 23:16:08 -0500 Subject: [PATCH 85/86] added processing of save files in batches; --- .../Importer/iCloud/iCloudSync.swift | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index dfd2b3561d..695bcd8314 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -133,6 +133,17 @@ class iCloudContainerSyncer: iCloudTypeSyncer { //no-op } + func prepareNextBatchToProcess() -> any Collection { + DLOG("newFiles: (\(newFiles.count)): \(newFiles)") + let nextFilesToProcess = newFiles.prefix(fileImportQueueMaxCount) + newFiles.subtract(nextFilesToProcess) + DLOG("newFiles minus processing files: (\(newFiles.count)): \(newFiles)") + if newFiles.isEmpty { + uploadedFiles.removeAll() + } + return nextFilesToProcess + } + func loadAllFromICloud(iterationComplete: (() -> Void)? = nil) -> Completable { return Completable.create { [weak self] completable in self?.setupObservers(completable: completable, iterationComplete: iterationComplete) @@ -531,7 +542,12 @@ class SaveStateSyncer: iCloudContainerSyncer { return } DLOG("downloaded save file: \(file)") - newFiles.insert(file) + queue.sync { [weak self] in + self?.newFiles.insert(file) + } + if newFiles.count >= fileImportQueueMaxCount { + importNewSaves() + } } override func deleteFromDatastore(_ file: URL) { @@ -598,14 +614,17 @@ class SaveStateSyncer: iCloudContainerSyncer { else { return } - guard pendingFilesToDownload.isEmpty - else { - return + queue.async(flags: .barrier) { [weak self] in + guard let jsonFiles = self?.prepareNextBatchToProcess(), + !jsonFiles.isEmpty + else { + return + } + self?.processJsonFiles(jsonFiles) } - let jsonFiles = newFiles - //TODO: process 10 at a time, do it like the RomsSyncer - newFiles.removeAll() - uploadedFiles.removeAll() + } + + func processJsonFiles(_ jsonFiles: any Collection) { //TODO: try to change this to a single task and can we do this on a background thread instead of the main? Task { Task.detached { // @MainActor in @@ -787,10 +806,6 @@ class RomsSyncer: iCloudContainerSyncer { else { return } -// guard pendingFilesToDownload.isEmpty -// else { -// return -// } Task { @MainActor in tryToImportNewRomFiles() } @@ -812,10 +827,7 @@ class RomsSyncer: iCloudContainerSyncer { } func importNewRomFiles() { - DLOG("newFiles: (\(newFiles.count)): \(newFiles)") - let nextFilesToProcess = newFiles.prefix(fileImportQueueMaxCount) - newFiles.subtract(nextFilesToProcess) - DLOG("newFiles minus processing files: (\(newFiles.count)): \(newFiles)") + let nextFilesToProcess = prepareNextBatchToProcess() DLOG("processingFiles: (\(processingFiles.count)): \(processingFiles)") processingFiles.formUnion(nextFilesToProcess) DLOG("processingFiles plus new files: (\(processingFiles.count)): \(processingFiles)") From d11db7320e2639c197344cc3a6dc4d13fa81215b Mon Sep 17 00:00:00 2001 From: pqsk <3964831+pabloarista@users.noreply.github.com> Date: Sun, 23 Feb 2025 01:03:13 -0500 Subject: [PATCH 86/86] updated error logs to include file and to use correct log function; updated to use barrier dispatch queue when modifying netFiles set --- .../Importer/iCloud/iCloudSync.swift | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift index 695bcd8314..ec7f80470f 100644 --- a/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift +++ b/PVLibrary/Sources/PVLibrary/Importer/iCloud/iCloudSync.swift @@ -247,7 +247,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { DLOG("Download started for: \(file)") } catch { self?.errorHandler.handleError(error, file: file) - DLOG("Failed to start download: \(error)") + ELOG("Failed to start download on file \(file): \(error)") } case NSMetadataUbiquitousItemDownloadingStatusCurrent: DLOG("item up to date: \(file)") @@ -289,7 +289,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { try fileManager.removeItem(atPath: existing.pathDecoded) } catch { errorHandler.handleError(error, file: existing) - ELOG("error deleting existing file that already exists in iCloud: \(existing), \(error)") + ELOG("error deleting existing file \(existing) that already exists in iCloud: \(error)") } }) { currentSource, currentDestination in try fileManager.setUbiquitous(true, itemAt: currentSource, destinationURL: currentDestination) @@ -348,7 +348,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { subdirectories = try fileManager.subpathsOfDirectory(atPath: source.pathDecoded) } catch { errorHandler.handleError(error, file: source) - ELOG("failed to get directory contents: \(error)") + ELOG("failed to get directory contents \(source): \(error)") return .saveFailure } DLOG("subdirectories of \(source): \(subdirectories)") @@ -366,7 +366,7 @@ class iCloudContainerSyncer: iCloudTypeSyncer { try fileManager.createDirectory(atPath: destination.pathDecoded, withIntermediateDirectories: true) } catch { errorHandler.handleError(error, file: destination) - DLOG("error creating directory: \(destination.pathDecoded), \(error)") + ELOG("error creating directory: \(destination), \(error)") } } if isDirectory.boolValue { @@ -542,10 +542,11 @@ class SaveStateSyncer: iCloudContainerSyncer { return } DLOG("downloaded save file: \(file)") - queue.sync { [weak self] in + let newFilesCount: Int = queue.sync { [weak self] in self?.newFiles.insert(file) + return self?.newFiles.count ?? 0 } - if newFiles.count >= fileImportQueueMaxCount { + if newFilesCount >= fileImportQueueMaxCount { importNewSaves() } } @@ -574,7 +575,7 @@ class SaveStateSyncer: iCloudContainerSyncer { } } catch { errorHandler.handleError(error, file: file) - ELOG("error delating from database: \(error)") + ELOG("error delating \(file) from database: \(error)") } } @@ -648,7 +649,7 @@ class SaveStateSyncer: iCloudContainerSyncer { } } catch { self?.errorHandler.handleError(error, file: json) - ELOG("Failed to update game: \(error)") + ELOG("Failed to update game \(json): \(error)") } } // TODO: Maybe any other missing data updates or update values in general? @@ -663,7 +664,7 @@ class SaveStateSyncer: iCloudContainerSyncer { } } catch { self?.errorHandler.handleError(error, file: json) - ELOG("error adding new save: \(error)") + ELOG("error adding new save \(json): \(error)") } } else { realm.add(newSave, update: .all) @@ -671,7 +672,7 @@ class SaveStateSyncer: iCloudContainerSyncer { ILOG("Added new save \(newSave.debugDescription)") } catch { self?.errorHandler.handleError(error, file: json) - ELOG("Decode error: \(error)") + ELOG("Decode error on \(json): \(error)") return } } @@ -722,7 +723,7 @@ class RomsSyncer: iCloudContainerSyncer { do { realm = try Realm() } catch { - ELOG("error removing game entries that do NOT exist in the cloud container") + ELOG("error removing game entries that do NOT exist in the cloud container \(romsPath)") return } var games = realm.objects(PVGame.self) @@ -766,10 +767,12 @@ class RomsSyncer: iCloudContainerSyncer { } } catch { errorHandler.handleError(error, file: file) - ELOG("error searching existing ROM: \(error)") + ELOG("error searching existing ROM \(file): \(error)") } DLOG("\(file) does NOT exist in database, adding to import set") - newFiles.insert(file) + queue.async(flags: .barrier) { [weak self] in + self?.newFiles.insert(file) + } } override func deleteFromDatastore(_ file: URL) { @@ -792,7 +795,7 @@ class RomsSyncer: iCloudContainerSyncer { try realm.deleteGame(game) } catch { errorHandler.handleError(error, file: file) - ELOG("error deleting ROM from database: \(error)") + ELOG("error deleting ROM \(file) from database: \(error)") } } @@ -817,9 +820,10 @@ class RomsSyncer: iCloudContainerSyncer { else { return } - Task { - importNewRomFiles() + queue.async(flags: .barrier) { [weak self] in + self?.importNewRomFiles() } + } func clearProcessedFiles() {