Merge branch 'master' into f-offline-mode

# Conflicts:
#	cachedDownloadSizes.json

.gitignore

@@ -38,6 +38,8 @@ travis_installer_staging
 .idea/misc.xml
 .idea/python-patcher.iml
 
+curl-ca-bundle.crt
+
 # Mod ignore folders
 INSTALLER_LOGS
 07th-mod-logs.zip

JSONValidator/Sources/JSONValidator/JSONValidator.swift

@@ -149,4 +149,6 @@ public struct ModOptionFileDefinition: Codable {
 	public var relativeExtractionPath: String?
 	/// The priority of this file (same system as above priorities)
 	public var priority: Int
+	/// A file or folder path relative to the *top-level game directory*, to be deleted before extraction
+	public var deletePath: String?
 }

common.py

@@ -130,6 +130,7 @@ class Globals:
 	ARIA_EXECUTABLE = None
 	SEVEN_ZIP_EXECUTABLE = None
 	CURL_EXECUTABLE = None # Not required, but if available will be used to download filenames on systems with old SSL versions
+	CURL_USE_BUNDLED_CERT = None
 
 	LOG_FOLDER = 'INSTALLER_LOGS'
 	LOG_BASENAME = datetime.datetime.now().strftime('MOD-INSTALLER-LOG-%Y-%m-%d_%H-%M-%S.txt')
@@ -191,9 +192,49 @@ class Globals:
 
 	@staticmethod
 	def scanForCURL():
-		# On Windows 10, default to system CURL (which uses Windows's certificates)
-		# If not available, use the curl bundled with the installer, which uses included cert file 'curl-ca-bundle.crt'
-		Globals.CURL_EXECUTABLE = findWorkingExecutablePath(["curl", "curl_bundled"], ["-I", "https://07th-mod.com/"])
+		# For now, just try to find a curl executable at all, don't check internet connectivity or if TLS is working
+		# Later in chooseCurlCertificate() we will check certs
+		Globals.CURL_EXECUTABLE = findWorkingExecutablePath(["curl", "curl_bundled"], ['-V'])
+
+	# this function must be run AFTER scanCertLocation()
+	@staticmethod
+	def chooseCurlCertificate():
+		# For now, this only executes if curl is available
+		if Globals.CURL_EXECUTABLE is None:
+			print("chooseCurlCertificate(): CURL not available: Not testing certificates.")
+			return
+
+		def testCurlHeaders(url, certPath):
+			args = [Globals.CURL_EXECUTABLE]
+
+			if certPath is not None:
+				args += ['--cacert', certificate_path]
+
+			args += ['-I', url]
+
+			with open(os.devnull, 'w') as os_devnull:
+				return subprocess.call(args, stdout=os_devnull, stderr=os_devnull) == 0
+
+		# Try:
+		# 1. Default Cert (whatever CURL uses when you don't specify argument)
+		# 2. On Linux, we scan for certs on the user's computer and store the first found one. Try this.
+		# 3. Try the certificate we bundle with the installer. We try this last becuase it might be out of date, depending on when the installer was last released.
+		paths_to_try = [None, Globals.CA_CERT_PATH, "curl-ca-bundle.crt"]
+
+		for certificate_path in paths_to_try:
+			if not testCurlHeaders('https://07th-mod.com/', certificate_path):
+				print("chooseCurlCertificate(): Failed to download headers using CURL from 07th-mod.com using cert {}".format(certificate_path))
+				continue
+
+			if not testCurlHeaders('https://github.com/', certificate_path):
+				print("chooseCurlCertificate(): Failed to download headers using CURL from github.com using cert {}".format(certificate_path))
+				continue
+
+			print("chooseCurlCertificate(): Successfully used certificate {} to download from 07th-mod and github".format(certificate_path))
+			Globals.CA_CERT_PATH = certificate_path
+			return
+
+		print("chooseCurlCertificate(): ERROR: No certificates were found to work, tried {} Probably can't use installer!".format(paths_to_try))
 
 	@staticmethod
 	def scanForAria():
@@ -1205,9 +1246,17 @@ def queryUsingCURL(queryUrl):
 
 			# On old SSL if we have curl use that instead
 			with open(os.devnull, 'w') as os_devnull:
+				# Build CURL arguments
+				subprocess_args = [Globals.CURL_EXECUTABLE]
+				if Globals.CA_CERT_PATH is not None:
+					subprocess_args += ['--cacert', Globals.CA_CERT_PATH]
+				subprocess_args += ["-fILX", "GET", queryUrl]
+
+				print("queryUsingCURL(): Using args {}".format(subprocess_args))
+
 				# Get the header, the -X GET is required because the github download links return a 403 if you try to send a HEAD request
-				headers = subprocess.check_output([Globals.CURL_EXECUTABLE, "-fILX", "GET", queryUrl],
-				                                  stderr=os_devnull).decode("utf-8")
+				headers = subprocess.check_output(subprocess_args, stderr=os_devnull).decode("utf-8")
+
 			# If there's redirects curl may print multiple headers with multiple content dispositions.  We want the last one
 			contentDisposition = re.findall("Content-Disposition: (.+)", headers, re.IGNORECASE)
 			contentDisposition = contentDisposition[-1].strip() if contentDisposition else None
@@ -1544,3 +1593,27 @@ def crc32_of_file(file_path):
 			fb = f.read(BLOCK_SIZE)
 
 	return "{:08x}".format(hash & 0xffffffff)
+
+def applyDeletions(installPath, optionParser):
+	#type: (str, installConfiguration.ModOptionParser) -> None
+	for opt in optionParser.downloadAndExtractOptionsByPriority:
+			if opt.deletePath is not None:
+				# Do not allow paths with '..' to avoid deleting stuff outside the install path
+				if '..' in opt.deletePath.replace('\\', '/').split('/'):
+					raise Exception("applyDeletions() Developer Error: You are not allowed to use '..' for the 'deletePath' option")
+
+				fullDeletePath = os.path.join(installPath, opt.deletePath)
+
+				# Do not allow deleting any paths outside of the streamingassets folder for now
+				if 'streamingassets' not in os.path.abspath(fullDeletePath).lower():
+					raise Exception("applyDeletions() Developer Error: You are not allowed to delete outside of the StreamingAssets folder deletePath. "
+					"NOTE: This assert should be removed if deletePath needs to be used for Umineko (and possibly some other check added)")
+
+				try:
+					print("applyDeletions(): Attempting to delete path {} due to option {}...".format(fullDeletePath, opt.name))
+					if os.path.isdir(fullDeletePath):
+						shutil.rmtree(fullDeletePath)
+					else:
+						os.remove(fullDeletePath)
+				except Exception as e:
+					print("applyDeletions(): Failed to delete path: {}".format(e))

fileVersionManagement.py

@@ -69,6 +69,7 @@ def parseRequirementsList(scanPath, requirementsListString):
 
 # TODO: also cache version data?
 class VersionManager:
+	cachedRemoteVersionInfo = {}
 	localVersionFileName = "installedVersionData.json"
 	def userDidPartialReinstall(self, gameInstallTimeProbePath):
 		"""
@@ -112,7 +113,11 @@ def __init__(self, fullInstallConfiguration, modFileList, localVersionFolder, da
 			self.remoteVersionInfo = _testRemoteSubModVersion
 		else:
 			try:
-				self.remoteVersionInfo = getRemoteVersion(self.targetID)
+				# Cache the remote version info to avoid continuously re-downloading it
+				self.remoteVersionInfo = VersionManager.cachedRemoteVersionInfo.get(self.targetID)
+				if self.remoteVersionInfo is None:
+					self.remoteVersionInfo = getRemoteVersion(self.targetID)
+					VersionManager.cachedRemoteVersionInfo[self.targetID] = self.remoteVersionInfo
 			except Exception as error:
 				self.remoteVersionInfo = None
 				print("VersionManager: Error while retrieving remote version information {}".format(error))

higurashiInstaller.py

@@ -90,6 +90,24 @@ def listInvalidUIFiles(folder):
 
 	return invalidUIFileList
 
+def copyFileIfSourceExistsAndDestDoesNot(sourcePath, destinationPath):
+	#type: (str, str) -> None
+	"""
+	Try to copy + move a file in a transactional way so you can't get a half-copied file.
+	The copy will only be performed if the source path exists, and the destination path doesn't already exist.
+	If the copy is not performed a warning message will be printed.
+	"""
+	if not path.exists(sourcePath):
+		print("WARNING: Not copying {} -> {} as source does not exist".format(sourcePath, destinationPath))
+		return
+
+	if path.exists(destinationPath):
+		print("WARNING: Not copying {} -> {} as destination already exists".format(sourcePath, destinationPath))
+		return
+
+	shutil.copy(sourcePath, destinationPath + '.temp')
+	os.rename(destinationPath + '.temp', destinationPath)
+
 class Installer:
 	def getDataDirectory(self, installPath):
 		if common.Globals.IS_MAC:
@@ -170,31 +188,41 @@ def __init__(self, fullInstallConfiguration, extractDirectlyToGameDirectory, mod
 
 		self.downloaderAndExtractor.printPreview()
 
-	def backupUI(self):
-		"""
-		Backs up the `sharedassets0.assets` file
-		Try to do this in a transactional way so you can't get a half-copied .backup file.
-		This is important since the .backup file is needed to determine which ui file to use on future updates
-
-		The file is not moved directly in case the installer is halted before the new UI file can be placed, resulting
-		in an install completely missing a sharedassets0.assets UI file.
-		"""
-		try:
-			uiPath = path.join(self.dataDirectory, "sharedassets0.assets")
-
+	def getBackupPath(self, relativePath):
 			# partialManualInstall is not really supported on MacOS, so just assume output folder is HigurashiEpX_Data
 			if self.forcedExtractDirectory is not None:
-				backupPath = path.join(self.forcedExtractDirectory, self.info.subModConfig.dataName, "sharedassets0.assets.backup")
+				return path.join(self.forcedExtractDirectory, self.info.subModConfig.dataName, relativePath + '.backup')
 			else:
-				backupPath = path.join(self.dataDirectory, "sharedassets0.assets.backup")
+				return path.join(self.dataDirectory, relativePath + '.backup')
 
-			if path.exists(uiPath) and not path.exists(backupPath):
-				shutil.copy(uiPath, backupPath + '.temp')
-				os.rename(backupPath + '.temp', backupPath)
+	def tryBackupFile(self, relativePath):
+		"""
+		Tries to backup a file relative to the dataDirectory of the game, unless a backup already exists.
+		"""
+		try:
+			sourcePath = path.join(self.dataDirectory, relativePath)
+			backupPath = self.getBackupPath(relativePath)
+			copyFileIfSourceExistsAndDestDoesNot(sourcePath, backupPath)
 		except Exception as e:
-			print('Error: Failed to backup sharedassets0.assets file: {} (need backup for future installs!)'.format(e))
+			print('Error: Failed to backup {} file: {}'.format(relativePath, e))
 			raise e
 
+
+	def backupFiles(self):
+		"""
+		Backs up various files necessary for the installer to operate
+		Usually this is to prevent the installer having issues if it fails or is stopped half-way
+		"""
+		# Backs up the `sharedassets0.assets` file
+		# Try to do this in a transactional way so you can't get a half-copied .backup file.
+		# This is important since the .backup file is needed to determine which ui file to use on future updates
+		# The file is not moved directly in case the installer is halted before the new UI file can be placed, resulting
+		# in an install completely missing a sharedassets0.assets UI file.
+		self.tryBackupFile('sharedassets0.assets')
+		# Backs up the `resources.assets` file
+		# The backup (resources.assets.backup) will be deleted on a successful install
+		self.tryBackupFile('resources.assets')
+
 	def clearCompiledScripts(self):
 		compiledScriptsPattern = path.join(self.assetsDir, "CompiledUpdateScripts/*.mg")
 
@@ -296,13 +324,18 @@ def _applyLanguageSpecificSharedAssets(self, folderToApply):
 		Returns False if there was an error during the proccess.
 		If no asset file was found to apply, this is not considered an error
 		(it's assumed the existing sharedassets0.assets is the correct one)"""
-		# If don't know own unity version, don't attempt to apply any UI
-		if self.info.unityVersion is None:
+		# Get the unity version (again) from the existing resources.assets file
+		# We don't use the version stored in self.info.unityVersion because on certain configurations,
+		# the mod itself updates the unity version, causing it to change mid-install.
+		try:
+			versionString = installConfiguration.getUnityVersion(self.dataDirectory, ignoreBackupAssets=True)
+		except Exception as e:
+			# If don't know own unity version, don't attempt to apply any UI
+			print("ERROR (_applyLanguageSpecificSharedAssets()): Failed to retrieve unity version from resources.assets as {}".format(e))
 			print("ERROR: can't apply UI file as don't know own unity version!")
 			return False
 
 		# Use the sharedassets file with matching os/unityversion if provided by the language patch
-		versionString = self.info.unityVersion
 		osString = common.Globals.OS_STRING
 		if self.isWine:
 			osString = "windows"
@@ -439,6 +472,16 @@ def cleanup(self, cleanExtractionDirectory, cleanDownloadDirectory=True):
 			# Removes the quarantine attribute from the game (which could cause it to get launched read-only, breaking the script compiler)
 			subprocess.call(["xattr", "-d", "com.apple.quarantine", self.directory])
 
+	def removeResourcesAssetsBackup(self):
+		# Remove the resources.assets.backup file if install succeeds
+		# This must be done immediately after extracting all files successfully (and before any languageSpecificAssets are applied)
+		resourcesBackupPath = self.getBackupPath('resources.assets')
+		try:
+			if os.path.exists(resourcesBackupPath):
+				forceRemove(resourcesBackupPath)
+		except Exception as e:
+			print("Warning: Failed to remove `{}`. Updating the mod may not work correctly unless this file is deleted.".format(resourcesBackupPath))
+
 	def saveFileVersionInfoStarted(self):
 		self.fileVersionManager.saveVersionInstallStarted()
 
@@ -453,7 +496,7 @@ def main(fullInstallConfiguration):
 
 	isVoiceOnly = fullInstallConfiguration.subModConfig.subModName == 'voice-only'
 	if isVoiceOnly:
-		print("Performing Voice-Only Install - backupUI() and cleanOld() will NOT be performed.")
+		print("Performing Voice-Only Install - backupFiles() and cleanOld() will NOT be performed.")
 
 	modOptionParser = installConfiguration.ModOptionParser(fullInstallConfiguration)
 	skipDownload = modOptionParser.downloadManually
@@ -465,6 +508,7 @@ def main(fullInstallConfiguration):
 		installer = Installer(fullInstallConfiguration, extractDirectlyToGameDirectory=False, modOptionParser=modOptionParser, forcedExtractDirectory=extractDir)
 		installer.download()
 		installer.extractFiles()
+		installer.removeResourcesAssetsBackup()
 		if installer.optionParser.installSteamGrid:
 			steamGridExtractor.extractSteamGrid(installer.downloadDir)
 		installer.applyLanguagePatchFixesIfNecessary()
@@ -478,11 +522,14 @@ def main(fullInstallConfiguration):
 		installer.download()
 		installer.saveFileVersionInfoStarted()
 		if not isVoiceOnly:
-			installer.backupUI()
+			installer.backupFiles()
 			installer.cleanOld()
+		# If any mod options request deletion of a folder, do it before the extraction
+		common.applyDeletions(fullInstallConfiguration.installPath, modOptionParser)
 		print("Extracting...")
 		installer.extractFiles()
 		commandLineParser.printSeventhModStatusUpdate(97, "Cleaning up...")
+		installer.removeResourcesAssetsBackup()
 		if installer.optionParser.installSteamGrid:
 			steamGridExtractor.extractSteamGrid(installer.downloadDir)
 		installer.applyLanguagePatchFixesIfNecessary()
@@ -497,10 +544,13 @@ def main(fullInstallConfiguration):
 		installer.extractFiles()
 		commandLineParser.printSeventhModStatusUpdate(85, "Moving files into place...")
 		if not isVoiceOnly:
-			installer.backupUI()
+			installer.backupFiles()
 			installer.cleanOld()
+		# If any mod options request deletion of a folder, do it before the extraction
+		common.applyDeletions(fullInstallConfiguration.installPath, modOptionParser)
 		installer.moveFilesIntoPlace()
 		commandLineParser.printSeventhModStatusUpdate(97, "Cleaning up...")
+		installer.removeResourcesAssetsBackup()
 		if installer.optionParser.installSteamGrid:
 			steamGridExtractor.extractSteamGrid(installer.downloadDir)
 		installer.applyLanguagePatchFixesIfNecessary()

httpGUI/index.html

@@ -48,7 +48,7 @@ <h3>Error while checking if installer is latest</h3>
 		            </div>
 		            <div class="module-group right">
 		                <div class="module left">
-							<dropdown-game-menu v-bind:handles="subModList"></dropdown-game-menu>
+							<dropdown-game-menu v-bind:handles="subModList" v-bind:on-download-logs="getLogsZip"></dropdown-game-menu>
 		                </div>
 		            </div>
 		        </div>

httpGUI/installer.html

@@ -57,7 +57,7 @@ <h4 class="panel-title">
 		            </div>
 		            <div class="module-group right">
 		                <div class="module left">
-							<dropdown-game-menu v-bind:handles="subModList"></dropdown-game-menu>
+							<dropdown-game-menu v-bind:handles="subModList" v-bind:on-download-logs="() => getLogsZip(selectedSubMod, selectedInstallPath)"></dropdown-game-menu>
 		                </div>
 		            </div>
 		        </div>

httpGUI/loading_screen.html

@@ -42,7 +42,7 @@ <h4 class="panel-title">
 						</div>
 					</div>
 					<a class="btn" v-on:click="installErrorDescription = ''">OK</a>
-					<a class="btn" v-on:click="getLogsZip(selectedSubMod, selectedInstallPath)">Download Install Logs</a>
+					<a class="btn" v-on:click="getLogsZip()">Download Install Logs</a>
 					<a class="btn" href="https://discord.gg/pf5VhF9" target="_blank">Visit 07th-mod Discord server (send install log here!)</a>
 				</modal>
 			</div>

httpGUI/python-patcher-index.js

@@ -119,6 +119,10 @@ window.onload = function onWindowLoaded() {
         doPost('clearLatestInstallerWarning', [], () => {});
         this.modalVisible = false;
       },
+      getLogsZip(subModToInstall, installPath) {
+        // Calls the function with same name in python-patcher-rest-lib.js
+        getLogsZip(subModToInstall, installPath);
+      }
     },
     computed: {
       versionInfoAvailable() {

httpGUI/python-patcher-lib.js

@@ -237,32 +237,6 @@ Continue install anyway?`)) {
           app.selectedInstallPath = pathToInstall;
         }
       },
-      // If either argument is null, will just get the installer logs and not the game logs
-      getLogsZip(subModToInstall, installPath) {
-        let errorMessage = "Failed to download log zip - Please see 'Getting Installer Log Files' at https://07th-mod.com/wiki/Installer/support/"
-
-        doPost('troubleshoot', { action: 'getLogsZip', subMod: subModToInstall, installPath }, (responseData) => {
-          console.log(responseData);
-
-          // Show an error message if couldn't generate logs zip
-          if (responseData.filePath === null) {
-            alert(errorMessage);
-            return;
-          }
-
-          // Before navigating to download, check that the link would not 404
-          // This prevents navigating away from the page to a 404 if the download does not exist
-          fetch(responseData.filePath,
-            { method: "HEAD" }
-          ).then((res) => {
-            if (res.ok) {
-              window.location.href = responseData.filePath;
-            } else {
-              alert(errorMessage);
-            }
-          });
-        });
-      },
       openSaveFolder(subModToInstall, installPath) {
         doPost('troubleshoot', { action: 'openSaveFolder', subMod: subModToInstall, installPath }, () => {});
       },
@@ -396,6 +370,10 @@ Continue install anyway?`)) {
 
         app.getLogsZip(app.selectedSubMod, app.selectedInstallPath);
       },
+      getLogsZip(subModToInstall, installPath) {
+        // Calls the function with same name in python-patcher-rest-lib.js
+        getLogsZip(subModToInstall, installPath);
+      }
     },
     computed: {
       modHandles() {