From 9f1414284831d4e42579bf2a9850851f059f8c98 Mon Sep 17 00:00:00 2001 From: Danielle Voznyy Date: Wed, 12 Feb 2025 02:01:39 -0500 Subject: [PATCH] fix: Far more reliable downloads fix: Account for program stopping mid download by writing to partial file, then renaming it fix: Set a default maximum of 4 concurrent download tasks fix: Don't check last changed headers for updates by default, reduces number of http calls when caching greatly chore: better rclone command not found error message --- gradle.properties | 2 +- .../keepup/api/KeepupDownloader.kt | 29 +++++++++-------- .../keepup/downloads/http/HttpDownloader.kt | 32 +++++++++++++------ .../downloads/rclone/RcloneDownloader.kt | 8 ++++- 4 files changed, 47 insertions(+), 24 deletions(-) diff --git a/gradle.properties b/gradle.properties index 2b54f14..4a661b0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ kotlin.code.style=official group=com.mineinabyss -version=3.2.0-alpha.4 +version=3.2.0-alpha.5 idofrontVersion=0.25.6 diff --git a/keepup-api/src/main/kotlin/com/mineinabyss/keepup/api/KeepupDownloader.kt b/keepup-api/src/main/kotlin/com/mineinabyss/keepup/api/KeepupDownloader.kt index 41cd85a..f53091a 100644 --- a/keepup-api/src/main/kotlin/com/mineinabyss/keepup/api/KeepupDownloader.kt +++ b/keepup-api/src/main/kotlin/com/mineinabyss/keepup/api/KeepupDownloader.kt @@ -7,12 +7,11 @@ import com.mineinabyss.keepup.downloads.parsing.DownloadSource import com.mineinabyss.keepup.similarfiles.SimilarFileChecker import com.mineinabyss.keepup.type_checker.FileTypeChecker.SYSTEM_SUPPORTS_FILE import io.ktor.client.* -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.* import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.channels.produce -import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import java.nio.file.Path import kotlin.io.path.absolute import kotlin.io.path.createDirectories @@ -22,13 +21,17 @@ class KeepupDownloader( val http: HttpClient, val config: KeepupDownloaderConfig, val githubConfig: GithubConfig, + val downloadDispatcher: CoroutineDispatcher = Dispatchers.IO, + val maxConcurrentDownloads: Int = 4, ) { + private val concurrentDownloads = Semaphore(maxConcurrentDownloads) + @OptIn(ExperimentalCoroutinesApi::class) fun download( vararg sources: DownloadSource, dest: Path, scope: CoroutineScope, - ): ReceiveChannel = scope.produce(Dispatchers.IO) { + ): ReceiveChannel = scope.produce { SYSTEM_SUPPORTS_FILE // check if system supports file command val similarFileChecker = if (config.ignoreSimilar) SimilarFileChecker(dest) else null val downloader = DownloadParser( @@ -37,16 +40,16 @@ class KeepupDownloader( githubConfig = githubConfig, similarFileChecker = similarFileChecker, ) - sources.map { source -> - val downloadPathForKey = (config.downloadCache / source.keyInConfig).absolute() - downloadPathForKey.createDirectories() - launch { - downloader - .download(source, downloadPathForKey) - .forEach { channel.send(it) } + launch(downloadDispatcher) { + concurrentDownloads.withPermit { + val downloadPathForKey = (config.downloadCache / source.keyInConfig).absolute() + downloadPathForKey.createDirectories() + downloader.download(source, downloadPathForKey) + .forEach { channel.send(it) } + } } - } + }.joinAll() } } diff --git a/keepup-api/src/main/kotlin/com/mineinabyss/keepup/downloads/http/HttpDownloader.kt b/keepup-api/src/main/kotlin/com/mineinabyss/keepup/downloads/http/HttpDownloader.kt index ec6c6c5..4fdc8fc 100644 --- a/keepup-api/src/main/kotlin/com/mineinabyss/keepup/downloads/http/HttpDownloader.kt +++ b/keepup-api/src/main/kotlin/com/mineinabyss/keepup/downloads/http/HttpDownloader.kt @@ -19,29 +19,43 @@ class HttpDownloader( val source: DownloadSource, val targetDir: Path, val fileName: String = source.query.substringAfterLast("/"), + val overrideWhenHeadersChange: Boolean = false, ) : Downloader { - override suspend fun download(): List { - val cacheFile = targetDir.resolve("$fileName.cache") - val targetFile = targetDir.resolve(fileName) + suspend fun getCacheString(query: String): String { val headers = client.head(source.query) val length = headers.contentLength() val lastModified = headers.lastModified() + return "Last-Modified: $lastModified, Content-Length: $length" + } - val cache = "Last-Modified: $lastModified, Content-Length: $length" - if (targetFile.exists() && cacheFile.exists() && cacheFile.readText() == cache) + override suspend fun download(): List { + val cacheFile = targetDir.resolve("$fileName.cache") + val targetFile = targetDir.resolve(fileName) + val partial = targetDir.resolve("$fileName.partial") + val cacheString = if (overrideWhenHeadersChange) getCacheString(source.query) else null + + // Check if target already exists and skip if it does, check last modified headers if overrideWhenHeadersChange is true + if (targetFile.exists() && (cacheString == null || (cacheFile.exists() && cacheFile.readText() == cacheString))) return listOf(DownloadResult.SkippedBecauseCached(targetFile, source.keyInConfig)) - client.get(source.query) { + // Write to partial file, then move it to target once download is complete + partial.deleteIfExists() + client.prepareGet { + url(source.query) timeout { requestTimeoutMillis = 30.seconds.inWholeMilliseconds } + }.execute { + it.bodyAsChannel() + .copyAndClose(partial.toFile().writeChannel()) } - .bodyAsChannel() - .copyAndClose(targetFile.toFile().writeChannel()) + + targetFile.deleteIfExists() + partial.moveTo(targetFile) // Only mark as cached after download is complete cacheFile.deleteIfExists() - cacheFile.createFile().writeText(cache) + cacheFile.createFile().writeText(cacheString ?: getCacheString(source.query)) return listOf(DownloadResult.Downloaded(targetFile, source.keyInConfig)) } diff --git a/keepup-api/src/main/kotlin/com/mineinabyss/keepup/downloads/rclone/RcloneDownloader.kt b/keepup-api/src/main/kotlin/com/mineinabyss/keepup/downloads/rclone/RcloneDownloader.kt index 016d982..bdfd505 100644 --- a/keepup-api/src/main/kotlin/com/mineinabyss/keepup/downloads/rclone/RcloneDownloader.kt +++ b/keepup-api/src/main/kotlin/com/mineinabyss/keepup/downloads/rclone/RcloneDownloader.kt @@ -1,5 +1,6 @@ package com.mineinabyss.keepup.downloads.rclone +import com.lordcodes.turtle.ShellCommandNotFoundException import com.mineinabyss.keepup.downloads.DownloadResult import com.mineinabyss.keepup.downloads.Downloader import com.mineinabyss.keepup.downloads.parsing.DownloadSource @@ -11,7 +12,12 @@ class RcloneDownloader( val targetDir: Path, ) : Downloader { override suspend fun download(): List { - val downloadPath = Rclone.sync(source.query, targetDir) + val downloadPath = runCatching { Rclone.sync(source.query, targetDir) } + .onFailure { + if (it is ShellCommandNotFoundException) + return listOf(DownloadResult.Failure("rclone command not found", source.keyInConfig)) + } + .getOrThrow() return listOf(DownloadResult.Downloaded(downloadPath, source.keyInConfig, overrideInfoMsg = MSG.rclone)) } }