From e4e5704e627f1ee6661e112e6bd96ec863b3700b Mon Sep 17 00:00:00 2001 From: darken Date: Sat, 25 May 2024 11:28:47 +0200 Subject: [PATCH] Analyzer: Speed up scans further Introduce new gateway method to just look op sizes like `du`. More efficient than doing complete lookups that are then summed. --- .../files/local/ipc/FileOpsConnection.aidl | 2 + .../sdmse/common/files/APathExtensions.kt | 14 ++- .../darken/sdmse/common/files/APathGateway.kt | 9 ++ .../common/files/APathLookupExtensions.kt | 5 ++ .../sdmse/common/files/GatewaySwitch.kt | 8 ++ .../common/files/local/EscalatingWalker.kt | 4 +- .../sdmse/common/files/local/LocalGateway.kt | 89 +++++++++++++++++-- .../common/files/local/LocalPathExtensions.kt | 1 - .../common/files/local/ipc/FileOpsClient.kt | 6 ++ .../common/files/local/ipc/FileOpsHost.kt | 10 ++- .../sdmse/common/files/saf/SAFGateway.kt | 36 ++++++++ .../core/storage/AppStorageScanner.kt | 27 ++++-- .../core/storage/StorageScannerExtensions.kt | 19 ++++ 13 files changed, 208 insertions(+), 22 deletions(-) diff --git a/app-common-io/src/main/aidl/eu/darken/sdmse/common/files/local/ipc/FileOpsConnection.aidl b/app-common-io/src/main/aidl/eu/darken/sdmse/common/files/local/ipc/FileOpsConnection.aidl index 37be638cd..508d618c6 100644 --- a/app-common-io/src/main/aidl/eu/darken/sdmse/common/files/local/ipc/FileOpsConnection.aidl +++ b/app-common-io/src/main/aidl/eu/darken/sdmse/common/files/local/ipc/FileOpsConnection.aidl @@ -34,6 +34,8 @@ interface FileOpsConnection { RemoteInputStream walkStream(in LocalPath path, in List pathDoesNotContain); + long du(in LocalPath path); + boolean createSymlink(in LocalPath linkPath, in LocalPath targetPath); boolean setModifiedAt(in LocalPath path, in long modifiedAt); diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathExtensions.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathExtensions.kt index 52fdb0b94..73b9e725c 100644 --- a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathExtensions.kt +++ b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathExtensions.kt @@ -2,12 +2,12 @@ package eu.darken.sdmse.common.files import eu.darken.sdmse.common.debug.logging.Logging.Priority.VERBOSE import eu.darken.sdmse.common.debug.logging.log -import eu.darken.sdmse.common.files.local.* +import eu.darken.sdmse.common.files.local.LocalPath import eu.darken.sdmse.common.files.local.crumbsTo import eu.darken.sdmse.common.files.local.isAncestorOf import eu.darken.sdmse.common.files.local.isParentOf import eu.darken.sdmse.common.files.local.startsWith -import eu.darken.sdmse.common.files.saf.* +import eu.darken.sdmse.common.files.saf.SAFPath import eu.darken.sdmse.common.files.saf.crumbsTo import eu.darken.sdmse.common.files.saf.isAncestorOf import eu.darken.sdmse.common.files.saf.isParentOf @@ -18,7 +18,7 @@ import okio.Source import java.io.File import java.io.IOException import java.time.Instant -import java.util.* +import java.util.Collections import eu.darken.sdmse.common.files.local.removePrefix as removePrefixLocalPath import eu.darken.sdmse.common.files.saf.removePrefix as removePrefixSafPath @@ -47,6 +47,14 @@ suspend fun

, PLE : APathLookupExtended

, GT : return gateway.walk(this, options) } + +suspend fun

, PLE : APathLookupExtended

, GT : APathGateway> P.du( + gateway: GT, + options: APathGateway.DuOptions = APathGateway.DuOptions() +): Long { + return gateway.du(this, options) +} + suspend fun T.exists(gateway: APathGateway, out APathLookupExtended>): Boolean { return gateway.exists(this) } diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathGateway.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathGateway.kt index 5ce6c2608..077bca919 100644 --- a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathGateway.kt +++ b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathGateway.kt @@ -38,6 +38,15 @@ interface APathGateway< get() = onFilter == null && onError == null } + suspend fun du( + path: P, + options: DuOptions = DuOptions() + ): Long + + data class DuOptions

>( + val abortOnError: Boolean = false, + ) + suspend fun exists(path: P): Boolean suspend fun canWrite(path: P): Boolean diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathLookupExtensions.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathLookupExtensions.kt index a7824b514..cf5c6b401 100644 --- a/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathLookupExtensions.kt +++ b/app-common-io/src/main/java/eu/darken/sdmse/common/files/APathLookupExtensions.kt @@ -21,6 +21,11 @@ suspend fun

, PLE : APathLookupExtended

, GT : options: APathGateway.WalkOptions = APathGateway.WalkOptions() ): Flow = lookedUp.walk(gateway, options) +suspend fun

, PLE : APathLookupExtended

, GT : APathGateway> PL.du( + gateway: GT, + options: APathGateway.DuOptions = APathGateway.DuOptions() +): Long = lookedUp.du(gateway, options) + suspend fun

> PL.exists( gateway: APathGateway, out APathLookupExtended

> ): Boolean = lookedUp.exists(gateway) diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/GatewaySwitch.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/GatewaySwitch.kt index 973c84bb9..38850b4d2 100644 --- a/app-common-io/src/main/java/eu/darken/sdmse/common/files/GatewaySwitch.kt +++ b/app-common-io/src/main/java/eu/darken/sdmse/common/files/GatewaySwitch.kt @@ -143,6 +143,14 @@ class GatewaySwitch @Inject constructor( return useGateway(path) { walk(path, options) } } + + override suspend fun du( + path: APath, + options: APathGateway.DuOptions> + ): Long { + return useGateway(path) { du(path, options) } + } + override suspend fun listFiles(path: APath): Collection { return useGateway(path) { listFiles(path) } } diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/EscalatingWalker.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/EscalatingWalker.kt index a4739477a..eadd20d89 100644 --- a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/EscalatingWalker.kt +++ b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/EscalatingWalker.kt @@ -21,10 +21,10 @@ import java.util.LinkedList /** * Prevents unnecessary lookups in Mode.NORMAL for nested directories */ -class EscalatingWalker constructor( +class EscalatingWalker( private val gateway: LocalGateway, private val start: LocalPath, - private val options: APathGateway.WalkOptions, + private val options: APathGateway.WalkOptions = APathGateway.WalkOptions() ) : AbstractFlow() { private val tag = "$TAG#${hashCode()}" diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalGateway.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalGateway.kt index 1b166b5f5..faffe57e2 100644 --- a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalGateway.kt +++ b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalGateway.kt @@ -3,11 +3,19 @@ package eu.darken.sdmse.common.files.local import eu.darken.sdmse.common.coroutine.AppScope import eu.darken.sdmse.common.coroutine.DispatcherProvider import eu.darken.sdmse.common.debug.Bugs -import eu.darken.sdmse.common.debug.logging.Logging.Priority.* +import eu.darken.sdmse.common.debug.logging.Logging.Priority.INFO +import eu.darken.sdmse.common.debug.logging.Logging.Priority.VERBOSE +import eu.darken.sdmse.common.debug.logging.Logging.Priority.WARN import eu.darken.sdmse.common.debug.logging.asLog import eu.darken.sdmse.common.debug.logging.log import eu.darken.sdmse.common.debug.logging.logTag -import eu.darken.sdmse.common.files.* +import eu.darken.sdmse.common.files.APathGateway +import eu.darken.sdmse.common.files.Ownership +import eu.darken.sdmse.common.files.Permissions +import eu.darken.sdmse.common.files.ReadException +import eu.darken.sdmse.common.files.WriteException +import eu.darken.sdmse.common.files.asFile +import eu.darken.sdmse.common.files.callbacks import eu.darken.sdmse.common.files.core.local.createSymlink import eu.darken.sdmse.common.files.core.local.isReadable import eu.darken.sdmse.common.files.core.local.listFiles2 @@ -26,14 +34,15 @@ import eu.darken.sdmse.common.shizuku.service.runModuleAction import eu.darken.sdmse.common.storage.StorageEnvironment import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.plus import kotlinx.coroutines.withContext -import okio.* +import okio.Sink +import okio.Source +import okio.sink +import okio.source import java.io.File import java.io.IOException import java.time.Instant -import java.util.* import javax.inject.Inject import javax.inject.Singleton @@ -409,7 +418,7 @@ class LocalGateway @Inject constructor( } canRead && mode == Mode.AUTO -> { - log(TAG, VERBOSE) { "walk($mode->NORMAL, direct): $path" } + log(TAG, VERBOSE) { "walk($mode->NORMAL, escalating): $path" } EscalatingWalker( gateway = this@LocalGateway, start = path, @@ -453,15 +462,77 @@ class LocalGateway @Inject constructor( else -> throw IOException("No matching mode available.") } - .onEach { -// if (Bugs.isTrace) log(TAG, VERBOSE) { "Walked $it" } - } } catch (e: IOException) { log(TAG, WARN) { "walk(path=$path, mode=$mode) failed:\n${e.asLog()}" } throw ReadException(path, cause = e) } } + override suspend fun du( + path: LocalPath, + options: APathGateway.DuOptions, + ): Long = du(path, options, Mode.AUTO) + + suspend fun du( + path: LocalPath, + options: APathGateway.DuOptions = APathGateway.DuOptions(), + mode: Mode = Mode.AUTO, + ): Long = runIO { + try { + val javaFile = path.asFile() + val canRead = when (mode) { + Mode.AUTO, Mode.NORMAL -> if (javaFile.canRead()) { + try { + javaFile.length() + true + } catch (e: IOException) { + false + } + } else { + false + } + + else -> false + } + + when { + mode == Mode.NORMAL -> { + log(TAG, VERBOSE) { "walk($mode->NORMAL, direct): $path" } + if (!canRead) throw ReadException(path) + javaFile.walkTopDown().map { it.length() }.sum() + } + + canRead && mode == Mode.AUTO -> { + log(TAG, VERBOSE) { "du($mode->AUTO, escalating): $path" } + try { + du(path, mode = Mode.NORMAL) + } catch (e: ReadException) { + when { + hasRoot() -> du(path, mode = Mode.ROOT) + hasShizuku() -> du(path, mode = Mode.ADB) + else -> throw e + } + } + } + + hasRoot() && (mode == Mode.ROOT || !canRead && mode == Mode.AUTO) -> { + log(TAG, VERBOSE) { "du($mode->ROOT): $path" } + rootOps { it.du(path) } + } + + hasShizuku() && (mode == Mode.ADB || !canRead && mode == Mode.AUTO) -> { + log(TAG, VERBOSE) { "du($mode->ADB): $path" } + adbOps { it.du(path) } + } + + else -> throw IOException("No matching mode available.") + } + } catch (e: IOException) { + log(TAG, WARN) { "du(path=$path, mode=$mode) failed:\n${e.asLog()}" } + throw ReadException(path, cause = e) + } + } + override suspend fun exists(path: LocalPath): Boolean = exists(path, Mode.AUTO) suspend fun exists(path: LocalPath, mode: Mode = Mode.AUTO): Boolean = runIO { diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalPathExtensions.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalPathExtensions.kt index e95907a27..3b9467a69 100644 --- a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalPathExtensions.kt +++ b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/LocalPathExtensions.kt @@ -45,7 +45,6 @@ fun LocalPath.performLookup(): LocalPathLookup { lookedUp = this, size = file.length(), modifiedAt = Instant.ofEpochMilli(file.lastModified()), - target = file.readLink()?.let { LocalPath.build(it) } ) } diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsClient.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsClient.kt index 1a27e215a..1a8b29592 100644 --- a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsClient.kt +++ b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsClient.kt @@ -91,6 +91,12 @@ class FileOpsClient @AssistedInject constructor( return output.toLocalPathLookupFlow() } + fun du(path: LocalPath): Long = try { + fileOpsConnection.du(path) + } catch (e: Exception) { + throw e.toFakeIOException() + } + fun readFile(path: LocalPath): Source = try { fileOpsConnection.readFile(path).source() } catch (e: Exception) { diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsHost.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsHost.kt index eaf176443..f323a1b80 100644 --- a/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsHost.kt +++ b/app-common-io/src/main/java/eu/darken/sdmse/common/files/local/ipc/FileOpsHost.kt @@ -44,7 +44,7 @@ class FileOpsHost @Inject constructor( private val ipcFunnel: IPCFunnel, ) : FileOpsConnection.Stub(), IpcHostModule { - private fun listFiles(path: LocalPath): List = path.asFile().listFiles2().map { LocalPath.build(it) } + private fun listFiles(path: LocalPath): List = path.asFile().listFiles2().map { LocalPath.build(it) } override fun listFilesStream(path: LocalPath): RemoteInputStream = try { if (Bugs.isTrace) log(TAG, VERBOSE) { "listFilesStream($path)..." } @@ -130,6 +130,14 @@ class FileOpsHost @Inject constructor( throw wrapPropagating(e) } + override fun du(path: LocalPath): Long = try { + if (Bugs.isTrace) log(TAG, VERBOSE) { "du($path)..." } + runBlocking { path.asFile().walkTopDown().map { it.length() }.sum() } + } catch (e: Exception) { + log(TAG, ERROR) { "exists(path=$path) failed\n${e.asLog()}" } + throw wrapPropagating(e) + } + override fun readFile(path: LocalPath): RemoteInputStream = try { if (Bugs.isTrace) log(TAG, VERBOSE) { "readFile($path)..." } FileInputStream(path.asFile()).remoteInputStream() diff --git a/app-common-io/src/main/java/eu/darken/sdmse/common/files/saf/SAFGateway.kt b/app-common-io/src/main/java/eu/darken/sdmse/common/files/saf/SAFGateway.kt index eee7cc5d2..bfc1af0bb 100644 --- a/app-common-io/src/main/java/eu/darken/sdmse/common/files/saf/SAFGateway.kt +++ b/app-common-io/src/main/java/eu/darken/sdmse/common/files/saf/SAFGateway.kt @@ -323,6 +323,42 @@ class SAFGateway @Inject constructor( } + override suspend fun du( + path: SAFPath, + options: APathGateway.DuOptions, + ): Long = runIO { + try { + val start = lookup(path) + log(TAG, VERBOSE) { "du($path) -> $start" } + + if (start.isFile) return@runIO start.size + + var total = start.size + + val queue = LinkedList(listOf(start)) + while (!queue.isEmpty()) { + val lookUp = queue.removeFirst() + + val newBatch = try { + lookupFiles(lookUp.lookedUp) + } catch (e: IOException) { + log(TAG, ERROR) { "Failed to read $lookUp: $e" } + emptyList() + } + + newBatch.forEach { child -> + total += child.size + if (child.isDirectory) queue.addFirst(child) + } + } + + total + } catch (e: Exception) { + log(TAG, WARN) { "du($path) failed." } + throw ReadException(path, cause = e) + } + } + override suspend fun read(path: SAFPath): Source = runIO { try { val docFile = findDocFile(path) diff --git a/app/src/main/java/eu/darken/sdmse/analyzer/core/storage/AppStorageScanner.kt b/app/src/main/java/eu/darken/sdmse/analyzer/core/storage/AppStorageScanner.kt index 5cd87a6aa..57f1bf504 100644 --- a/app/src/main/java/eu/darken/sdmse/analyzer/core/storage/AppStorageScanner.kt +++ b/app/src/main/java/eu/darken/sdmse/analyzer/core/storage/AppStorageScanner.kt @@ -179,24 +179,39 @@ class AppStorageScanner @AssistedInject constructor( val appMediaGroup = publicMediaPaths.value() .map { it.child(request.pkg.packageName) } .filter { gatewaySwitch.exists(it, type = GatewaySwitch.Type.AUTO) } - .map { it.walkContentItem(gatewaySwitch) } + .mapNotNull { path -> + try { + if (request.shallow) { + path.sizeContentItem(gatewaySwitch) + } else { + path.walkContentItem(gatewaySwitch) + } + } catch (e: ReadException) { + null + } + } .toSet() .takeIf { it.isNotEmpty() } - ?.let { contentSet -> + ?.let { ContentGroup( label = R.string.analyzer_storage_content_app_media_label.toCaString(), - contents = contentSet, + contents = it, ) } - val extraData = request.extraData + val extraDataGroup = request.extraData .mapNotNull { path -> try { - path.walkContentItem(gatewaySwitch) + if (request.shallow) { + path.sizeContentItem(gatewaySwitch) + } else { + path.walkContentItem(gatewaySwitch) + } } catch (e: ReadException) { null } } + .toSet() .takeIf { it.isNotEmpty() } ?.let { ContentGroup( @@ -212,7 +227,7 @@ class AppStorageScanner @AssistedInject constructor( appCode = appCodeGroup, appData = appDataGroup, appMedia = appMediaGroup, - extraData = extraData, + extraData = extraDataGroup, ), ) } diff --git a/app/src/main/java/eu/darken/sdmse/analyzer/core/storage/StorageScannerExtensions.kt b/app/src/main/java/eu/darken/sdmse/analyzer/core/storage/StorageScannerExtensions.kt index 2bd72aee5..64d9ef20d 100644 --- a/app/src/main/java/eu/darken/sdmse/analyzer/core/storage/StorageScannerExtensions.kt +++ b/app/src/main/java/eu/darken/sdmse/analyzer/core/storage/StorageScannerExtensions.kt @@ -11,6 +11,7 @@ import eu.darken.sdmse.common.files.FileType import eu.darken.sdmse.common.files.GatewaySwitch import eu.darken.sdmse.common.files.ReadException import eu.darken.sdmse.common.files.Segments +import eu.darken.sdmse.common.files.du import eu.darken.sdmse.common.files.walk import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.toList @@ -108,4 +109,22 @@ internal suspend fun APath.walkContentItem(gatewaySwitch: GatewaySwitch): Conten } } +internal suspend fun APath.sizeContentItem(gatewaySwitch: GatewaySwitch): ContentItem { + log(TAG, VERBOSE) { "Sizing content items for $this" } + + val lookup = gatewaySwitch.lookup(this, type = GatewaySwitch.Type.AUTO) + + val extraSize = if (lookup.fileType == FileType.DIRECTORY) { + try { + lookup.du(gatewaySwitch) + } catch (e: ReadException) { + log(TAG, WARN) { "Failed to du $this: ${e.asLog()}" } + 0L + } + } else { + 0L + } + return ContentItem.fromInaccessible(this, lookup.size + extraSize) +} + private val TAG = logTag("Analyzer", "Storage", "Scanner", "Extensions")