Skip to content

Commit

Permalink
Analyzer: Speed up scans
Browse files Browse the repository at this point in the history 
Defer scanning apps in details until the user actually needs it.
During the first pass of all apps, we only do a "shallow" scan, and then only if the enters the app details, then we do a deep scan.
  • Loading branch information
@d4rken
d4rken committed May 24, 2024
1 parent 3d69d6c commit 6473e13
Show file tree
Hide file tree
Showing 6 changed files with 187 additions and 48 deletions.
42 changes: 40 additions & 2 deletions app/src/main/java/eu/darken/sdmse/analyzer/core/Analyzer.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import eu.darken.sdmse.analyzer.core.content.ContentGroup
import eu.darken.sdmse.analyzer.core.device.DeviceStorage
import eu.darken.sdmse.analyzer.core.device.DeviceStorageScanTask
import eu.darken.sdmse.analyzer.core.device.DeviceStorageScanner
import eu.darken.sdmse.analyzer.core.storage.AppDeepScanTask
import eu.darken.sdmse.analyzer.core.storage.StorageScanTask
import eu.darken.sdmse.analyzer.core.storage.StorageScanner
import eu.darken.sdmse.analyzer.core.storage.categories.AppCategory
Expand All @@ -20,7 +21,9 @@ import eu.darken.sdmse.analyzer.core.storage.toFlatContent
import eu.darken.sdmse.analyzer.core.storage.toNestedContent
import eu.darken.sdmse.common.collections.mutate
import eu.darken.sdmse.common.coroutine.AppScope
import eu.darken.sdmse.common.debug.logging.Logging.Priority.*
import eu.darken.sdmse.common.debug.logging.Logging.Priority.INFO
import eu.darken.sdmse.common.debug.logging.Logging.Priority.VERBOSE
import eu.darken.sdmse.common.debug.logging.Logging.Priority.WARN
import eu.darken.sdmse.common.debug.logging.log
import eu.darken.sdmse.common.debug.logging.logTag
import eu.darken.sdmse.common.files.GatewaySwitch
Expand All @@ -30,7 +33,10 @@ import eu.darken.sdmse.common.files.isAncestorOf
import eu.darken.sdmse.common.files.matches
import eu.darken.sdmse.common.flow.replayingShare
import eu.darken.sdmse.common.getQuantityString2
import eu.darken.sdmse.common.progress.*
import eu.darken.sdmse.common.progress.Progress
import eu.darken.sdmse.common.progress.updateProgressPrimary
import eu.darken.sdmse.common.progress.updateProgressSecondary
import eu.darken.sdmse.common.progress.withProgress
import eu.darken.sdmse.common.sharedresource.SharedResource
import eu.darken.sdmse.common.storage.StorageId
import eu.darken.sdmse.main.core.SDMTool
Expand All @@ -42,6 +48,7 @@ import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import javax.inject.Inject
Expand Down Expand Up @@ -111,6 +118,7 @@ class Analyzer @Inject constructor(
is DeviceStorageScanTask -> scanStorageDevices(task)
is StorageScanTask -> scanStorageContents(task)
is ContentDeleteTask -> deleteContent(task)
is AppDeepScanTask -> deepScanApp(task)
else -> throw UnsupportedOperationException("Unsupported task: $task")
}

Expand Down Expand Up @@ -235,6 +243,36 @@ class Analyzer @Inject constructor(
)
}

private suspend fun deepScanApp(task: AppDeepScanTask): AppDeepScanTask.Result {
log(TAG, VERBOSE) { "deepScanApp(): $task" }

if (!appInventorySetupModule.isComplete()) {
log(TAG, WARN) { "SetupModule INVENTORY is not complete" }
throw IncompleteSetupException(SetupModule.Type.INVENTORY)
}

val targetStorage = storageDevices.value.singleOrNull { it.id == task.storageId }
?: throw IllegalStateException("Couldn't find ${task.storageId}")
val targetCategory = storageCategories.first()[targetStorage.id]!!.filterIsInstance<AppCategory>().single()
val targetApp = targetCategory.pkgStats[task.installId]!!

val start = System.currentTimeMillis()

val updatedApp = storageScanner.get().withProgress(this) { deepScanApp(targetStorage, targetApp) }

val stop = System.currentTimeMillis()
log(TAG) { "deepScanApp() took ${stop - start}ms" }

storageCategories.value = storageCategories.value.mutate {
this[targetStorage.id] = this[targetStorage.id]!!.map { category ->
if (category !is AppCategory) return@map category
category.copy(pkgStats = category.pkgStats.mutate { replace(task.installId, updatedApp) })
}
}

return AppDeepScanTask.Result(true)
}

data class State(
val data: Data,
val progress: Progress.Data?,
Expand Down
23 changes: 23 additions & 0 deletions app/src/main/java/eu/darken/sdmse/analyzer/core/storage/AppDeepScanTask.kt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package eu.darken.sdmse.analyzer.core.storage

import eu.darken.sdmse.analyzer.core.AnalyzerTask
import eu.darken.sdmse.common.ca.CaString
import eu.darken.sdmse.common.ca.toCaString
import eu.darken.sdmse.common.pkgs.features.Installed
import eu.darken.sdmse.common.storage.StorageId
import kotlinx.parcelize.Parcelize

@Parcelize
data class AppDeepScanTask(
val storageId: StorageId,
val installId: Installed.InstallId,
) : AnalyzerTask {

@Parcelize
data class Result(
private val success: Boolean,
) : AnalyzerTask.Result {
override val primaryInfo: CaString
get() = eu.darken.sdmse.common.R.string.general_result_success_message.toCaString()
}
}
88 changes: 53 additions & 35 deletions app/src/main/java/eu/darken/sdmse/analyzer/core/storage/AppStorageScanner.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import eu.darken.sdmse.analyzer.core.device.DeviceStorage
import eu.darken.sdmse.analyzer.core.storage.categories.AppCategory
import eu.darken.sdmse.common.areas.DataArea
import eu.darken.sdmse.common.ca.toCaString
import eu.darken.sdmse.common.clutter.Marker
import eu.darken.sdmse.common.coroutine.SuspendingLazy
import eu.darken.sdmse.common.debug.logging.Logging.Priority.ERROR
import eu.darken.sdmse.common.debug.logging.Logging.Priority.WARN
Expand All @@ -22,7 +21,6 @@ import eu.darken.sdmse.common.files.GatewaySwitch
import eu.darken.sdmse.common.files.ReadException
import eu.darken.sdmse.common.files.core.local.File
import eu.darken.sdmse.common.files.local.LocalPath
import eu.darken.sdmse.common.forensics.OwnerInfo
import eu.darken.sdmse.common.hasApiLevel
import eu.darken.sdmse.common.pkgs.features.Installed
import eu.darken.sdmse.common.pkgs.getPrivateDataDirs
Expand Down Expand Up @@ -73,22 +71,21 @@ class AppStorageScanner @AssistedInject constructor(
.toSet()
}

suspend fun processPkg(
pkg: Installed,
topLevelDirs: Set<OwnerInfo>,
suspend fun process(
request: Request,
): Result {
val appStorStats = try {
statsManager.queryStatsForPkg(storage.id, pkg)
statsManager.queryStatsForPkg(storage.id, request.pkg)
} catch (e: Exception) {
log(TAG, WARN) { "Failed to query stats for ${pkg.id} due to $e" }
log(TAG, WARN) { "Failed to query stats for ${request.pkg.id} due to $e" }
null
}

val appCodeGroup = when {
storage.type != DeviceStorage.Type.PRIMARY -> null
pkg.isSystemApp && !pkg.isUpdatedSystemApp -> null
request.pkg.isSystemApp && !request.pkg.isUpdatedSystemApp -> null
else -> {
val appCode = pkg.applicationInfo?.sourceDir
val appCode = request.pkg.applicationInfo?.sourceDir
?.let {
when {
it.endsWith("base.apk") -> File(it).parent
Expand All @@ -97,7 +94,7 @@ class AppStorageScanner @AssistedInject constructor(
}
?.let { LocalPath.build(it) }
?.let { codeDir ->
if (useRoot) {
if (!request.shallow && useRoot) {
try {
return@let codeDir.walkContentItem(gatewaySwitch)
} catch (e: ReadException) {
Expand All @@ -108,7 +105,7 @@ class AppStorageScanner @AssistedInject constructor(
if (appStorStats != null) {
return@let ContentItem.fromInaccessible(
codeDir,
if (pkg.userHandle == currentUser) appStorStats.appBytes else 0L
if (request.pkg.userHandle == currentUser) appStorStats.appBytes else 0L
)
}

Expand All @@ -124,12 +121,14 @@ class AppStorageScanner @AssistedInject constructor(

// TODO: For root we look up /data/media?

// Android/data/<pkg>
// Android/data/<request.pkg>
val dataDirPubs = publicDataPaths.value()
.map { it.child(pkg.packageName) }
.map { it.child(request.pkg.packageName) }
.mapNotNull { pubData ->
when {
hasApiLevel(33) && !useRoot && !useShizuku -> ContentItem.fromInaccessible(pubData)
request.shallow || (hasApiLevel(33) && !useRoot && !useShizuku) -> {
ContentItem.fromInaccessible(pubData)
}

gatewaySwitch.exists(pubData, type = GatewaySwitch.Type.AUTO) -> try {
pubData.walkContentItem(gatewaySwitch)
Expand All @@ -145,28 +144,28 @@ class AppStorageScanner @AssistedInject constructor(
val dataDirPrivs = run {
if (storage.type != DeviceStorage.Type.PRIMARY) return@run emptySet()

if (dataAreas.any { it.type == DataArea.Type.PRIVATE_DATA }) {
if (!request.shallow && dataAreas.any { it.type == DataArea.Type.PRIVATE_DATA }) {
try {
return@run pkg
return@run request.pkg
.getPrivateDataDirs(dataAreas)
.filter { gatewaySwitch.exists(it, type = GatewaySwitch.Type.CURRENT) }
.map { it.walkContentItem(gatewaySwitch) }
} catch (e: ReadException) {
log(TAG, ERROR) { "Failed to read private data dirs for $pkg: ${e.asLog()}" }
log(TAG, ERROR) { "Failed to read private data dirs for $request.pkg: ${e.asLog()}" }
}
}

if (appStorStats != null) {
return@run setOfNotNull(pkg.applicationInfo?.dataDir).map {
return@run setOfNotNull(request.pkg.applicationInfo?.dataDir).map {
ContentItem.fromInaccessible(
LocalPath.build(it),
// This is a simplification, because storage stats don't provider more fine grained infos
// This is a simplification, because storage stats don't provide more fine grained infos
appStorStats.dataBytes - (dataDirPubs.firstOrNull()?.size ?: 0L)
)
}
}

setOfNotNull(pkg.applicationInfo?.dataDir).map {
setOfNotNull(request.pkg.applicationInfo?.dataDir).map {
ContentItem.fromInaccessible(LocalPath.build(it))
}
}
Expand All @@ -176,9 +175,9 @@ class AppStorageScanner @AssistedInject constructor(
contents = dataDirPrivs + dataDirPubs,
)

// Android/media/<pkg>
// Android/media/<request.pkg>
val appMediaGroup = publicMediaPaths.value()
.map { it.child(pkg.packageName) }
.map { it.child(request.pkg.packageName) }
.filter { gatewaySwitch.exists(it, type = GatewaySwitch.Type.AUTO) }
.map { it.walkContentItem(gatewaySwitch) }
.toSet()
Expand All @@ -190,17 +189,10 @@ class AppStorageScanner @AssistedInject constructor(
)
}

val consumed = mutableSetOf<OwnerInfo>()
val extraData = topLevelDirs
.filter {
val owner = it.getOwner(pkg.id) ?: return@filter false
!owner.hasFlag(Marker.Flag.CUSTODIAN) && !owner.hasFlag(Marker.Flag.COMMON)
}
.mapNotNull { ownerInfo ->
val extraData = request.extraData
.mapNotNull { path ->
try {
ownerInfo.areaInfo.file.walkContentItem(gatewaySwitch).also {
consumed.add(ownerInfo)
}
path.walkContentItem(gatewaySwitch)
} catch (e: ReadException) {
null
}
Expand All @@ -215,19 +207,45 @@ class AppStorageScanner @AssistedInject constructor(

return Result(
pkgStat = AppCategory.PkgStat(
pkg = pkg,
pkg = request.pkg,
isShallow = request.shallow,
appCode = appCodeGroup,
appData = appDataGroup,
appMedia = appMediaGroup,
extraData = extraData,
),
consumedTopLevelDirs = consumed,
)
}

sealed interface Request {
val pkg: Installed
val shallow: Boolean
val extraData: Collection<APath>

data class Initial(
override val pkg: Installed,
override val extraData: Collection<APath>,
) : Request {
override val shallow: Boolean
get() = true
}

data class Reprocessing(
val pkgStat: AppCategory.PkgStat,
) : Request {
override val pkg: Installed
get() = pkgStat.pkg

override val shallow: Boolean
get() = false

override val extraData: Collection<APath>
get() = pkgStat.extraData?.contents?.map { it.path } ?: emptySet()
}
}

data class Result(
val pkgStat: AppCategory.PkgStat,
val consumedTopLevelDirs: Set<OwnerInfo>,
)

@AssistedFactory
Expand Down
Loading

0 comments on commit 6473e13

Please sign in to comment.