From 3318485ad0ba7b8a7c7af097e897c00c3407468e Mon Sep 17 00:00:00 2001 From: Jalil Arfaoui Date: Sun, 22 Aug 2021 23:21:39 +0200 Subject: [PATCH] poc/introduce_webview_analyzing Code is not final, this is more a proof of concept that we can find the currently displayed amazon product title. --- .../scout/accessibility/BackgroundService.kt | 160 +++++++++++++++++- 1 file changed, 155 insertions(+), 5 deletions(-) diff --git a/android/app/src/main/java/com/dismoi/scout/accessibility/BackgroundService.kt b/android/app/src/main/java/com/dismoi/scout/accessibility/BackgroundService.kt index a86fef5..4101875 100644 --- a/android/app/src/main/java/com/dismoi/scout/accessibility/BackgroundService.kt +++ b/android/app/src/main/java/com/dismoi/scout/accessibility/BackgroundService.kt @@ -10,6 +10,7 @@ import android.content.Context import android.content.Intent import android.os.* import android.provider.Settings.canDrawOverlays +import android.util.Log import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo import androidx.annotation.RequiresApi @@ -25,6 +26,8 @@ class BackgroundService : AccessibilityService() { private val NOTIFICATION_TIMEOUT: Long = 500 + private val TAG = "Accessibility" + private val handler = Handler(Looper.getMainLooper()) private val runnableCode: Runnable = object : Runnable { override fun run() { @@ -83,7 +86,7 @@ class BackgroundService : AccessibilityService() { */ info.feedbackType = AccessibilityServiceInfo.FEEDBACK_VISUAL - info.flags = AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS + info.flags = AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS or AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS /* The minimal period in milliseconds between two accessibility events of @@ -116,19 +119,166 @@ class BackgroundService : AccessibilityService() { return "com.android.launcher3" == packageName } + private fun getEventType(event: AccessibilityEvent): String { + when (event.eventType) { + AccessibilityEvent.TYPE_VIEW_CLICKED -> return "TYPE_VIEW_CLICKED" + AccessibilityEvent.TYPE_VIEW_FOCUSED -> return "TYPE_VIEW_FOCUSED" + AccessibilityEvent.TYPE_VIEW_SELECTED -> return "TYPE_VIEW_SELECTED" + AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED -> return "TYPE_WINDOW_STATE_CHANGED" + AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED -> return "TYPE_WINDOW_CONTENT_CHANGED" + AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED -> return "TYPE_VIEW_TEXT_CHANGED" + AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED -> return "TYPE_VIEW_TEXT_SELECTION_CHANGED" + AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED -> return "TYPE_VIEW_ACCESSIBILITY_FOCUSED" + AccessibilityEvent.TYPE_WINDOWS_CHANGED -> return "TYPE_WINDOWS_CHANGED" + AccessibilityEvent.TYPE_ANNOUNCEMENT -> return "TYPE_ANNOUNCEMENT" + } + return event.eventType.toString() + } + + @RequiresApi(Build.VERSION_CODES.O) + private fun findByClassName(node: AccessibilityNodeInfo, className: String, level: Int = 0): AccessibilityNodeInfo? { + node.refresh() + val count = node.childCount + for (i in 0 until count) { + val child = node.getChild(i) + if (child != null) { + if (child.className.toString() == className) { + return child + } + val foundInChild = findByClassName(child, className, level + 1) + if (foundInChild != null) return foundInChild + } + } + return null + } + + @RequiresApi(Build.VERSION_CODES.P) + private fun findHeading(node: AccessibilityNodeInfo, level: Int = 0): AccessibilityNodeInfo? { + node.refresh() + val count = node.childCount + for (i in 0 until count) { + val child = node.getChild(i) + if (child != null) { + if (child?.isHeading) { + return child + } + val foundInChild = findHeading(child, level + 1) + if (foundInChild != null) return foundInChild + } + } + return null + } + + + @RequiresApi(Build.VERSION_CODES.O) + private fun findById(node: AccessibilityNodeInfo, id: String, level: Int = 0): AccessibilityNodeInfo? { + val count = node.childCount + for (i in 0 until count) { + val child = node.getChild(i) + + if (child != null) { + if (child?.viewIdResourceName?.toString() == id) { + return child + } + val foundInChild = findById(child, id, level + 1) + if (foundInChild != null) return foundInChild + } + } + return null + } + + + @RequiresApi(Build.VERSION_CODES.O) + private fun findTexts(node: AccessibilityNodeInfo, level: Int = 0): String { + val count = node.childCount + var texts = "" + for (i in 0 until count) { + val child = node.getChild(i) + + texts += child.text?.toString() ?: child.contentDescription?.toString() ?: "" + texts += "\n" + findTexts(child, level + 1) + "\n" + } + return texts + } + + @RequiresApi(Build.VERSION_CODES.O) + private fun findWebview(node: AccessibilityNodeInfo): AccessibilityNodeInfo? { + return findByClassName(node, "android.webkit.WebView") + } + + @RequiresApi(30) + private fun logHierarchy(node: AccessibilityNodeInfo, level: Int = 0) { + node.refresh() + val id = node.viewIdResourceName + val text = node.text?.toString() + val content = node.contentDescription?.toString() + val avExtras = node.availableExtraData?.joinToString(", ") + val count = node.childCount + + val extras = node.extras + val extrasList = mutableListOf() + for (key in extras.keySet()) { + extrasList.add("$key: ${extras.get(key)}") + } + val allExtras = extrasList.joinToString(" / ") + + Log.d(TAG, "${" ".repeat(level)} ($level) " + + "className: ${node.className}, " + + "id: ${id ?: "NO ID"}, " + + "text: $text, " + + "content: $content, " + + "extras: $allExtras, " + + "avExtras: $avExtras, " + + "hint: ${node.hintText}, " + + "heading: ${node.isHeading}, " + + "inputType: ${node.inputType}, " + + "state: ${node.stateDescription}" + ) + + for (i in 0 until count) { + val child = node.getChild(i) + if (child != null) { + logHierarchy(child, level + 1) + } + } + } + + private fun getTextAndContent(node: AccessibilityNodeInfo?): String? { + return if (node != null) "${node.text?.toString()} / ${node.contentDescription?.toString()}" else null + } + /* This method is called back by the system when it detects an AccessibilityEvent that matches the event filtering parameters specified by your accessibility service */ - @RequiresApi(Build.VERSION_CODES.JELLY_BEAN_MR2) + @RequiresApi(30) override fun onAccessibilityEvent(event: AccessibilityEvent) { - if (rootInActiveWindow == null) { + Log.d(TAG, "Event : ${getEventType(event)}, Package: ${event.packageName}, Source: ${event.source?.className.toString()}") + + val root = rootInActiveWindow + + if (root == null) { return } - if (overlayIsActivated(applicationContext)) { - val packageName = event.packageName.toString() + if (root.packageName?.toString() == "com.android.chrome" && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && root != null) { + Log.d(TAG, "Active window packageName : ${root.packageName}, className: ${root.className}") + } + + // Temporary demo code specific to Amazon + val webview = findWebview(root) + if (webview != null) { + val titleExpanderContent = findById(webview, "titleExpanderContent") + if (titleExpanderContent != null) { + val titleView = findHeading(titleExpanderContent) + val title = titleView?.text ?: titleView?.contentDescription + Log.d(TAG, "Found Amazon page title : $title") + } + } + + if (overlayIsActivated(applicationContext)) {applicationContext + val packageName = event.packageName?.toString() ?: "NO PACKAGE" if (getEventType(event) == "TYPE_WINDOW_STATE_CHANGED" && packageName != "com.android.chrome") { if (packageName.contains("com.google.android.inputmethod") ||