Skip to content

Commit

Permalink
poc/introduce_webview_analyzing
Browse files Browse the repository at this point in the history 
Code is not final, this is more a proof of concept that we can find the currently displayed amazon product title.
  • Loading branch information
@JalilArfaoui
JalilArfaoui committed Aug 22, 2021
1 parent c7c5d13 commit 3318485
Showing 1 changed file with 155 additions and 5 deletions.
160 changes: 155 additions & 5 deletions android/app/src/main/java/com/dismoi/scout/accessibility/BackgroundService.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import android.content.Context
import android.content.Intent
import android.os.*
import android.provider.Settings.canDrawOverlays
import android.util.Log
import android.view.accessibility.AccessibilityEvent
import android.view.accessibility.AccessibilityNodeInfo
import androidx.annotation.RequiresApi
Expand All @@ -25,6 +26,8 @@ class BackgroundService : AccessibilityService() {

private val NOTIFICATION_TIMEOUT: Long = 500

private val TAG = "Accessibility"

private val handler = Handler(Looper.getMainLooper())
private val runnableCode: Runnable = object : Runnable {
override fun run() {
Expand Down Expand Up @@ -83,7 +86,7 @@ class BackgroundService : AccessibilityService() {
*/

info.feedbackType = AccessibilityServiceInfo.FEEDBACK_VISUAL
info.flags = AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS
info.flags = AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS or AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS

/*
The minimal period in milliseconds between two accessibility events of
Expand Down Expand Up @@ -116,19 +119,166 @@ class BackgroundService : AccessibilityService() {
return "com.android.launcher3" == packageName
}

private fun getEventType(event: AccessibilityEvent): String {
when (event.eventType) {
AccessibilityEvent.TYPE_VIEW_CLICKED -> return "TYPE_VIEW_CLICKED"
AccessibilityEvent.TYPE_VIEW_FOCUSED -> return "TYPE_VIEW_FOCUSED"
AccessibilityEvent.TYPE_VIEW_SELECTED -> return "TYPE_VIEW_SELECTED"
AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED -> return "TYPE_WINDOW_STATE_CHANGED"
AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED -> return "TYPE_WINDOW_CONTENT_CHANGED"
AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED -> return "TYPE_VIEW_TEXT_CHANGED"
AccessibilityEvent.TYPE_VIEW_TEXT_SELECTION_CHANGED -> return "TYPE_VIEW_TEXT_SELECTION_CHANGED"
AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED -> return "TYPE_VIEW_ACCESSIBILITY_FOCUSED"
AccessibilityEvent.TYPE_WINDOWS_CHANGED -> return "TYPE_WINDOWS_CHANGED"
AccessibilityEvent.TYPE_ANNOUNCEMENT -> return "TYPE_ANNOUNCEMENT"
}
return event.eventType.toString()
}

@RequiresApi(Build.VERSION_CODES.O)
private fun findByClassName(node: AccessibilityNodeInfo, className: String, level: Int = 0): AccessibilityNodeInfo? {
node.refresh()
val count = node.childCount
for (i in 0 until count) {
val child = node.getChild(i)
if (child != null) {
if (child.className.toString() == className) {
return child
}
val foundInChild = findByClassName(child, className, level + 1)
if (foundInChild != null) return foundInChild
}
}
return null
}

@RequiresApi(Build.VERSION_CODES.P)
private fun findHeading(node: AccessibilityNodeInfo, level: Int = 0): AccessibilityNodeInfo? {
node.refresh()
val count = node.childCount
for (i in 0 until count) {
val child = node.getChild(i)
if (child != null) {
if (child?.isHeading) {
return child
}
val foundInChild = findHeading(child, level + 1)
if (foundInChild != null) return foundInChild
}
}
return null
}


@RequiresApi(Build.VERSION_CODES.O)
private fun findById(node: AccessibilityNodeInfo, id: String, level: Int = 0): AccessibilityNodeInfo? {
val count = node.childCount
for (i in 0 until count) {
val child = node.getChild(i)

if (child != null) {
if (child?.viewIdResourceName?.toString() == id) {
return child
}
val foundInChild = findById(child, id, level + 1)
if (foundInChild != null) return foundInChild
}
}
return null
}


@RequiresApi(Build.VERSION_CODES.O)
private fun findTexts(node: AccessibilityNodeInfo, level: Int = 0): String {
val count = node.childCount
var texts = ""
for (i in 0 until count) {
val child = node.getChild(i)

texts += child.text?.toString() ?: child.contentDescription?.toString() ?: ""
texts += "\n" + findTexts(child, level + 1) + "\n"
}
return texts
}

@RequiresApi(Build.VERSION_CODES.O)
private fun findWebview(node: AccessibilityNodeInfo): AccessibilityNodeInfo? {
return findByClassName(node, "android.webkit.WebView")
}

@RequiresApi(30)
private fun logHierarchy(node: AccessibilityNodeInfo, level: Int = 0) {
node.refresh()
val id = node.viewIdResourceName
val text = node.text?.toString()
val content = node.contentDescription?.toString()
val avExtras = node.availableExtraData?.joinToString(", ")
val count = node.childCount

val extras = node.extras
val extrasList = mutableListOf<String>()
for (key in extras.keySet()) {
extrasList.add("$key: ${extras.get(key)}")
}
val allExtras = extrasList.joinToString(" / ")

Log.d(TAG, "${" ".repeat(level)} ($level) " +
"className: ${node.className}, " +
"id: ${id ?: "NO ID"}, " +
"text: $text, " +
"content: $content, " +
"extras: $allExtras, " +
"avExtras: $avExtras, " +
"hint: ${node.hintText}, " +
"heading: ${node.isHeading}, " +
"inputType: ${node.inputType}, " +
"state: ${node.stateDescription}"
)

for (i in 0 until count) {
val child = node.getChild(i)
if (child != null) {
logHierarchy(child, level + 1)
}
}
}

private fun getTextAndContent(node: AccessibilityNodeInfo?): String? {
return if (node != null) "${node.text?.toString()} / ${node.contentDescription?.toString()}" else null
}

/*
This method is called back by the system when it detects an
AccessibilityEvent that matches the event filtering parameters
specified by your accessibility service
*/
@RequiresApi(Build.VERSION_CODES.JELLY_BEAN_MR2)
@RequiresApi(30)
override fun onAccessibilityEvent(event: AccessibilityEvent) {
if (rootInActiveWindow == null) {
Log.d(TAG, "Event : ${getEventType(event)}, Package: ${event.packageName}, Source: ${event.source?.className.toString()}")

val root = rootInActiveWindow

if (root == null) {
return
}

if (overlayIsActivated(applicationContext)) {
val packageName = event.packageName.toString()
if (root.packageName?.toString() == "com.android.chrome" && Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && root != null) {
Log.d(TAG, "Active window packageName : ${root.packageName}, className: ${root.className}")
}

// Temporary demo code specific to Amazon
val webview = findWebview(root)
if (webview != null) {
val titleExpanderContent = findById(webview, "titleExpanderContent")
if (titleExpanderContent != null) {
val titleView = findHeading(titleExpanderContent)
val title = titleView?.text ?: titleView?.contentDescription
Log.d(TAG, "Found Amazon page title : $title")
}
}

if (overlayIsActivated(applicationContext)) {applicationContext
val packageName = event.packageName?.toString() ?: "NO PACKAGE"

if (getEventType(event) == "TYPE_WINDOW_STATE_CHANGED" && packageName != "com.android.chrome") {
if (packageName.contains("com.google.android.inputmethod") ||
Expand Down

0 comments on commit 3318485

Please sign in to comment.