Add debug tracing to scheduling service

embrace-android-core/src/main/kotlin/io/embrace/android/embracesdk/internal/injection/DeliveryModuleImpl.kt

@@ -187,7 +187,8 @@ internal class DeliveryModuleImpl(
                 workerThreadModule.backgroundWorker(Worker.Background.IoRegWorker),
                 workerThreadModule.backgroundWorker(Worker.Background.HttpRequestWorker),
                 initModule.clock,
-                initModule.logger
+                initModule.logger,
+                deliveryTracer
             )
         }
     }

embrace-android-delivery/src/main/kotlin/io/embrace/android/embracesdk/internal/delivery/debug/DeliveryTraceState.kt

@@ -50,14 +50,16 @@
      * The payload store returned a list of payloads by priority
      */
     internal class GetPayloadsByPriority(val payloads: List<StoredTelemetryMetadata>) : DeliveryTraceState() {
-        override fun toString(): String = "GetPayloadsByPriority, count=${payloads.size},\n${payloads.reportListString()}"
+        override fun toString(): String =
+            "GetPayloadsByPriority, count=${payloads.size},\n${payloads.reportListString()}"
     }
 
     /**
      * The payload store returned a list of undelivered payloads
      */
     internal class GetUndeliveredPayloads(val payloads: List<StoredTelemetryMetadata>) : DeliveryTraceState() {
-        override fun toString(): String = "GetUndeliveredPayloads, count=${payloads.size},\n${payloads.reportListString()}"
+        override fun toString(): String =
+            "GetUndeliveredPayloads, count=${payloads.size},\n${payloads.reportListString()}"
     }
 
     /**
@@ -92,6 +94,43 @@
         override fun toString(): String = "SessionCacheAttempt"
     }
 
+    /**
+     * The delivery loop was started
+     */
+    internal class StartDeliveryLoop(private val loopAlreadyActive: Boolean) : DeliveryTraceState() {
+        override fun toString(): String = "StartDeliveryLoop loopAlreadyActive=$loopAlreadyActive"
+    }
+
+    /**
+     * A payload was enqueued for delivery
+     */
+    class PayloadEnqueued(private val payload: StoredTelemetryMetadata) : DeliveryTraceState() {
+        override fun toString(): String = "PayloadEnqueued, ${payload.toReportString()}"
+    }
+
+    /**
+     * A payload result was obtained
+     */
+    class PayloadResult(
+        private val payload: StoredTelemetryMetadata,
+        private val result: ExecutionResult,
+    ) : DeliveryTraceState() {
+        override fun toString(): String = "PayloadResult, ${payload.toReportString()}, result=${result.javaClass.simpleName}"
+    }
+
+    /**
+     * A payload queue was created
+     */
+    class PayloadQueueCreated(
+        private val payloadsByPriority: List<StoredTelemetryMetadata>,
+        private val payloadsToSend: List<StoredTelemetryMetadata>,
+    ) : DeliveryTraceState() {
+        override fun toString(): String {
+            return "PayloadQueueCreated, payloadsByPriority=${payloadsByPriority.map { it.uuid }}, " +
+                "payloadsToSend=${payloadsToSend.map { it.uuid }}"
+        }
+    }
+
     internal fun StoredTelemetryMetadata.toReportString(): String {
         return "$timestamp, $uuid, $payloadType, complete=$complete"
     }

embrace-android-delivery/src/main/kotlin/io/embrace/android/embracesdk/internal/delivery/debug/DeliveryTracer.kt

@@ -59,4 +59,23 @@ class DeliveryTracer {
                 "#$k, $state"
             }.joinToString("\n")
     }
+
+    fun onStartDeliveryLoop(sendLoopActive: Boolean) {
+        events.add(DeliveryTraceState.StartDeliveryLoop(sendLoopActive))
+    }
+
+    fun onPayloadQueueCreated(
+        payloadsByPriority: List<StoredTelemetryMetadata>,
+        payloadsToSend: List<StoredTelemetryMetadata>,
+    ) {
+        events.add(DeliveryTraceState.PayloadQueueCreated(payloadsByPriority, payloadsToSend))
+    }
+
+    fun onPayloadEnqueued(payload: StoredTelemetryMetadata) {
+        events.add(DeliveryTraceState.PayloadEnqueued(payload))
+    }
+
+    fun onPayloadResult(payload: StoredTelemetryMetadata, result: ExecutionResult) {
+        events.add(DeliveryTraceState.PayloadResult(payload, result))
+    }
 }

embrace-android-delivery/src/main/kotlin/io/embrace/android/embracesdk/internal/delivery/intake/IntakeServiceImpl.kt

@@ -60,8 +60,8 @@ class IntakeServiceImpl(
             val lastReference = cacheReferences[metadata.envelopeType]
 
             if (metadata.complete) {
-                schedulingService.onPayloadIntake()
                 deliveryTracer?.onPayloadIntake(metadata)
+                schedulingService.onPayloadIntake()
             } else {
                 cacheReferences[metadata.envelopeType] = metadata
                 if (!cacheableEnvelopeTypes.contains(metadata.envelopeType)) {

embrace-android-delivery/src/main/kotlin/io/embrace/android/embracesdk/internal/delivery/scheduling/SchedulingServiceImpl.kt

@@ -4,6 +4,7 @@ import io.embrace.android.embracesdk.internal.clock.Clock
 import io.embrace.android.embracesdk.internal.comms.api.Endpoint
 import io.embrace.android.embracesdk.internal.comms.delivery.NetworkStatus
 import io.embrace.android.embracesdk.internal.delivery.StoredTelemetryMetadata
+import io.embrace.android.embracesdk.internal.delivery.debug.DeliveryTracer
 import io.embrace.android.embracesdk.internal.delivery.execution.ExecutionResult
 import io.embrace.android.embracesdk.internal.delivery.execution.RequestExecutionService
 import io.embrace.android.embracesdk.internal.delivery.storage.PayloadStorageService
@@ -25,7 +26,8 @@ class SchedulingServiceImpl(
     private val schedulingWorker: BackgroundWorker,
     private val deliveryWorker: BackgroundWorker,
     private val clock: Clock,
-    private val logger: EmbLogger
+    private val logger: EmbLogger,
+    private val deliveryTracer: DeliveryTracer? = null,
 ) : SchedulingService {
 
     private val blockedEndpoints: MutableMap<Endpoint, Long> = ConcurrentHashMap()
@@ -65,9 +67,12 @@ class SchedulingServiceImpl(
         // When a payload arrives, check to see if there's already an active job try to deliver payloads
         // If not, schedule job. If so, do nothing.
         if (sendLoopActive.compareAndSet(false, true)) {
+            deliveryTracer?.onStartDeliveryLoop(true)
             schedulingWorker.submit {
                 deliveryLoop()
             }
+        } else {
+            deliveryTracer?.onStartDeliveryLoop(false)
         }
     }
 
@@ -120,13 +125,20 @@ class SchedulingServiceImpl(
         }
     }
 
-    private fun createPayloadQueue(exclude: Set<StoredTelemetryMetadata> = emptySet()) = LinkedList(
-        storageService.getPayloadsByPriority()
+    private fun createPayloadQueue(exclude: Set<StoredTelemetryMetadata> = emptySet()): LinkedList<StoredTelemetryMetadata> {
+        val payloadsByPriority = storageService.getPayloadsByPriority()
+        val payloadsToSend = payloadsByPriority
             .filter { it.shouldSendPayload() && !exclude.contains(it) }
             .sortedWith(storedTelemetryComparator)
-    )
+        deliveryTracer?.onPayloadQueueCreated(
+            payloadsByPriority,
+            payloadsToSend,
+        )
+        return LinkedList(payloadsToSend)
+    }
 
     private fun queueDelivery(payload: StoredTelemetryMetadata): Future<ExecutionResult> {
+        deliveryTracer?.onPayloadEnqueued(payload)
         activeSends.add(payload)
         return deliveryWorker.submit<ExecutionResult> {
             val result: ExecutionResult =
@@ -149,6 +161,8 @@ class SchedulingServiceImpl(
                     ExecutionResult.Incomplete(exception = t, retry = false)
                 }
 
+            deliveryTracer?.onPayloadResult(payload, result)
+
             with(result) {
                 if (!shouldRetry) {
                     // If the response is such that we should not ever retry the delivery of this payload,