assessing SparkListenerSQLExecution events

nubank · Feb 5, 2025 · 9998c4a · 9998c4a
1 parent 96ee599
commit 9998c4a
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/.../src/main/java/io/openlineage/spark/agent/lifecycle/SparkApplicationExecutionContext.java b/.../src/main/java/io/openlineage/spark/agent/lifecycle/SparkApplicationExecutionContext.java
@@ -101,7 +101,6 @@ public void start(SparkListenerApplicationStart applicationStart) {
 
   @Override
   public void end(SparkListenerApplicationEnd applicationEnd) {
-    log.info("SparkListenerApplicationEnd: {}", applicationEnd.toString());
     String applicationId =
         olContext.getSparkContext().map(context -> context.applicationId()).orElse(null);
     log.debug("SparkListenerApplicationEnd - applicationId: {}", applicationId);

diff --git a/...park/app/src/main/java/io/openlineage/spark/agent/lifecycle/SparkSQLExecutionContext.java b/...park/app/src/main/java/io/openlineage/spark/agent/lifecycle/SparkSQLExecutionContext.java
@@ -79,6 +79,9 @@ public SparkSQLExecutionContext(
 
   @Override
   public void start(SparkListenerSQLExecutionStart startEvent) {
+    log.info("SparkListenerSQLExecutionStart - executionId: {}", startEvent.executionId());
+    log.info("SparkListenerSQLExecutionStart - event: {}", startEvent.toString());
+    log.info("SparkListenerSQLExecutionStart - event.sparkPlanInfo: {}", startEvent.sparkPlanInfo());
     if (log.isDebugEnabled()) {
       log.debug("SparkListenerSQLExecutionStart - executionId: {}", startEvent.executionId());
     }
@@ -110,12 +113,14 @@ public void start(SparkListenerSQLExecutionStart startEvent) {
                 .jobFacetsBuilder(getJobFacetsBuilder(olContext.getQueryExecution().get()))
                 .build());
 
-    log.debug("Posting event for start {}: {}", executionId, event);
+    log.info("Posting event for start {}: {}", executionId, OpenLineageClientUtils.toJson(event));
     NuEventEmitter.emit(event, eventEmitter);
   }
 
   @Override
   public void end(SparkListenerSQLExecutionEnd endEvent) {
+    log.info("SparkListenerSQLExecutionEnd - executionId: {}", endEvent.executionId());
+    log.info("SparkListenerSQLExecutionEnd - event: {}", endEvent.toString());
     if (log.isDebugEnabled()) {
       log.debug("SparkListenerSQLExecutionEnd - executionId: {}", endEvent.executionId());
     }
@@ -160,6 +165,7 @@ public void end(SparkListenerSQLExecutionEnd endEvent) {
     if (log.isDebugEnabled()) {
       log.debug("Posting event for end {}: {}", executionId, OpenLineageClientUtils.toJson(event));
     }
+    log.info("Posting event for end {}: {}", executionId, OpenLineageClientUtils.toJson(event));
     NuEventEmitter.emit(event, eventEmitter);
   }