unable to emit metric should not fail the pipeline

data-integrations · Oct 27, 2023 · 1c2fae6 · 1c2fae6
1 parent 0c1f8e9
commit 1c2fae6
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 2 deletions.
diff --git a/pom.xml b/pom.xml
@@ -20,7 +20,7 @@
 
   <groupId>io.cdap.plugin</groupId>
   <artifactId>google-cloud</artifactId>
-  <version>0.22.4</version>
+  <version>0.22.5</version>
   <name>Google Cloud Plugins</name>
   <packaging>jar</packaging>
   <description>Plugins for Google Big Query</description>

diff --git a/src/main/java/io/cdap/plugin/gcp/bigquery/action/BigQueryExecute.java b/src/main/java/io/cdap/plugin/gcp/bigquery/action/BigQueryExecute.java
@@ -169,14 +169,24 @@ public void run(ActionContext context) throws Exception {
       }
     }
 
+    context.getMetrics().gauge(RECORDS_PROCESSED, rows);
+    try {
+      recordBytesProcessedMetric(context, queryJob);
+    } catch (Exception exception) {
+      // log the exception but not fail the pipeline
+      LOG.warn("Exception while trying to emit bytes processed metric.",
+          exception);
+    }
+  }
+
+  private void recordBytesProcessedMetric(ActionContext context, Job queryJob) {
     long processedBytes =
         ((JobStatistics.QueryStatistics) queryJob.getStatistics()).getTotalBytesProcessed();
     LOG.info("Job {} processed {} bytes", queryJob.getJobId(), processedBytes);
     Map<String, String> tags = new ImmutableMap.Builder<String, String>()
         .put(Constants.Metrics.Tag.APP_ENTITY_TYPE, Action.PLUGIN_TYPE)
         .put(Constants.Metrics.Tag.APP_ENTITY_TYPE_NAME, BigQueryExecute.NAME)
         .build();
-    context.getMetrics().gauge(RECORDS_PROCESSED, rows);
     context.getMetrics().child(tags).countLong(BigQuerySinkUtils.BYTES_PROCESSED_METRIC,
         processedBytes);
   }