diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index c6b1eb5c..32dd10b6 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -315,7 +315,7 @@ "schema_filepath": "/home/airflow/gcs/dags/schemas/", "sentry_dsn": "https://9e0a056541c3445083329b072f2df690@o14203.ingest.us.sentry.io/6190849", "sentry_environment": "development", - "stellar_etl_internal_image_name": "stellar/stellar-etl:204d343fa", + "stellar_etl_internal_image_name": " amishastellar/stellar-etl-internal:ab0a13897", "table_ids": { "accounts": "accounts", "assets": "history_assets", diff --git a/dags/external_data_dag.py b/dags/external_data_dag.py index bea102f1..51e60f54 100644 --- a/dags/external_data_dag.py +++ b/dags/external_data_dag.py @@ -3,11 +3,13 @@ It is scheduled to export information to BigQuery at regular intervals. """ +import logging from ast import literal_eval from datetime import datetime from json import loads from airflow import DAG +from airflow.configuration import conf from airflow.models.variable import Variable from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from kubernetes.client import models as k8s