From 7ee9e0d09dfa4a13725fd709bc4fc11c8f4bb6dc Mon Sep 17 00:00:00 2001 From: Amisha Singla Date: Fri, 13 Dec 2024 15:41:10 -0600 Subject: [PATCH] moar refactor update image --- airflow_variables_dev.json | 3 ++- airflow_variables_prod.json | 5 +++++ dags/external_data_dag.py | 18 ++++++++++++------ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json index 22f97fb1..714c8bff 100644 --- a/airflow_variables_dev.json +++ b/airflow_variables_dev.json @@ -316,7 +316,7 @@ "schema_filepath": "/home/airflow/gcs/dags/schemas/", "sentry_dsn": "https://9e0a056541c3445083329b072f2df690@o14203.ingest.us.sentry.io/6190849", "sentry_environment": "development", - "stellar_etl_internal_image_name": "amishastellar/stellar-etl-internal:e3b9a2ea7", + "stellar_etl_internal_image_name": "amishastellar/stellar-etl-internal:cd53bcf70", "table_ids": { "accounts": "accounts", "assets": "history_assets", @@ -331,6 +331,7 @@ "liquidity_pools": "liquidity_pools", "offers": "offers", "operations": "history_operations", + "retool_entity_data": "retool_entity_data", "signers": "account_signers", "trades": "history_trades", "transactions": "history_transactions", diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json index fa03f153..46669b27 100644 --- a/airflow_variables_prod.json +++ b/airflow_variables_prod.json @@ -309,10 +309,12 @@ } } }, + "retool_api_key": "test-api-key", "sandbox_dataset": "crypto_stellar_internal_sandbox", "schema_filepath": "/home/airflow/gcs/dags/schemas/", "sentry_dsn": "https://94027cdcc4c9470f9dafa2c0b456c2c9@o14203.ingest.us.sentry.io/5806618", "sentry_environment": "production", + "stellar_etl_internal_image_name": "amishastellar/stellar-etl-internal:e3b9a2ea7", "table_ids": { "accounts": "accounts", "assets": "history_assets", @@ -327,6 +329,7 @@ "liquidity_pools": "liquidity_pools", "offers": "offers", "operations": "history_operations", + "retool_entity_data": "retool_entity_data", "signers": "account_signers", "trades": "history_trades", "transactions": "history_transactions", @@ -347,9 +350,11 @@ "create_sandbox": 1020, "current_state": 1200, "default": 60, + "del_ins_retool_entity_data_task": 720, "elementary_dbt_data_quality": 2100, "elementary_generate_report": 1200, "enriched_history_operations": 1800, + "export_retool_data": 720, "fee_stats": 360, "history_assets": 360, "liquidity_pool_trade_volume": 1200, diff --git a/dags/external_data_dag.py b/dags/external_data_dag.py index f8e11b3d..16720102 100644 --- a/dags/external_data_dag.py +++ b/dags/external_data_dag.py @@ -25,6 +25,12 @@ init_sentry() +EXTERNAL_DATA_TABLE_NAMES = Variable.get("table_ids", deserialize_json=True) +EXTERNAL_DATA_PROJECT_NAME = Variable.get("bq_project") +EXTERNAL_DATA_DATASET_NAME = Variable.get("bq_dataset") +RETOOL_TABLE_NAME = EXTERNAL_DATA_TABLE_NAMES["retool_entity_data"] +RETOOL_EXPORT_TASK_ID = "export_retool_data" + # Initialize the DAG dag = DAG( "external_data_dag", @@ -50,7 +56,7 @@ retool_export_task = build_export_task( dag, - "export_retool_data", + RETOOL_EXPORT_TASK_ID, command="export-retool", cmd_args=[ "--start-time", @@ -101,14 +107,14 @@ def get_insert_to_bq_task( retool_insert_to_bq_task = get_insert_to_bq_task( - table_name="retool_entity_data", - project="test-hubble-319619", - dataset="test_crypto_stellar_internal", - export_task_id="export_retool_data", + table_name=RETOOL_TABLE_NAME, + project=EXTERNAL_DATA_PROJECT_NAME, + dataset=EXTERNAL_DATA_DATASET_NAME, + export_task_id=RETOOL_EXPORT_TASK_ID, source_object_suffix="", partition=False, cluster=False, - table_id="test-hubble-319619.test_crypto_stellar_internal.retool_entity_data", + table_id=f"{EXTERNAL_DATA_PROJECT_NAME}.{EXTERNAL_DATA_DATASET_NAME}.{RETOOL_TABLE_NAME}", ) retool_export_task >> retool_insert_to_bq_task