Determine data refresh pool by environment variable with default (#2352)

WordPress · Jun 14, 2023 · 41b0cdb · 41b0cdb
1 parent c20f275
commit 41b0cdb
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/catalog/dags/data_refresh/data_refresh_task_factory.py b/catalog/dags/data_refresh/data_refresh_task_factory.py
@@ -65,7 +65,7 @@
 logger = logging.getLogger(__name__)
 
 
-DATA_REFRESH_POOL = "data_refresh"
+DATA_REFRESH_POOL = os.getenv("DATA_REFRESH_POOL", "data_refresh")
 
 
 def create_data_refresh_task_group(

diff --git a/catalog/env.template b/catalog/env.template
@@ -106,6 +106,9 @@ OPENVERSE_BUCKET=openverse-storage
 # Seconds to wait before poking for availability of the data refresh pool when running a data_refresh
 # DAG. Used to shorten the time for testing purposes.
 DATA_REFRESH_POKE_INTERVAL=5
+# The pool to use for the data refresh tasks. Defaults to `data_refresh` in production but
+# we override this locally to prevent having to frequently recreate it.
+DATA_REFRESH_POOL=default_pool
 # Number of Retries if DAG task fails to run
 DEFAULT_RETRY_COUNT = 2
 # Whether to enable catchup for dated DAGs, allowing automatic backfill.