diff --git a/.github/workflows/dlt_deploy.yml b/.github/workflows/dlt_deploy.yml index bc85705..dd85d0e 100644 --- a/.github/workflows/dlt_deploy.yml +++ b/.github/workflows/dlt_deploy.yml @@ -35,6 +35,17 @@ jobs: # See https://github.com/databricks/setup-cli - uses: databricks/setup-cli@main + - name: Replace pipeline name and schema + working-directory: . + env: + PIPELINE_ORIGIN_NAME: dlt_sample_pipeline + PIPELINE_ORIGIN_SCHEMA: base_schema_name + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + run: > + BRANCH_NAME=`echo "$BRANCH_NAME" | sed -r 's/[/]+/_/g'` + sed -i -e s/${PIPELINE_ORIGIN_NAME}/${PIPELINE_ORIGIN_NAME}_${BRANCH_NAME}/g resources/dlt_sample_pipeline.yml + sed -i -e s/${PIPELINE_ORIGIN_SCHEMA}/${PIPELINE_ORIGIN_SCHEMA}_${BRANCH_NAME}/g resources/dlt_sample_pipeline.yml + # Deploy the bundle to the "qa" target as defined # in the bundle's settings file. - run: databricks bundle deploy diff --git a/resources/dlt_sample_job.yml b/resources/dlt_sample_job.yml index 5eeed22..02df905 100644 --- a/resources/dlt_sample_job.yml +++ b/resources/dlt_sample_job.yml @@ -18,15 +18,7 @@ resources: notebook_task: notebook_path: ../src/notebook.ipynb - - task_key: refresh_pipeline - depends_on: - - task_key: notebook_task - pipeline_task: - pipeline_id: ${resources.pipelines.dlt_sample_pipeline.id} - - task_key: main_task - depends_on: - - task_key: refresh_pipeline job_cluster_key: job_cluster python_wheel_task: package_name: dlt_sample diff --git a/resources/dlt_sample_pipeline.yml b/resources/dlt_sample_pipeline.yml index 5dc57e9..2332aa8 100644 --- a/resources/dlt_sample_pipeline.yml +++ b/resources/dlt_sample_pipeline.yml @@ -2,9 +2,9 @@ resources: pipelines: dlt_sample_pipeline: - name: dlt_sample_pipeline [${bundle.git.branch}] - catalog: "ema_rina" - target: dlt_sample_pr_${bundle.git.branch} + name: dlt_sample_pipeline + catalog: ema_rina + target: base_schema_name libraries: - notebook: path: ../src/dlt_sample/DLT_1.py