Skip to content

Commit

Permalink
add noop for first etl job
Browse files Browse the repository at this point in the history
  • Loading branch information
nathandf committed Dec 7, 2023
1 parent 2cde257 commit d85e878
Showing 1 changed file with 31 additions and 8 deletions.
39 changes: 31 additions & 8 deletions src/api/src/backend/views/ETLPipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,25 +213,48 @@ def post(self, request, group_id, *_, **__):
# The first tapis job should be dependent on the gen-inbound-manifests task
last_task_id = "gen-inbound-manifests"

# No-op condition
# Create a condition that should only be applied to the first Tapis Job
# in the series to ensure that when there is no data to process, the first
# job (and all subsequent jobs) do not run
no_op_condition = {
"ne": [
{
"task_output": {
"task_id": last_task_id,
"output_id": "ACTIVE_MANIFEST"
}
},
None
]
}

# Create a tapis job task for each job provided in the request.
tasks = []
try:
for i, job in enumerate(body.jobs, start=1):
# Set up the conditions for the job to run
conditions = [
{
"eq": [
{"args": "RESUBMIT_OUTBOUND"},
None
]
}
]

# Add no-op condition for only first task
if i == 1:
conditions.append(no_op_condition)

task_id = f"etl-job-{i}"
tasks.append(
TapisJobTask(**{
"id": task_id,
"type": "tapis_job",
"tapis_job_def": job,
"depends_on": [{"id": last_task_id}],
"conditions": [
{
"eq": [
{"args": "RESUBMIT_OUTBOUND"},
None
]
}
]
"conditions": conditions
})
)
last_task_id = task_id
Expand Down

0 comments on commit d85e878

Please sign in to comment.