From 30143401bb58c8a93e967aa5f1d2b48870743f5b Mon Sep 17 00:00:00 2001 From: Marcos Prieto Date: Thu, 16 Nov 2023 14:56:11 +0100 Subject: [PATCH] Include the since/until parameters for H slim task The dates here take into account the progress made so far and move the window forward a few months to see the impact on performance. In theory a very big window will degrade performance significantly as it will need to visit more annotations. --- h_periodic/h_beat.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/h_periodic/h_beat.py b/h_periodic/h_beat.py index 99678dd..e8fc5f6 100644 --- a/h_periodic/h_beat.py +++ b/h_periodic/h_beat.py @@ -17,6 +17,9 @@ print("h_beat disabled by DISABLE_H_BEAT environment variable") sys.exit() +SLIM_TASK_SINCE = "2016-01-01" +SLIM_TASK_UNTIL = "2018-03-31" + celery = Celery("h") celery.conf.update( beat_schedule_filename="h-celerybeat-schedule", @@ -55,14 +58,22 @@ "fill-annotation-slim-first-batch": { "options": {"expires": 30}, "task": "h.tasks.annotations.fill_annotation_slim", - "schedule": crontab(hour="7", minute="*/3"), - "kwargs": {"batch_size": 250}, + "schedule": crontab(hour="7", minute="*/2"), + "kwargs": { + "batch_size": 250, + "since": SLIM_TASK_SINCE, + "until": SLIM_TASK_UNTIL, + }, }, "fill-annotation-slim": { "options": {"expires": 30}, "task": "h.tasks.annotations.fill_annotation_slim", "schedule": crontab(hour="8-15", minute="*/2"), - "kwargs": {"batch_size": 3000}, + "kwargs": { + "batch_size": 3000, + "since": SLIM_TASK_SINCE, + "until": SLIM_TASK_UNTIL, + }, }, "report-sync-annotations-queue-length": { "options": {"expires": 30},