articles: add index command as task, batched

Signed-off-by: pamfilos <[email protected]>

Author: pamfilos

scoap3/articles/tasks.py

@@ -162,3 +162,22 @@ def compliance_checks(article_id):
     report.save()
     logger.info("Compliance checks completed for article %s", article_id)
     return f"Compliance checks completed for article {article_id}"
+
+
+from django.core.paginator import Paginator
+from scoap3.articles.models import Article
+from django_opensearch_dsl.registries import registry
+
+@shared_task
+def index_article_batch(article_ids):
+    articles = Article.objects.filter(id__in=article_ids)
+    for article in articles:
+        registry.update(article)
+
+def index_all_articles(batch_size=100):
+    all_articles = Article.objects.all().values_list('id', flat=True)
+    paginator = Paginator(all_articles, batch_size)
+
+    for page_number in paginator.page_range:
+        page = paginator.page(page_number)
+        index_article_batch.delay(list(page.object_list))

scoap3/management/commands/articles_index.py

@@ -0,0 +1,30 @@
+import math
+
+from django.core.files.storage import storages
+from django.core.management.base import BaseCommand, CommandParser
+
+from scoap3.articles.tasks import index_all_articles
+
+
+class Command(BaseCommand):
+    help = "Article indexing commands"
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "--from-id",
+            type=int,
+            default=None,
+            required=False,
+            help="Article id to start the indexing FROM.",
+        )
+
+        parser.add_argument(
+            "--batch-size",
+            type=int,
+            default=1000,
+            required=False,
+            help="Batchsize to migrate per task.",
+        )
+
+    def handle(self, *args, **options):
+        index_all_articles(options['batch_size'])