add ! aurora import

.pre-commit-config.yaml

@@ -5,25 +5,25 @@ repos:
       - id: isort
         stages: [pre-commit]
   - repo: https://github.com/ambv/black
-    rev: 24.4.2
+    rev: 24.10.0
     hooks:
       - id: black
         args: [--config=pyproject.toml]
         exclude: "migrations|snapshots"
         stages: [pre-commit]
   - repo: https://github.com/PyCQA/flake8
-    rev: 7.1.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         args: [--config=.flake8]
         stages: [ pre-commit ]
   - repo: https://github.com/PyCQA/bandit
-    rev: '1.7.9' # Update me!
+    rev: '1.7.10' # Update me!
     hooks:
       - id: bandit
         args: ["-c", "bandit.yaml"]
   - repo: https://github.com/twisted/towncrier
-    rev: 23.11.0
+    rev: 24.8.0
     hooks:
       - id: towncrier-check
 

pyproject.toml

@@ -21,7 +21,7 @@ dependencies = [
     "django-celery-beat>=2.6.0",
     "django-celery-boost>=0.2.0",
     "django-celery-results>=2.5.1",
-    "django-constance>=3.1.0",
+    "django-constance>=4.1.3",
     "django-csp",
     "django-debug-toolbar>=4.4.2",
     "django-environ>=0.11.2",

src/country_workspace/admin/job.py

@@ -1,5 +1,6 @@
 from django.contrib import admin
 
+from adminfilters.autocomplete import AutoCompleteFilter
 from django_celery_boost.admin import CeleryTaskModelAdmin
 
 from ..models import AsyncJob
@@ -9,6 +10,16 @@
 @admin.register(AsyncJob)
 class AsyncJobAdmin(CeleryTaskModelAdmin, BaseModelAdmin):
     list_display = (
+        "id",
         "program",
         "type",
+        "task_status",
     )
+    list_filter = ("type", ("program", AutoCompleteFilter))
+    ordering = ("id",)
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False

src/country_workspace/config/__init__.py

@@ -52,6 +52,8 @@ class Group(Enum):
         True,
         setting("allowed-hosts"),
     ),
+    "AURORA_API_TOKEN": (str, "", "", True, "Aurora API token"),
+    "AURORA_API_URL": (str, "", "", True, "Aurora API url"),
     "CACHE_URL": (str, "", "redis://localhost:6379/0", True, setting("cache-url")),
     "CELERY_BROKER_URL": (
         str,
@@ -82,7 +84,7 @@ class Group(Enum):
         celery_doc("#broker-transport-options"),
     ),
     "CSRF_COOKIE_SECURE": (bool, True, False, True, setting("csrf-cookie-secure")),
-    "CSRF_TRUSTED_ORIGINS": (list, "localhost", "", True, ""),
+    "CSRF_TRUSTED_ORIGINS": (list, ["http://localhost"], "", True, ""),
     "DATABASE_URL": (
         str,
         SmartEnv.NOTSET,

src/country_workspace/config/fragments/app.py

@@ -6,6 +6,9 @@
 TENANT_TENANT_MODEL = "country_workspace.Office"
 TENANT_HQ = "= HQ ="
 
+AURORA_API_TOKEN = env("AURORA_API_TOKEN")
+AURORA_API_URL = env("AURORA_API_URL")
+
 HOPE_API_TOKEN = env("HOPE_API_TOKEN")
 HOPE_API_URL = env("HOPE_API_URL")
 

src/country_workspace/config/fragments/constance.py

@@ -1,6 +1,10 @@
-from django.conf import settings
-
-from ..settings import NEW_USER_DEFAULT_GROUP
+from country_workspace.config.fragments.app import (
+    AURORA_API_TOKEN,
+    AURORA_API_URL,
+    HOPE_API_TOKEN,
+    HOPE_API_URL,
+    NEW_USER_DEFAULT_GROUP,
+)
 
 CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend"
 
@@ -44,22 +48,22 @@
         "Group to assign to any new user",
         "group_select",
     ),
-    "HOPE_API_URL": (settings.HOPE_API_URL, "HOPE API Server address", str),
-    "HOPE_API_TOKEN": (settings.HOPE_API_TOKEN, "HOPE API Access Token", "write_only_input"),
-    "AURORA_API_URL": ("https://register.unicef.org/api/", "Aurora API Server address", str),
-    "AURORA_API_TOKEN": ("", "Aurora API Access Token", "write_only_input"),
-    "KOBO_API_URL": ("", "Kobo API Server address", str),
+    "AURORA_API_TOKEN": (AURORA_API_TOKEN, "Aurora API Access Token", "write_only_input"),
+    "AURORA_API_URL": (AURORA_API_URL, "Aurora API Server address", str),
+    "HOPE_API_TOKEN": (HOPE_API_TOKEN, "HOPE API Access Token", "write_only_input"),
+    "HOPE_API_URL": (HOPE_API_URL, "HOPE API Server address", str),
     "KOBO_API_TOKEN": ("", "Kobo API Access Token", "write_only_input"),
+    "KOBO_API_URL": ("", "Kobo API Server address", str),
 }
 
 CONSTANCE_CONFIG_FIELDSETS = {
     "New User Options": ("NEW_USER_IS_STAFF", "NEW_USER_DEFAULT_GROUP"),
     "Remote System Tokens": (
-        "HOPE_API_URL",
-        "HOPE_API_TOKEN",
-        "AURORA_API_URL",
         "AURORA_API_TOKEN",
-        "KOBO_API_URL",
+        "AURORA_API_URL",
+        "HOPE_API_TOKEN",
+        "HOPE_API_URL",
         "KOBO_API_TOKEN",
+        "KOBO_API_URL",
     ),
 }

src/country_workspace/config/settings.py

@@ -50,6 +50,7 @@
     "country_workspace.security",
     "country_workspace.apps.Config",
     "country_workspace.workspaces.apps.Config",
+    "country_workspace.contrib.aurora.apps.Config",
     "country_workspace.versioning",
     *env("EXTRA_APPS"),
 )

src/country_workspace/constants.py

@@ -0,0 +1,5 @@
+from typing import Final
+
+from django.utils import timezone
+
+BATCH_NAME_DEFAULT: Final[str] = f"Batch {timezone.now()}"

src/country_workspace/contrib/aurora/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class Config(AppConfig):
+    name = __name__.rpartition(".")[0]
+    verbose_name = "Country Workspace | Aurora"

src/country_workspace/contrib/aurora/client.py

@@ -0,0 +1,64 @@
+from typing import Any, Generator
+from urllib.parse import urljoin
+
+import requests
+from constance import config
+
+
+class AuroraClient:
+    """
+    A client for interacting with the Aurora API.
+
+    Provides methods to fetch data from the Aurora API with authentication.
+    Handles pagination automatically for large datasets.
+    """
+
+    def __init__(self, token: str | None = None) -> None:
+        """
+        Initialize the AuroraClient.
+
+        Args:
+            token (str | None): An optional API token for authentication. If not provided,
+                the token is retrieved from the Constance configuration (config.AURORA_API_TOKEN).
+        """
+        self.token = token or config.AURORA_API_TOKEN
+
+    def _get_url(self, path: str) -> str:
+        """
+        Construct a fully qualified URL for the Aurora API.
+
+        Args:
+            path (str): The relative API path.
+
+        Returns:
+            str: The full URL, ensuring it ends with a trailing slash.
+        """
+        url = urljoin(config.AURORA_API_URL, path)
+        if not url.endswith("/"):
+            url = url + "/"
+        return url
+
+    def get(self, path: str) -> Generator[dict[str, Any], None, None]:
+        """
+        Fetch records from the Aurora API with automatic pagination.
+
+        Args:
+            path (str): The relative API path to fetch data from.
+
+        Yields:
+            dict[str, Any]: Individual records from the API.
+
+        Raises:
+            Exception: If the API response has a status code other than 200.
+        """
+        url = self._get_url(path)
+        while url:
+            ret = requests.get(url, headers={"Authorization": f"Token {self.token}"}, timeout=10)
+            if ret.status_code != 200:
+                raise Exception(f"Error {ret.status_code} fetching {url}")
+            data = ret.json()
+
+            for record in data["results"]:
+                yield record
+
+            url = data.get("next")

src/country_workspace/contrib/aurora/forms.py

@@ -0,0 +1,16 @@
+from django import forms
+
+
+class ImportAuroraForm(forms.Form):
+
+    batch_name = forms.CharField(required=False, help_text="Label for this batch")
+
+    check_before = forms.BooleanField(required=False, help_text="Prevent import if errors")
+
+    household_name_column = forms.CharField(
+        required=False,
+        initial="family_name",
+        help_text="Which Individual's column contains the Household's name",
+    )
+
+    fail_if_alien = forms.BooleanField(required=False)

src/country_workspace/contrib/aurora/sync.py

@@ -0,0 +1,125 @@
+from typing import Any
+
+from django.db.transaction import atomic
+
+from country_workspace.constants import BATCH_NAME_DEFAULT
+from country_workspace.contrib.aurora.client import AuroraClient
+from country_workspace.models import AsyncJob, Batch, Household, Individual, User
+from country_workspace.utils.fields import clean_field_name
+
+
+def sync_aurora_job(job: AsyncJob) -> dict[str, Any]:
+    """
+    Synchronizes data from the Aurora system into the database for the given job within an atomic transaction.
+
+    Args:
+        job (AsyncJob): The job instance containing configuration and context for synchronization.
+
+    Returns:
+        dict[str, Any]: A dictionary with counts of households and individuals created, e.g.,
+        {"households": total_hh, "individuals": total_ind}.
+    """
+    total_hh = total_ind = 0
+    client = AuroraClient()
+    with atomic():
+        job.batch = _create_batch(job)
+        job.save()
+        # TODO: Duplicate import ?
+        for record in client.get("record"):
+            for f_name, f_value in record["fields"].items():
+                if f_name == "household":
+                    hh = _create_household(job, f_value[0])
+                    total_hh += 1
+                elif f_name == "individuals":
+                    total_ind += len(_create_individuals(job, hh, f_value))
+        return {"households": total_hh, "individuals": total_ind}
+
+
+def _create_batch(job: AsyncJob) -> Batch:
+    """
+    Creates a batch entity associated with the given job.
+
+    Args:
+        job (AsyncJob): The job instance containing the configuration for the batch creation.
+
+    Returns:
+        Batch: The newly created batch instance.
+    """
+    return Batch.objects.create(
+        name=job.config.get("batch_name") or BATCH_NAME_DEFAULT,
+        program=job.program,
+        country_office=job.program.country_office,
+        imported_by=User.objects.get(pk=job.config.get("imported_by_id")),
+    )
+
+
+def _create_household(job: AsyncJob, fields: dict[str, Any]) -> Household:
+    """
+    Creates a household entity associated with the given job and batch.
+
+    Args:
+        job (AsyncJob): The job instance containing context for household creation.
+        fields (dict[str, Any]): A dictionary containing household data fields.
+
+    Returns:
+        Household: The newly created household instance.
+    """
+    return job.program.households.create(
+        batch=job.batch, flex_fields={clean_field_name(k): v for k, v in fields.items()}
+    )
+
+
+def _create_individuals(
+    job: AsyncJob,
+    household: Household,
+    fields: list[dict[str, Any]],
+) -> list[Individual]:
+    """
+    Creates individuals associated with a household and updates the household name if necessary.
+
+    Args:
+        job (AsyncJob): The job instance containing configuration and context for individual creation.
+        household (Household): The household to associate with the individuals.
+        fields (list[dict[str, Any]]): A list of dictionaries containing individual data fields.
+
+    Returns:
+        list[Individual]: The list of newly created individual instances.
+    """
+    individuals = []
+    for individual in fields:
+
+        _update_household_name_from_individual(job, household, individual)
+
+        fullname = next((k for k in individual if k.startswith("given_name")), None)
+        individuals.append(
+            Individual(
+                batch=job.batch,
+                household_id=household.pk,
+                name=individual.get(fullname, ""),
+                flex_fields={clean_field_name(k): v for k, v in individual.items()},
+            )
+        )
+
+    return job.program.individuals.bulk_create(individuals)
+
+
+def _update_household_name_from_individual(job: AsyncJob, household: Household, individual: dict[str, Any]) -> None:
+    """
+    Updates the household name based on an individual's relationship and name field.
+
+    This method checks if the individual is marked as the head of the household
+    and updates the household name accordingly.
+
+    Args:
+        job (AsyncJob): The job instance containing configuration details.
+        household (Household): The household to update.
+        individual (dict[str, Any]): The individual data containing potential household name information.
+
+    Returns:
+        None
+    """
+    if any(individual.get(k) == "head" for k in individual if k.startswith("relationship")):
+        for k, v in individual.items():
+            if clean_field_name(k) == job.config["household_name_column"]:
+                job.program.households.filter(pk=household.pk).update(name=v)
+                break

src/country_workspace/models/jobs.py

@@ -7,6 +7,7 @@ class AsyncJob(CeleryTaskModel, models.Model):
     class JobType(models.TextChoices):
         BULK_UPDATE_HH = "BULK_UPDATE_HH"
         BULK_UPDATE_IND = "BULK_UPDATE_IND"
+        AURORA_SYNC = "AURORA_SYNC"
 
     type = models.CharField(max_length=50, choices=JobType.choices)
     program = models.ForeignKey("Program", related_name="jobs", on_delete=models.CASCADE)

src/country_workspace/tasks.py

@@ -6,6 +6,7 @@
 from hope_smart_import.readers import open_xls
 
 from country_workspace.config.celery import app
+from country_workspace.contrib.aurora.sync import sync_aurora_job
 from country_workspace.models import AsyncJob
 
 
@@ -14,6 +15,8 @@ def sync_job_task(pk: int, version: int) -> dict[str, Any]:
     job: AsyncJob = AsyncJob.objects.select_related("program").get(pk=pk, version=version)
     if job.type == AsyncJob.JobType.BULK_UPDATE_IND:
         return bulk_update_individual(job)
+    elif job.type == AsyncJob.JobType.AURORA_SYNC:
+        return sync_aurora_job(job)
 
 
 def bulk_update_individual(job: AsyncJob) -> dict[str, Any]: