Generic tasks view refactor

rocky/assets/js/renderNormalizerOutputOOIs.js

@@ -11,11 +11,14 @@ buttons.forEach((button) => {
     .closest("tr")
     .getAttribute("data-task-id")
     .replace(/-/g, "");
+  const organization =
+    organization_code ||
+    button.closest("tr").getAttribute("data-organization-code");
   const json_url =
     "/" +
     language +
     "/" +
-    organization_code +
+    organization +
     "/tasks/normalizers/" +
     encodeURI(task_id);
 
@@ -79,7 +82,7 @@ buttons.forEach((button) => {
             "/" +
             language +
             "/" +
-            escapeHTMLEntities(encodeURIComponent(organization_code));
+            escapeHTMLEntities(encodeURIComponent(organization));
           let object_list = "";
           // set the observed at time a fews seconds into the future, as the job finish time is not the same as the ooi-creation time. Due to async reasons the object might be a bit slow.
           data["timestamp"] = Date.parse(data["valid_time"] + "Z");

rocky/katalogus/templates/partials/single_action_form.html

@@ -1,4 +1,6 @@
-<form method="post" class="inline">
+<form method="post"
+      class="inline"
+      {% if url %}action="{{ url }}"{% endif %}>
     {% csrf_token %}
     <input type="hidden" name="action" value="{{ action }}">
     {% if key %}<input type="hidden" name="{{ key }}" value="{{ value }}">{% endif %}

rocky/rocky/scheduler.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+import collections
 import datetime
-import json
+import logging
 import uuid
 from enum import Enum
 from functools import cached_property
@@ -162,6 +163,7 @@ def __getitem__(self, key) -> list[Task]:
         else:
             raise TypeError("Invalid slice argument type.")
 
+        logging.info("Getting max %s lazy items at offset %s with filter %s", limit, offset, self.kwargs)
         res = self.scheduler_client.list_tasks(
             limit=limit,
             offset=offset,
@@ -214,7 +216,7 @@ class SchedulerHTTPError(SchedulerError):
 
 
 class SchedulerClient:
-    def __init__(self, base_uri: str, organization_code: str):
+    def __init__(self, base_uri: str, organization_code: str | None):
         self._client = httpx.Client(base_url=base_uri)
         self.organization_code = organization_code
 
@@ -223,31 +225,28 @@ def list_tasks(
         **kwargs,
     ) -> PaginatedTasksResponse:
         try:
-            kwargs = {k: v for k, v in kwargs.items() if v is not None}  # filter Nones from kwargs
-            res = self._client.get("/tasks", params=kwargs)
+            filter_key = "filters"
+            params = {k: v for k, v in kwargs.items() if v is not None if k != filter_key}  # filter Nones from kwargs
+            endpoint = "/tasks"
+            res = self._client.post(endpoint, params=params, json=kwargs.get(filter_key, None))
             return PaginatedTasksResponse.model_validate_json(res.content)
         except ValidationError:
             raise SchedulerValidationError(extra_message=_("Task list: "))
         except ConnectError:
             raise SchedulerConnectError(extra_message=_("Task list: "))
 
     def get_task_details(self, task_id: str) -> Task:
-        try:
-            res = self._client.get(f"/tasks/{task_id}")
-            res.raise_for_status()
-            task_details = Task.model_validate_json(res.content)
+        task_details = Task.model_validate_json(self._get(f"/tasks/{task_id}", "content"))
 
-            if task_details.type == "normalizer":
-                organization = task_details.data.raw_data.boefje_meta.organization
-            else:
-                organization = task_details.data.organization
+        if task_details.type == "normalizer":
+            organization = task_details.data.raw_data.boefje_meta.organization
+        else:
+            organization = task_details.data.organization
 
-            if organization != self.organization_code:
-                raise SchedulerTaskNotFound()
+        if organization != self.organization_code:
+            raise SchedulerTaskNotFound()
 
-            return task_details
-        except ConnectError:
-            raise SchedulerConnectError()
+        return task_details
 
     def push_task(self, item: Task) -> None:
         try:
@@ -270,24 +269,45 @@ def push_task(self, item: Task) -> None:
             raise SchedulerError()
 
     def health(self) -> ServiceHealth:
-        try:
-            health_endpoint = self._client.get("/health")
-            health_endpoint.raise_for_status()
-            return ServiceHealth.model_validate_json(health_endpoint.content)
-        except HTTPError:
-            raise SchedulerHTTPError()
-        except ConnectError:
-            raise SchedulerConnectError()
+        return ServiceHealth.model_validate_json(self._get("/health", return_type="content"))
+
+    def _get_task_stats(self, scheduler_id: str) -> dict:
+        """Return task stats for specific scheduler."""
+        return self._get(f"/tasks/stats/{scheduler_id}")  # type: ignore
 
     def get_task_stats(self, task_type: str) -> dict:
+        """Return task stats for specific task type."""
+        return self._get_task_stats(scheduler_id=f"{task_type}-{self.organization_code}")
+
+    @staticmethod
+    def _merge_stat_dicts(dicts: list[dict]) -> dict:
+        """Merge multiple stats dicts."""
+        stat_sum: dict[str, collections.Counter] = collections.defaultdict(collections.Counter)
+        for dct in dicts:
+            for timeslot, counts in dct.items():
+                stat_sum[timeslot].update(counts)
+        return dict(stat_sum)
+
+    def get_combined_schedulers_stats(self, scheduler_ids: list) -> dict:
+        """Return merged stats for a set of scheduler ids."""
+        return SchedulerClient._merge_stat_dicts(
+            dicts=[self._get_task_stats(scheduler_id=scheduler_id) for scheduler_id in scheduler_ids]
+        )
+
+    def _get(self, path: str, return_type: str = "json") -> dict | bytes:
+        """Helper to do a get request and raise warning for path."""
         try:
-            res = self._client.get(f"/tasks/stats/{task_type}-{self.organization_code}")
+            res = self._client.get(path)
             res.raise_for_status()
-        except ConnectError:
-            raise SchedulerConnectError(extra_message=_("Task statistics: "))
-        task_stats = json.loads(res.content)
-        return task_stats
+        except HTTPError as exc:
+            raise SchedulerError(path) from exc
+        except ConnectError as exc:
+            raise SchedulerConnectError(path) from exc
+
+        if return_type == "content":
+            return res.content
+        return res.json()
 
 
-def scheduler_client(organization_code: str) -> SchedulerClient:
+def scheduler_client(organization_code: str | None) -> SchedulerClient:
     return SchedulerClient(settings.SCHEDULER_API, organization_code)

rocky/rocky/settings.py

@@ -78,7 +78,7 @@
     "loggers": {
         "root": {
             "handlers": ["console"],
-            "level": "INFO",
+            "level": env("LOG_LEVEL", default="WARNING").upper(),
         },
     },
 }

rocky/rocky/templates/header.html

@@ -32,6 +32,11 @@
                                 <a href="{{ index_url }}"
                                    {% if index_url == request.path|urlencode %}aria-current="page"{% endif %}>{% translate "Crisis room" %}</a>
                             </li>
+                            <li>
+                                {% url "all_task_list" as index_url %}
+                                <a href="{{ index_url }}"
+                                   {% if index_url in request.path|urlencode %}aria-current="page"{% endif %}>{% translate "Tasks" %}</a>
+                            </li>
                         {% else %}
                             <li>
                                 {% url "organization_crisis_room" organization.code as index_url %}

rocky/rocky/templates/tasks/boefjes.html

@@ -25,25 +25,35 @@ <h1>{% translate "Boefjes" %}</h1>
                         <table rf-selector="table-scan-history">
                             <thead>
                                 <tr>
+                                    {% if not organization.code %}
+                                        <th scope="col">{% translate "Organization Code" %}</th>
+                                    {% endif %}
                                     <th scope="col">{% translate "Boefje" %}</th>
                                     <th scope="col">{% translate "Status" %}</th>
                                     <th scope="col">{% translate "Created date" %}</th>
+                                    <th scope="col">{% translate "Modified date" %}</th>
                                     <th scope="col">{% translate "Input Object" %}</th>
                                     <th scope="col" class="visually-hidden">{% translate "Details" %}</th>
                                 </tr>
                             </thead>
                             <tbody>
                                 {% for task in task_list %}
                                     <tr data-task-id="{{ task.id }}">
+                                        {% if not organization.code %}
+                                            <td>
+                                                <a href="{% url "organization_crisis_room" task.data.organization %}">{{ task.data.organization }}</a>
+                                            </td>
+                                        {% endif %}
                                         <td>
-                                            <a href="{% url "boefje_detail" organization.code task.data.boefje.id %}">{{ task.data.boefje.name }}</a>
+                                            <a href="{% url "boefje_detail" task.data.organization task.data.boefje.id %}">{{ task.data.boefje.name }}</a>
                                         </td>
                                         <td class="nowrap">
                                             <i class="icon {{ task.status.value }}"></i>&nbsp;{{ task.status.value|capfirst }}
                                         </td>
                                         <td>{{ task.created_at }}</td>
+                                        <td>{{ task.modified_at }}</td>
                                         <td>
-                                            <a href="{% ooi_url "ooi_detail" task.data.input_ooi organization.code observed_at=task.created_at|date:'c' %}">{{ task.data.input_ooi }}</a>
+                                            <a href="{% ooi_url "ooi_detail" task.data.input_ooi task.data.organization observed_at=task.created_at|date:'c' %}">{{ task.data.input_ooi }}</a>
                                         </td>
                                         <td>
                                             <button type="button"
@@ -56,7 +66,7 @@ <h1>{% translate "Boefjes" %}</h1>
                                         </td>
                                     </tr>
                                     <tr class="expando-row">
-                                        <td colspan="5">
+                                        <td colspan="7">
                                             {% include "tasks/partials/task_actions.html" %}
 
                                         </td>

rocky/rocky/templates/tasks/normalizers.html

@@ -26,29 +26,40 @@ <h1>{% translate "Normalizers" %}</h1>
                         <table rf-selector="table-scan-history">
                             <thead>
                                 <tr>
+                                    {% if not organization.code %}
+                                        <th scope="col">{% translate "Organization Code" %}</th>
+                                    {% endif %}
                                     <th scope="col">{% translate "Normalizer" %}</th>
                                     <th scope="col">{% translate "Status" %}</th>
                                     <th scope="col">{% translate "Created date" %}</th>
+                                    <th scope="col">{% translate "Modified date" %}</th>
                                     <th scope="col">{% translate "Boefje" %}</th>
                                     <th scope="col">{% translate "Boefje input OOI" %}</th>
                                     <th scope="col" class="visually-hidden">{% translate "Details" %}</th>
                                 </tr>
                             </thead>
                             <tbody>
                                 {% for task in task_list %}
-                                    <tr data-task-id="{{ task.id }}">
+                                    <tr data-task-id="{{ task.id }}"
+                                        {% if not organization %}data-organization-code="{{ task.data.raw_data.boefje_meta.organization }}"{% endif %}>
+                                        {% if not organization %}
+                                            <td>
+                                                <a href="{% url "organization_crisis_room" task.data.raw_data.boefje_meta.organization %}">{{ task.data.raw_data.boefje_meta.organization }}</a>
+                                            </td>
+                                        {% endif %}
                                         <td>
-                                            <a href="{% url "normalizer_detail" organization.code task.data.normalizer.id %}">{{ task.data.normalizer.id }}</a>
+                                            <a href="{% url "normalizer_detail" task.data.raw_data.boefje_meta.organization task.data.normalizer.id %}">{{ task.data.normalizer.id }}</a>
                                         </td>
                                         <td class="nowrap">
                                             <i class="icon {{ task.status.value }}"></i>&nbsp;{{ task.status.value|capfirst }}
                                         </td>
                                         <td>{{ task.created_at }}</td>
+                                        <td>{{ task.modified_at }}</td>
                                         <td>
-                                            <a href="{% url "boefje_detail" organization.code task.data.raw_data.boefje_meta.boefje.id %}">{{ task.data.raw_data.boefje_meta.boefje.id }}</a>
+                                            <a href="{% url "boefje_detail" task.data.raw_data.boefje_meta.organization task.data.raw_data.boefje_meta.boefje.id %}">{{ task.data.raw_data.boefje_meta.boefje.id }}</a>
                                         </td>
                                         <td>
-                                            <a href="{% ooi_url "ooi_detail" task.data.raw_data.boefje_meta.input_ooi organization.code observed_at=task.created_at|date:'c' %}">{{ task.data.raw_data.boefje_meta.input_ooi }}</a>
+                                            <a href="{% ooi_url "ooi_detail" task.data.raw_data.boefje_meta.input_ooi task.data.raw_data.boefje_meta.organization observed_at=task.created_at|date:'c' %}">{{ task.data.raw_data.boefje_meta.input_ooi }}</a>
                                         </td>
                                         <td>
                                             <button type="button"
@@ -61,7 +72,7 @@ <h1>{% translate "Normalizers" %}</h1>
                                         </td>
                                     </tr>
                                     <tr class="expando-row">
-                                        <td colspan="7">
+                                        <td colspan="8">
                                             {% include "tasks/partials/task_actions.html" %}
 
                                         </td>

rocky/rocky/templates/tasks/partials/tab_navigation.html

@@ -3,10 +3,18 @@
 <nav class="tabs" aria-label="{% translate "List of tasks" %}">
     <ul>
         <li {% if view == "boefjes_tasks" %}aria-current="page"{% endif %}>
-            <a href="{% url 'boefjes_task_list' organization.code %}">{% translate "Boefjes" %}</a>
+            {% if organization %}
+                <a href="{% url 'boefjes_task_list' organization.code %}">{% translate "Boefjes" %}</a>
+            {% else %}
+                <a href="{% url 'all_boefjes_task_list' %}">{% translate "Boefjes" %}</a>
+            {% endif %}
         </li>
         <li {% if view == "normalizers_tasks" %}aria-current="page"{% endif %}>
-            <a href="{% url 'normalizers_task_list' organization.code %}">{% translate "Normalizers" %}</a>
+            {% if organization %}
+                <a href="{% url 'normalizers_task_list' organization.code %}">{% translate "Normalizers" %}</a>
+            {% else %}
+                <a href="{% url 'all_normalizers_task_list' %}">{% translate "Normalizers" %}</a>
+            {% endif %}
         </li>
     </ul>
 </nav>

rocky/rocky/templates/tasks/partials/task_actions.html

@@ -13,18 +13,25 @@ <h2>{% translate "Yielded objects" %}</h2>
         {% if task.status.value in "completed,failed" %}
             {% if task.type == "normalizer" %}
                 <a class="button"
-                   href="{% url 'bytes_raw' organization_code=organization.code boefje_meta_id=task.data.raw_data.boefje_meta.id %}"><span class="icon ti-download"></span>{% translate "Download meta and raw data" %}</a>
-                {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %}
+                   href="{% url 'bytes_raw' organization_code=task.data.raw_data.boefje_meta.organization boefje_meta_id=task.data.raw_data.boefje_meta.id %}"><span class="icon ti-download"></span>{% translate "Download meta and raw data" %}</a>
+                {% url 'task_list' organization_code=task.data.raw_data.boefje_meta.organization as taskurl %}
+                {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" url=taskurl value=task.id %}
 
             {% elif task.type == "boefje" %}
                 <a class="button"
-                   href="{% url 'bytes_raw' organization_code=organization.code boefje_meta_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download meta and raw data" %}</a>
-                {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %}
+                   href="{% url 'bytes_raw' organization_code=task.data.organization boefje_meta_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download meta and raw data" %}</a>
+                {% url 'task_list' organization_code=task.data.organization as taskurl %}
+                {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" url=taskurl value=task.id %}
 
             {% endif %}
         {% else %}
-            <a class="button"
-               href="{% url 'download_task_meta' organization_code=organization.code task_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download task data" %}</a>
+            {% if task.type == "normalizer" %}
+                <a class="button"
+                   href="{% url 'download_task_meta' organization_code=task.data.raw_data.boefje_meta.organization task_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download task data" %}</a>
+            {% elif task.type == "boefje" %}
+                <a class="button"
+                   href="{% url 'download_task_meta' organization_code=task.data.organization task_id=task.id %}"><span class="icon ti-download"></span>{% translate "Download task data" %}</a>
+            {% endif %}
         {% endif %}
     </div>
 </section>