diff --git a/rocky/assets/js/renderNormalizerOutputOOIs.js b/rocky/assets/js/renderNormalizerOutputOOIs.js index 1e7f71d37ea..e7d7fdcbefe 100644 --- a/rocky/assets/js/renderNormalizerOutputOOIs.js +++ b/rocky/assets/js/renderNormalizerOutputOOIs.js @@ -11,11 +11,14 @@ buttons.forEach((button) => { .closest("tr") .getAttribute("data-task-id") .replace(/-/g, ""); + const organization = + organization_code || + button.closest("tr").getAttribute("data-organization-code"); const json_url = "/" + language + "/" + - organization_code + + organization + "/tasks/normalizers/" + encodeURI(task_id); @@ -79,7 +82,7 @@ buttons.forEach((button) => { "/" + language + "/" + - escapeHTMLEntities(encodeURIComponent(organization_code)); + escapeHTMLEntities(encodeURIComponent(organization)); let object_list = ""; // set the observed at time a fews seconds into the future, as the job finish time is not the same as the ooi-creation time. Due to async reasons the object might be a bit slow. data["timestamp"] = Date.parse(data["valid_time"] + "Z"); diff --git a/rocky/katalogus/templates/partials/single_action_form.html b/rocky/katalogus/templates/partials/single_action_form.html index 7aada262413..cdcd087a872 100644 --- a/rocky/katalogus/templates/partials/single_action_form.html +++ b/rocky/katalogus/templates/partials/single_action_form.html @@ -1,4 +1,6 @@ -
+ {% csrf_token %} {% if key %}{% endif %} diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 3bf00674a16..6d1c6ed2ae2 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -4456,10 +4456,6 @@ msgstr "" msgid "Task list: " msgstr "" -#: rocky/scheduler.py -msgid "Task statistics: " -msgstr "" - #: rocky/settings.py msgid "Blue light" msgstr "" @@ -6088,12 +6084,20 @@ msgstr "" msgid "List of tasks for boefjes" msgstr "" +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +msgid "Organization Code" +msgstr "" + #: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html #: rocky/templates/tasks/ooi_detail_task_list.html #: rocky/templates/tasks/plugin_detail_task_list.html msgid "Created date" msgstr "" +#: rocky/templates/tasks/boefjes.html rocky/templates/tasks/normalizers.html +msgid "Modified date" +msgstr "" + #: rocky/templates/tasks/normalizers.html msgid "There are no tasks for normalizers" msgstr "" @@ -6704,6 +6708,15 @@ msgid "" "refresh of the page may be needed to show the results." msgstr "" +#: rocky/views/tasks.py +#, python-brace-format +msgid "Fetching tasks failed: no connection with scheduler: {error}" +msgstr "" + +#: rocky/views/tasks.py +msgid "All Tasks" +msgstr "" + #: rocky/views/upload_csv.py msgid "" "For URL object type, a column 'raw' with URL values is required, starting " diff --git a/rocky/rocky/scheduler.py b/rocky/rocky/scheduler.py index 416aebbc648..f6c11d1cea0 100644 --- a/rocky/rocky/scheduler.py +++ b/rocky/rocky/scheduler.py @@ -1,7 +1,8 @@ from __future__ import annotations +import collections import datetime -import json +import logging import uuid from enum import Enum from functools import cached_property @@ -162,6 +163,7 @@ def __getitem__(self, key) -> list[Task]: else: raise TypeError("Invalid slice argument type.") + logging.info("Getting max %s lazy items at offset %s with filter %s", limit, offset, self.kwargs) res = self.scheduler_client.list_tasks( limit=limit, offset=offset, @@ -214,7 +216,7 @@ class SchedulerHTTPError(SchedulerError): class SchedulerClient: - def __init__(self, base_uri: str, organization_code: str): + def __init__(self, base_uri: str, organization_code: str | None): self._client = httpx.Client(base_url=base_uri) self.organization_code = organization_code @@ -223,8 +225,10 @@ def list_tasks( **kwargs, ) -> PaginatedTasksResponse: try: - kwargs = {k: v for k, v in kwargs.items() if v is not None} # filter Nones from kwargs - res = self._client.get("/tasks", params=kwargs) + filter_key = "filters" + params = {k: v for k, v in kwargs.items() if v is not None if k != filter_key} # filter Nones from kwargs + endpoint = "/tasks" + res = self._client.post(endpoint, params=params, json=kwargs.get(filter_key, None)) return PaginatedTasksResponse.model_validate_json(res.content) except ValidationError: raise SchedulerValidationError(extra_message=_("Task list: ")) @@ -232,22 +236,19 @@ def list_tasks( raise SchedulerConnectError(extra_message=_("Task list: ")) def get_task_details(self, task_id: str) -> Task: - try: - res = self._client.get(f"/tasks/{task_id}") - res.raise_for_status() - task_details = Task.model_validate_json(res.content) + if self.organization_code is None: + raise SchedulerTaskNotFound("No organization defined in client.") + task_details = Task.model_validate_json(self._get(f"/tasks/{task_id}", "content")) - if task_details.type == "normalizer": - organization = task_details.data.raw_data.boefje_meta.organization - else: - organization = task_details.data.organization + if task_details.type == "normalizer": + organization = task_details.data.raw_data.boefje_meta.organization + else: + organization = task_details.data.organization - if organization != self.organization_code: - raise SchedulerTaskNotFound() + if organization != self.organization_code: + raise SchedulerTaskNotFound() - return task_details - except ConnectError: - raise SchedulerConnectError() + return task_details def push_task(self, item: Task) -> None: try: @@ -270,24 +271,45 @@ def push_task(self, item: Task) -> None: raise SchedulerError() def health(self) -> ServiceHealth: - try: - health_endpoint = self._client.get("/health") - health_endpoint.raise_for_status() - return ServiceHealth.model_validate_json(health_endpoint.content) - except HTTPError: - raise SchedulerHTTPError() - except ConnectError: - raise SchedulerConnectError() + return ServiceHealth.model_validate_json(self._get("/health", return_type="content")) + + def _get_task_stats(self, scheduler_id: str) -> dict: + """Return task stats for specific scheduler.""" + return self._get(f"/tasks/stats/{scheduler_id}") # type: ignore def get_task_stats(self, task_type: str) -> dict: + """Return task stats for specific task type.""" + return self._get_task_stats(scheduler_id=f"{task_type}-{self.organization_code}") + + @staticmethod + def _merge_stat_dicts(dicts: list[dict]) -> dict: + """Merge multiple stats dicts.""" + stat_sum: dict[str, collections.Counter] = collections.defaultdict(collections.Counter) + for dct in dicts: + for timeslot, counts in dct.items(): + stat_sum[timeslot].update(counts) + return dict(stat_sum) + + def get_combined_schedulers_stats(self, scheduler_ids: list) -> dict: + """Return merged stats for a set of scheduler ids.""" + return SchedulerClient._merge_stat_dicts( + dicts=[self._get_task_stats(scheduler_id=scheduler_id) for scheduler_id in scheduler_ids] + ) + + def _get(self, path: str, return_type: str = "json") -> dict | bytes: + """Helper to do a get request and raise warning for path.""" try: - res = self._client.get(f"/tasks/stats/{task_type}-{self.organization_code}") + res = self._client.get(path) res.raise_for_status() - except ConnectError: - raise SchedulerConnectError(extra_message=_("Task statistics: ")) - task_stats = json.loads(res.content) - return task_stats + except HTTPError as exc: + raise SchedulerError(path) from exc + except ConnectError as exc: + raise SchedulerConnectError(path) from exc + + if return_type == "content": + return res.content + return res.json() -def scheduler_client(organization_code: str) -> SchedulerClient: +def scheduler_client(organization_code: str | None) -> SchedulerClient: return SchedulerClient(settings.SCHEDULER_API, organization_code) diff --git a/rocky/rocky/settings.py b/rocky/rocky/settings.py index 497bb9ee2c1..6040c32ca5c 100644 --- a/rocky/rocky/settings.py +++ b/rocky/rocky/settings.py @@ -78,7 +78,7 @@ "loggers": { "root": { "handlers": ["console"], - "level": "INFO", + "level": env("LOG_LEVEL", default="INFO").upper(), }, }, } diff --git a/rocky/rocky/templates/header.html b/rocky/rocky/templates/header.html index f96979c6c04..ca685e775f4 100644 --- a/rocky/rocky/templates/header.html +++ b/rocky/rocky/templates/header.html @@ -32,6 +32,11 @@ {% translate "Crisis room" %} +
  • + {% url "all_task_list" as index_url %} + {% translate "Tasks" %} +
    • {% else %}
  • {% url "organization_crisis_room" organization.code as index_url %} diff --git a/rocky/rocky/templates/tasks/boefjes.html b/rocky/rocky/templates/tasks/boefjes.html index b9a1920a992..e786c0acba9 100644 --- a/rocky/rocky/templates/tasks/boefjes.html +++ b/rocky/rocky/templates/tasks/boefjes.html @@ -25,9 +25,13 @@

    {% translate "Boefjes" %}

    + {% if not organization.code %} + + {% endif %} + @@ -35,15 +39,21 @@

    {% translate "Boefjes" %}

    {% for task in task_list %} + {% if not organization.code %} + + {% endif %} + - diff --git a/rocky/rocky/templates/tasks/normalizers.html b/rocky/rocky/templates/tasks/normalizers.html index 9062d970129..096327e328c 100644 --- a/rocky/rocky/templates/tasks/normalizers.html +++ b/rocky/rocky/templates/tasks/normalizers.html @@ -26,9 +26,13 @@

    {% translate "Normalizers" %}

    {% translate "Organization Code" %}{% translate "Boefje" %} {% translate "Status" %} {% translate "Created date" %}{% translate "Modified date" %} {% translate "Input Object" %}
    + {{ task.data.organization }} + - {{ task.data.boefje.name }} + {{ task.data.boefje.name }}  {{ task.status.value|capfirst }} {{ task.created_at }}{{ task.modified_at }} - {{ task.data.input_ooi }} + {{ task.data.input_ooi }}
    + {% include "tasks/partials/task_actions.html" %}
    + {% if not organization.code %} + + {% endif %} + @@ -36,19 +40,26 @@

    {% translate "Normalizers" %}

    {% for task in task_list %} - + + {% if not organization %} + + {% endif %} + - diff --git a/rocky/rocky/templates/tasks/partials/tab_navigation.html b/rocky/rocky/templates/tasks/partials/tab_navigation.html index dec34aab028..16754c651a4 100644 --- a/rocky/rocky/templates/tasks/partials/tab_navigation.html +++ b/rocky/rocky/templates/tasks/partials/tab_navigation.html @@ -3,10 +3,18 @@ diff --git a/rocky/rocky/templates/tasks/partials/task_actions.html b/rocky/rocky/templates/tasks/partials/task_actions.html index 1a894e2b364..017cf65b7fc 100644 --- a/rocky/rocky/templates/tasks/partials/task_actions.html +++ b/rocky/rocky/templates/tasks/partials/task_actions.html @@ -13,18 +13,25 @@

    {% translate "Yielded objects" %}

    {% if task.status.value in "completed,failed" %} {% if task.type == "normalizer" %} {% translate "Download meta and raw data" %} - {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %} + href="{% url 'bytes_raw' organization_code=task.data.raw_data.boefje_meta.organization boefje_meta_id=task.data.raw_data.boefje_meta.id %}">{% translate "Download meta and raw data" %} + {% url 'task_list' organization_code=task.data.raw_data.boefje_meta.organization as taskurl %} + {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" url=taskurl value=task.id %} {% elif task.type == "boefje" %} {% translate "Download meta and raw data" %} - {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" value=task.id %} + href="{% url 'bytes_raw' organization_code=task.data.organization boefje_meta_id=task.id %}">{% translate "Download meta and raw data" %} + {% url 'task_list' organization_code=task.data.organization as taskurl %} + {% include "partials/single_action_form.html" with btn_text=_("Reschedule") btn_class="ghost" btn_icon="icon ti-refresh" action="reschedule_task" key="task_id" url=taskurl value=task.id %} {% endif %} {% else %} - {% translate "Download task data" %} + {% if task.type == "normalizer" %} + {% translate "Download task data" %} + {% elif task.type == "boefje" %} + {% translate "Download task data" %} + {% endif %} {% endif %} diff --git a/rocky/rocky/urls.py b/rocky/rocky/urls.py index bb1157c8409..1397c325969 100644 --- a/rocky/rocky/urls.py +++ b/rocky/rocky/urls.py @@ -40,7 +40,12 @@ from rocky.views.scan_profile import ScanProfileDetailView, ScanProfileResetView from rocky.views.scans import ScanListView from rocky.views.task_detail import BoefjeTaskDetailView, DownloadTaskDetail, NormalizerTaskJSONView -from rocky.views.tasks import BoefjesTaskListView, NormalizersTaskListView +from rocky.views.tasks import ( + AllBoefjesTaskListView, + AllNormalizersTaskListView, + BoefjesTaskListView, + NormalizersTaskListView, +) from rocky.views.upload_csv import UploadCSV from rocky.views.upload_raw import UploadRaw @@ -72,6 +77,9 @@ PrivacyStatementView.as_view(), name="privacy_statement", ), + path("tasks/", AllBoefjesTaskListView.as_view(), name="all_task_list"), + path("tasks/boefjes", AllBoefjesTaskListView.as_view(), name="all_boefjes_task_list"), + path("tasks/normalizers", AllNormalizersTaskListView.as_view(), name="all_normalizers_task_list"), path( "/settings/indemnifications/", IndemnificationAddView.as_view(), diff --git a/rocky/rocky/views/tasks.py b/rocky/rocky/views/tasks.py index 698b1e15d7e..3f6b9acfa3c 100644 --- a/rocky/rocky/views/tasks.py +++ b/rocky/rocky/views/tasks.py @@ -5,9 +5,10 @@ from django.utils.translation import gettext_lazy as _ from django.views.generic.list import ListView from httpx import HTTPError +from tools.forms.scheduler import TaskFilterForm from rocky.paginator import RockyPaginator -from rocky.scheduler import SchedulerError +from rocky.scheduler import LazyTaskList, SchedulerError, scheduler_client from rocky.views.page_actions import PageActionsView from rocky.views.scheduler import SchedulerView @@ -63,3 +64,52 @@ class BoefjesTaskListView(TaskListView): class NormalizersTaskListView(TaskListView): template_name = "tasks/normalizers.html" task_type = "normalizer" + + +class AllTaskListView(SchedulerListView, PageActionsView): + paginator_class = RockyPaginator + paginate_by = 20 + context_object_name = "task_list" + client = scheduler_client(None) + task_filter_form = TaskFilterForm + + def get_queryset(self): + task_type = self.request.GET.get("type", self.task_type) + self.schedulers = [f"{task_type}-{o.code}" for o in self.request.user.organizations] + form_data = self.task_filter_form(self.request.GET).data.dict() + kwargs = {k: v for k, v in form_data.items() if v} + + try: + return LazyTaskList( + self.client, + task_type=task_type, + filters={"filters": [{"column": "scheduler_id", "operator": "in", "value": self.schedulers}]}, + **kwargs, + ) + + except HTTPError as error: + error_message = _(f"Fetching tasks failed: no connection with scheduler: {error}") + messages.add_message(self.request, messages.ERROR, error_message) + return [] + except SchedulerError as error: + messages.add_message(self.request, messages.ERROR, str(error)) + return [] + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context["task_filter_form"] = self.task_filter_form(self.request.GET) + context["stats"] = self.client.get_combined_schedulers_stats(scheduler_ids=self.schedulers) + context["breadcrumbs"] = [ + {"url": reverse("all_task_list", kwargs={}), "text": _("All Tasks")}, + ] + return context + + +class AllBoefjesTaskListView(AllTaskListView): + template_name = "tasks/boefjes.html" + task_type = "boefje" + + +class AllNormalizersTaskListView(AllTaskListView): + template_name = "tasks/normalizers.html" + task_type = "normalizer"
    {% translate "Organization Code" %}{% translate "Normalizer" %} {% translate "Status" %} {% translate "Created date" %}{% translate "Modified date" %} {% translate "Boefje" %} {% translate "Boefje input OOI" %}
    + {{ task.data.raw_data.boefje_meta.organization }} + - {{ task.data.normalizer.id }} + {{ task.data.normalizer.id }}  {{ task.status.value|capfirst }} {{ task.created_at }}{{ task.modified_at }} - {{ task.data.raw_data.boefje_meta.boefje.id }} + {{ task.data.raw_data.boefje_meta.boefje.id }} - {{ task.data.raw_data.boefje_meta.input_ooi }} + {{ task.data.raw_data.boefje_meta.input_ooi }}
    + {% include "tasks/partials/task_actions.html" %}