Merge pull request #243 from aldbr/main_FEAT_pagination

feat(jobs): pagination

diracx-cli/src/diracx/cli/jobs.py

@@ -4,7 +4,8 @@
 __all__ = ("app",)
 
 import json
-from typing import Annotated
+import re
+from typing import Annotated, cast
 
 from rich.console import Console
 from rich.table import Table
@@ -52,29 +53,66 @@ async def search(
     ],
     condition: Annotated[list[SearchSpec], Option(parser=parse_condition)] = [],
     all: bool = False,
+    page: int = 1,
+    per_page: int = 10,
 ):
     async with DiracClient() as api:
-        jobs = await api.jobs.search(
+        jobs, content_range = await api.jobs.search(
             parameters=None if all else parameter,
             search=condition if condition else None,
+            page=page,
+            per_page=per_page,
+            cls=lambda _, jobs, headers: (
+                jobs,
+                ContentRange(headers.get("Content-Range", "jobs")),
+            ),
         )
-    display(jobs, "jobs")
 
-
-def display(data, unit: str):
+    display(jobs, cast(ContentRange, content_range))
+
+
+class ContentRange:
+    unit: str | None = None
+    start: int | None = None
+    end: int | None = None
+    total: int | None = None
+
+    def __init__(self, header: str):
+        if match := re.fullmatch(r"(\w+) (\d+-\d+|\*)/(\d+|\*)", header):
+            self.unit, range, total = match.groups()
+            self.total = int(total)
+            if range != "*":
+                self.start, self.end = map(int, range.split("-"))
+        elif match := re.fullmatch(r"\w+", header):
+            self.unit = match.group()
+
+    @property
+    def caption(self):
+        if self.start is None and self.end is None:
+            range_str = "all"
+        else:
+            range_str = (
+                f"{self.start if self.start is not None else 'unknown'}-"
+                f"{self.end if self.end is not None else 'unknown'} "
+                f"of {self.total or 'unknown'}"
+            )
+        return f"Showing {range_str} {self.unit}"
+
+
+def display(data, content_range: ContentRange):
     output_format = get_diracx_preferences().output_format
     match output_format:
         case OutputFormats.JSON:
             print(json.dumps(data, indent=2))
         case OutputFormats.RICH:
-            display_rich(data, unit)
+            display_rich(data, content_range)
         case _:
             raise NotImplementedError(output_format)
 
 
-def display_rich(data, unit: str) -> None:
+def display_rich(data, content_range: ContentRange) -> None:
     if not data:
-        print(f"No {unit} found")
+        print(f"No {content_range.unit} found")
         return
 
     console = Console()
@@ -83,7 +121,7 @@ def display_rich(data, unit: str) -> None:
         table = Table(
             "Parameter",
             "Value",
-            caption=f"Showing {len(data)} of {len(data)} {unit}",
+            caption=content_range.caption,
             caption_justify="right",
         )
         for job in data:
@@ -93,7 +131,7 @@ def display_rich(data, unit: str) -> None:
     else:
         table = Table(
             *columns,
-            caption=f"Showing {len(data)} of {len(data)} {unit}",
+            caption=content_range.caption,
             caption_justify="right",
         )
         for job in data:

diracx-cli/tests/test_jobs.py

@@ -1,13 +1,122 @@
 from __future__ import annotations
 
 import json
+import os
+import tempfile
+
+import pytest
+from pytest import raises
 
 from diracx import cli
+from diracx.core.models import ScalarSearchSpec
+from diracx.core.preferences import get_diracx_preferences
 
+TEST_JDL = """
+    Arguments = "jobDescription.xml -o LogLevel=INFO";
+    Executable = "dirac-jobexec";
+    JobGroup = jobGroup;
+    JobName = jobName;
+    JobType = User;
+    LogLevel = INFO;
+    OutputSandbox =
+        {
+            Script1_CodeOutput.log,
+            std.err,
+            std.out
+        };
+    Priority = 1;
+    Site = ANY;
+    StdError = std.err;
+    StdOutput = std.out;
+"""
+
+
+@pytest.fixture
+async def jdl_file():
+    with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as temp_file:
+        temp_file.write(TEST_JDL)
+        temp_file.flush()
+        yield temp_file.name
+
+
+async def test_submit(with_cli_login, jdl_file, capfd):
+    """Test submitting a job using a JDL file."""
+
+    with open(jdl_file, "r") as temp_file:
+        await cli.jobs.submit([temp_file])
 
-async def test_search(with_cli_login, capfd):
-    await cli.jobs.search()
     cap = capfd.readouterr()
     assert cap.err == ""
+    assert "Inserted 1 jobs with ids" in cap.out
+
+
+async def test_search(with_cli_login, jdl_file, capfd):
+    """Test searching for jobs."""
+
+    # Submit 20 jobs
+    with open(jdl_file, "r") as temp_file:
+        await cli.jobs.submit([temp_file] * 20)
+
+    cap = capfd.readouterr()
+
     # By default the output should be in JSON format as capfd is not a TTY
-    json.loads(cap.out)
+    await cli.jobs.search()
+    cap = capfd.readouterr()
+    assert cap.err == ""
+    jobs = json.loads(cap.out)
+
+    # There should be 10 jobs by default
+    assert len(jobs) == 10
+    assert "JobID" in jobs[0]
+    assert "JobGroup" in jobs[0]
+
+    # Change per-page to a very large number to get all the jobs at once: the caption should change
+    await cli.jobs.search(per_page=9999)
+    cap = capfd.readouterr()
+    assert cap.err == ""
+    jobs = json.loads(cap.out)
+
+    # There should be 20 jobs at least now
+    assert len(jobs) >= 20
+    assert "JobID" in cap.out
+    assert "JobGroup" in cap.out
+
+    # Search for a job that doesn't exist
+    condition = ScalarSearchSpec(parameter="Status", operator="eq", value="nonexistent")
+    await cli.jobs.search(condition=[condition])
+    cap = capfd.readouterr()
+    assert cap.err == ""
+    assert "[]" == cap.out.strip()
+
+    # Switch to RICH output
+    get_diracx_preferences.cache_clear()
+    os.environ["DIRACX_OUTPUT_FORMAT"] = "RICH"
+
+    await cli.jobs.search()
+    cap = capfd.readouterr()
+    assert cap.err == ""
+
+    with raises(json.JSONDecodeError):
+        json.loads(cap.out)
+
+    assert "JobID" in cap.out
+    assert "JobGroup" in cap.out
+    assert "Showing 0-9 of " in cap.out
+
+    # Change per-page to a very large number to get all the jobs at once: the caption should change
+    await cli.jobs.search(per_page=9999)
+    cap = capfd.readouterr()
+    assert cap.err == ""
+
+    with raises(json.JSONDecodeError):
+        json.loads(cap.out)
+
+    assert "JobID" in cap.out
+    assert "JobGroup" in cap.out
+    assert "Showing all jobs" in cap.out
+
+    # Search for a job that doesn't exist
+    await cli.jobs.search(condition=[condition])
+    cap = capfd.readouterr()
+    assert cap.err == ""
+    assert "No jobs found" in cap.out

diracx-client/src/diracx/client/__init__.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/_client.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/_configuration.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/_vendor.py

@@ -1,5 +1,5 @@
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/aio/__init__.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/aio/_client.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/aio/_configuration.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/aio/_vendor.py

@@ -1,5 +1,5 @@
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/aio/operations/__init__.py

@@ -1,6 +1,6 @@
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 

diracx-client/src/diracx/client/aio/operations/_operations.py

@@ -1,7 +1,7 @@
 # pylint: disable=too-many-lines,too-many-statements
 # coding=utf-8
 # --------------------------------------------------------------------------
-# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].9)
+# Code generated by Microsoft (R) AutoRest Code Generator (autorest: 3.10.2, generator: @autorest/[email protected].17)
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 from io import IOBase
@@ -2072,7 +2072,7 @@ async def search(
         self,
         body: Optional[_models.JobSearchParams] = None,
         *,
-        page: int = 0,
+        page: int = 1,
         per_page: int = 100,
         content_type: str = "application/json",
         **kwargs: Any,
@@ -2085,7 +2085,7 @@ async def search(
 
         :param body: Default value is None.
         :type body: ~client.models.JobSearchParams
-        :keyword page: Default value is 0.
+        :keyword page: Default value is 1.
         :paramtype page: int
         :keyword per_page: Default value is 100.
         :paramtype per_page: int
@@ -2102,7 +2102,7 @@ async def search(
         self,
         body: Optional[IO[bytes]] = None,
         *,
-        page: int = 0,
+        page: int = 1,
         per_page: int = 100,
         content_type: str = "application/json",
         **kwargs: Any,
@@ -2115,7 +2115,7 @@ async def search(
 
         :param body: Default value is None.
         :type body: IO[bytes]
-        :keyword page: Default value is 0.
+        :keyword page: Default value is 1.
         :paramtype page: int
         :keyword per_page: Default value is 100.
         :paramtype per_page: int
@@ -2132,7 +2132,7 @@ async def search(
         self,
         body: Optional[Union[_models.JobSearchParams, IO[bytes]]] = None,
         *,
-        page: int = 0,
+        page: int = 1,
         per_page: int = 100,
         **kwargs: Any,
     ) -> List[JSON]:
@@ -2144,7 +2144,7 @@ async def search(
 
         :param body: Is either a JobSearchParams type or a IO[bytes] type. Default value is None.
         :type body: ~client.models.JobSearchParams or IO[bytes]
-        :keyword page: Default value is 0.
+        :keyword page: Default value is 1.
         :paramtype page: int
         :keyword per_page: Default value is 100.
         :paramtype per_page: int
@@ -2199,18 +2199,27 @@ async def search(
 
         response = pipeline_response.http_response
 
-        if response.status_code not in [200]:
+        if response.status_code not in [200, 206]:
             if _stream:
                 await response.read()  # Load the body in memory and close the socket
             map_error(
                 status_code=response.status_code, response=response, error_map=error_map
             )
             raise HttpResponseError(response=response)
 
-        deserialized = self._deserialize("[object]", pipeline_response)
+        response_headers = {}
+        if response.status_code == 200:
+            deserialized = self._deserialize("[object]", pipeline_response)
+
+        if response.status_code == 206:
+            response_headers["Content-Range"] = self._deserialize(
+                "str", response.headers.get("Content-Range")
+            )
+
+            deserialized = self._deserialize("[object]", pipeline_response)
 
         if cls:
-            return cls(pipeline_response, deserialized, {})  # type: ignore
+            return cls(pipeline_response, deserialized, response_headers)  # type: ignore
 
         return deserialized  # type: ignore