Support delimiter for ls_iterate API

tosfs/core.py

@@ -401,6 +401,7 @@ def ls_iterate(
         detail: bool = False,
         versions: bool = False,
         batch_size: int = LS_OPERATION_DEFAULT_MAX_ITEMS,
+        delimiter: str = "/",
         **kwargs: Union[str, bool, float, None],
     ) -> Generator[Union[dict, str], None, None]:
         """List objects under the given path in batches then returns an iterator.
@@ -415,6 +416,8 @@ def ls_iterate(
             Whether to list object versions (default is False).
         batch_size : int, optional
             The number of items to fetch in each batch (default is 1000).
+        delimiter : str, optional
+            The delimiter to use for the list operation (default is '/').
         **kwargs : dict, optional
             Additional arguments.
 
@@ -450,7 +453,7 @@ def _call_list_objects_type2(
                     bucket,
                     prefix,
                     start_after=prefix,
-                    delimiter="/",
+                    delimiter=delimiter,
                     max_keys=batch_size,
                     continuation_token=continuation_token,
                 )

tosfs/tests/test_tosfs.py

@@ -97,6 +97,21 @@ def test_ls_iterate(
         result.append(batch)
     assert len(result) == len([dir_name, another_dir_name])
 
+    # test list recursively
+    expected = [
+        f"{bucket}/{temporary_workspace}/{dir_name}",
+        f"{bucket}/{temporary_workspace}/{dir_name}/{file_name}",
+        f"{bucket}/{temporary_workspace}/{dir_name}/{sub_dir_name}",
+        f"{bucket}/{temporary_workspace}/{dir_name}/{sub_dir_name}/{sub_file_name}",
+        f"{bucket}/{temporary_workspace}/{another_dir_name}",
+    ]
+    result = list(
+        tosfs.ls_iterate(
+            f"{bucket}/{temporary_workspace}", delimiter="", recursive=True
+        )
+    )
+    assert result == expected
+
 
 def test_inner_rm(tosfs: TosFileSystem, bucket: str, temporary_workspace: str) -> None:
     file_name = random_str()