Functionality test

tests/fs/test_docker_fsspec_cat.py

@@ -11,6 +11,9 @@
 
 LOGGER = logging.getLogger(__name__)
 
+FILE_PATH = "/opt/alluxio/ufs/test.csv"
+ALLUXIO_PREFIX = "alluxio::"
+
 
 def validate_read_range(
     alluxio_file_system: AlluxioFileSystem,
@@ -45,8 +48,6 @@ def validate_read_range(
 def alluxio_fsspec_cat_file(alluxio_file_system, alluxio_path, local_path):
     file_size = os.path.getsize(local_path)
 
-    alluxio_file_system.ls(alluxio_path)
-
     # Validate normal case
     max_length = 13 * 1024
     for _ in range(NUM_TESTS):
@@ -86,8 +87,11 @@ def test_alluxio_fsspec_cat_file(alluxio_file_system: AlluxioFileSystem):
         alluxio_file_system, ALLUXIO_FILE_PATH, LOCAL_FILE_PATH
     )
     alluxio_fsspec_cat_file(
-        alluxio_file_system, "alluxio::" + ALLUXIO_FILE_PATH, LOCAL_FILE_PATH
+        alluxio_file_system,
+        ALLUXIO_PREFIX + ALLUXIO_FILE_PATH,
+        LOCAL_FILE_PATH,
     )
+    alluxio_fsspec_cat_file(alluxio_file_system, FILE_PATH, LOCAL_FILE_PATH)
 
 
 def test_etcd_alluxio_fsspec_cat_file(

tests/fs/test_docker_fsspec_dir_function.py

@@ -0,0 +1,52 @@
+import logging
+import os
+
+from alluxiofs import AlluxioFileSystem
+from tests.conftest import LOCAL_FILE_PATH
+from tests.fs.test_docker_fsspec_cat import ALLUXIO_PREFIX
+from tests.fs.test_docker_fsspec_file_function import check_file_info
+
+LOGGER = logging.getLogger(__name__)
+
+DIR_PATH = "/opt/alluxio/ufs"
+SUB_DIR_PATH = "/opt/alluxio/ufs/hash_res"
+FILE_PREFIX = "file://"
+
+
+def check_dir_info(dir_info, dir_path):
+    assert dir_info.get("name") == dir_path
+    assert dir_info.get("type") == "directory"
+    assert not dir_info.get("size")
+
+
+def alluxio_fsspec_test_dir(alluxio_file_system, alluxio_dir_path):
+    file_size = os.path.getsize(LOCAL_FILE_PATH)
+
+    file_list = alluxio_file_system.ls(alluxio_dir_path)
+    assert len(file_list) == 2
+    if file_list[0].get("type") == "file":
+        check_file_info(file_list[0], file_size)
+        check_dir_info(file_list[1], SUB_DIR_PATH)
+    elif file_list[0].get("type") == "directory":
+        check_dir_info(file_list[0], SUB_DIR_PATH)
+        check_file_info(file_list[1], file_size)
+    else:
+        raise AssertionError("File type is invalid.")
+    dir_info = alluxio_file_system.info(alluxio_dir_path)
+    check_dir_info(dir_info, DIR_PATH)
+    assert alluxio_file_system.isdir(alluxio_dir_path)
+    assert not alluxio_file_system.isfile(alluxio_dir_path)
+
+
+def test_alluxio_fsspec_dir_function(alluxio_file_system: AlluxioFileSystem):
+    alluxio_fsspec_test_dir(alluxio_file_system, DIR_PATH)
+    alluxio_fsspec_test_dir(alluxio_file_system, FILE_PREFIX + DIR_PATH)
+    alluxio_fsspec_test_dir(
+        alluxio_file_system, ALLUXIO_PREFIX + FILE_PREFIX + DIR_PATH
+    )
+
+
+def test_etcd_alluxio_fsspec_dir_function(
+    etcd_alluxio_file_system: AlluxioFileSystem,
+):
+    test_alluxio_fsspec_dir_function(etcd_alluxio_file_system)

tests/fs/test_docker_fsspec_file_function.py

@@ -0,0 +1,53 @@
+import logging
+import os
+
+from alluxiofs import AlluxioFileSystem
+from tests.conftest import ALLUXIO_FILE_PATH
+from tests.conftest import LOCAL_FILE_PATH
+from tests.fs.test_docker_fsspec_cat import ALLUXIO_PREFIX
+from tests.fs.test_docker_fsspec_cat import FILE_PATH
+
+LOGGER = logging.getLogger(__name__)
+
+
+def check_file_info(file_info, file_size):
+    assert file_info.get("name") == FILE_PATH
+    assert file_info.get("type") == "file"
+    assert file_info.get("size") == file_size
+
+
+def alluxio_fsspec_test_file(alluxio_file_system, alluxio_path, local_path):
+    file_size = os.path.getsize(local_path)
+
+    file_list = alluxio_file_system.ls(alluxio_path)
+    assert len(file_list) == 1
+    check_file_info(file_list[0], file_size)
+    file_info = alluxio_file_system.info(alluxio_path)
+    check_file_info(file_info, file_size)
+    assert not alluxio_file_system.isdir(alluxio_path)
+    assert alluxio_file_system.isfile(alluxio_path)
+
+    with alluxio_file_system.open(alluxio_path) as f:
+        alluxio_file_data = f.read()
+
+    with open(local_path, "rb") as local_file:
+        local_file_data = local_file.read()
+    assert local_file_data == alluxio_file_data
+
+
+def test_alluxio_fsspec_file_function(alluxio_file_system: AlluxioFileSystem):
+    alluxio_fsspec_test_file(
+        alluxio_file_system, ALLUXIO_FILE_PATH, LOCAL_FILE_PATH
+    )
+    alluxio_fsspec_test_file(
+        alluxio_file_system,
+        ALLUXIO_PREFIX + ALLUXIO_FILE_PATH,
+        LOCAL_FILE_PATH,
+    )
+    alluxio_fsspec_test_file(alluxio_file_system, FILE_PATH, LOCAL_FILE_PATH)
+
+
+def test_etcd_alluxio_fsspec_file_function(
+    etcd_alluxio_file_system: AlluxioFileSystem,
+):
+    test_alluxio_fsspec_file_function(etcd_alluxio_file_system)