ci: fix ci failure(#58)

fix: #58
Currently some of the tests relied on the test bucket status. Because the test
files were created manually several months ago. We need to decouple
this to make our tests more robust.

Create test files before the tests session begin.

Author: karajan1001

tests/conftest.py

@@ -4,10 +4,12 @@
 # pylint: disable=missing-function-docstring
 # pylint: disable=redefined-outer-name
 import os
+import pathlib
 import subprocess
 import time
 import uuid
 
+import oss2
 import pytest
 import requests
 
@@ -16,37 +18,46 @@
 PORT = 5555
 AccessKeyId = os.environ.get("OSS_ACCESS_KEY_ID", "")
 AccessKeySecret = os.environ.get("OSS_SECRET_ACCESS_KEY", "")
+LICENSE_PATH = os.path.join(
+    pathlib.Path(__file__).parent.parent.resolve(), "LICENSE"
+)
+NUMBERS = b"1234567890\n"
 
 
 test_id = uuid.uuid4()
 
 
-@pytest.fixture()
+@pytest.fixture(scope="session")
 def emulator_endpoint():
-    return "http://127.0.0.1:%s/" % PORT
+    return f"http://127.0.0.1:{PORT}/"
 
 
-@pytest.fixture()
+@pytest.fixture(scope="session")
 def endpoint():
     return os.environ.get("OSS_ENDPOINT")
 
 
-@pytest.fixture()
+@pytest.fixture(scope="session")
 def test_bucket_name():
     return os.environ.get("OSS_TEST_BUCKET_NAME")
 
 
-@pytest.fixture()
-def test_path(test_bucket_name):
-    return f"/{test_bucket_name}/ossfs_test/{test_id}"
+@pytest.fixture(scope="session")
+def test_directory():
+    return f"ossfs_test/{test_id}"
+
+
+@pytest.fixture(scope="session")
+def test_path(test_bucket_name, test_directory):
+    return f"/{test_bucket_name}/{test_directory}"
 
 
 @pytest.fixture()
 def oss_emulator_server_start(emulator_endpoint):
     """
     Start a local emulator server
     """
-    with subprocess.Popen("ruby bin/emulator -r store -p {}".format(PORT)):
+    with subprocess.Popen(f"ruby bin/emulator -r store -p {PORT}"):
 
         timeout = 5
         while timeout > 0:
@@ -61,7 +72,7 @@ def oss_emulator_server_start(emulator_endpoint):
         yield
 
 
-@pytest.fixture()
+@pytest.fixture(scope="session")
 def init_config(endpoint):
     result = {}
     result["key"] = AccessKeyId
@@ -73,3 +84,37 @@ def init_config(endpoint):
 @pytest.fixture()
 def ossfs(init_config):
     return OSSFileSystem(**init_config)
+
+
+def put_object(endpoint, bucket_name, filename, contents):
+    auth = oss2.Auth(AccessKeyId, AccessKeySecret)
+    bucket = oss2.Bucket(auth, endpoint, bucket_name)
+    bucket.put_object(filename, contents)
+
+
+def put_file(endpoint, bucket_name, key, filename):
+    auth = oss2.Auth(AccessKeyId, AccessKeySecret)
+    bucket = oss2.Bucket(auth, endpoint, bucket_name)
+    bucket.put_object_from_file(key, filename)
+
+
+@pytest.fixture(scope="session")
+def file_in_anonymous(endpoint, test_directory):
+    bucket = "dvc-anonymous"
+    file = f"{test_directory}/file"
+    put_object(endpoint, bucket, file, "foobar")
+    return f"/{bucket}/{file}"
+
+
+@pytest.fixture(scope="session")
+def number_file(test_bucket_name, endpoint, test_directory):
+    filename = f"{test_directory}/number"
+    put_object(endpoint, test_bucket_name, filename, NUMBERS)
+    return f"/{test_bucket_name}/{filename}"
+
+
+@pytest.fixture(scope="session")
+def license_file(test_bucket_name, endpoint, test_directory):
+    filename = f"{test_directory}/LICENSE"
+    put_file(endpoint, test_bucket_name, filename, LICENSE_PATH)
+    return f"/{test_bucket_name}/{filename}"

tests/test_file.py

@@ -8,17 +8,7 @@
 import os
 
 import pytest
-
-files = {
-    "LICENSE": (
-        b"                                 Apache License\n"
-        b"                           Version 2.0, January 2004\n"
-        b"                        http://www.apache.org/licenses/\n"
-    ),
-    "number": (b"1234567890\n"),
-}
-
-glob_files = {"file.dat": b"", "filexdat": b""}
+from conftest import LICENSE_PATH, NUMBERS
 
 
 def test_simple(ossfs, test_path):
@@ -62,35 +52,36 @@ def test_seek(ossfs, test_path):
             assert f.seek(i) == i
 
 
-def test_read_small(ossfs, test_bucket_name):
-    fn = test_bucket_name + "/number"
-    with ossfs.open(fn, "rb", block_size=3) as f:
+def test_read_small(ossfs, number_file):
+    with ossfs.open(number_file, "rb", block_size=3) as f:
         out = []
         while True:
             data = f.read(2)
             if data == b"":
                 break
             out.append(data)
-        assert ossfs.cat(fn) == b"".join(out)
+        assert ossfs.cat(number_file) == b"".join(out)
 
 
-def test_read_ossfs_block(ossfs, test_bucket_name):
-    data = files["LICENSE"]
+def test_read_ossfs_block(ossfs, license_file, number_file):
+    with open(LICENSE_PATH, "rb") as f_r:
+        data = f_r.read()
     lines = io.BytesIO(data).readlines()
-    path = test_bucket_name + "/LICENSE"
-    assert ossfs.read_block(path, 0, 10, b"\n") == lines[0]
-    assert ossfs.read_block(path, 40, 10, b"\n") == lines[1]
-    assert ossfs.read_block(path, 0, 80, b"\n") == lines[0] + lines[1]
-    assert ossfs.read_block(path, 0, 120, b"\n") == data
+    assert ossfs.read_block(license_file, 0, 10, b"\n") == lines[0]
+    assert ossfs.read_block(license_file, 40, 10, b"\n") == lines[1]
+    assert ossfs.read_block(license_file, 0, 80, b"\n") == lines[0] + lines[1]
+    assert ossfs.read_block(license_file, 0, 120, b"\n") == b"".join(
+        [lines[0], lines[1], lines[2]]
+    )
 
-    data = files["number"]
-    lines = io.BytesIO(data).readlines()
-    path = test_bucket_name + "/number"
-    assert len(ossfs.read_block(path, 0, 5)) == 5
-    assert len(ossfs.read_block(path, 4, 150)) == len(data) - 4
-    assert ossfs.read_block(path, 20, 25) == b""
+    lines = io.BytesIO(NUMBERS).readlines()
+    assert len(ossfs.read_block(number_file, 0, 5)) == 5
+    assert len(ossfs.read_block(number_file, 4, 150)) == len(NUMBERS) - 4
+    assert ossfs.read_block(number_file, 20, 25) == b""
 
-    assert ossfs.read_block(path, 5, None) == ossfs.read_block(path, 5, 25)
+    assert ossfs.read_block(number_file, 5, None) == ossfs.read_block(
+        number_file, 5, 25
+    )
 
 
 @pytest.mark.parametrize("size", [2 ** 10, 2 ** 20, 10 * 2 ** 20])
@@ -134,15 +125,18 @@ def test_write_blocks(ossfs, test_path):
     assert ossfs.info(file)["Size"] == 15 * 2 ** 20
 
 
-def test_readline(ossfs, test_bucket_name):
-    all_items = files.items()
-    for k, data in all_items:
-        with ossfs.open("/".join([test_bucket_name, k]), "rb") as f:
-            result = f.readline()
-            expected = data.split(b"\n")[0] + (
-                b"\n" if data.count(b"\n") else b""
-            )
-            assert result == expected
+def test_readline(ossfs, number_file, license_file):
+    with ossfs.open("/".join([number_file]), "rb") as f_r:
+        result = f_r.readline()
+        expected = NUMBERS
+        assert result == expected
+
+    with ossfs.open("/".join([license_file]), "rb") as f_r, open(
+        LICENSE_PATH, "rb"
+    ) as f_l:
+        result = f_r.readline()
+        expected = f_l.readline()
+        assert result == expected
 
 
 def test_readline_empty(ossfs, test_path):
@@ -174,10 +168,10 @@ def test_readline_blocksize(ossfs, test_path):
         assert result == expected
 
 
-def test_next(ossfs, test_bucket_name):
-    expected = files["LICENSE"].split(b"\n")[0] + b"\n"
-    with ossfs.open(test_bucket_name + "/LICENSE") as f:
-        result = next(f)
+def test_next(ossfs, license_file):
+    with open(LICENSE_PATH, "rb") as f_l, ossfs.open(license_file) as f_r:
+        expected = f_l.readline()
+        result = next(f_r)
         assert result == expected
 
 
@@ -219,7 +213,7 @@ def test_file_status(ossfs, test_path):
 @pytest.mark.parametrize("data_size", [0, 20, 10 * 2 ** 20])
 @pytest.mark.parametrize("append_size", [0, 20, 10 * 2 ** 20])
 def test_append(ossfs, test_path, data_size, append_size):
-    file = test_path + "/test_append/file_{}_{}".format(data_size, append_size)
+    file = test_path + f"/test_append/file_{data_size}_{append_size}"
     data = os.urandom(data_size)
     extra = os.urandom(append_size)
     with ossfs.open(file, "wb") as f:
@@ -230,8 +224,8 @@ def test_append(ossfs, test_path, data_size, append_size):
     assert ossfs.cat(file) == data + extra
 
 
-def test_bigger_than_block_read(ossfs, test_bucket_name):
-    with ossfs.open(test_bucket_name + "/number", "rb", block_size=3) as f:
+def test_bigger_than_block_read(ossfs, number_file):
+    with ossfs.open(number_file, "rb", block_size=3) as f:
         out = []
         while True:
             data = f.read(4)

tests/test_login.py

@@ -73,6 +73,6 @@ def test_env_endpoint(endpoint, test_bucket_name, monkeypatch):
     ossfs.ls(test_bucket_name)
 
 
-def test_anonymous_login():
-    ossfs = OSSFileSystem(endpoint="http://oss-cn-hangzhou.aliyuncs.com")
-    ossfs.cat("/dvc-test-anonymous/LICENSE", connect_timeout=600)
+def test_anonymous_login(file_in_anonymous, endpoint):
+    ossfs = OSSFileSystem(endpoint=endpoint)
+    ossfs.cat(f"{file_in_anonymous}")

tests/test_object.py

@@ -132,14 +132,13 @@ def test_bulk_delete(ossfs, test_path):
     )
 
 
-def test_ossfs_file_access(ossfs, test_bucket_name):
-    fn = test_bucket_name + "/number"
+def test_ossfs_file_access(ossfs, number_file):
     data = b"1234567890\n"
-    assert ossfs.cat(fn) == data
-    assert ossfs.head(fn, 3) == data[:3]
-    assert ossfs.tail(fn, 3) == data[-3:]
-    assert ossfs.tail(fn, 10000) == data
-    assert ossfs.info(fn)["Size"] == len(data)
+    assert ossfs.cat(number_file) == data
+    assert ossfs.head(number_file, 3) == data[:3]
+    assert ossfs.tail(number_file, 3) == data[-3:]
+    assert ossfs.tail(number_file, 10000) == data
+    assert ossfs.info(number_file)["Size"] == len(data)
 
 
 def test_du(ossfs, test_path):
@@ -173,16 +172,16 @@ def test_ossfs_ls(ossfs, test_path):
     assert all(isinstance(item, dict) for item in L)
 
 
-def test_ossfs_big_ls(ossfs, test_bucket_name):
-    path = test_bucket_name + "/test_ossfs_big_ls"
+def test_ossfs_big_ls(ossfs, test_path):
+    path = test_path + "/test_ossfs_big_ls"
     if not ossfs.exists(path):
         for x in range(1200):
-            ossfs.touch(path + "/%i.part" % x)
+            ossfs.touch(path + f"/{x}.part")
     files = ossfs.find(path, connect_timeout=600)
     for x in range(1200):
-        file = path + "/%i.part" % x
+        file = path + f"/{x}.part"
         if file not in files:
-            ossfs.touch(path + "/%i.part" % x)
+            ossfs.touch(path + f"/{x}.part")
 
     assert len(ossfs.find(path, connect_timeout=600)) == 1200
 
@@ -210,15 +209,13 @@ def test_ossfs_glob(ossfs, test_path):
     assert fn2 not in ossfs.glob(path + "/nested/file.*")
 
 
-def test_copy(ossfs, test_bucket_name, test_path):
-    number_file = test_bucket_name + "/number"
+def test_copy(ossfs, number_file, test_path):
     new_file = test_path + "/test_copy/file"
     ossfs.copy(number_file, new_file)
     assert ossfs.cat(number_file) == ossfs.cat(new_file)
 
 
-def test_move(ossfs, test_bucket_name, test_path):
-    number_file = test_bucket_name + "/number"
+def test_move(ossfs, number_file, test_path):
     from_file = test_path + "/test_move/from"
     to_file = test_path + "/test_move/to"
     ossfs.copy(number_file, from_file)
@@ -378,7 +375,7 @@ def test_get_file_info_with_selector(ossfs, test_path):
         elif info["name"].rstrip("/").endswith(dir_a):
             assert info["type"] == "directory"
         else:
-            raise ValueError("unexpected path {}".format(info["name"]))
+            raise ValueError(f"unexpected path {info['name']}")
 
 
 def test_same_name_but_no_exact(ossfs, test_path):
@@ -419,8 +416,8 @@ def test_leading_forward_slash(ossfs, test_path):
     assert ossfs.exists("/" + path + "/some/file")
 
 
-def test_find_with_prefix(ossfs, test_bucket_name):
-    path = test_bucket_name + "/test_find_with_prefix"
+def test_find_with_prefix(ossfs, test_path):
+    path = test_path + "/test_find_with_prefix"
     if not ossfs.exists(path):
         for cursor in range(100):
             ossfs.touch(path + f"/prefixes/test_{cursor}")
@@ -448,10 +445,10 @@ def test_find_with_prefix(ossfs, test_bucket_name):
 READ_BLOCK_SIZE = 2 ** 14  # 16KB blocks
 
 
-def test_get_put_file(ossfs, tmpdir, test_bucket_name):
+def test_get_put_file(ossfs, tmpdir, test_path):
     src_file = str(tmpdir / "source")
     src2_file = str(tmpdir / "source_2")
-    dest_file = test_bucket_name + "/get_put_file/dest"
+    dest_file = test_path + "/get_put_file/dest"
 
     data = b"test" * 2 ** 20