Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplfy random file name in test_fsspec.py #104

Merged
merged 1 commit into from
Sep 21, 2024
Merged

Simplfy random file name in test_fsspec.py #104

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 22 additions & 25 deletions tosfs/tests/test_fsspec.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@
# limitations under the License.

import io
import random
import string
from typing import Any

import pytest

from tosfs.utils import random_str


def test_ls(fsspecfs: Any, bucket: str, temporary_workspace: str):
path_to_find = f"{bucket}/{temporary_workspace}"
Expand All @@ -29,7 +29,7 @@ def test_ls(fsspecfs: Any, bucket: str, temporary_workspace: str):

def test_copy(fsspecfs: Any, bucket: str, temporary_workspace: str):
# Create a temporary directory and files
dir_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
dir_name = random_str()
subdir_path = f"{bucket}/{temporary_workspace}/{dir_name}"
fsspecfs.mkdir(subdir_path)
file1_path = f"{subdir_path}/file1.txt"
Expand Down Expand Up @@ -122,7 +122,7 @@ def test_info(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_write_and_read_bytes(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path_to_write = f"{bucket}/{temporary_workspace}/{file_name}.bin"
data_to_write = b"Hello, World!"

Expand All @@ -140,7 +140,7 @@ def test_write_and_read_bytes(fsspecfs: Any, bucket: str, temporary_workspace: s


def test_write_and_read_text(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path_to_write = f"{bucket}/{temporary_workspace}/{file_name}.txt"
data_to_write = "Hello, World!"

Expand All @@ -158,7 +158,7 @@ def test_write_and_read_text(fsspecfs: Any, bucket: str, temporary_workspace: st


def test_with_size(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path = f"{bucket}/{temporary_workspace}/{file_name}.txt"

expected_file_size = 10
Expand All @@ -175,7 +175,7 @@ def test_with_size(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_simple(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path = f"{bucket}/{temporary_workspace}/{file_name}.txt"
data = b"a" * (10 * 1**10)

Expand All @@ -189,7 +189,7 @@ def test_simple(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_write_large(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path = f"{bucket}/{temporary_workspace}/{file_name}.txt"
mb = 2**20
payload_size = int(2.5 * 1 * mb)
Expand All @@ -202,7 +202,7 @@ def test_write_large(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_write_limit(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path = f"{bucket}/{temporary_workspace}/{file_name}.txt"
mb = 2**20
block_size = 1 * mb
Expand All @@ -216,7 +216,7 @@ def test_write_limit(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_readline(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path = f"{bucket}/{temporary_workspace}/{file_name}.txt"

lines_to_write = [b"First line\n", b"Second line\n", b"Third line"]
Expand All @@ -236,7 +236,7 @@ def test_readline(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_readline_empty(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path = f"{bucket}/{temporary_workspace}/{file_name}.txt"
data = b""
with fsspecfs.open(path, "wb") as f:
Expand All @@ -247,7 +247,7 @@ def test_readline_empty(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_readline_blocksize(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path = f"{bucket}/{temporary_workspace}/{file_name}.txt"
data = b"ab\n" + b"a" * (1 * 2**20) + b"\nab"
with fsspecfs.open(path, "wb") as f:
Expand All @@ -267,7 +267,7 @@ def test_readline_blocksize(fsspecfs: Any, bucket: str, temporary_workspace: str


def test_next(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path = f"{bucket}/{temporary_workspace}/{file_name}.csv"

csv_content = b"name,amount,id\nAlice,100,1\nBob,200,2\nCharlie,300,3\n"
Expand All @@ -285,7 +285,7 @@ def test_next(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_iterable(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path = f"{bucket}/{temporary_workspace}/{file_name}"
data = b"abc\n123"
with fsspecfs.open(path, "wb") as f:
Expand All @@ -310,7 +310,7 @@ def test_iterable(fsspecfs: Any, bucket: str, temporary_workspace: str):
def test_write_read_without_protocol(
fsspecfs: Any, bucket: str, temporary_workspace: str
):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path_to_write = f"{bucket}/{temporary_workspace}/{file_name}.bin"
path_without_protocol = fsspecfs._strip_protocol(path_to_write)
data_to_write = b"Hello, World!"
Expand Down Expand Up @@ -557,8 +557,8 @@ def test_du(fsspecfs: Any, bucket: str, temporary_workspace: str):

def test_isdir(fsspecfs: Any, bucket: str, temporary_workspace: str):
# Setup
dir_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
dir_name = random_str()
file_name = random_str()
dir_path = f"{bucket}/{temporary_workspace}/{dir_name}"
file_path = f"{bucket}/{temporary_workspace}/{file_name}.txt"
fsspecfs.mkdir(dir_path)
Expand All @@ -580,8 +580,8 @@ def test_isdir(fsspecfs: Any, bucket: str, temporary_workspace: str):

def test_isfile(fsspecfs: Any, bucket: str, temporary_workspace: str):
# Setup
dir_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
dir_name = random_str()
file_name = random_str()
dir_path = f"{bucket}/{temporary_workspace}/{dir_name}"
file_path = f"{bucket}/{temporary_workspace}/{file_name}.txt"
fsspecfs.mkdir(dir_path)
Expand All @@ -602,10 +602,7 @@ def test_isfile(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_rm(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_names = [
"".join(random.choices(string.ascii_letters + string.digits, k=10))
for _ in range(5)
]
file_names = [random_str() for _ in range(5)]
paths_to_remove = [
f"{bucket}/{temporary_workspace}/{file_name}.txt" for file_name in file_names
]
Expand Down Expand Up @@ -633,7 +630,7 @@ def test_rm(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_cat_file(fsspecfs: Any, bucket: str, temporary_workspace: str):
file_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
file_name = random_str()
path_to_write = f"{bucket}/{temporary_workspace}/{file_name}.bin"
content = b"Hello, World! This is a test file."
with fsspecfs.open(path_to_write, "wb") as f:
Expand All @@ -657,7 +654,7 @@ def test_cat_file(fsspecfs: Any, bucket: str, temporary_workspace: str):


def test_cat(fsspecfs: Any, bucket: str, temporary_workspace: str):
dir_name = "".join(random.choices(string.ascii_letters + string.digits, k=10))
dir_name = random_str()
subdir_path = f"{bucket}/{temporary_workspace}/{dir_name}"
fsspecfs.mkdir(subdir_path)
file1_path = f"{subdir_path}/file1.txt"
Expand Down