Skip to content

Commit

Permalink
chore: Create the directory automatically if it doesn't exist #70 (#73)
Browse files Browse the repository at this point in the history
Create PR for #70

This PR is add decorator and test code for create new folder if it dose
not exist.
```python
def create_folder_if_not_exists(func: Any) -> Callable[..., int]:
    """Decorator to create folder if it does not exist."""

    def wrapper(*args: Any, **kwargs: Any) -> int:
        try:
            filepath = kwargs["filepath"]
        except KeyError:
            filepath = args[0]
        folder = os.path.dirname(filepath)
        if not os.path.exists(folder) and folder != "":
            os.makedirs(folder)
        return func(*args, **kwargs)

    return wrapper
```
The reason why the try except statement is used is because other methods
that call the write_csv function sometimes pass the filepath into args
and sometimes kwarg.

usage
```python
@create_folder_if_not_exists
def write_csv(filepath: Path, records: List[dict], schema: dict, **kwargs: Any) -> int:
    """Write a CSV file."""
    if "properties" not in schema:
        raise ValueError("Stream's schema has no properties defined.")

    keys: List[str] = list(schema["properties"].keys())
    with open(filepath, "w", encoding="utf-8", newline="") as fp:
        writer = csv.DictWriter(fp, fieldnames=keys, dialect="excel", **kwargs)
        writer.writeheader()
        for record_count, record in enumerate(records, start=1):
            writer.writerow(record)

    return record_count
```
If there is a better way please let me know

---------

Co-authored-by: Edgar Ramírez Mondragón <[email protected]>
  • Loading branch information
HyoungSooo and edgarrmondragon authored Feb 19, 2024
1 parent 50637da commit 6cae335
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 1 deletion.
20 changes: 19 additions & 1 deletion target_csv/serialization.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,26 @@
import csv # noqa: D100
from pathlib import Path
from typing import Any, List
from typing import Any, List, Callable
import os


def create_folder_if_not_exists(func: Any) -> Callable[..., int]:
"""Decorator to create folder if it does not exist."""

def wrapper(*args: Any, **kwargs: Any) -> int:
try:
filepath = kwargs["filepath"]
except KeyError:
filepath = args[0]
folder = os.path.dirname(filepath)
if not os.path.exists(folder) and folder != "":
os.makedirs(folder)
return func(*args, **kwargs)

return wrapper


@create_folder_if_not_exists
def write_csv(filepath: Path, records: List[dict], schema: dict, **kwargs: Any) -> int:
"""Write a CSV file."""
if "properties" not in schema:
Expand Down
19 changes: 19 additions & 0 deletions tests/test_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,30 @@ def output_filepath(output_dir) -> Path:
return result


@pytest.fixture
def test_file_paths(output_dir) -> List[Path]:
paths = []
for dir in range(4):
path = Path(output_dir / f"test-dir-{dir}/csv-test-output-{dir}.csv")
if path.exists():
path.unlink()

paths.append(path)

return paths


def test_csv_write(output_filepath) -> None:
for schema, records in SAMPLE_DATASETS:
write_csv(filepath=output_filepath, records=records, schema=schema)


def test_csv_write_if_not_exists(test_file_paths) -> None:
for path in test_file_paths:
for schema, records in SAMPLE_DATASETS:
write_csv(filepath=path, records=records, schema=schema)


def test_csv_roundtrip(output_filepath) -> None:
for schema, records in SAMPLE_DATASETS:
write_csv(filepath=output_filepath, records=records, schema=schema)
Expand Down

0 comments on commit 6cae335

Please sign in to comment.