From 9801241d3333cd0cbb189c9ff7320d8bb844f07c Mon Sep 17 00:00:00 2001 From: Matthew Watkins Date: Mon, 13 May 2024 14:20:15 +0100 Subject: [PATCH] Chore: Fix multiple linting errors Signed-off-by: Matthew Watkins --- .pre-commit-config.yaml | 5 +++-- README.md | 40 +++++++++++++++++++++--------------- osc_ingest_trino/__init__.py | 13 ++++++++++-- pyproject.toml | 2 +- 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e4bb04..6214cf1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,7 +60,7 @@ repos: # Lint: Markdown - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.39.0 + rev: v0.40.0 hooks: - id: markdownlint args: ["--fix"] @@ -96,6 +96,7 @@ repos: rev: 6.3.0 hooks: - id: pydocstyle + additional_dependencies: ["tomli"] - repo: https://github.com/PyCQA/isort rev: 5.13.2 @@ -117,7 +118,7 @@ repos: - --ignore=E2, W5, F401, E401 - --select=E, W, F, N - --max-line-length=120 - additional_dependencies: + additional_dependencies: - pep8-naming - repo: https://github.com/pre-commit/mirrors-mypy diff --git a/README.md b/README.md index e3e1d10..354927c 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ # osc-ingest-tools + python tools to assist with standardized data ingestion workflows -### Install from PyPi +## Install from PyPi -``` +```console pip install osc-ingest-tools ``` @@ -42,10 +43,10 @@ pip install osc-ingest-tools RangeIndex: 3 entries, 0 to 2 Data columns (total 2 columns): - # Column Non-Null Count Dtype ---- ------ -------------- ----- + # Column Non-Null Count Dtype +--- ------ -------------- ----- 0 first_name 3 non-null string - 1 age_in_years 3 non-null Int64 + 1 age_in_years 3 non-null Int64 dtypes: Int64(1), string(1) memory usage: 179.0 bytes @@ -55,10 +56,11 @@ memory usage: 179.0 bytes first_name varchar, age_in_years bigint ->>> +>>> ``` #### Adding custom type mappings to `create_table_schema_pairs` + ```python >>> df = pd.DataFrame(data, columns = ['First Name', 'Age In Years']) @@ -68,10 +70,10 @@ memory usage: 179.0 bytes RangeIndex: 3 entries, 0 to 2 Data columns (total 2 columns): - # Column Non-Null Count Dtype ---- ------ -------------- ----- + # Column Non-Null Count Dtype +--- ------ -------------- ----- 0 first_name 3 non-null object - 1 age_in_years 3 non-null int64 + 1 age_in_years 3 non-null int64 dtypes: int64(1), object(1) memory usage: 176.0+ bytes @@ -87,7 +89,7 @@ memory usage: 176.0+ bytes ### Development Patches may be contributed via pull requests to -https://github.com/os-climate/osc-ingest-tools. +. All changes must pass the automated test suite, along with various static checks. @@ -97,13 +99,15 @@ checks. Enabling automatic formatting via [pre-commit](https://pre-commit.com/) is recommended: -``` + +```console pip install black isort pre-commit pre-commit install ``` To ensure compliance with static check tools, developers may wish to run; -``` + +```console pip install black isort # auto-sort imports isort . @@ -111,8 +115,9 @@ isort . black . ``` -Code can then be tested using tox. -``` +Code can then be tested using tox: + +```console # run static checks and tests tox # run only tests @@ -124,14 +129,17 @@ tox -e cov ``` ### Releasing + To release a new version of this library, authorized developers should; + - Prepare a signed release commit updating `version` in setup.py - Tag the commit using [Semantic Versioning](https://semver.org/spec/v2.0.0.html) -prepended with "v" + prepended with "v" - Push the tag E.g., -``` + +```console git commit -sm "Release v0.3.4" git tag v0.3.4 git push --follow-tags diff --git a/osc_ingest_trino/__init__.py b/osc_ingest_trino/__init__.py index c76ac45..b4e11d4 100644 --- a/osc_ingest_trino/__init__.py +++ b/osc_ingest_trino/__init__.py @@ -1,8 +1,17 @@ from .boto3_utils import attach_s3_bucket, upload_directory_to_s3 from .dotenv_utils import load_credentials_dotenv -from .sqlcols import enforce_partition_column_order, enforce_sql_column_names, sql_compliant_name +from .sqlcols import ( + enforce_partition_column_order, + enforce_sql_column_names, + sql_compliant_name, +) from .sqltypes import create_table_schema_pairs, pandas_type_to_sql -from .trino_utils import TrinoBatchInsert, _do_sql, attach_trino_engine, fast_pandas_ingest_via_hive +from .trino_utils import ( + TrinoBatchInsert, + _do_sql, + attach_trino_engine, + fast_pandas_ingest_via_hive, +) __all__ = [ "sql_compliant_name", diff --git a/pyproject.toml b/pyproject.toml index 8cd3fa2..d1ac480 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ test = ["pdm[pytest]", "pytest-cov"] tox = ["tox", "tox-pdm>=0.5"] docs = ["sphinx>=7.2.6", "sphinx-copybutton>=0.5.2"] dev = ["tox>=4.11.3", "tox-pdm>=0.7.0"] -lint = ["pre-commit"] +lint = ["pre-commit", "tomli"] [tool.pytest.ini_options] testpaths = "tests/"