From 403e70b005b454e54f89fb9de04754bf19c989ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hilko=20Jan=C3=9Fen?= Date: Mon, 23 Sep 2024 13:05:29 +0200 Subject: [PATCH] Added pre-commit --- .github/workflows/main.yml | 2 +- .pre-commit-config.yaml | 22 ++++++++ .pre-commit-hooks.yaml | 2 +- CHANGELOG | 2 +- README.md | 2 +- clean_dotenv/__main__.py | 2 +- clean_dotenv/_main.py | 35 ++++++------- clean_dotenv/_parser.py | 63 ++++++++++------------- requirements-dev.txt | 2 +- requirements.txt | 2 +- tests/main_test.py | 101 +++++++++++++++++++------------------ tox.ini | 2 +- 12 files changed, 129 insertions(+), 108 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9152bcb..426b139 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,4 +16,4 @@ jobs: uses: asottile/workflows/.github/workflows/tox.yml@v1.6.0 with: env: '["py39", "py310", "py311", "py312", "py313"]' - os: ubuntu-latest \ No newline at end of file + os: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..5d45785 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: double-quote-string-fixer + - repo: https://github.com/asottile/reorder-python-imports + rev: v3.13.0 + hooks: + - id: reorder-python-imports + args: [--py39-plus] + - repo: https://github.com/asottile/add-trailing-comma + rev: v3.1.0 + hooks: + - id: add-trailing-comma + - repo: https://github.com/asottile/pyupgrade + rev: v3.17.0 + hooks: + - id: pyupgrade + args: [--py39-plus] diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index b407c33..acff639 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -4,4 +4,4 @@ entry: clean-dotenv language: python always_run: true - pass_filenames: false \ No newline at end of file + pass_filenames: false diff --git a/CHANGELOG b/CHANGELOG index 15cb45c..97170a2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,4 +2,4 @@ - Allow values for named keys to be kept (#5) * v0.0.6 -- Preserve comments in the produced .env.example file (#3) \ No newline at end of file +- Preserve comments in the produced .env.example file (#3) diff --git a/README.md b/README.md index 60ff2d0..4e1691d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ S3_BUCKET_NAME="" ``` [^1]: https://www.zdnet.com/article/botnets-have-been-silently-mass-scanning-the-internet-for-unsecured-env-files/ - + ## Installation ```bash diff --git a/clean_dotenv/__main__.py b/clean_dotenv/__main__.py index 47d03a1..fc6a611 100644 --- a/clean_dotenv/__main__.py +++ b/clean_dotenv/__main__.py @@ -2,5 +2,5 @@ from clean_dotenv._main import main -if __name__ == "__main__": +if __name__ == '__main__': raise SystemExit(main()) diff --git a/clean_dotenv/_main.py b/clean_dotenv/_main.py index 6030357..68a1b3f 100644 --- a/clean_dotenv/_main.py +++ b/clean_dotenv/_main.py @@ -1,6 +1,7 @@ -import os import argparse +import os from collections.abc import Iterator + import clean_dotenv._parser as DotEnvParser @@ -10,16 +11,16 @@ def _clean_env(path_to_env: str, values_to_keep: list[str] = []): dotenv_elements = DotEnvParser.parse_stream(open(path_to_env)) # Create new filename for the .env file --> test.env becomes test.env.example - path_to_example_file = path_to_env + ".example" + path_to_example_file = path_to_env + '.example' # Write .example file - with open(path_to_example_file, "w") as example_env_f: + with open(path_to_example_file, 'w') as example_env_f: # We now iterate through the original .env file and write everything except for the value into the new file for i, dotenv_element in enumerate(dotenv_elements): if dotenv_element.multiline_whitespace: - print(dotenv_element.multiline_whitespace, end="", file=example_env_f) + print(dotenv_element.multiline_whitespace, end='', file=example_env_f) if dotenv_element.export: # e.g. export AWS_KEY=... - print(dotenv_element.export, end="", file=example_env_f) + print(dotenv_element.export, end='', file=example_env_f) if dotenv_element.key: print( ( @@ -27,19 +28,19 @@ def _clean_env(path_to_env: str, values_to_keep: list[str] = []): if dotenv_element.key in values_to_keep else f"{dotenv_element.key}={dotenv_element.separator}{dotenv_element.separator}" ), - end="", + end='', file=example_env_f, ) if dotenv_element.comment: - print(dotenv_element.comment, end="", file=example_env_f) + print(dotenv_element.comment, end='', file=example_env_f) if dotenv_element.end_of_line: - print(dotenv_element.end_of_line, end="", file=example_env_f) + print(dotenv_element.end_of_line, end='', file=example_env_f) def _find_dotenv_files(path_to_root: str) -> Iterator[str]: # Finds and yields .env files in the path_to_root for entry in os.scandir(path_to_root): - if entry.name.endswith(".env") and entry.is_file(): + if entry.name.endswith('.env') and entry.is_file(): # Create a cleaned .env.example file for the found .env file yield entry.path @@ -53,19 +54,19 @@ def _main(path_to_root: str, values_to_keep: list[str] = []): def main(): parser = argparse.ArgumentParser( - description="Automatically creates an .env.example which creates the same keys as your .env file, but without the values" + description='Automatically creates an .env.example which creates the same keys as your .env file, but without the values', ) parser.add_argument( - "--root_path", + '--root_path', type=str, - help="Root path in which .env files shall be looked for", + help='Root path in which .env files shall be looked for', default=os.getcwd(), ) parser.add_argument( - "-k", - "--keep", - nargs="*", - help="Variables which shall not be cleaned by clean-dotenv. Separate values by space.", + '-k', + '--keep', + nargs='*', + help='Variables which shall not be cleaned by clean-dotenv. Separate values by space.', default=[], ) @@ -73,5 +74,5 @@ def main(): _main(path_to_root=args.root_path, values_to_keep=args.keep) -if __name__ == "__main__": +if __name__ == '__main__': raise SystemExit(main()) diff --git a/clean_dotenv/_parser.py b/clean_dotenv/_parser.py index 44aa9da..e9d48a1 100644 --- a/clean_dotenv/_parser.py +++ b/clean_dotenv/_parser.py @@ -1,22 +1,16 @@ # This is the original python-dotenv parser with minor changes to work in clean-dotenv # Find the copyright and license below: - # Copyright (c) 2014, Saurabh Kumar (python-dotenv), 2013, Ted Tieken (django-dotenv-rw), 2013, Jacob Kaplan-Moss (django-dotenv) - # Redistribution and use in source and binary forms, with or without modification, # are permitted provided that the following conditions are met: - # - Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. - # - Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. - # - Neither the name of django-dotenv nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. - # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -28,35 +22,34 @@ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - import codecs import re -from typing import ( - IO, - NamedTuple, - Optional, -) -from collections.abc import Iterator, Sequence -from re import Match, Pattern +from collections.abc import Iterator +from collections.abc import Sequence +from re import Match +from re import Pattern +from typing import IO +from typing import NamedTuple +from typing import Optional def make_regex(string: str, extra_flags: int = 0) -> Pattern[str]: return re.compile(string, re.UNICODE | extra_flags) -_newline = make_regex(r"(\r\n|\n|\r)") -_multiline_whitespace = make_regex(r"(\s*)", extra_flags=re.MULTILINE) -_whitespace = make_regex(r"([^\S\r\n]*)") -_export = make_regex(r"(export[^\S\r\n]+)?") +_newline = make_regex(r'(\r\n|\n|\r)') +_multiline_whitespace = make_regex(r'(\s*)', extra_flags=re.MULTILINE) +_whitespace = make_regex(r'([^\S\r\n]*)') +_export = make_regex(r'(export[^\S\r\n]+)?') _single_quoted_key = make_regex(r"'([^']+)'") -_unquoted_key = make_regex(r"([^=\#\s]+)") -_equal_sign = make_regex(r"(=[^\S\r\n]*)") +_unquoted_key = make_regex(r'([^=\#\s]+)') +_equal_sign = make_regex(r'(=[^\S\r\n]*)') _single_quoted_value = make_regex(r"'((?:\\'|[^'])*)'") _double_quoted_value = make_regex(r'"((?:\\"|[^"])*)"') -_unquoted_value = make_regex(r"([^\r\n]*)") -_comment = make_regex(r"([^\S\r\n]*#[^\r\n]*)?") -_end_of_line = make_regex(r"[^\S\r\n]*(\r\n|\n|\r|$)") -_rest_of_line = make_regex(r"[^\r\n]*(?:\r|\n|\r\n)?") +_unquoted_value = make_regex(r'([^\r\n]*)') +_comment = make_regex(r'([^\S\r\n]*#[^\r\n]*)?') +_end_of_line = make_regex(r'[^\S\r\n]*(\r\n|\n|\r|$)') +_rest_of_line = make_regex(r'[^\r\n]*(?:\r|\n|\r\n)?') _double_quote_escapes = make_regex(r"\\[\\'\"abfnrtv]") _single_quote_escapes = make_regex(r"\\[\\']") @@ -84,10 +77,10 @@ def __init__(self, chars: int, line: int) -> None: self.line = line @classmethod - def start(cls) -> "Position": + def start(cls) -> 'Position': return cls(chars=0, line=1) - def set(self, other: "Position") -> None: + def set(self, other: 'Position') -> None: self.chars = other.chars self.line = other.line @@ -124,28 +117,28 @@ def peek(self, count: int) -> str: def read(self, count: int) -> str: result = self.string[self.position.chars : self.position.chars + count] if len(result) < count: - raise Error("read: End of string") + raise Error('read: End of string') self.position.advance(result) return result def read_regex(self, regex: Pattern[str]) -> Sequence[str]: match = regex.match(self.string, self.position.chars) if match is None: - raise Error("read_regex: Pattern not found") + raise Error('read_regex: Pattern not found') self.position.advance(self.string[match.start() : match.end()]) return match.groups() def decode_escapes(regex: Pattern[str], string: str) -> str: def decode_match(match: Match[str]) -> str: - return codecs.decode(match.group(0), "unicode-escape") # type: ignore + return codecs.decode(match.group(0), 'unicode-escape') # type: ignore return regex.sub(decode_match, string) def parse_key(reader: Reader) -> Optional[str]: char = reader.peek(1) - if char == "#": + if char == '#': return None elif char == "'": (key,) = reader.read_regex(_single_quoted_key) @@ -156,7 +149,7 @@ def parse_key(reader: Reader) -> Optional[str]: def parse_unquoted_value(reader: Reader) -> str: (part,) = reader.read_regex(_unquoted_value) - return re.sub(r"\s+#.*", "", part).rstrip() + return re.sub(r'\s+#.*', '', part).rstrip() def parse_value(reader: Reader) -> tuple[str, str]: @@ -167,10 +160,10 @@ def parse_value(reader: Reader) -> tuple[str, str]: elif char == '"': (value,) = reader.read_regex(_double_quoted_value) return decode_escapes(_double_quote_escapes, value), '"' - elif char in ("", "\n", "\r"): - return "", "" + elif char in ('', '\n', '\r'): + return '', '' else: - return parse_unquoted_value(reader), "" + return parse_unquoted_value(reader), '' def parse_binding(reader: Reader) -> Binding: @@ -192,7 +185,7 @@ def parse_binding(reader: Reader) -> Binding: (export,) = reader.read_regex(_export) key = parse_key(reader) reader.read_regex(_whitespace) - if reader.peek(1) == "=": + if reader.peek(1) == '=': reader.read_regex(_equal_sign) value, separator = parse_value(reader) else: diff --git a/requirements-dev.txt b/requirements-dev.txt index 7150cde..883a414 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -r requirements.txt pytest coverage -covdefaults \ No newline at end of file +covdefaults diff --git a/requirements.txt b/requirements.txt index 3e338bf..566cccb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -python-dotenv \ No newline at end of file +python-dotenv diff --git a/tests/main_test.py b/tests/main_test.py index b132ac4..5dd8621 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -1,9 +1,14 @@ +import shutil +import tempfile from os import DirEntry -from unittest.mock import MagicMock, call, mock_open, patch +from unittest.mock import call +from unittest.mock import MagicMock +from unittest.mock import mock_open +from unittest.mock import patch + import pytest + from clean_dotenv import _main as clean_dotenv -import tempfile -import shutil class DirEntry: @@ -20,18 +25,18 @@ def is_file(self): @pytest.mark.parametrize( - ("s", "expected"), + ('s', 'expected'), ( - pytest.param("#test", "#test", id="comment-only"), + pytest.param('#test', '#test', id='comment-only'), pytest.param( "export AWS_PROFILE='test' #exporttest", "export AWS_PROFILE='' #exporttest", - id="export", + id='export', ), pytest.param( 'AWS_PROFILE="test" #double', 'AWS_PROFILE="" #double', - id="double quotes", + id='double quotes', ), pytest.param( """ @@ -42,7 +47,7 @@ def is_file(self): AWS_PROFILE="" AWS_KEY="" """, - id="multiple", + id='multiple', ), pytest.param( """ @@ -54,7 +59,7 @@ def is_file(self): AWS_PROFILE="" AWS_KEY="" """, - id="multiline", + id='multiline', ), pytest.param( """ @@ -65,7 +70,7 @@ def is_file(self): A="" B='' """, - id="mixed separator", + id='mixed separator', ), pytest.param( """# Copy and paste the credentials here. @@ -74,8 +79,8 @@ def is_file(self): AWS_PROFILE="default" - # Alternatively, if you wish to use keys directly, uncomment the - # following lines and provide the values. Comment out or remove the + # Alternatively, if you wish to use keys directly, uncomment the + # following lines and provide the values. Comment out or remove the # AWS_PROFILE line above. # AWS_ACCESS_KEY_ID="" @@ -88,15 +93,15 @@ def is_file(self): AWS_PROFILE="" - # Alternatively, if you wish to use keys directly, uncomment the - # following lines and provide the values. Comment out or remove the + # Alternatively, if you wish to use keys directly, uncomment the + # following lines and provide the values. Comment out or remove the # AWS_PROFILE line above. # AWS_ACCESS_KEY_ID="" # AWS_SECRET_ACCESS_KEY="" # AWS_SESSION_TOKEN="" """, - id="GitHub Issue #3 Example", + id='GitHub Issue #3 Example', ), ), ) @@ -104,24 +109,24 @@ def test_clean_function(s, expected): # First we create a temp directory in which we store the .env file tmpdir = tempfile.mkdtemp() # We write the content into a .env - with open(f"{tmpdir}/.env", "w") as f: - print(s, end="", file=f) + with open(f"{tmpdir}/.env", 'w') as f: + print(s, end='', file=f) clean_dotenv._clean_env(f"{tmpdir}/.env") # We now get the cleaned file - with open(f"{tmpdir}/.env.example", "r") as f: + with open(f"{tmpdir}/.env.example") as f: output = f.read() shutil.rmtree(tmpdir) assert output == expected @pytest.mark.parametrize( - ("s", "expected", "keep"), + ('s', 'expected', 'keep'), ( pytest.param( "export AWS_PROFILE='test' #exporttest", "export AWS_PROFILE='test' #exporttest", - ["AWS_PROFILE"], - id="single-keep", + ['AWS_PROFILE'], + id='single-keep', ), pytest.param( """ @@ -132,8 +137,8 @@ def test_clean_function(s, expected): AWS_PROFILE="123" AWS_KEY="123" """, - ["AWS_PROFILE", "AWS_KEY"], - id="multi-keep", + ['AWS_PROFILE', 'AWS_KEY'], + id='multi-keep', ), pytest.param( """ @@ -146,8 +151,8 @@ def test_clean_function(s, expected): 34" AWS_KEY="123" """, - ["AWS_PROFILE", "AWS_KEY"], - id="multi-keep", + ['AWS_PROFILE', 'AWS_KEY'], + id='multi-keep', ), pytest.param( """ @@ -158,8 +163,8 @@ def test_clean_function(s, expected): AWS_PROFILE="" AWS_KEY="123" """, - ["aws_profile", "AWS_KEY"], - id="case-sensitive", + ['aws_profile', 'AWS_KEY'], + id='case-sensitive', ), pytest.param( """ @@ -170,8 +175,8 @@ def test_clean_function(s, expected): password= url=www.google.com """, - ["url"], - id="GitHub Issue #5 Example", + ['url'], + id='GitHub Issue #5 Example', ), ), ) @@ -179,55 +184,55 @@ def test_clean_function_with_values_to_keep(s, expected, keep): # First we create a temp directory in which we store the .env file tmpdir = tempfile.mkdtemp() # We write the content into a .env - with open(f"{tmpdir}/.env", "w") as f: - print(s, end="", file=f) + with open(f"{tmpdir}/.env", 'w') as f: + print(s, end='', file=f) clean_dotenv._clean_env(f"{tmpdir}/.env", values_to_keep=keep) # We now get the cleaned file - with open(f"{tmpdir}/.env.example", "r") as f: + with open(f"{tmpdir}/.env.example") as f: output = f.read() shutil.rmtree(tmpdir) assert output == expected -@patch("os.scandir") +@patch('os.scandir') def test_find_dotenv_files(mock_scandir): # Mock os.scandir() for files mock_scandir.return_value = [ DirEntry(filename) - for filename in ["test.py", "abba", "env", "test.env", ".env"] + for filename in ['test.py', 'abba', 'env', 'test.env', '.env'] ] - assert list(clean_dotenv._find_dotenv_files(None)) == ["test.env", ".env"] + assert list(clean_dotenv._find_dotenv_files(None)) == ['test.env', '.env'] # Mock os.scandir() for directories mock_scandir.return_value = [ DirEntry(filename, is_file=False) - for filename in ["test.py", "abba", "env", "test.env", ".env", "env"] + for filename in ['test.py', 'abba', 'env', 'test.env', '.env', 'env'] ] assert list(clean_dotenv._find_dotenv_files(None)) == [] def test_find_dotenv_files_function(): - with patch("os.scandir") as mock_scandir: - mock_scandir.return_value = [DirEntry("test.env")] + with patch('os.scandir') as mock_scandir: + mock_scandir.return_value = [DirEntry('test.env')] - result = list(clean_dotenv._find_dotenv_files("path_to_root")) + result = list(clean_dotenv._find_dotenv_files('path_to_root')) - assert result == ["test.env"] + assert result == ['test.env'] -@patch("argparse.ArgumentParser.parse_args") -@patch("clean_dotenv._main._main") +@patch('argparse.ArgumentParser.parse_args') +@patch('clean_dotenv._main._main') def test_main(mock_main, mock_parse_args): - mock_parse_args.return_value = MagicMock(root_path="test_rpath", keep=[]) + mock_parse_args.return_value = MagicMock(root_path='test_rpath', keep=[]) clean_dotenv.main() - mock_main.assert_called_once_with(path_to_root="test_rpath", values_to_keep=[]) + mock_main.assert_called_once_with(path_to_root='test_rpath', values_to_keep=[]) def test__main(): # Mock _find_dotenv_files - mm_find_dotenv = MagicMock(return_value=[".env", "test.env"]) + mm_find_dotenv = MagicMock(return_value=['.env', 'test.env']) clean_dotenv._find_dotenv_files = mm_find_dotenv # Mock _clean_env @@ -235,11 +240,11 @@ def test__main(): clean_dotenv._clean_env = mm_clean_env # Call main method - clean_dotenv._main(path_to_root="test_directory", values_to_keep=[]) + clean_dotenv._main(path_to_root='test_directory', values_to_keep=[]) # Detection should be called once - mm_find_dotenv.assert_called_once_with("test_directory") + mm_find_dotenv.assert_called_once_with('test_directory') # The creation of new .env file should be called twice, last with "test.env" assert mm_clean_env.call_count == 2 - mm_clean_env.assert_called_with(path_to_env="test.env", values_to_keep=[]) + mm_clean_env.assert_called_with(path_to_env='test.env', values_to_keep=[]) diff --git a/tox.ini b/tox.ini index 47c1005..80c6170 100644 --- a/tox.ini +++ b/tox.ini @@ -6,4 +6,4 @@ deps = -rrequirements-dev.txt commands = coverage erase coverage run -m pytest {posargs:tests} - coverage report \ No newline at end of file + coverage report