From 09b3df74aeedfc9f3985825347f672a8b2fb4ead Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Thu, 4 Jan 2024 09:50:15 -0800 Subject: [PATCH 1/4] Add code style and error check. --- .codespellrc | 4 + .flake8 | 26 ++ .github/workflows/test.yml | 40 ++ .gitignore | 60 +++ .isort.cfg | 19 + .pylintrc | 494 +++++++++++++++++++++++++ dev-requirements.txt | 19 + eachdist.ini | 12 + pyproject.toml | 11 + scripts/check_for_valid_readme.py | 50 +++ scripts/eachdist.py | 591 ++++++++++++++++++++++++++++++ scripts/update_sha.py | 67 ++++ tox.ini | 38 ++ 13 files changed, 1431 insertions(+) create mode 100644 .codespellrc create mode 100644 .flake8 create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 .isort.cfg create mode 100644 .pylintrc create mode 100644 dev-requirements.txt create mode 100644 eachdist.ini create mode 100644 pyproject.toml create mode 100644 scripts/check_for_valid_readme.py create mode 100755 scripts/eachdist.py create mode 100644 scripts/update_sha.py create mode 100644 tox.ini diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 000000000..bdc1157d0 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,4 @@ +[codespell] +# skipping auto generated folders +skip = ./.tox,./.mypy_cache,./target,*/LICENSE,./venv +ignore-words-list = ot diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..e7484981f --- /dev/null +++ b/.flake8 @@ -0,0 +1,26 @@ +[flake8] +ignore = + # line too long, defer to black + E501 + + # allow line breaks before binary ops + W503 + + # allow line breaks after binary ops + W504 + + # allow whitespace before ':' (https://github.com/psf/black#slices) + E203 + +exclude = + .bzr + .git + .hg + .svn + .tox + CVS + .venv*/ + venv*/ + target + __pycache__ + scripts/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..921febff9 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,40 @@ +name: AWS Distro Tests + +on: + push: + pull_request: +env: + CORE_REPO_SHA: da48e0b131ff34ff382b7d1206f71b2e31929cab + CONTRIB_REPO_SHA: 7c12ad9844ac179e3f6a493491707a9bafd06f6b + +jobs: + misc: + strategy: + fail-fast: false + matrix: + tox-environment: ["spellcheck", "lint"] + name: ${{ matrix.tox-environment }} + runs-on: ubuntu-20.04 + steps: + - name: Checkout Contrib Repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v2 + - name: Set up Python 3.10 + uses: actions/setup-python@v2 + with: + python-version: "3.10" + - name: Install tox + run: pip install tox==3.27.1 + - name: Install libsnappy-dev + if: ${{ matrix.tox-environment == 'lint' }} + run: sudo apt-get install -y libsnappy-dev + - name: Cache tox environment + # Preserves .tox directory between runs for faster installs + uses: actions/cache@v1 + with: + path: | + .tox + ~/.cache/pip + key: v7-misc-tox-cache-${{ matrix.tox-environment }}-${{ hashFiles('tox.ini', + 'dev-requirements.txt') }} + - name: run tox + run: tox -e ${{ matrix.tox-environment }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..c359e1372 --- /dev/null +++ b/.gitignore @@ -0,0 +1,60 @@ +*.py[cod] +*.sw[op] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +dist-info +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +pyvenv.cfg +lib +lib64 +__pycache__ +venv*/ +.venv*/ + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +coverage.xml +.coverage +.nox +.tox +.cache +htmlcov + +# Translations +*.mo + +# Mac +.DS_Store + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# JetBrains +.idea + +# VSCode +.vscode + +# Sphinx +_build/ + +# mypy +.mypy_cache/ +target diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000..afe42d3d4 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,19 @@ +[settings] +include_trailing_comma=True +force_grid_wrap=0 +use_parentheses=True +line_length=79 +profile=black + +; 3 stands for Vertical Hanging Indent, e.g. +; from third_party import ( +; lib1, +; lib2, +; lib3, +; ) +; docs: https://github.com/timothycrosley/isort#multi-line-output-modes +multi_line_output=3 +skip=target +skip_glob=**/gen/*,.venv*/*,venv*/*,.tox/* +known_first_party=opentelemetry +known_third_party=psutil,pytest,redis,redis_opentracing diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000..f906c45df --- /dev/null +++ b/.pylintrc @@ -0,0 +1,494 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-whitelist=cassandra + +# Add list of files or directories to be excluded. They should be base names, not +# paths. +ignore=CVS,gen,Dockerfile,docker-compose.yml,README.md,requirements.txt + +# Add files or directories matching the regex patterns to be excluded. The +# regex matches against base names, not paths. +ignore-patterns= +ignore-paths= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use. +jobs=0 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins=pylint.extensions.no_self_use + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=missing-docstring, + fixme, # Warns about FIXME, TODO, etc. comments. + too-few-public-methods, # Might be good to re-enable this later. + too-many-instance-attributes, + too-many-arguments, + duplicate-code, + ungrouped-imports, # Leave this up to isort + wrong-import-order, # Leave this up to isort + line-too-long, # Leave this up to black + exec-used, + super-with-arguments, # temp-pylint-upgrade + isinstance-second-argument-not-valid-type, # temp-pylint-upgrade + raise-missing-from, # temp-pylint-upgrade + unused-argument, # temp-pylint-upgrade + protected-access, # temp-pylint-upgrade + super-init-not-called, # temp-pylint-upgrade + invalid-overridden-method, # temp-pylint-upgrade + missing-module-docstring, # temp-pylint-upgrade + import-error, # needed as a workaround as reported here: https://github.com/open-telemetry/opentelemetry-python-contrib/issues/290 + cyclic-import, + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +# enable=c-extension-no-member + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +#evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format=text + +# Tells whether to display a full report or only the messages. +#reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit + + +[LOGGING] + +# Format style used to check logging format string. `old` means using % +# formatting, while `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package.. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members=types_pb2.* + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +#ignore-mixin-members=yes + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +#ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +#ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore. +ignored-argument-names=_.*|^ignored_|^unused_|^kwargs|^args + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format=LF + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=79 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. +#class-attribute-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=any + +# Regular expression matching correct constant names. Overrides const-naming- +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=_, + log, + logger + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=yes + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. +variable-rgx=(([a-z_][a-z0-9_]{1,})|(_[a-z0-9_]*)|(__[a-z][a-z0-9_]+__))$ + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=yes + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled). +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled). +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library=six + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make, + _Span + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=cls + + +[DESIGN] + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement. +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception". +overgeneral-exceptions=Exception diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 000000000..fffb4c445 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,19 @@ +pylint==3.0.2 +flake8==6.1.0 +isort==5.12.0 +black==22.3.0 +httpretty==1.1.4 +mypy==0.931 +sphinx==7.1.2 +sphinx-rtd-theme==2.0.0rc4 +sphinx-autodoc-typehints==1.25.2 +pytest==7.1.3 +pytest-cov==4.1.0 +readme-renderer==42.0 +bleach==4.1.0 # transient dependency for readme-renderer +protobuf~=3.13 +markupsafe>=2.0.1 +codespell==2.1.0 +requests==2.31.0 +ruamel.yaml==0.17.21 +flaky==3.7.0 diff --git a/eachdist.ini b/eachdist.ini new file mode 100644 index 000000000..a0a315e34 --- /dev/null +++ b/eachdist.ini @@ -0,0 +1,12 @@ +# These will be sorted first in that order. +# All packages that are depended upon by others should be listed here. +[DEFAULT] + +[lintroots] +extraroots=scripts/ +subglob=*.py,tests/,test/,src/* +ignore=sklearn + +[testroots] +extraroots=tests/ +subglob=tests/,test/ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..c1a64c524 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.black] +line-length = 79 +exclude = ''' +( + \.git + | \.tox + | venv + | build + | dist +) +''' diff --git a/scripts/check_for_valid_readme.py b/scripts/check_for_valid_readme.py new file mode 100644 index 000000000..42446dd74 --- /dev/null +++ b/scripts/check_for_valid_readme.py @@ -0,0 +1,50 @@ +"""Test script to check given paths for valid README.rst files.""" +import argparse +import sys +from pathlib import Path + +import readme_renderer.rst + + +def is_valid_rst(path): + """Checks if RST can be rendered on PyPI.""" + with open(path, encoding="utf-8") as readme_file: + markup = readme_file.read() + return readme_renderer.rst.render(markup, stream=sys.stderr) is not None + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Checks README.rst file in path for syntax errors." + ) + parser.add_argument( + "paths", nargs="+", help="paths containing a README.rst to test" + ) + parser.add_argument("-v", "--verbose", action="store_true") + return parser.parse_args() + + +def main(): + args = parse_args() + error = False + + for path in map(Path, args.paths): + readme = path / "README.rst" + try: + if not is_valid_rst(readme): + error = True + print("FAILED: RST syntax errors in", readme) + continue + except FileNotFoundError: + print("FAILED: README.rst not found in", path) + continue + if args.verbose: + print("PASSED:", readme) + + if error: + sys.exit(1) + print("All clear.") + + +if __name__ == "__main__": + main() diff --git a/scripts/eachdist.py b/scripts/eachdist.py new file mode 100755 index 000000000..068b42380 --- /dev/null +++ b/scripts/eachdist.py @@ -0,0 +1,591 @@ +#!/usr/bin/env python3 + +import argparse +import os +import shlex +import shutil +import subprocess +import sys +from configparser import ConfigParser +from inspect import cleandoc +from itertools import chain +from pathlib import Path, PurePath + +DEFAULT_ALLSEP = " " +DEFAULT_ALLFMT = "{rel}" + +NON_SRC_DIRS = ["build", "dist", "__pycache__", "lib", "venv", ".tox"] + + +def unique(elems): + seen = set() + for elem in elems: + if elem not in seen: + yield elem + seen.add(elem) + + +subprocess_run = subprocess.run + + +def extraargs_help(calledcmd): + return cleandoc( + f""" + Additional arguments to pass on to {calledcmd}. + + This is collected from any trailing arguments passed to `%(prog)s`. + Use an initial `--` to separate them from regular arguments. + """ + ) + + +def parse_args(args=None): + parser = argparse.ArgumentParser(description="Development helper script.") + parser.set_defaults(parser=parser) + parser.add_argument( + "--dry-run", + action="store_true", + help="Only display what would be done, don't actually do anything.", + ) + subparsers = parser.add_subparsers(metavar="COMMAND") + subparsers.required = True + + excparser = subparsers.add_parser( + "exec", + help="Run a command for each or all targets.", + formatter_class=argparse.RawTextHelpFormatter, + description=cleandoc( + """Run a command according to the `format` argument for each or all targets. + + This is an advanced command that is used internally by other commands. + + For example, to install all distributions in this repository + editable, you could use: + + scripts/eachdist.py exec "python -m pip install -e {}" + + This will run pip for all distributions which is quite slow. It gets + a bit faster if we only invoke pip once but with all the paths + gathered together, which can be achieved by using `--all`: + + scripts/eachdist.py exec "python -m pip install {}" --all "-e {}" + + The sortfirst option in the DEFAULT section of eachdist.ini makes + sure that dependencies are installed before their dependents. + + Search for usages of `parse_subargs` in the source code of this script + to see more examples. + + This command first collects target paths and then executes + commands according to `format` and `--all`. + + Target paths are initially all Python distribution root paths + (as determined by the existence of pyproject.toml, etc. files). + They are then augmented according to the section of the + `PROJECT_ROOT/eachdist.ini` config file specified by the `--mode` option. + + The following config options are available (and processed in that order): + + - `extraroots`: List of project root-relative glob expressions. + The resulting paths will be added. + - `sortfirst`: List of glob expressions. + Any matching paths will be put to the front of the path list, + in the same order they appear in this option. If more than one + glob matches, ordering is according to the first. + - `subglob`: List of glob expressions. Each path added so far is removed + and replaced with the result of all glob expressions relative to it (in + order of the glob expressions). + + After all this, any duplicate paths are removed (the first occurrence remains). + """ + ), + ) + excparser.set_defaults(func=execute_args) + excparser.add_argument( + "format", + help=cleandoc( + """Format string for the command to execute. + + The available replacements depend on whether `--all` is specified. + If `--all` was specified, there is only a single replacement, + `{}`, that is replaced with the string that is generated from + joining all targets formatted with `--all` to a single string + with the value of `--allsep` as separator. + + If `--all` was not specified, the following replacements are available: + + - `{}`: the absolute path to the current target in POSIX format + (with forward slashes) + - `{rel}`: like `{}` but relative to the project root. + - `{raw}`: the absolute path to the current target in native format + (thus exactly the same as `{}` on Unix but with backslashes on Windows). + - `{rawrel}`: like `{raw}` but relative to the project root. + + The resulting string is then split according to POSIX shell rules + (so you can use quotation marks or backslashes to handle arguments + containing spaces). + + The first token is the name of the executable to run, the remaining + tokens are the arguments. + + Note that a shell is *not* involved by default. + You can add bash/sh/cmd/powershell yourself to the format if you want. + + If `--all` was specified, the resulting command is simply executed once. + Otherwise, the command is executed for each found target. In both cases, + the project root is the working directory. + """ + ), + ) + excparser.add_argument( + "--all", + nargs="?", + const=DEFAULT_ALLFMT, + metavar="ALLFORMAT", + help=cleandoc( + """Instead of running the command for each target, join all target + paths together to run a single command. + + This option optionally takes a format string to apply to each path. The + available replacements are the ones that would be available for `format` + if `--all` was not specified. + + Default ALLFORMAT if this flag is specified: `%(const)s`. + """ + ), + ) + excparser.add_argument( + "--allsep", + help=cleandoc( + """Separator string for the strings resulting from `--all`. + Only valid if `--all` is specified. + """ + ), + ) + excparser.add_argument( + "--allowexitcode", + type=int, + action="append", + default=[0], + help=cleandoc( + """The given command exit code is treated as success and does not abort execution. + Can be specified multiple times. + """ + ), + ) + excparser.add_argument( + "--mode", + "-m", + default="DEFAULT", + help=cleandoc( + """Section of config file to use for target selection configuration. + See description of exec for available options.""" + ), + ) + + instparser = subparsers.add_parser( + "install", help="Install all distributions." + ) + + def setup_instparser(instparser): + instparser.set_defaults(func=install_args) + instparser.add_argument( + "pipargs", nargs=argparse.REMAINDER, help=extraargs_help("pip") + ) + + setup_instparser(instparser) + instparser.add_argument("--editable", "-e", action="store_true") + instparser.add_argument("--with-test-deps", action="store_true") + instparser.add_argument("--with-dev-deps", action="store_true") + instparser.add_argument("--eager-upgrades", action="store_true") + + devparser = subparsers.add_parser( + "develop", + help="Install all distributions editable + dev dependencies.", + ) + setup_instparser(devparser) + devparser.set_defaults( + editable=True, + with_dev_deps=True, + eager_upgrades=True, + with_test_deps=True, + ) + + lintparser = subparsers.add_parser( + "lint", help="Lint everything, autofixing if possible." + ) + lintparser.add_argument("--check-only", action="store_true") + lintparser.set_defaults(func=lint_args) + + testparser = subparsers.add_parser( + "test", + help="Test everything (run pytest yourself for more complex operations).", + ) + testparser.set_defaults(func=test_args) + testparser.add_argument( + "pytestargs", nargs=argparse.REMAINDER, help=extraargs_help("pytest") + ) + + fmtparser = subparsers.add_parser( + "format", + help="Formats all source code with black and isort.", + ) + fmtparser.set_defaults(func=format_args) + fmtparser.add_argument( + "--path", + required=False, + help="Format only this path instead of entire repository", + ) + + return parser.parse_args(args) + + +def find_projectroot(search_start=Path(".")): + root = search_start.resolve() + for root in chain((root,), root.parents): + if any((root / marker).exists() for marker in (".git", "tox.ini")): + return root + return None + + +def find_targets_unordered(rootpath): + for subdir in rootpath.iterdir(): + if not subdir.is_dir(): + continue + if subdir.name.startswith(".") or subdir.name.startswith("venv"): + continue + if any((subdir / marker).exists() for marker in ("pyproject.toml",)): + yield subdir + else: + yield from find_targets_unordered(subdir) + + +def getlistcfg(strval): + return [ + val.strip() + for line in strval.split("\n") + for val in line.split(",") + if val.strip() + ] + + +def find_targets(mode, rootpath): + if not rootpath: + sys.exit("Could not find a root directory.") + + cfg = ConfigParser() + cfg.read(str(rootpath / "eachdist.ini")) + mcfg = cfg[mode] + + targets = list(find_targets_unordered(rootpath)) + if "extraroots" in mcfg: + targets += [ + path + for extraglob in getlistcfg(mcfg["extraroots"]) + for path in rootpath.glob(extraglob) + ] + if "sortfirst" in mcfg: + sortfirst = getlistcfg(mcfg["sortfirst"]) + + def keyfunc(path): + path = path.relative_to(rootpath) + for idx, pattern in enumerate(sortfirst): + if path.match(pattern): + return idx + return float("inf") + + targets.sort(key=keyfunc) + if "ignore" in mcfg: + ignore = getlistcfg(mcfg["ignore"]) + + def filter_func(path): + path = path.relative_to(rootpath) + for pattern in ignore: + if path.match(pattern): + return False + return True + + filtered = filter(filter_func, targets) + targets = list(filtered) + + subglobs = getlistcfg(mcfg.get("subglob", "")) + if subglobs: + targets = [ + newentry + for newentry in ( + target / subdir + for target in targets + for subglob in subglobs + # We need to special-case the dot, because glob fails to parse that with an IndexError. + for subdir in ( + (target,) if subglob == "." else target.glob(subglob) + ) + ) + if ".egg-info" not in str(newentry) and newentry.exists() + ] + + return list(unique(targets)) + + +def runsubprocess(dry_run, params, *args, **kwargs): + cmdstr = join_args(params) + if dry_run: + print(cmdstr) + return None + + # Py < 3.6 compat. + cwd = kwargs.get("cwd") + if cwd and isinstance(cwd, PurePath): + kwargs["cwd"] = str(cwd) + + check = kwargs.pop("check") # Enforce specifying check + + print(">>>", cmdstr, file=sys.stderr, flush=True) + + # This is a workaround for subprocess.run(['python']) leaving the virtualenv on Win32. + # The cause for this is that when running the python.exe in a virtualenv, + # the wrapper executable launches the global python as a subprocess and the search sequence + # for CreateProcessW which subprocess.run and Popen use is a follows + # (https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw): + # > 1. The directory from which the application loaded. + # This will be the directory of the global python.exe, not the venv directory, due to the suprocess mechanism. + # > 6. The directories that are listed in the PATH environment variable. + # Only this would find the "correct" python.exe. + + params = list(params) + executable = shutil.which(params[0]) + if executable: + params[0] = executable + try: + return subprocess_run(params, *args, check=check, **kwargs) + except OSError as exc: + raise ValueError( + "Failed executing " + repr(params) + ": " + str(exc) + ) from exc + + +def execute_args(args): + if args.allsep and not args.all: + args.parser.error("--allsep specified but not --all.") + + if args.all and not args.allsep: + args.allsep = DEFAULT_ALLSEP + + rootpath = find_projectroot() + targets = find_targets(args.mode, rootpath) + if not targets: + print(f"Info: No targets selected (root: {rootpath})") + return + + def fmt_for_path(fmt, path): + return fmt.format( + path.as_posix(), + rel=path.relative_to(rootpath).as_posix(), + raw=path, + rawrel=path.relative_to(rootpath), + ) + + def _runcmd(cmd): + result = runsubprocess( + args.dry_run, shlex.split(cmd), cwd=rootpath, check=False + ) + if result is not None and result.returncode not in args.allowexitcode: + print( + f"'{cmd}' failed with code {result.returncode}", + file=sys.stderr, + ) + sys.exit(result.returncode) + + if args.all: + allstr = args.allsep.join( + fmt_for_path(args.all, path) for path in targets + ) + cmd = args.format.format(allstr) + _runcmd(cmd) + else: + for target in targets: + cmd = fmt_for_path(args.format, target) + _runcmd(cmd) + + +def clean_remainder_args(remainder_args): + if remainder_args and remainder_args[0] == "--": + del remainder_args[0] + + +def join_args(arglist): + return " ".join(map(shlex.quote, arglist)) + + +def install_args(args): + clean_remainder_args(args.pipargs) + if args.eager_upgrades: + args.pipargs += ["--upgrade-strategy=eager"] + + if args.with_dev_deps: + runsubprocess( + args.dry_run, + [ + "python", + "-m", + "pip", + "install", + "--upgrade", + "pip", + "setuptools", + "wheel", + ] + + args.pipargs, + check=True, + ) + + allfmt = "-e 'file://{}" if args.editable else "'file://{}" + # packages should provide an extra_requires that is named + # 'test', to denote test dependencies. + extras = [] + if args.with_test_deps: + extras.append("test") + if extras: + allfmt += f"[{','.join(extras)}]" + # note the trailing single quote, to close the quote opened above. + allfmt += "'" + + execute_args( + parse_subargs( + args, + ( + "exec", + "python -m pip install {} " + join_args(args.pipargs), + "--all", + allfmt, + ), + ) + ) + if args.with_dev_deps: + rootpath = find_projectroot() + runsubprocess( + args.dry_run, + [ + "python", + "-m", + "pip", + "install", + "--upgrade", + "-r", + str(rootpath / "dev-requirements.txt"), + ] + + args.pipargs, + check=True, + ) + + +def parse_subargs(parentargs, args): + subargs = parse_args(args) + subargs.dry_run = parentargs.dry_run or subargs.dry_run + return subargs + + +def lint_args(args): + rootdir = str(find_projectroot()) + + runsubprocess( + args.dry_run, + ("black", "--config", f"{rootdir}/pyproject.toml", ".") + + (("--diff", "--check") if args.check_only else ()), + cwd=rootdir, + check=True, + ) + runsubprocess( + args.dry_run, + ("isort", "--settings-path", f"{rootdir}/.isort.cfg", ".") + + (("--diff", "--check-only") if args.check_only else ()), + cwd=rootdir, + check=True, + ) + runsubprocess( + args.dry_run, + ("flake8", "--config", f"{rootdir}/.flake8", rootdir), + check=True, + ) + execute_args( + parse_subargs( + args, ("exec", "pylint {}", "--all", "--mode", "lintroots") + ) + ) + execute_args( + parse_subargs( + args, + ( + "exec", + "python scripts/check_for_valid_readme.py {}", + "--all", + ), + ) + ) + + +def find(name, path): + non_src_dirs = [os.path.join(path, nsd) for nsd in NON_SRC_DIRS] + + def _is_non_src_dir(root) -> bool: + for nsd in non_src_dirs: + if root.startswith(nsd): + return True + return False + + for root, _, files in os.walk(path): + if _is_non_src_dir(root): + continue + if name in files: + return os.path.join(root, name) + return None + + +def test_args(args): + clean_remainder_args(args.pytestargs) + execute_args( + parse_subargs( + args, + ( + "exec", + "pytest {} " + join_args(args.pytestargs), + "--mode", + "testroots", + ), + ) + ) + + +def format_args(args): + format_dir = str(find_projectroot()) + if args.path: + format_dir = os.path.join(format_dir, args.path) + root_dir = str(find_projectroot()) + runsubprocess( + args.dry_run, + ("black", "--config", f"{root_dir}/pyproject.toml", "."), + cwd=format_dir, + check=True, + ) + runsubprocess( + args.dry_run, + ( + "isort", + "--settings-path", + f"{root_dir}/.isort.cfg", + "--profile", + "black", + ".", + ), + cwd=format_dir, + check=True, + ) + + +def main(): + args = parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/update_sha.py b/scripts/update_sha.py new file mode 100644 index 000000000..951380304 --- /dev/null +++ b/scripts/update_sha.py @@ -0,0 +1,67 @@ +import argparse + +import requests +from ruamel.yaml import YAML + +CORE_API_URL = ( + "https://api.github.com/repos/open-telemetry/opentelemetry-python/commits/" +) +CONTRIB_API_URL = "https://api.github.com/repos/open-telemetry/opentelemetry-python-contrib/commits/" +WORKFLOW_FILE = ".github/workflows/test.yml" + + +def get_core_sha(branch): + url = CORE_API_URL + branch + response = requests.get(url, timeout=15) + response.raise_for_status() + return response.json()["sha"] + + +def get_contrib_sha(branch): + url = CONTRIB_API_URL + branch + response = requests.get(url, timeout=15) + response.raise_for_status() + return response.json()["sha"] + + +def update_core_sha(sha): + yaml = YAML() + yaml.preserve_quotes = True + with open(WORKFLOW_FILE, "r", encoding="utf-8") as file: + workflow = yaml.load(file) + workflow["env"]["CORE_REPO_SHA"] = sha + with open(WORKFLOW_FILE, "w", encoding="utf-8") as file: + yaml.dump(workflow, file) + + +def update_contrib_sha(sha): + yaml = YAML() + yaml.preserve_quotes = True + with open(WORKFLOW_FILE, "r", encoding="utf-8") as file: + workflow = yaml.load(file) + workflow["env"]["CONTRIB_REPO_SHA"] = sha + with open(WORKFLOW_FILE, "w", encoding="utf-8") as file: + yaml.dump(workflow, file) + + +def main(): + args = parse_args() + core_sha = get_core_sha(args.core_branch) + contrib_sha = get_contrib_sha(args.contrib_branch) + update_core_sha(core_sha) + update_contrib_sha(contrib_sha) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Updates the SHA in the workflow file" + ) + parser.add_argument("-cb", "--core-branch", help="core branch to use") + parser.add_argument( + "-tb", "--contrib-branch", help="contrib branch to use" + ) + return parser.parse_args() + + +if __name__ == "__main__": + main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..4a35b46b8 --- /dev/null +++ b/tox.ini @@ -0,0 +1,38 @@ +[tox] +isolated_build = True +skipsdist = True +skip_missing_interpreters = True +envlist = + lint + spellcheck + +[testenv] +deps = + -c dev-requirements.txt + test: pytest + test: pytest-benchmark + +commands_pre = +; Install without -e to test the actual installation + py3{7,8,9,10,11}: python -m pip install -U pip setuptools wheel + +commands = + test: pytest {posargs} + +[testenv:spellcheck] +basepython: python3 +recreate = True +deps = + codespell + +commands = + codespell + +[testenv:lint] +basepython: python3 +recreate = True +deps = + -r dev-requirements.txt + +commands = + python scripts/eachdist.py lint --check-only From ff6e1e279f312018b193592df1633e7826bb698b Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Thu, 4 Jan 2024 16:29:12 -0800 Subject: [PATCH 2/4] Delete unuseful pkg. --- .flake8 | 1 - .github/workflows/test.yml | 2 ++ .isort.cfg | 2 +- dev-requirements.txt | 3 --- eachdist.ini | 1 - 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.flake8 b/.flake8 index e7484981f..38499702b 100644 --- a/.flake8 +++ b/.flake8 @@ -23,4 +23,3 @@ exclude = venv*/ target __pycache__ - scripts/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 921febff9..ce5a37bbd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,3 +1,5 @@ +# This workflow check the code style for the repo +# TODO: add unit test check after adding AWS Distro project name: AWS Distro Tests on: diff --git a/.isort.cfg b/.isort.cfg index afe42d3d4..a64b63937 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -16,4 +16,4 @@ multi_line_output=3 skip=target skip_glob=**/gen/*,.venv*/*,venv*/*,.tox/* known_first_party=opentelemetry -known_third_party=psutil,pytest,redis,redis_opentracing +known_third_party=psutil,pytest diff --git a/dev-requirements.txt b/dev-requirements.txt index fffb4c445..db1c84ac5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,9 +4,6 @@ isort==5.12.0 black==22.3.0 httpretty==1.1.4 mypy==0.931 -sphinx==7.1.2 -sphinx-rtd-theme==2.0.0rc4 -sphinx-autodoc-typehints==1.25.2 pytest==7.1.3 pytest-cov==4.1.0 readme-renderer==42.0 diff --git a/eachdist.ini b/eachdist.ini index a0a315e34..9fa0ae9a7 100644 --- a/eachdist.ini +++ b/eachdist.ini @@ -5,7 +5,6 @@ [lintroots] extraroots=scripts/ subglob=*.py,tests/,test/,src/* -ignore=sklearn [testroots] extraroots=tests/ From ac81e4eb213ff146a6684b681355e9f634900c3b Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Fri, 5 Jan 2024 10:35:54 -0800 Subject: [PATCH 3/4] Remove update_sha and use the latest commit for pkgs dependencies, update README for code style check and unit test. --- .github/workflows/test.yml | 3 -- README.md | 22 +++++++++++++ scripts/update_sha.py | 67 -------------------------------------- 3 files changed, 22 insertions(+), 70 deletions(-) delete mode 100644 scripts/update_sha.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce5a37bbd..c2c132940 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,9 +5,6 @@ name: AWS Distro Tests on: push: pull_request: -env: - CORE_REPO_SHA: da48e0b131ff34ff382b7d1206f71b2e31929cab - CONTRIB_REPO_SHA: 7c12ad9844ac179e3f6a493491707a9bafd06f6b jobs: misc: diff --git a/README.md b/README.md index 9e0988e5a..0411db2d6 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,28 @@ This project provide AWS Distro base on [OpenTelemetry Python Contrib](https://g preconfigured for use with AWS services. Please check out that project too to get a better understanding of the underlying internals. +## Code Style Check + +This package applys code style check automatically when created a push/pull request to the project repository. You can apply style check locally before submitting the PR by following: +1. Install related packages: +```sh +pip install isort pylint black flake8 +``` +2. Check code style errors: +```sh +python scripts/eachdist.py lint --check-only +``` +3. Apply the fix for the errors automatically: +```sh +python scripts/eachdist.py lint +``` + +## Unit test +This package detects all the unit tests defined in folder with naming "tests"/"test" under the same directory as pyproject.toml file. Please make sure to add unit test everytime a new feature added. +The workflow will run the test tox environment automatically whenever there is a push/pull request. Please make sure you install the related package needed for the unit tests in `commands_pre`. + +If you want to test a specific component/feature, please add a new environment in tox.ini file, and add related workflow as needed. + ## License This project is licensed under the Apache-2.0 License. diff --git a/scripts/update_sha.py b/scripts/update_sha.py deleted file mode 100644 index 951380304..000000000 --- a/scripts/update_sha.py +++ /dev/null @@ -1,67 +0,0 @@ -import argparse - -import requests -from ruamel.yaml import YAML - -CORE_API_URL = ( - "https://api.github.com/repos/open-telemetry/opentelemetry-python/commits/" -) -CONTRIB_API_URL = "https://api.github.com/repos/open-telemetry/opentelemetry-python-contrib/commits/" -WORKFLOW_FILE = ".github/workflows/test.yml" - - -def get_core_sha(branch): - url = CORE_API_URL + branch - response = requests.get(url, timeout=15) - response.raise_for_status() - return response.json()["sha"] - - -def get_contrib_sha(branch): - url = CONTRIB_API_URL + branch - response = requests.get(url, timeout=15) - response.raise_for_status() - return response.json()["sha"] - - -def update_core_sha(sha): - yaml = YAML() - yaml.preserve_quotes = True - with open(WORKFLOW_FILE, "r", encoding="utf-8") as file: - workflow = yaml.load(file) - workflow["env"]["CORE_REPO_SHA"] = sha - with open(WORKFLOW_FILE, "w", encoding="utf-8") as file: - yaml.dump(workflow, file) - - -def update_contrib_sha(sha): - yaml = YAML() - yaml.preserve_quotes = True - with open(WORKFLOW_FILE, "r", encoding="utf-8") as file: - workflow = yaml.load(file) - workflow["env"]["CONTRIB_REPO_SHA"] = sha - with open(WORKFLOW_FILE, "w", encoding="utf-8") as file: - yaml.dump(workflow, file) - - -def main(): - args = parse_args() - core_sha = get_core_sha(args.core_branch) - contrib_sha = get_contrib_sha(args.contrib_branch) - update_core_sha(core_sha) - update_contrib_sha(contrib_sha) - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Updates the SHA in the workflow file" - ) - parser.add_argument("-cb", "--core-branch", help="core branch to use") - parser.add_argument( - "-tb", "--contrib-branch", help="contrib branch to use" - ) - return parser.parse_args() - - -if __name__ == "__main__": - main() From 2c6b245ae135a56b7326481d4153f71520fdddfb Mon Sep 17 00:00:00 2001 From: Zhonghao Zhao Date: Fri, 5 Jan 2024 10:40:37 -0800 Subject: [PATCH 4/4] Apply codespell. --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0411db2d6..2665191af 100644 --- a/README.md +++ b/README.md @@ -8,22 +8,24 @@ understanding of the underlying internals. ## Code Style Check -This package applys code style check automatically when created a push/pull request to the project repository. You can apply style check locally before submitting the PR by following: +This package applies code style check automatically when created a push/pull request to the project repository. You can apply style check locally before submitting the PR by following: 1. Install related packages: ```sh -pip install isort pylint black flake8 +pip install isort pylint black flake8 codespell ``` -2. Check code style errors: +2. Check code style errors using codespell and lint: ```sh +codespell python scripts/eachdist.py lint --check-only ``` 3. Apply the fix for the errors automatically: ```sh +codespell . --write-changes python scripts/eachdist.py lint ``` ## Unit test -This package detects all the unit tests defined in folder with naming "tests"/"test" under the same directory as pyproject.toml file. Please make sure to add unit test everytime a new feature added. +This package detects all the unit tests defined in folder with naming "tests"/"test" under the same directory as pyproject.toml file. Please make sure to add unit test every time a new feature added. The workflow will run the test tox environment automatically whenever there is a push/pull request. Please make sure you install the related package needed for the unit tests in `commands_pre`. If you want to test a specific component/feature, please add a new environment in tox.ini file, and add related workflow as needed.