diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..325e79b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,58 @@ +default_language_version: + python: python3.8 + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.5.0 + hooks: + - id: trailing-whitespace + language_version: python3 + - id: end-of-file-fixer + language_version: python3 + - id: check-yaml + language_version: python3 + - id: debug-statements + language_version: python3 +- repo: https://github.com/ambv/black + rev: 20.8b1 + hooks: + - id: black + language_version: python3 + args: ["--target-version", "py38"] +- repo: https://github.com/pycqa/flake8 + rev: 3.8.3 + hooks: + - id: flake8 + language_version: python3 + args: ['--config=setup.cfg'] +#- repo: https://github.com/pre-commit/mirrors-autopep8 +# rev: v1.4.4 +# hooks: +# - id: autopep8 +# args: ['--global-config=setup.cfg','--in-place'] +# - repo: https://github.com/timothycrosley/isort +# rev: 5.0.7 +# hooks: +# - id: isort +# language_version: python3 +# args: ['--profile', 'black'] +#- repo: https://github.com/pycqa/pydocstyle +# rev: 5.0.2 +# hooks: +# - id: pydocstyle +# args: ["--conventions=numpy"] +# - repo: https://github.com/asottile/pyupgrade +# rev: v2.4.1 +# hooks: +# - id: pyupgrade +# language_version: python3 +# - repo: meta +# hooks: +# - id: check-hooks-apply +# - id: check-useless-excludes +# - repo: https://github.com/kynan/nbstripout +# rev: 0.3.7 +# hooks: +# - id: nbstripout +# language_version: python3 +# files: ".ipynb" diff --git a/.travis.yml b/.travis.yml index 6a373af..9561ca2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ matrix: env: - CONDA_FN="Miniconda3-latest-Linux-x86_64.sh" - PEP8=true + - BLACK=true - PYTHON_DESIRED=3.8 sudo: false @@ -47,10 +48,11 @@ install: - conda env update -f environment.yml - source activate rooki # Packages for testing - - pip install pytest nbval flake8 + - pip install pytest nbval flake8 black # Install package - python setup.py install script: - make test - make test-nb - if [[ $PEP8 == true ]]; then flake8 rooki tests; fi + - if [[ $BLACK == true ]]; then black --check --target-version py38 rooki tests; fi diff --git a/docs/source/development.rst b/docs/source/development.rst index a4614f7..8700980 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -20,11 +20,12 @@ Install additional dependencies: $ pip install -r requirements_dev.txt -When you're done making changes, check that your changes pass `flake8` and the tests: +When you're done making changes, check that your changes pass `black`, `flake8` and the tests: .. code-block:: console - $ flake8 rooki + $ black rooki tests + $ flake8 rooki tests $ pytest tests Or use the Makefile: @@ -34,6 +35,19 @@ Or use the Makefile: $ make lint $ make test + +Add pre-commit hooks +-------------------- + +Before committing your changes, we ask that you install `pre-commit` in your environment. +`Pre-commit` runs git hooks that ensure that your code resembles that of the project +and catches and corrects any small errors or inconsistencies when you `git commit`: + +.. code-block:: console + + $ conda install -c conda-forge pre_commit + $ pre-commit install + Write Documentation ------------------- diff --git a/requirements_dev.txt b/requirements_dev.txt index bf73b86..7ef41a6 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ -pip>=18.1 +pip>=20.0 bumpversion>=0.5.3 wheel>=0.32.1 watchdog>=0.9.0 @@ -8,7 +8,8 @@ coverage>=4.5.1 Sphinx>=1.8.1 nbsphinx>=0.7.0 twine>=1.12.1 - +pre-commit>=2.7.1 +black>=20.0 pytest>=3.8.2 pytest-runner>=4.2 nbval>=0.9.6 diff --git a/rooki/__init__.py b/rooki/__init__.py index c4ad269..2f3dfee 100644 --- a/rooki/__init__.py +++ b/rooki/__init__.py @@ -6,5 +6,5 @@ __all__ = [ - 'rooki', + "rooki", ] diff --git a/rooki/client.py b/rooki/client.py index 1fdf3a1..df9e796 100644 --- a/rooki/client.py +++ b/rooki/client.py @@ -9,10 +9,10 @@ class Rooki(WPSClient): def __init__(self, url=None, mode=None, verify=None): - self._url = url or config.get_config_value('service', 'url') - self._mode = mode or config.get_config_value('service', 'mode') + self._url = url or config.get_config_value("service", "url") + self._mode = mode or config.get_config_value("service", "mode") if verify is None: - self._verify = config.get_config_value('service', 'ssl_verify') + self._verify = config.get_config_value("service", "ssl_verify") else: self._verify = verify progress = self.mode == ASYNC diff --git a/rooki/config.py b/rooki/config.py index fada2d7..2a275fc 100644 --- a/rooki/config.py +++ b/rooki/config.py @@ -3,9 +3,11 @@ import logging ROOKI_HOME = os.path.abspath(os.path.dirname(__file__)) -DEFAULT_CFG = os.path.join(ROOKI_HOME, 'default.cfg') +DEFAULT_CFG = os.path.join(ROOKI_HOME, "default.cfg") -RAW_OPTIONS = [('logging', 'format'), ] +RAW_OPTIONS = [ + ("logging", "format"), +] CONFIG = None LOGGER = logging.getLogger("ROOKI") @@ -23,7 +25,7 @@ def get_config_value(section, option): if not CONFIG: load_configuration() - value = '' + value = "" if CONFIG.has_section(section): if CONFIG.has_option(section, option): @@ -47,32 +49,32 @@ def load_configuration(cfgfiles=None): """ global CONFIG - LOGGER.info('loading configuration') + LOGGER.info("loading configuration") CONFIG = configparser.ConfigParser(os.environ) - LOGGER.debug('setting default values') - CONFIG.add_section('service') - CONFIG.set('service', 'url', 'http://localhost:5000/wps') + LOGGER.debug("setting default values") + CONFIG.add_section("service") + CONFIG.set("service", "url", "http://localhost:5000/wps") config_files = _get_default_config_files_location() if cfgfiles: config_files.extend(cfgfiles) - if 'ROOKI_CFG' in os.environ: - config_files.append(os.environ['ROOKI_CFG']) + if "ROOKI_CFG" in os.environ: + config_files.append(os.environ["ROOKI_CFG"]) loaded_files = CONFIG.read(config_files) if loaded_files: - LOGGER.info('Configuration file(s) {} loaded'.format(loaded_files)) + LOGGER.info("Configuration file(s) {} loaded".format(loaded_files)) else: - LOGGER.info('No configuration files loaded. Using default values') + LOGGER.info("No configuration files loaded. Using default values") # dirty hack to set rook url on binder - if 'ROOK_URL' in os.environ: - CONFIG.set('service', 'url', os.environ['ROOK_URL']) - if 'ROOK_MODE' in os.environ: - CONFIG.set('service', 'mode', os.environ['ROOK_MODE']) - if 'ROOK_SSL_VERIFY' in os.environ: - CONFIG.set('service', 'ssl_verify', os.environ['ROOK_SSL_VERIFY']) + if "ROOK_URL" in os.environ: + CONFIG.set("service", "url", os.environ["ROOK_URL"]) + if "ROOK_MODE" in os.environ: + CONFIG.set("service", "mode", os.environ["ROOK_MODE"]) + if "ROOK_SSL_VERIFY" in os.environ: + CONFIG.set("service", "ssl_verify", os.environ["ROOK_SSL_VERIFY"]) def _get_default_config_files_location(): @@ -83,8 +85,11 @@ def _get_default_config_files_location(): :returns: configuration files :rtype: list of strings """ - LOGGER.debug('trying to estimate the default location') - cfgfiles = [DEFAULT_CFG, "/etc/rooki.cfg", ] - if 'HOME' in os.environ: - cfgfiles.append(os.path.join(os.environ['HOME'], ".rooki.cfg")) + LOGGER.debug("trying to estimate the default location") + cfgfiles = [ + DEFAULT_CFG, + "/etc/rooki.cfg", + ] + if "HOME" in os.environ: + cfgfiles.append(os.path.join(os.environ["HOME"], ".rooki.cfg")) return cfgfiles diff --git a/rooki/operators.py b/rooki/operators.py index b6310b6..b8a6588 100644 --- a/rooki/operators.py +++ b/rooki/operators.py @@ -7,7 +7,6 @@ class Operator: - def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs @@ -15,19 +14,19 @@ def __init__(self, *args, **kwargs): def _tree(self, tree=defaultdict(dict)): # args = [arg._tree(tree) for arg in self.args] [arg._tree(tree) for arg in self.args] - tree['steps'][self.method_key] = { - 'run': self.METHOD, - 'in': { - 'collection': _unpack_if_single([arg.collection for arg in self.args]), + tree["steps"][self.method_key] = { + "run": self.METHOD, + "in": { + "collection": _unpack_if_single([arg.collection for arg in self.args]), **self.kwargs, - } + }, } - tree['outputs']['output'] = self.collection + tree["outputs"]["output"] = self.collection return tree - def _serialise(self, doc='workflow'): + def _serialise(self, doc="workflow"): tree = self._tree() - tree['doc'] = doc + tree["doc"] = doc return json.dumps(tree) def orchestrate(self): @@ -36,7 +35,7 @@ def orchestrate(self): @property def method_key(self): methods = self._get_methods([]) - return f'{self.METHOD}_{self.variable}_{methods.count(self.METHOD)}' + return f"{self.METHOD}_{self.variable}_{methods.count(self.METHOD)}" def _get_methods(self, methods): # method_names = [arg._get_methods(methods) for arg in self.args] @@ -46,7 +45,7 @@ def _get_methods(self, methods): @property def collection(self): - return f'{self.method_key}/output' + return f"{self.method_key}/output" @property def variable(self): @@ -54,21 +53,20 @@ def variable(self): class Input: - def __init__(self, variable, dataset): self.variable = variable self.dataset = dataset - def _serialise(self, doc='workflow'): + def _serialise(self, doc="workflow"): tree = self._tree() - tree['doc'] = doc + tree["doc"] = doc return json.dumps(tree) def orchestrate(self): return rooki.orchestrate(workflow=self._serialise()) def _tree(self, tree=defaultdict(dict)): - tree['inputs'][self.variable] = self.dataset + tree["inputs"][self.variable] = self.dataset return tree def _get_methods(self, methods): @@ -76,37 +74,37 @@ def _get_methods(self, methods): @property def collection(self): - return f'inputs/{self.variable}' + return f"inputs/{self.variable}" class Average(Operator): - METHOD = 'average' + METHOD = "average" class Subset(Operator): - METHOD = 'subset' + METHOD = "subset" class Diff(Operator): - METHOD = 'diff' + METHOD = "diff" def _tree(self, tree=defaultdict(dict)): # args = [arg._tree(tree) for arg in self.args] [arg._tree(tree) for arg in self.args] - tree['steps'][self.method_key] = { - 'run': self.METHOD, - 'in': { - 'collection_a': _unpack_if_single([self.args[0].collection]), - 'collection_b': _unpack_if_single([self.args[1].collection]), + tree["steps"][self.method_key] = { + "run": self.METHOD, + "in": { + "collection_a": _unpack_if_single([self.args[0].collection]), + "collection_b": _unpack_if_single([self.args[1].collection]), **self.kwargs, - } + }, } - tree['outputs']['output'] = self.collection + tree["outputs"]["output"] = self.collection return tree @property def variable(self): - return 'var' + return "var" def _unpack_if_single(list): diff --git a/rooki/results.py b/rooki/results.py index 304e9dd..38a2b3d 100644 --- a/rooki/results.py +++ b/rooki/results.py @@ -23,7 +23,7 @@ def status(self): if self.ok: status = self.response.status else: - status = '; '.join([error.text.strip() for error in self.response.errors]) + status = "; ".join([error.text.strip() for error in self.response.errors]) return status @property @@ -42,7 +42,7 @@ def xml(self): @property def doc(self): if not self._doc: - self._doc = BeautifulSoup(self.xml, 'xml') + self._doc = BeautifulSoup(self.xml, "xml") return self._doc @property @@ -50,7 +50,7 @@ def size(self): """total size of all files in bytes.""" if self._size is None: total_size = 0 - for size in self.doc.find_all('size'): + for size in self.doc.find_all("size"): total_size += int(size.text) self._size = total_size return self._size @@ -69,15 +69,16 @@ def size_in_gb(self): @property def num_files(self): if self._num_files is None: - self._num_files = len(self.doc.find_all('file')) + self._num_files = len(self.doc.find_all("file")) return self._num_files def download_urls(self): - return [url.text for url in self.doc.find_all('metaurl')] + return [url.text for url in self.doc.find_all("metaurl")] def download(self): try: import metalink.download + files = metalink.download.get(self.url, self.outdir, segmented=False) except Exception: files = [] @@ -86,6 +87,7 @@ def download(self): def datasets(self): try: import xarray as xr + datasets = [xr.open_dataset(file) for file in self.download()] except Exception: datasets = [] diff --git a/tests/common.py b/tests/common.py index 3cc10f8..de761e7 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,3 +1,3 @@ from rooki import config -ROOK_URL = config.get_config_value('service', 'url') +ROOK_URL = config.get_config_value("service", "url") diff --git a/tests/conftest.py b/tests/conftest.py index 5ce21b6..d4d8f7d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,5 +5,5 @@ @pytest.fixture(scope="session") def rooki(): - rooki = Rooki(mode='async', verify=False) + rooki = Rooki(mode="async", verify=False) return rooki diff --git a/tests/test_rooki.py b/tests/test_rooki.py index d793b42..700fef9 100644 --- a/tests/test_rooki.py +++ b/tests/test_rooki.py @@ -8,14 +8,15 @@ def test_rooki_settings(rooki): assert rooki.url == ROOK_URL - assert rooki.mode == 'async' + assert rooki.mode == "async" assert rooki.verify is False def test_rooki_subset(rooki): resp = rooki.subset( - collection='c3s-cmip5.output1.ICHEC.EC-EARTH.historical.day.atmos.day.r1i1p1.tas.latest', - time='1860-01-01/1900-12-30') + collection="c3s-cmip5.output1.ICHEC.EC-EARTH.historical.day.atmos.day.r1i1p1.tas.latest", + time="1860-01-01/1900-12-30", + ) assert resp.ok is True assert resp.num_files == 1 assert len(resp.download_urls()) == 1