major refactoring, rewrite testing setup to use clisops, rename proje…

…ct to birdhouse-raven, update pre-commit

.github/workflows/main.yml

@@ -70,7 +70,6 @@ jobs:
           environment-file: environment.yml
           create-args: >-
             python=${{ matrix.python-version }}
-          micromamba-version: 1.5.10-0 # Pin micromamba version because of following issue: https://github.com/mamba-org/setup-micromamba/issues/225
       - name: Install requirements 📦
         run: |
           python -m pip install -e ".[dev]"

.pre-commit-config.yaml

@@ -6,40 +6,17 @@ repos:
     rev: v5.0.0
     hooks:
       - id: trailing-whitespace
-        exclude: setup.cfg
       - id: end-of-file-fixer
-        exclude: setup.cfg
       - id: check-yaml
       - id: debug-statements
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.19.0
+    hooks:
+      - id: pyupgrade
   - repo: https://github.com/ambv/black
     rev: 24.10.0
     hooks:
       - id: black
-        args: ["--target-version", "py39"]
-  - repo: https://github.com/pycqa/flake8
-    rev: 7.1.1
-    hooks:
-      - id: flake8
-        args: ['--config=setup.cfg']
-#-   repo: https://github.com/pre-commit/mirrors-autopep8
-#    rev: v1.4.4
-#    hooks:
-#    -   id: autopep8
-#        args: ['--global-config=setup.cfg','--in-place']
-#-   repo: https://github.com/timothycrosley/isort
-#    rev: 5.6.4
-#    hooks:
-#    -   id: isort
-#        args: ['--profile', 'black']
-#-   repo: https://github.com/pycqa/pydocstyle
-#    rev: 5.0.2
-#    hooks:
-#    -   id: pydocstyle
-#        args: ["--conventions=numpy"]
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.17.0
-    hooks:
-      - id: pyupgrade
   - repo: meta
     hooks:
       - id: check-hooks-apply

Makefile

@@ -99,12 +99,11 @@ clean-docs: ## remove documentation artifacts
 	@-rm -f docs/modules.rst
 	$(MAKE) -C docs clean
 
-lint/flake8: ## check style with flake8
-	@echo "Running flake8 code style checks ..."
-	@bash -c 'flake8 src/rook tests'
-	@#bash -c 'ruff check src/rook tests'
+lint/ruff: ## check style with flake8
+	@echo "Running ruff code style checks ..."
+	@bash -c 'ruff check src/rook tests'
 
-lint: lint/flake8 ## check style
+lint: lint/ruff ## check style
 
 ## Testing targets:
 

environment.yml

@@ -32,3 +32,4 @@ dependencies:
   - aiohttp
   # dashboard
   - bokeh
+  - humanize

notebooks/decadal_dynamic.py

@@ -1,6 +1,7 @@
-import xarray as xr
-import pandas as pd
 from pathlib import Path
+
+import pandas as pd
+import xarray as xr
 from memory_profiler import profile
 
 

notebooks/decadal_dynamic_agg.py

@@ -1,6 +1,7 @@
-import xarray as xr
-import pandas as pd
 from pathlib import Path
+
+import pandas as pd
+import xarray as xr
 from memory_profiler import profile
 
 

notebooks/decadal_static.py

@@ -1,6 +1,7 @@
-import xarray as xr
-import pandas as pd
 from pathlib import Path
+
+import pandas as pd
+import xarray as xr
 from memory_profiler import profile
 
 

pyproject.toml

@@ -3,7 +3,7 @@ requires = ["flit_core >=3.9,<4"]
 build-backend = "flit_core.buildapi"
 
 [project]
-name = "rook"
+name = "birdhouse-rook"
 authors = [
   {name = "Carsten Ehbrecht", email = "[email protected]"}
 ]
@@ -36,24 +36,53 @@ dependencies = [
   "click >=8.1.7",
   "jinja2 >=3.1.4",
   "psutil >=6.0.0",
-  "pywps >=4.5.1,<4.6"
+  "pywps >=4.5.2,<4.7",
+  "daops >=0.11.0,<0.12.0",
+  "clisops >=0.14.0",
+  "roocs-utils >=0.6.9,<0.7.0",
+  "roocs_grids >=0.1.2",
+  "xarray >=0.21,<2023.3.0", # https://github.com/pydata/xarray/issues/7794
+  "cf-xarray >=0.7",
+  "cftime>=1.2.1",
+  "dask[complete] >=2021.12",
+  "netcdf4>=1.4",
+  # workflow
+  "networkx",
+  # provenance
+  "prov >=2.0.0",
+  "pydot",
+  # catalog
+  "intake",
+  "pandas >=1.5.3",
+  "SQLAlchemy >=1.4.45,<2.0",
+  "aiohttp",
+  # dashboard
+  "bokeh",
+  "humanize"
 ]
 
 [project.optional-dependencies]
 dev = [
   "pip >=24.2.0",
+  "beautifulsoup4",
+  "black >=24.10.0",
   "bump-my-version >=0.26.0",
   "coverage >=7.5.0",
   "cruft >=2.15.0",
   "flit >=3.9.0,<4.0",
+  "ipython>=8.13.0",
+  "nbconvert >=7.10.0",
   "nbsphinx >=0.9.5",
   "nbval >=0.10.0",
+  "pre-commit >=3.5.0",
   "pytest >=8.0.0",
   "pytest-cov >=5.0.0",
   "ruff >=0.5.7",
   "sphinx >=7.0.0",
   "tox >=4.18.0",
-  "watchdog >=4.0.0"
+  "twine >=4.0.0",
+  "watchdog >=4.0.0",
+  "wheel >=0.40.0"
 ]
 
 [project.scripts]
@@ -66,6 +95,15 @@ rook = "rook.cli:cli"
 
 [tool]
 
+[tool.black]
+target-version = [
+  "py39",
+  "py310",
+  "py311",
+  "py312"
+]
+
+
 [tool.bumpversion]
 current_version = "0.14.0"
 commit = true
@@ -97,6 +135,9 @@ omit = ["tests/*.py"]
 relative_files = true
 source = ["rook"]
 
+[tool.flit.module]
+name = "rook"
+
 [tool.flit.sdist]
 include = [
   ".zenodo.json",
@@ -184,10 +225,20 @@ extend-select = [
   "RUF022" # unsorted-dunder-all
 ]
 ignore = [
+  "BLE001", # blind-except
   "COM", # commas
+  "D100", # undocumented-public-module
+  "D101", # undocumented-public-class
+  "D102", # undocumented-public-method
+  "D103", # undocumented-public-function
+  "D104", # undocumented-public-package
   "D205", # blank-line-after-summary
   "D400", # ends-in-period
-  "D401" # non-imperative-mood
+  "D401", # non-imperative-mood
+  "G004", # logging-fstring-interpolation
+  "PERF203", # try-except-in-loop
+  "S110", # try-except-pass
+  "S112", # try-except-continue
 ]
 preview = true
 select = [

requirements_upstream.txt

@@ -0,0 +1,3 @@
+daops @ git+https://github.com/roocs/daops.git@regrid-main-ce#egg=daops
+clisops @ git+https://github.com/roocs/clisops.git@master#egg=clisops
+roocs-utils @ git+https://github.com/roocs/roocs-utils.git@master#egg=roocs_utils

src/rook/__init__.py

@@ -3,7 +3,7 @@
 ###################################################################################
 # Apache Software License 2.0
 #
-# Copyright (c) 2024, Carsten Ehbrecht, Eleanor Smith, Ag Stephens
+# Copyright (c) 2024, Carsten Ehbrecht, Eleanor Smith, Ag Stephens, Trevor James Smith
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,10 +18,10 @@
 # limitations under the License.
 ###################################################################################
 
-from .__version__ import __author__, __email__, __version__  # noqa: F401
-
 from roocs_utils.config import get_config
 
+from .__version__ import __author__, __email__, __version__  # noqa: F401
+
 
 # Workaround for roocs_utils to not re-import rook
 class Package:
@@ -31,4 +31,4 @@ class Package:
 package = Package()
 CONFIG = get_config(package)
 
-from .wsgi import application  # noqa: F401
+from .wsgi import application  # noqa: E402

src/rook/catalog/__init__.py

@@ -1,8 +1,9 @@
 from roocs_utils.exceptions import InvalidCollection
 
-from .db import DBCatalog
 from rook import CONFIG
 
+from .db import DBCatalog
+
 
 def get_catalog(project):
     if CONFIG[f"project:{project}"].get("use_catalog"):
@@ -14,6 +15,6 @@ def get_catalog(project):
 
 
 __all__ = [
-    "get_catalog",
     "DBCatalog",
+    "get_catalog",
 ]

src/rook/catalog/db.py

@@ -1,16 +1,10 @@
 import pandas as pd
 import sqlalchemy
-from pywps.dblog import get_session
-from sqlalchemy.types import DateTime
-from sqlalchemy.types import Integer
-from sqlalchemy.types import String
-from sqlalchemy.types import Text
-
 from daops.catalog.base import Catalog
 from daops.catalog.intake import IntakeCatalog
-from daops.catalog.util import MAX_DATETIME
-from daops.catalog.util import MIN_DATETIME
-from daops.catalog.util import parse_time
+from daops.catalog.util import MAX_DATETIME, MIN_DATETIME, parse_time
+from pywps.dblog import get_session
+from sqlalchemy.types import DateTime, Integer, String, Text
 
 
 class DBCatalog(Catalog):
@@ -62,24 +56,24 @@ def to_db(self):
             session.close()
 
     def _query(self, collection, time=None, time_components=None):
-        """
-        https://stackoverflow.com/questions/8603088/sqlalchemy-in-clause
-        """
+        """https://stackoverflow.com/questions/8603088/sqlalchemy-in-clause"""
         self.update()
         start, end = parse_time(time, time_components)
 
         session = get_session()
         try:
             if len(collection) > 1:
+                # FIXME: This is vulnerable to SQL injection
                 query_ = (
-                    f"SELECT * FROM {self.table_name} WHERE ds_id IN {tuple(collection)} "
-                    f"and end_time>='{start}' and start_time<='{end}'"
+                    f"SELECT * FROM {self.table_name} WHERE ds_id IN {tuple(collection)} "  # noqa: S608
+                    f"and end_time>='{start}' and start_time<='{end}'"  # noqa: S608
                 )
 
             else:
-                query_ = (
-                    f"SELECT * FROM {self.table_name} WHERE ds_id='{collection[0]}' "
-                    f"and end_time>='{start}' and start_time<='{end}'"
+                # FIXME: This is vulnerable to SQL injection
+                query_ = (  # noqa: S608
+                    f"SELECT * FROM {self.table_name} WHERE ds_id='{collection[0]}' "  # noqa: S608
+                    f"and end_time>='{start}' and start_time<='{end}'"  # noqa: S608
                 )
             result = session.execute(query_).fetchall()