Merge pull request #490 from sentinel-hub/develop

Release version 1.3.0

.flake8

@@ -1,8 +1,9 @@
 [flake8]
-ignore = E203, W503
+ignore = E203, W503, C408
 exclude = .git, __pycache__, build, dist
 max-line-length= 120
 max-complexity = 15
+min_python_version = 3.7.0
 per-file-ignores =
     # imported but unused
     __init__.py: F401

.github/workflows/ci_action.yml

@@ -4,60 +4,76 @@ on:
   pull_request:
   push:
     branches:
-      - 'master'
-      - 'develop'
+      - "master"
+      - "develop"
   schedule:
-    - cron: '0 0 * * *'
+    - cron: "0 0 * * *"
 
 env:
   # The only way to simulate if-else statement
   CHECKOUT_BRANCH: ${{ github.event_name == 'schedule' && 'develop' || github.ref }}
 
-
 jobs:
-
-  check-code-black-isort-flake8:
+  check-pre-commit-hooks:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
         uses: actions/checkout@v2
         with:
           ref: ${{ env.CHECKOUT_BRANCH }}
-    
+
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
           architecture: x64
 
-      - name: Prepare linters
-        run: pip install black[jupyter] isort flake8 nbqa
+      - name: Prepare pre-commit validators
+        run: |
+          pip install pre-commit
 
-      - name: Check code compliance with black
-        run: black . --check --diff
+      - name: Check code compliance with pre-commit validators
+        run: pre-commit run --all-files
 
-      - name: Check code compliance with isort
-        run: |
-          isort . --check --diff
-          nbqa isort . --nbqa-diff
+  check-code-pylint-and-mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ env.CHECKOUT_BRANCH }}
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+          architecture: x64
 
-      - name: Check code compliance with flake8
+      - name: Install packages
         run: |
-          flake8 .
-          nbqa flake8 . --nbqa-exclude=examples/core/CoreOverview.ipynb
+          sudo apt-get update
+          sudo apt-get install -y build-essential libgdal-dev graphviz proj-bin gcc libproj-dev libspatialindex-dev
+          pip install -r requirements-dev.txt --upgrade
+          python install_all.py -e
+
+      - name: Run pylint
+        run: make pylint
+
+      - name: Run mypy
+        run: mypy core/eolearn/core
 
   test-on-github:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version:
-          - '3.7'
-          - '3.9'
-          - '3.10'
-        include: 
+          - "3.7"
+          - "3.9"
+          - "3.10"
+        include:
           # A flag marks whether full or partial tests should be run
           # We don't run integration tests on pull requests from outside repos, because they don't have secrets
-          - python-version: '3.8'
+          - python-version: "3.8"
             full_test_suite: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
     steps:
       - name: Checkout branch
@@ -86,10 +102,9 @@ jobs:
             --sh_client_secret "${{ secrets.SH_CLIENT_SECRET }}"
           pytest --cov --cov-report=term --cov-report=xml
 
-      - name: Run pylint and reduced tests
+      - name: Run reduced tests
         if: ${{ !matrix.full_test_suite }}
         run: |
-          make pylint
           pytest -m "not sh_integration"
 
       - name: Upload code coverage

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+      - id: debug-statements
+      - id: check-json
+      - id: check-toml
+      - id: check-yaml
+      - id: check-merge-conflict
+      - id: debug-statements
+
+  - repo: https://github.com/psf/black
+    rev: 22.8.0
+    hooks:
+      - id: black
+        language_version: python3
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        name: isort (python)
+
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v1.5.3
+    hooks:
+      - id: autoflake
+        args:
+          [
+            --remove-all-unused-imports,
+            --in-place,
+            --ignore-init-module-imports,
+          ]
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8
+        additional_dependencies:
+          - flake8-bugbear
+          - flake8-comprehensions
+          - flake8-simplify
+          - flake8-typing-imports
+
+  - repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.4.0
+    hooks:
+      - id: nbqa-black
+      - id: nbqa-isort
+      - id: nbqa-flake8

CHANGELOG.md

@@ -1,7 +1,20 @@
+## [Version 1.3.0] - 2022-10-06
+
+- (**codebreaking**) Adapted Sentinel Hub tasks to `sentinelhub-py 3.8.0` which switched to Catalog 1.0.0.
+- (**codebreaking**) Removed support for loading pickled objects in EOPatches (deprecated since version 1.0.0).
+- (**codebreaking**) Various improvements of `FeatureIO` class. Only affects direct use of class.
+- Added type annotations to majority of `eolearn.core`. The types are now exposed via `py.typed` file, which enables use of `mypy`. Added type-checking to CI for the `core` module.
+- Numpy-array based features can now save and load `object` populated arrays.
+- Improved documentation building, fixed links to GitHub.
+- Improved test coverage.
+- Added pre-commit hooks to repository for easier development.
+- Various minor improvements.
+
+
 ## [Version 1.2.1] - 2022-09-12
 
 - Corrected the default for `no_data_value` in `ImportFromTiffTask` and `ExportToTiffTask` to `None`. The previous default of `0` was a poor choice in many scenarios. The switch might alter behavior in existing code.
-- Changed the way `SpatialResizeTask` accepts parameters for the final image size. Now supports resizing by using resolution.  
+- Changed the way `SpatialResizeTask` accepts parameters for the final image size. Now supports resizing by using resolution.
 - Added `ExplodeBandsTask` that explodes a multi-band feature into multiple features.
 - Exposed resampling parameters in Sentinel Hub tasks and included a `geometry` execution parameter.
 - Reworked internal classes `FeatureIO` and `_FeatureDict` to improve types and maintainability.
@@ -38,9 +51,9 @@
 - Large improvements of parallelization in EOExecutor. Introduced the `eolearn.core.utils.parallelize` module, featuring tools for different parallelization modes.
 - Added support for session sharing in `SentinelHubInputTask`, `SentinelHubEvalscriptTask` and `SentinelHubDemTask` by adding a `session_loader` parameter. Session sharing of `sentinelhub-py` is explained [here](https://github.com/sentinel-hub/sentinelhub-py/blob/master/examples/session_sharing.ipynb).
 - Added `SpatialResizeTask` to `eolearn.features.feature_manipulation` for spatially resizing EOPatch features.
-- Improved how `ImportFromTiffTask` reads from remote filesystems. 
+- Improved how `ImportFromTiffTask` reads from remote filesystems.
 - Switched to non-structural hashing of `EONode` class to avoid massive slowdowns in large workflows.
-- Improved procedure for building documentation and displaying of type annotations. 
+- Improved procedure for building documentation and displaying of type annotations.
 - Various minor improvements.
 
 
@@ -182,7 +195,7 @@
 - Added `eolearn.features.DoublyLogisticApproximationTask`, contributed by @bsircelj.
 - Optional parameter `config` for `SaveTask` and `LoadTask` to enable defining custom AWS credentials.
 - Fixed a bug in `eolearn.features.ValueFilloutTask`.
-- Started releasing `eo-learn` (sub)packages also as wheels. 
+- Started releasing `eo-learn` (sub)packages also as wheels.
 - Minor improvements and fixes.
 
 ## [Version 0.7.7] - 2020-08-03
@@ -228,4 +241,4 @@
 - `eolearn.core.EOWorkflow`: fixed generating task dependencies.
 ### Added
 - Processing API docs generation.
-- Introduced CHANGELOG.md.
+- Introduced CHANGELOG.md.

CONTRIBUTING.md

@@ -146,7 +146,7 @@ class FooTask(EOTask):
 When creating a new task, bear in mind the following:
 
 * Tasks should be as modular as possible, facilitating task re-use and sharing.
-* An `EOTask` should perform a well-defined operation on the input eopatch(es). If the operation could be split into atomic sub-operations that could be used separately, then consider splitting the task into multiple tasks. Similarly, if tasks share the bulk of the implementation but differ in a minority of implementation, consider using Base classes and inheritance. The interpolation tasks represent a good example of this.  
+* An `EOTask` should perform a well-defined operation on the input eopatch(es). If the operation could be split into atomic sub-operations that could be used separately, then consider splitting the task into multiple tasks. Similarly, if tasks share the bulk of the implementation but differ in a minority of implementation, consider using Base classes and inheritance. The interpolation tasks represent a good example of this.
 * Tasks should be as generalizable as possible, therefore hard-coding of task parameters or `EOPatch` feature types should be avoided. Use the `EOTask._parse_features` method to parse input features in a task, and pass task parameters as arguments, either in the constructor, or at run-time.
 * If in doubt on whether a task is general enough to be of interest to the community, or you are not sure to which sub-package to contribute your task to, send us an email or open a [feature request](#feature-requests).
 

README.md

@@ -90,9 +90,9 @@ Some subpackages contain extension modules under `extra` subfolder. Those module
 
 ### Conda Forge distribution
 
-The package requires a Python environment **>=3.7**. 
+The package requires a Python environment **>=3.7**.
 
-Thanks to the maintainers of the conda forge feedstock (@benhuff, @dcunn, @mwilson8, @oblute, @rluria14), `eo-learn` can 
+Thanks to the maintainers of the conda forge feedstock (@benhuff, @dcunn, @mwilson8, @oblute, @rluria14), `eo-learn` can
 be installed using `conda-forge` as follows:
 
 ```bash
@@ -123,7 +123,7 @@ docker pull sentinelhub/eolearn:latest
 docker run -p 8888:8888 sentinelhub/eolearn:latest
 ```
 
-An extended version of the `latest` image additionally contains all example notebooks and data to get you started with `eo-learn`. Run it with: 
+An extended version of the `latest` image additionally contains all example notebooks and data to get you started with `eo-learn`. Run it with:
 
 ```bash
 docker pull sentinelhub/eolearn:latest-examples
@@ -164,8 +164,8 @@ If you would like to contribute to `eo-learn`, check out our [contribution guide
  * [Tracking a rapidly changing planet](https://medium.com/@developmentseed/tracking-a-rapidly-changing-planet-bc02efe3545d) (by Development Seed)
  * [Land Cover Monitoring System](https://medium.com/sentinel-hub/land-cover-monitoring-system-84406e3019ae) (by Jovan Visnjic and Matej Aleksandrov)
  * [eo-learn Webinar](https://www.youtube.com/watch?v=Rv-yK7Vbk4o) (by Anze Zupanc)
- * [Cloud Masks at Your Service](https://medium.com/sentinel-hub/cloud-masks-at-your-service-6e5b2cb2ce8a) 
- * [ML examples for Common Agriculture Policy](https://medium.com/sentinel-hub/area-monitoring-concept-effc2c262583) 
+ * [Cloud Masks at Your Service](https://medium.com/sentinel-hub/cloud-masks-at-your-service-6e5b2cb2ce8a)
+ * [ML examples for Common Agriculture Policy](https://medium.com/sentinel-hub/area-monitoring-concept-effc2c262583)
    * [High-Level Concept](https://medium.com/sentinel-hub/area-monitoring-concept-effc2c262583)
    * [Data Handling](https://medium.com/sentinel-hub/area-monitoring-data-handling-c255b215364f)
    * [Outlier detection](https://medium.com/sentinel-hub/area-monitoring-observation-outlier-detection-34f86b7cc63)
@@ -183,8 +183,8 @@ If you would like to contribute to `eo-learn`, check out our [contribution guide
    * [The Challenge of Small Parcels](https://medium.com/sentinel-hub/area-monitoring-the-challenge-of-small-parcels-96121e169e5b)
    * [Traffic Light System](https://medium.com/sentinel-hub/area-monitoring-traffic-light-system-4a1348481c40)
    * [Expert Judgement Application](https://medium.com/sentinel-hub/expert-judgement-application-67a07f2feac4)
- * [Scale-up your eo-learn workflow using Batch Processing API](https://medium.com/sentinel-hub/scale-up-your-eo-learn-workflow-using-batch-processing-api-d183b70ea237) (by Maxim Lamare) 
- 
+ * [Scale-up your eo-learn workflow using Batch Processing API](https://medium.com/sentinel-hub/scale-up-your-eo-learn-workflow-using-batch-processing-api-d183b70ea237) (by Maxim Lamare)
+
 
 ## Questions and Issues
 

core/MANIFEST.in

@@ -1,6 +1,7 @@
 include requirements*.txt
 include LICENSE
 include README.md
+include eolearn/core/py.typed
 exclude eolearn/tests/*
 exclude eolearn/tests/test_extra/*
 exclude eolearn/tests/test_utils/*

core/eolearn/core/__init__.py

@@ -32,4 +32,4 @@
 from .utils.parallelize import execute_with_mp_lock, join_futures, join_futures_iter, parallelize
 from .utils.parsing import FeatureParser
 
-__version__ = "1.2.1"
+__version__ = "1.3.0"