CI: chain requirements to ensure compatibility (#3112)

*Issue #, if available:* atm, all install requirements are installed as stand-alone, which means if there is a version collision, `pip` does not care about it, and so for eventual version compatibility, a collision may not be discovered *Description of changes:* just install all needed requirements for the particular job at once so pip sees all required packages and finds an intersection among all :flags: By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. cc: @jaheba @kashif @lostella kind ling as it seems there are no revivers by default 🐿️ **Please tag this pr with at least one of these labels to make our release process faster:** BREAKING, new feature, bug fix, other change, dev setup
awslabs · Feb 7, 2024 · bff165f · bff165f
1 parent 9ea944b
commit bff165f
Show file tree

Hide file tree

Showing 15 changed files with 70 additions and 84 deletions.
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -27,13 +27,12 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install -U pip
-        pip install ".[arrow]"
-        pip install -r requirements/requirements-docs.txt
-        pip install -r requirements/requirements-mxnet.txt
-        pip install -r requirements/requirements-pytorch.txt
-        pip install -r requirements/requirements-rotbaum.txt
-        pip install -r requirements/requirements-extras-statsforecast.txt
+        pip install ".[arrow]" \
+          -r requirements/requirements-docs.txt \
+          -r requirements/requirements-mxnet.txt \
+          -r requirements/requirements-pytorch.txt \
+          -r requirements/requirements-rotbaum.txt \
+          -r requirements/requirements-extras-statsforecast.txt
 
     - name: Install Pandoc
       run: |

diff --git a/.github/workflows/doctests.yaml b/.github/workflows/doctests.yaml
@@ -21,10 +21,9 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install other dependencies
       run: |
-        python -m pip install -U pip
-        pip install -e .
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-arrow.txt
+        pip install -e . \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-arrow.txt
     - name: Test with pytest
       run: |
         pytest -n2 --doctest-modules \

diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml
@@ -14,7 +14,6 @@ jobs:
         python-version: '3.8'
     - name: Install dependencies
       run: |
-        python -m pip install -U pip
         pip install ruff
         pip install docformatter[tomli]==1.5.0
     - name: Ruff (Flake8)

diff --git a/.github/workflows/mxnet_nightly.yml b/.github/workflows/mxnet_nightly.yml
@@ -24,17 +24,16 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install pip==20.2
+        pip install pip==20.2
         pip install --pre "mxnet<2" -f https://dist.mxnet.io/python
-        pip install torch==1.10.0+cpu --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html
-        pip install pytorch-lightning~=1.5
-        pip install -e ".[shell]"
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-extras-sagemaker-sdk.txt
-        pip install -r requirements/requirements-extras-m-competitions.txt
-        pip install -r requirements/requirements-rotbaum.txt
-        pip install -r requirements/requirements-extras-anomaly-evaluation.txt
-        pip install -r requirements/requirements-extras-autogluon.txt
+        pip install "torch==1.10.0" "pytorch-lightning~=1.5" -f https://download.pytorch.org/whl/cpu/torch_stable.html
+        pip install -e ".[shell]" \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-extras-sagemaker-sdk.txt \
+          -r requirements/requirements-extras-m-competitions.txt \
+          -r requirements/requirements-rotbaum.txt \
+          -r requirements/requirements-extras-anomaly-evaluation.txt \
+          -r requirements/requirements-extras-autogluon.txt
     - name: Test with pytest
       run: |
         pytest -m 'not (gpu or serial)' --cov src/gluonts --cov-report=term --cov-report xml test
diff --git a/.github/workflows/style_type_checks.yml b/.github/workflows/style_type_checks.yml
@@ -17,7 +17,6 @@ jobs:
         python-version: '3.8'
     - name: Install dependencies
       run: |
-        python -m pip install -U pip
         pip install .
         pip install click black mypy
         pip install types-python-dateutil

diff --git a/.github/workflows/test_release_unix_nightly.yml b/.github/workflows/test_release_unix_nightly.yml
@@ -23,17 +23,16 @@ jobs:
       run: |
         git clone https://github.com/awslabs/gluonts --branch $(curl https://api.github.com/repos/awslabs/gluonts/releases/latest | grep tag_name | cut -d : -f 2,3 | tr -d \"\ | tr -d \,\ )
         cd gluon-ts
-        python -m pip install pip==20.2
-        pip install mxnet~=1.8.0
-        pip install torch==1.10.0+cpu --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html
-        pip install pytorch-lightning~=1.5
-        pip install -e ".[shell]"
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-extras-sagemaker-sdk.txt
-        pip install -r requirements/requirements-extras-m-competitions.txt
-        pip install -r requirements/requirements-rotbaum.txt
-        pip install -r requirements/requirements-extras-anomaly-evaluation.txt
-        pip install -r requirements/requirements-extras-autogluon.txt
+        pip install pip==20.2
+        pip install "mxnet~=1.8.0" "torch==1.10.0" "pytorch-lightning~=1.5" \
+          -f https://download.pytorch.org/whl/cpu/torch_stable.html
+        pip install -e ".[shell]" \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-extras-sagemaker-sdk.txt \
+          -r requirements/requirements-extras-m-competitions.txt \
+          -r requirements/requirements-rotbaum.txt \
+          -r requirements/requirements-extras-anomaly-evaluation.txt \
+          -r requirements/requirements-extras-autogluon.txt
     - name: Test with pytest
       run: |
         cd gluon-ts

diff --git a/.github/workflows/test_release_win32_nightly.yml b/.github/workflows/test_release_win32_nightly.yml
@@ -25,17 +25,14 @@ jobs:
         $tmp=$tmp.psobject.properties.value.trim() 
         git clone https://github.com/awslabs/gluonts --branch $tmp
         cd gluon-ts
-        python -m pip install -U pip
-        pip install mxnet~=1.7.0
-        pip install torch
-        pip install pytorch-lightning~=1.5
-        pip install -e ".[shell]"
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-extras-sagemaker-sdk.txt
-        pip install -r requirements/requirements-extras-m-competitions.txt
-        pip install -r requirements/requirements-rotbaum.txt
-        pip install -r requirements/requirements-extras-anomaly-evaluation.txt
-        pip install -r requirements/requirements-extras-autogluon.txt
+        pip install "mxnet~=1.7.0" torch "pytorch-lightning~=1.5"
+        pip install -e ".[shell]" \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-extras-sagemaker-sdk.txt \
+          -r requirements/requirements-extras-m-competitions.txt \
+          -r requirements/requirements-rotbaum.txt \
+          -r requirements/requirements-extras-anomaly-evaluation.txt \
+          -r requirements/requirements-extras-autogluon.txt
       shell: powershell
     - name: Test with pytest
       run: |

diff --git a/.github/workflows/tests-nixtla.yaml b/.github/workflows/tests-nixtla.yaml
@@ -21,11 +21,10 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install other dependencies
       run: |
-        python -m pip install -U pip
-        pip install .
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-extras-statsforecast.txt
-        pip install -r requirements/requirements-extras-hierarchicalforecast.txt
+        pip install . \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-extras-statsforecast.txt \
+          -r requirements/requirements-extras-hierarchicalforecast.txt
     - name: Test with pytest
       run: |
         pytest -n2 --doctest-modules test/ext/statsforecast

diff --git a/.github/workflows/tests-nursery.yml b/.github/workflows/tests-nursery.yml
@@ -29,15 +29,16 @@ jobs:
       run: pip install mxnet~=1.7.0
     - name: Install other dependencies
       run: |
-        python -m pip install -U pip
-        pip install torch==1.10.0+cpu --no-cache-dir -f https://download.pytorch.org/whl/cpu/torch_stable.html
-        pip install pytorch-lightning~=1.5
-        pip install -e ".[shell]"
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-extras-sagemaker-sdk.txt
-        pip install -r requirements/requirements-extras-m-competitions.txt
-        pip install -r requirements/requirements-extras-anomaly-evaluation.txt
-        pip install -r requirements/requirements-extras-autogluon.txt
+        pip install "torch==1.10.0" "pytorch-lightning~=1.5" \
+          -f https://download.pytorch.org/whl/cpu/torch_stable.html
+        pip install 
+        pip install -e ".[shell]" \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-extras-sagemaker-sdk.txt \
+          -r requirements/requirements-extras-m-competitions.txt \
+          -r requirements/requirements-rotbaum.txt \
+          -r requirements/requirements-extras-anomaly-evaluation.txt \
+          -r requirements/requirements-extras-autogluon.txt
     - name: Test with pytest
       run: |
         pytest -n2 --doctest-modules test/nursery
diff --git a/.github/workflows/tests-prophet.yaml b/.github/workflows/tests-prophet.yaml
@@ -21,10 +21,9 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install other dependencies
       run: |
-        python -m pip install -U pip
-        pip install .
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-extras-prophet.txt
+        pip install . \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-extras-prophet.txt
     - name: Test with pytest
       run: |
         pytest -n2 --doctest-modules test/ext/prophet
diff --git a/.github/workflows/tests-r.yml b/.github/workflows/tests-r.yml
@@ -38,10 +38,9 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install Python dependencies
       run: |
-        python -m pip install -U pip
-        pip install .
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-extras-r.txt
+        pip install . \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-extras-r.txt
     - name: Test with pytest
       run: |
         pytest -n2 test/ext/r_forecast
diff --git a/.github/workflows/tests-torch.yml b/.github/workflows/tests-torch.yml
@@ -21,12 +21,11 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install -U pip
-        pip install ".[arrow]"
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-extras-m-competitions.txt
-        pip install -r requirements/requirements-pytorch.txt
-        pip install -r requirements/requirements-extras-cpflows.txt
+        pip install ".[arrow]" \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-extras-m-competitions.txt \
+          -r requirements/requirements-pytorch.txt \
+          -r requirements/requirements-extras-cpflows.txt
     - name: Test with pytest
       run: |
         pytest -n2 --doctest-modules --ignore test/nursery test
diff --git a/.github/workflows/tests-xgboost.yaml b/.github/workflows/tests-xgboost.yaml
@@ -21,12 +21,11 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install other dependencies
       run: |
-        python -m pip install -U pip setuptools wheel
-        python -m pip install -U "Cython<3.0"
-        pip install .
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-rotbaum.txt
-        pip install -r requirements/requirements-rotbaum-extra-methods.txt
+        pip install "Cython<3.0"
+        pip install . \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-rotbaum.txt \
+          -r requirements/requirements-rotbaum-extra-methods.txt
     - name: Test with pytest
       run: |
         pytest -n2 --timeout 120 --doctest-modules test/ext/rotbaum
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -21,10 +21,9 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install other dependencies
       run: |
-        python -m pip install -U pip
-        pip install ".[mxnet,arrow,shell]"
-        pip install -r requirements/requirements-test.txt
-        pip install -r requirements/requirements-extras-m-competitions.txt
+        pip install ".[mxnet,arrow,shell]" \
+          -r requirements/requirements-test.txt \
+          -r requirements/requirements-extras-m-competitions.txt
     - name: Test with pytest
       run: |
         pytest -n2 --doctest-modules --ignore test/nursery test
diff --git a/requirements/requirements-rotbaum-extra-methods.txt b/requirements/requirements-rotbaum-extra-methods.txt
@@ -1,2 +1,2 @@
 lightgbm>=2.3.1
-scikit-garden>=0.1.2
+# scikit-garden @ https://github.com/scikit-garden/scikit-garden/archive/refs/heads/master.zip