Dependency cleanup and Python 3.12 support

.github/workflows/ci-fast.yml

@@ -87,7 +87,8 @@ jobs:
     strategy:
       matrix:
         os: [arc-runner-set]
-        python-version: ['3.10']
+        # TODO: REVERT
+        python-version: ['3.12']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
@@ -99,7 +100,8 @@ jobs:
     strategy:
       matrix:
         os: [arc-runner-set]
-        python-version: ['3.10']
+        # TODO: REVERT
+        python-version: ['3.12']
         test_environment: [default, docker-server-docker-orchestrator-mysql]
       fail-fast: false
     uses: ./.github/workflows/integration-test-fast.yml

.github/workflows/ci-slow.yml

@@ -199,7 +199,9 @@ jobs:
     strategy:
       matrix:
         os: [arc-runner-set]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        # python-version: ['3.8', '3.9', '3.10', '3.11']
+        # TODO: REVERT
+        python-version: ['3.12']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
@@ -212,7 +214,9 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        # python-version: ['3.8', '3.9', '3.10', '3.11']
+        # TODO: REVERT
+        python-version: ['3.12']
       fail-fast: false
     uses: ./.github/workflows/unit-test.yml
     with:
@@ -225,7 +229,9 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        # python-version: ['3.8', '3.9', '3.10', '3.11']
+        # TODO: REVERT
+        python-version: ['3.12']
         # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64)
         # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760
         exclude:
@@ -246,7 +252,9 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        # python-version: ['3.8', '3.9', '3.10', '3.11']
+        # TODO: REVERT
+        python-version: ['3.12']
         test_environment: [default]
       fail-fast: false
     uses: ./.github/workflows/integration-test-slow.yml
@@ -261,7 +269,9 @@ jobs:
     strategy:
       matrix:
         os: [macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        # python-version: ['3.8', '3.9', '3.10', '3.11']
+        # TODO: REVERT
+        python-version: ['3.12']
         # Python 3.8 and 3.9 is on macos-13 but not macos-latest (macos-14-arm64)
         # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760
         exclude:

.github/workflows/integration-test-fast.yml

@@ -36,9 +36,10 @@ on:
       python-version:
         description: Python version
         type: choice
-        options: ['3.8', '3.9', '3.10', '3.11']
+        # TODO: REVERT
+        options: ['3.12']
         required: false
-        default: '3.8'
+        default: '3.12'
       test_environment:
         description: The test environment
         type: choice

.github/workflows/integration-test-slow.yml

@@ -36,9 +36,10 @@ on:
       python-version:
         description: Python version
         type: choice
-        options: ['3.8', '3.9', '3.10', '3.11']
+        # TODO: REVERT
+        options: ['3.12']
         required: false
-        default: '3.8'
+        default: '3.12'
       test_environment:
         description: The test environment
         type: choice

.github/workflows/templates-test.yml

@@ -22,9 +22,10 @@ on:
       python-version:
         description: Python version
         type: choice
-        options: ['3.8', '3.9', '3.10', '3.11']
+        # TODO: REVERT
+        options: ['3.12']
         required: false
-        default: '3.8'
+        default: '3.12'
 jobs:
   all-template-tests:
     name: all-template-tests

.github/workflows/unit-test.yml

@@ -37,9 +37,10 @@ on:
       python-version:
         description: Python version
         type: choice
-        options: ['3.8', '3.9', '3.10', '3.11']
+        # TODO: REVERT
+        options: ['3.12']
         required: false
-        default: '3.8'
+        default: '3.12'
       enable_tmate:
         description: Enable tmate session for debugging
         type: choice

pyproject.toml

@@ -44,26 +44,20 @@ zenml = "zenml.cli.cli:cli"
 alembic = { version = "~1.8.1" }
 bcrypt = { version = "4.0.1" }
 click = "^8.0.1,<8.1.4"
-click-params = "^0.3.0"
 cloudpickle = ">=2.0.0,<3"
 distro = "^1.6.0"
 docker = "~7.1.0"
-httplib2 = "<0.20,>=0.19.1"
 gitpython = "^3.1.18"
-numpy = "<2.0.0"
 packaging = ">=24.1"
-pandas = ">=1.1.5"
 passlib = { extras = ["bcrypt"], version = "~1.7.4" }
 psutil = ">=5.0.0"
 pydantic = "~2.7"
 pydantic-settings = "~2.2.1"
 pymysql = { version = "~1.1.0,>=1.1.1" }
-pyparsing = ">=2.4.0"
-python = ">=3.8,<3.12"
+python = ">=3.8,<3.13"
 python-dateutil = "^2.8.1"
 pyyaml = ">=6.0.1"
 rich = { extras = ["jupyter"], version = ">=12.0.0" }
-secure = "~0.3.0"
 sqlalchemy = "^2.0.0"
 sqlalchemy_utils = "*"
 sqlmodel = "0.0.18"
@@ -77,6 +71,7 @@ pyjwt = { extras = ["crypto"], version = "2.7.*", optional = true }
 orjson = { version = "~3.10.0", optional = true }
 Jinja2 = { version = "*", optional = true }
 ipinfo = { version = ">=4.4.3", optional = true }
+secure = { version = "~0.3.0" , optional = true }
 
 # Optional dependencies for project templates
 copier = { version = ">=8.1.0", optional = true }
@@ -137,6 +132,7 @@ typing-extensions = { version = ">=3.7.4", optional = true }
 darglint = { version = "^1.8.1", optional = true }
 ruff = { version = ">=0.1.7", optional = true }
 yamlfix = { version = "^1.16.0", optional = true }
+maison= { version = "^1.4.3", optional = true }
 
 # pytest
 pytest = { version = "^7.4.0", optional = true }
@@ -153,6 +149,7 @@ mkdocs-material = { version = "^8.1.7", optional = true }
 mkdocs-awesome-pages-plugin = { version = "^2.6.1", optional = true }
 mkdocstrings = { version = "^0.17.0", optional = true }
 mike = { version = "^1.1.2", optional = true }
+
 # mypy type stubs
 types-certifi = { version = "^2021.10.8.0", optional = true }
 types-croniter = { version = "^1.0.2", optional = true }
@@ -186,6 +183,7 @@ server = [
     "orjson",
     "Jinja2",
     "ipinfo",
+    "secure",
 ]
 templates = ["copier", "jinja2-time", "ruff", "pyyaml-include"]
 terraform = ["python-terraform"]
@@ -237,6 +235,7 @@ dev = [
     "mkdocs-awesome-pages-plugin",
     "mkdocstrings",
     "mike",
+    "maison",
     "types-certifi",
     "types-croniter",
     "types-futures",

scripts/install-zenml-dev.sh

@@ -28,15 +28,16 @@ parse_args () {
 
 install_zenml() {
     # install ZenML in editable mode
-    uv pip install $PIP_ARGS -e ".[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev,mlstacks]"
+    uv pip install $PIP_ARGS -e ".[server,templates,terraform,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,dev]"
 }
 
 install_integrations() {
 
     # figure out the python version
     python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")
 
-    ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure pigeon prodigy"
+    # TODO: Add mlflow, huggingface and tensorflow back in
+    ignore_integrations="feast label_studio bentoml seldon pycaret skypilot_aws skypilot_gcp skypilot_azure pigeon prodigy mlflow tensorflow huggingface"
 
     # turn the ignore integrations into a list of --ignore-integration args
     ignore_integrations_args=""

src/zenml/feature_stores/base_feature_store.py

@@ -14,9 +14,7 @@
 """The base class for feature stores."""
 
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Type, Union, cast
-
-import pandas as pd
+from typing import Any, Dict, List, Type, cast
 
 from zenml.enums import StackComponentType
 from zenml.stack import Flavor, StackComponent
@@ -42,10 +40,10 @@ def config(self) -> BaseFeatureStoreConfig:
     @abstractmethod
     def get_historical_features(
         self,
-        entity_df: Union[pd.DataFrame, str],
+        entity_df: Any,
         features: List[str],
         full_feature_names: bool = False,
-    ) -> pd.DataFrame:
+    ) -> Any:
         """Returns the historical features for training or batch scoring.
 
         Args:
@@ -54,7 +52,7 @@ def get_historical_features(
             full_feature_names: Whether to return the full feature names.
 
         Returns:
-            The historical features as a Pandas DataFrame.
+            The historical features.
         """
 
     @abstractmethod

src/zenml/integrations/__init__.py

@@ -48,7 +48,9 @@
 from zenml.integrations.mlflow import MlflowIntegration  # noqa
 from zenml.integrations.neptune import NeptuneIntegration  # noqa
 from zenml.integrations.neural_prophet import NeuralProphetIntegration  # noqa
-from zenml.integrations.openai import OpenAIIntegration  # noqa
+from zenml.integrations.numpy import NumpyIntegration  # noqa
+from zenml.integrations.openai import OpenAIIntegration  #
+from zenml.integrations.pandas import PandasIntegration  # noqa
 from zenml.integrations.pigeon import PigeonIntegration # noqa
 from zenml.integrations.pillow import PillowIntegration  # noqa
 from zenml.integrations.polars import PolarsIntegration  # noqa

src/zenml/integrations/constants.py

@@ -44,7 +44,9 @@
 MLFLOW = "mlflow"
 NEPTUNE = "neptune"
 NEURAL_PROPHET = "neural_prophet"
+NUMPY = "numpy"
 OPEN_AI = "openai"
+PANDAS = "pandas"
 PIGEON = "pigeon"
 PILLOW = "pillow"
 PLOTLY = "plotly"

src/zenml/integrations/databricks/__init__.py

@@ -30,7 +30,9 @@ class DatabricksIntegration(Integration):
     """Definition of Databricks Integration for ZenML."""
 
     NAME = DATABRICKS
-    REQUIREMENTS = ["databricks-sdk==0.28.0"]
+    REQUIREMENTS = ["databricks-sdk==0.28.0", "numpy<2.0.0", "pandas>=2.0.0"]
+
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["numpy", "pandas"]
 
     @classmethod
     def flavors(cls) -> List[Type[Flavor]]:

src/zenml/integrations/deepchecks/__init__.py

@@ -38,10 +38,13 @@ class DeepchecksIntegration(Integration):
     REQUIREMENTS = [
         "deepchecks[vision]==0.8.0",
         "torchvision>=0.14.0",
-        "pandas<2.0.0",
         "opencv-python==4.5.5.64",  # pin to same version
         "opencv-python-headless==4.5.5.64",  # pin to same version
         "tenacity!=8.4.0",  # https://github.com/jd/tenacity/issues/471
+        # Deepchecks materializer inherits from the PandasMaterializer, thus
+        # we need to put the requirements of the pandas integration here as
+        # well.
+        "pandas>=2.0.0",
     ]
     APT_PACKAGES = ["ffmpeg", "libsm6", "libxext6"]
     REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas","torchvision","tenacity"]

src/zenml/integrations/deepchecks/materializers/deepchecks_dataset_materializer.py

@@ -18,7 +18,9 @@
 from deepchecks.tabular import Dataset
 
 from zenml.enums import ArtifactType, VisualizationType
-from zenml.materializers.pandas_materializer import PandasMaterializer
+from zenml.integrations.pandas.materializers.pandas_materializer import (
+    PandasMaterializer,
+)
 
 if TYPE_CHECKING:
     from zenml.metadata.metadata_types import MetadataType

src/zenml/integrations/evidently/__init__.py

@@ -56,8 +56,9 @@ class EvidentlyIntegration(Integration):
     REQUIREMENTS = [
         "evidently>=0.4.16,<=0.4.22",
         "tenacity!=8.4.0",  # https://github.com/jd/tenacity/issues/471
+        "pandas>=2.0.0",
     ]
-    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["tenacity"]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["tenacity", "pandas"]
 
     @classmethod
     def flavors(cls) -> List[Type[Flavor]]:

src/zenml/integrations/facets/__init__.py

@@ -21,7 +21,7 @@ class FacetsIntegration(Integration):
     """Definition of Facets integration for ZenML."""
 
     NAME = FACETS
-    REQUIREMENTS = ["facets-overview>=1.0.0", "pandas"]
+    REQUIREMENTS = ["facets-overview>=1.0.0", "pandas>=2.0.0"]
     REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas"]
 
     @staticmethod

src/zenml/integrations/feast/__init__.py

@@ -33,8 +33,8 @@ class FeastIntegration(Integration):
     NAME = FEAST
     # click is added to keep the feast click version
     # in sync with ZenML's click
-    REQUIREMENTS = ["feast<=0.37.1", "click>=8.0.1,<8.1.4"]
-    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["click"]
+    REQUIREMENTS = ["feast<=0.37.1", "click>=8.0.1,<8.1.4", "pandas>=2.0.0"]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["click", "pandas"]
 
     @classmethod
     def flavors(cls) -> List[Type[Flavor]]:

src/zenml/integrations/great_expectations/__init__.py

@@ -32,9 +32,12 @@ class GreatExpectationsIntegration(Integration):
 
     NAME = GREAT_EXPECTATIONS
     REQUIREMENTS = [
-        "great-expectations>=0.17.15,<1.0",
+        "great-expectations>=0.17.15,<1.0","pandas>=2.0.0"
     ]
 
+
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["pandas"]
+
     @staticmethod
     def activate() -> None:
         """Activate the Great Expectations integration."""

src/zenml/integrations/huggingface/materializers/huggingface_datasets_materializer.py

@@ -31,9 +31,11 @@
 from datasets.dataset_dict import DatasetDict
 
 from zenml.enums import ArtifactType, VisualizationType
+from zenml.integrations.pandas.materializers.pandas_materializer import (
+    PandasMaterializer,
+)
 from zenml.io import fileio
 from zenml.materializers.base_materializer import BaseMaterializer
-from zenml.materializers.pandas_materializer import PandasMaterializer
 from zenml.utils import io_utils
 
 if TYPE_CHECKING:

src/zenml/integrations/mlflow/__init__.py

@@ -36,6 +36,8 @@ class MlflowIntegration(Integration):
         "mlflow>=2.1.1,<3",
         "mlserver>=1.3.3",
         "mlserver-mlflow>=1.3.3",
+        "numpy<2.0.0",
+        "pandas>=2.0.0",
         # TODO: remove this requirement once rapidjson is fixed
         "python-rapidjson<1.15",
         # When you do:
@@ -48,7 +50,7 @@ class MlflowIntegration(Integration):
         "pydantic>=2.7.0,<2.8.0",
     ]
 
-    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["python-rapidjson", "pydantic"]
+    REQUIREMENTS_IGNORED_ON_UNINSTALL = ["python-rapidjson", "pydantic", 'numpy', 'pandas']
 
     @classmethod
     def activate(cls) -> None: