Refactor pipeline functions and update project naming conventions to …

…'gitguarden'

.github/workflows/run_train_deploy_pipeline.yml

@@ -0,0 +1,55 @@
+name: Staging Trigger Train and Deploy Pipeline
+on:
+  pull_request:
+    types: [opened, synchronize]
+    branches: [staging, main]
+    paths:
+      - 'train_and_deploy/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-staging-workflow:
+    runs-on: ubuntu-latest
+    env:
+      ZENML_HOST: ${{ secrets.ZENML_HOST }}
+      ZENML_API_KEY: ${{ secrets.ZENML_API_KEY }}
+      ZENML_STAGING_STACK: ef6c474d-b6e7-49a7-a046-0dab39f7969a
+      ZENML_GITHUB_SHA: ${{ github.event.pull_request.head.sha }}
+      ZENML_GITHUB_URL_PR: ${{ github.event.pull_request._links.html.href }}
+      ZENML_DEBUG: true
+      ZENML_ANALYTICS_OPT_IN: false
+      ZENML_LOGGING_VERBOSITY: INFO
+      ZENML_PROJECT_SECRET_NAME: llm-complete
+      ZENML_DISABLE_CLIENT_SERVER_MISMATCH_WARNING: True
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install requirements
+        working-directory: ./train_and_deploy
+        run: |
+          pip3 install -r requirements.txt
+          zenml integration install bentoml skypilot_kubernetes  s3 aws  evidently --uv -y
+
+      - name: Connect to ZenML server
+        working-directory: ./train_and_deploy
+        run: |
+          zenml init
+
+      - name: Set stack (Staging)
+        working-directory: ./train_and_deploy
+        run: |
+          zenml stack set ${{ env.ZENML_STAGING_STACK }}
+
+      - name: Run pipeline (Staging)
+        working-directory: ./train_and_deploy
+        run: |
+          python run.py --training

train_and_deploy/README.md

@@ -7,7 +7,7 @@ classification datasets provided by the scikit-learn library. The project was
 generated from the [E2E Batch ZenML project template](https://github.com/zenml-io/template-e2e-batch)
 with the following properties:
 - Project name: ZenML E2E project
-- Technical Name: e2e_use_case
+- Technical Name: gitguarden
 - Version: `0.0.1`
 - Licensed with apache to ZenML GmbH<>
 - Deployment environment: `staging`

train_and_deploy/configs/deployer_config.yaml

@@ -18,14 +18,13 @@
 # environment configuration
 settings:
   docker:
+    python_package_installer: uv
     required_integrations:
       - aws
-      - evidently
-      - mlflow
       - sklearn
-      - slack
       - bentoml
 
+
 # configuration of steps  
 steps:
   notify_on_success:
@@ -34,8 +33,8 @@ steps:
 
 # configuration of the Model Control Plane
 model:
-  name: e2e_use_case
-  version: production
+  name: gitguarden
+  version: staging
 
 # pipeline level extra configurations
 extra:

train_and_deploy/configs/inference_config.yaml

@@ -18,10 +18,8 @@
 # environment configuration
 settings:
   docker:
+    python_package_installer: uv
     required_integrations:
-      - gcp
-      - evidently
-      - mlflow
       - sklearn
       - slack
       - bentoml
@@ -34,7 +32,7 @@ steps:
 
 # configuration of the Model Control Plane
 model:
-  name: e2e_use_case
+  name: gitguarden
   version: staging
 
 # pipeline level extra configurations

train_and_deploy/configs/train_config.yaml

@@ -18,10 +18,8 @@
 # environment configuration
 settings:
   docker:
+    python_package_installer: uv
     required_integrations:
-      - gcp
-      - evidently
-      - mlflow
       - sklearn
       - slack
       - bentoml
@@ -30,19 +28,19 @@ settings:
 steps:
   model_trainer:
     parameters:
-      name: e2e_use_case
+      name: gitguarden
   promote_with_metric_compare:
     parameters:
-      mlflow_model_name: e2e_use_case
+      mlflow_model_name: gitguarden
   notify_on_success:
     parameters:
       notify_on_success: False
 
 # configuration of the Model Control Plane
 model:
-  name: e2e_use_case
+  name: gitguarden
   license: apache
-  description: e2e_use_case E2E Batch Use Case
+  description: gitguarden E2E Batch Use Case
   audience: All ZenML users
   use_cases: |
     The ZenML E2E project project demonstrates how the most important steps of 

train_and_deploy/pipelines/__init__.py

@@ -16,6 +16,6 @@
 #
 
 
-from .batch_inference import e2e_use_case_batch_inference
-from .training import e2e_use_case_training
-from .deployment import e2e_use_case_deployment
+from .batch_inference import gitguarden_batch_inference
+from .training import gitguarden_training
+from .local_deployment import gitguarden_local_deployment

train_and_deploy/pipelines/batch_inference.py

@@ -33,7 +33,7 @@
 
 
 @pipeline(on_failure=notify_on_failure)
-def e2e_use_case_batch_inference():
+def gitguarden_batch_inference():
     """
     Model batch inference pipeline.
 

train_and_deploy/pipelines/deployment.py → train_and_deploy/pipelines/local_deployment.py

@@ -21,7 +21,7 @@
 
 
 @pipeline(on_failure=notify_on_failure, enable_cache=False)
-def e2e_use_case_deployment():
+def gitguarden_local_deployment():
     """
     Model deployment pipeline.
 

train_and_deploy/pipelines/training.py

@@ -32,15 +32,14 @@
     train_data_preprocessor,
     train_data_splitter,
 )
-
 from zenml import pipeline
 from zenml.logger import get_logger
 
 logger = get_logger(__name__)
 
 
 @pipeline(on_failure=notify_on_failure)
-def e2e_use_case_training(
+def gitguarden_training(
     model_search_space: Dict[str, Any],
     target_env: str,
     test_size: float = 0.2,
@@ -51,8 +50,7 @@ def e2e_use_case_training(
     min_test_accuracy: float = 0.0,
     fail_on_accuracy_quality_gates: bool = False,
 ):
-    """
-    Model training pipeline.
+    """Model training pipeline.
 
     This is a pipeline that loads the data, processes it and splits
     it into train and test sets, then search for best hyperparameters,

train_and_deploy/requirements.txt

@@ -1,16 +1,4 @@
-zenml[server]==0.56.3
+zenml[server]==0.70.0
 bentoml>=1.0.10
 scikit-learn
-mlflow>=2.1.1,<=2.12.1
-mlserver>=1.3.3
-mlserver-mlflow>=1.3.3
-python-rapidjson<1.15
-kfp==1.8.22
-gcsfs
-google-cloud-secret-manager
-google-cloud-container>=2.21.0
-google-cloud-storage>=2.9.0
-google-cloud-aiplatform>=1.34.0
-google-cloud-build>=3.11.0
-kubernetes
-evidently>0.2.6,<0.4.5
+kubernetes

train_and_deploy/run.py

@@ -21,11 +21,10 @@
 
 import click
 from pipelines import (
-    e2e_use_case_batch_inference,
-    e2e_use_case_deployment,
-    e2e_use_case_training,
+    gitguarden_batch_inference,
+    gitguarden_local_deployment,
+    gitguarden_training,
 )
-
 from zenml.logger import get_logger
 
 logger = get_logger(__name__)
@@ -168,7 +167,6 @@ def main(
             not affect the pipeline.
         only_inference: If `True` only inference pipeline will be triggered.
     """
-
     # Run a pipeline with the required parameters. This executes
     # all steps in the pipeline in the correct order using the orchestrator
     # stack component that is configured in your active ZenML stack.
@@ -195,9 +193,9 @@ def main(
             "train_config.yaml",
         )
         pipeline_args["run_name"] = (
-            f"e2e_use_case_training_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}"
+            f"gitguarden_training_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}"
         )
-        e2e_use_case_training.with_options(**pipeline_args)(**run_args_train)
+        gitguarden_training.with_options(**pipeline_args)(**run_args_train)
         logger.info("Training pipeline finished successfully!")
 
     if deployment:
@@ -209,9 +207,9 @@ def main(
             "deployer_config.yaml",
         )
         pipeline_args["run_name"] = (
-            f"e2e_use_case_deployment_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}"
+            f"gitguarden_local_deployment_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}"
         )
-        e2e_use_case_deployment.with_options(**pipeline_args)(**run_args_inference)
+        gitguarden_local_deployment.with_options(**pipeline_args)(**run_args_inference)
 
     if inference:
         # Execute Batch Inference Pipeline
@@ -222,9 +220,9 @@ def main(
             "inference_config.yaml",
         )
         pipeline_args["run_name"] = (
-            f"e2e_use_case_batch_inference_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}"
+            f"gitguarden_batch_inference_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}"
         )
-        e2e_use_case_batch_inference.with_options(**pipeline_args)(
+        gitguarden_batch_inference.with_options(**pipeline_args)(
             **run_args_inference
         )
 

train_and_deploy/service.py

@@ -3,12 +3,12 @@
 import bentoml
 from bentoml.io import NumpyNdarray
 
-e2e_use_case_runner = bentoml.sklearn.get("e2e_use_case").to_runner()
+gitguarden_runner = bentoml.sklearn.get("gitguarden").to_runner()
 
-svc = bentoml.Service(name="e2e_use_case_service", runners=[e2e_use_case_runner])
+svc = bentoml.Service(name="gitguarden_service", runners=[gitguarden_runner])
 
 input_spec = NumpyNdarray(dtype="float", shape=(-1, 30))
 
 @svc.api(input=input_spec, output=NumpyNdarray())
 async def predict(input_arr):
-    return await e2e_use_case_runner.predict.async_run(input_arr)
+    return await gitguarden_runner.predict.async_run(input_arr)

train_and_deploy/steps/deployment/bento_builder.py

@@ -35,7 +35,7 @@
 def bento_builder() -> (
     Annotated[
         Optional[bento.Bento],
-        ArtifactConfig(name="mlflow_deployment", is_model_artifact=True),
+        ArtifactConfig(name="bentoml_deployment", is_model_artifact=True),
     ]
 ):
     """Predictions step.
@@ -72,6 +72,14 @@ def bento_builder() -> (
                 "bento_uri": os.path.join(get_step_context().get_output_artifact_uri(), DEFAULT_BENTO_FILENAME),
             },
             build_ctx=source_utils.get_source_root(),
+            python={
+                "packages": [
+                    "scikit-learn",
+                    "pandas",
+                    "numpy",
+                    "zenml"
+                ],
+            },
         )
     else:
         logger.warning("Skipping deployment as the orchestrator is not local.")

train_and_deploy/steps/deployment/deployment_deploy.py

@@ -18,27 +18,33 @@
 
 from typing import Optional
 
+from bentoml._internal.bento import bento
 from typing_extensions import Annotated
-
-from zenml import ArtifactConfig, get_step_context, step
+from zenml import (
+    ArtifactConfig,
+    Model,
+    get_step_context,
+    log_artifact_metadata,
+    step,
+)
 from zenml.client import Client
-from zenml.integrations.bentoml.services.bentoml_deployment import (
-    BentoMLDeploymentService,
+from zenml.integrations.bentoml.services.bentoml_container_deployment import (
+    BentoMLContainerDeploymentService,
+)
+from zenml.integrations.bentoml.services.deployment_type import (
+    BentoMLDeploymentType,
 )
-from zenml import Model, log_artifact_metadata
 from zenml.integrations.bentoml.steps import bentoml_model_deployer_step
 from zenml.logger import get_logger
 
-from bentoml._internal.bento import bento
-
 logger = get_logger(__name__)
 
 @step
 def deployment_deploy(
     bento: bento.Bento,
 ) -> (
     Annotated[
-        Optional[BentoMLDeploymentService],
+        Optional[BentoMLContainerDeploymentService],
         ArtifactConfig(name="bentoml_deployment", is_deployment_artifact=True),
     ]
 ):
@@ -68,9 +74,10 @@ def deployment_deploy(
         bentoml_deployment = bentoml_model_deployer_step.entrypoint(
             model_name=model.name,  # Name of the model
             port=3009,  # Port to be used by the http server
-            production=False,  # Deploy the model in production mode
+            production=True,  # Deploy the model in production mode
             timeout=1000,
             bento=bento,
+            deployment_type=BentoMLDeploymentType.CONTAINER,
         )
 
         bentoml_service = Client().get_service(name_id_or_prefix=bentoml_deployment.uuid)
@@ -82,7 +89,8 @@ def deployment_deploy(
                 "prediction_url": bentoml_service.prediction_url,
                 "health_check_url": bentoml_service.health_check_url,
                 "model_uri": model.get_artifact(name="model").uri,
-                "bento" : bentoml_service.config.get("bento"),
+                "bento_tag" : bentoml_service.config.get("bento_tag"),
+                "bentoml_model_image": bentoml_service.config.get("image"),
             }
         )
     else:

train_and_deploy/steps/hp_tuning/hp_tuning_select_best_model.py

@@ -50,7 +50,7 @@ def hp_tuning_select_best_model(
         hp_output = model.get_data_artifact("hp_result")
         model_: ClassifierMixin = hp_output.load()
         # fetch metadata we attached earlier
-        metric = float(hp_output.run_metadata["metric"].value)
+        metric = float(hp_output.run_metadata["metric"])
         if best_model is None or best_metric < metric:
             best_model = model_
     ### YOUR CODE ENDS HERE ###