From f937dfe95a144907ce66f951e2b5c12619966acb Mon Sep 17 00:00:00 2001 From: ElieTrigano <103178864+ElieTrigano@users.noreply.github.com> Date: Tue, 16 Apr 2024 10:38:16 +0200 Subject: [PATCH 1/2] feat: improve init (#173) Co-authored-by: Elie Trigano --- README.md | 8 +- .../components/dummy_component.py.jinja | 13 ++ .../_templates/configs/json_config.json.jinja | 3 + .../_templates/configs/python_config.py.jinja | 17 ++ .../_templates/configs/toml_config.toml.jinja | 1 + deployer/_templates/deployer.env.jinja | 14 ++ .../_templates/deployment/Dockerfile.jinja | 25 +++ .../deployment/build_base_image.sh.jinja | 17 ++ .../deployment/cloudbuild_local.yaml.jinja | 36 ++++ .../pipelines/pipeline_minimal.py.jinja | 9 + .../_templates/requirements-vertex.txt.jinja | 8 + deployer/cli.py | 180 +++++++++-------- deployer/constants.py | 62 ++++-- deployer/init_deployer.py | 182 ++++++++++++++++++ deployer/pipeline_checks.py | 4 +- deployer/pipeline_deployer.py | 3 +- deployer/settings.py | 21 +- deployer/utils/config.py | 6 +- deployer/utils/console.py | 9 +- deployer/utils/exceptions.py | 4 + docs/CLI_REFERENCE.md | 10 +- example/Makefile | 2 +- .../vertex/deployment/cloudbuild_local.yaml | 4 + pyproject.toml | 4 + tests/integration_tests/test_configuration.py | 2 +- tests/unit_tests/test_settings.py | 16 +- 26 files changed, 515 insertions(+), 145 deletions(-) create mode 100644 deployer/_templates/components/dummy_component.py.jinja create mode 100644 deployer/_templates/configs/json_config.json.jinja create mode 100644 deployer/_templates/configs/python_config.py.jinja create mode 100644 deployer/_templates/configs/toml_config.toml.jinja create mode 100644 deployer/_templates/deployer.env.jinja create mode 100644 deployer/_templates/deployment/Dockerfile.jinja create mode 100644 deployer/_templates/deployment/build_base_image.sh.jinja create mode 100644 deployer/_templates/deployment/cloudbuild_local.yaml.jinja create mode 100644 deployer/_templates/pipelines/pipeline_minimal.py.jinja create mode 100644 deployer/_templates/requirements-vertex.txt.jinja create mode 100644 deployer/init_deployer.py diff --git a/README.md b/README.md index 9c34660..142c31b 100644 --- a/README.md +++ b/README.md @@ -320,7 +320,6 @@ vertex-deployer deploy dummy_pipeline \ --upload \ --run \ --env-file example.env \ - --local-package-path . \ --tags my-tag \ --config-filepath vertex/configs/dummy_pipeline/config_test.json \ --experiment-name my-experiment \ @@ -414,8 +413,7 @@ This will overwrite default values. It can be useful if you always use the same ```toml [tool.vertex-deployer] -pipelines_root_path = "my/path/to/vertex/pipelines" -configs_root_path = "my/path/to/vertex/configs" +vertex_folder_path = "my/path/to/vertex" log_level = "INFO" [tool.vertex-deployer.deploy] @@ -427,8 +425,7 @@ You can display all the configurable parameterss with default values by running: $ vertex-deployer config --all '*' means the value was set in config file -* pipelines_root_path=my/path/to/vertex/pipelines -* config_root_path=my/path/to/vertex/configs +* vertex_folder_path=my/path/to/vertex * log_level=INFO deploy env_file=None @@ -444,7 +441,6 @@ deploy config_name=None enable_caching=False experiment_name=None - local_package_path=vertex/pipelines/compiled_pipelines check all=False config_filepath=None diff --git a/deployer/_templates/components/dummy_component.py.jinja b/deployer/_templates/components/dummy_component.py.jinja new file mode 100644 index 0000000..a907bc9 --- /dev/null +++ b/deployer/_templates/components/dummy_component.py.jinja @@ -0,0 +1,13 @@ +from kfp.dsl import component, Artifact, Input, Output + + +@component(base_image="python:3.10-slim-buster") +def dummy_component( + name: str, + my_input_artifact: Input[Artifact], + my_output_artifact: Output[Artifact], +): + """This component is a dummy""" + print(f"Hello, {name}!") + + my_output_artifact.metadata["name"] = name diff --git a/deployer/_templates/configs/json_config.json.jinja b/deployer/_templates/configs/json_config.json.jinja new file mode 100644 index 0000000..979adea --- /dev/null +++ b/deployer/_templates/configs/json_config.json.jinja @@ -0,0 +1,3 @@ +{ + "name": "John Doe" +} diff --git a/deployer/_templates/configs/python_config.py.jinja b/deployer/_templates/configs/python_config.py.jinja new file mode 100644 index 0000000..94e52b9 --- /dev/null +++ b/deployer/_templates/configs/python_config.py.jinja @@ -0,0 +1,17 @@ +import google.cloud.aiplatform as aip +from aip import Artifact + +# You can retrieve an existing Metadata Artifact given a resource name or ID. +# You can check aip documentation for more information: +# https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform.Artifact +artifact_name = "projects/123/locations/us-central1/metadataStores/default/artifacts/my-resource" +# or +artifact_name = "my-resource" + +my_artifact = Artifact(artifact_name=artifact_name, + project="my_project", + location="us-central1", + ) + +input_artifacts = {"my_input_artifact": my_artifact} +parameter_values = {"name": "John Doe in python config"} diff --git a/deployer/_templates/configs/toml_config.toml.jinja b/deployer/_templates/configs/toml_config.toml.jinja new file mode 100644 index 0000000..64a1f66 --- /dev/null +++ b/deployer/_templates/configs/toml_config.toml.jinja @@ -0,0 +1 @@ +name = "John Doe" diff --git a/deployer/_templates/deployer.env.jinja b/deployer/_templates/deployer.env.jinja new file mode 100644 index 0000000..f7ea74a --- /dev/null +++ b/deployer/_templates/deployer.env.jinja @@ -0,0 +1,14 @@ +PROJECT_ID= +GCP_REGION= + +TAG=latest + +# Google Artifact Registry +GAR_LOCATION= # Google Artifact Registry repo location +GAR_DOCKER_REPO_ID= +GAR_PIPELINES_REPO_ID= +GAR_VERTEX_BASE_IMAGE_NAME= + +# Vertex AI +VERTEX_STAGING_BUCKET_NAME= # without gs:// +VERTEX_SERVICE_ACCOUNT= # full service account email diff --git a/deployer/_templates/deployment/Dockerfile.jinja b/deployer/_templates/deployment/Dockerfile.jinja new file mode 100644 index 0000000..36d9814 --- /dev/null +++ b/deployer/_templates/deployment/Dockerfile.jinja @@ -0,0 +1,25 @@ +FROM python:3.10-slim-buster + +ARG PROJECT_ID +ARG GCP_REGION +ARG GAR_LOCATION +ARG GAR_PIPELINES_REPO_ID +ARG VERTEX_STAGING_BUCKET_NAME +ARG VERTEX_SERVICE_ACCOUNT + +ENV PROJECT_ID=${PROJECT_ID} +ENV GCP_REGION=${GCP_REGION} +ENV GAR_LOCATION=${GAR_LOCATION} +ENV GAR_PIPELINES_REPO_ID=${GAR_PIPELINES_REPO_ID} +ENV VERTEX_STAGING_BUCKET_NAME=${VERTEX_STAGING_BUCKET_NAME} +ENV VERTEX_SERVICE_ACCOUNT=${VERTEX_SERVICE_ACCOUNT} + +WORKDIR /app + +COPY deployer-requirements.txt . +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install -r deployer-requirements.txt + +ENV PYTHONPATH "${PYTHONPATH}:." + +COPY ./vertex /app/vertex diff --git a/deployer/_templates/deployment/build_base_image.sh.jinja b/deployer/_templates/deployment/build_base_image.sh.jinja new file mode 100644 index 0000000..51c5b85 --- /dev/null +++ b/deployer/_templates/deployment/build_base_image.sh.jinja @@ -0,0 +1,17 @@ +#!/bin/bash +ENV_FILENAME=deployer.env +source $ENV_FILENAME +echo "Using '$ENV_FILENAME' env variables..." + +SUBSTITUTIONS="\ +_GAR_LOCATION=${GAR_LOCATION},\ +_GAR_DOCKER_REPO_ID=${GAR_DOCKER_REPO_ID},\ +_GAR_VERTEX_BASE_IMAGE_NAME=${GAR_VERTEX_BASE_IMAGE_NAME},\ +_TAG=${TAG} +" + +gcloud builds submit \ + --config {{ cloud_build_path }} \ + --project=$PROJECT_ID \ + --region $GCP_REGION \ + --substitutions $SUBSTITUTIONS diff --git a/deployer/_templates/deployment/cloudbuild_local.yaml.jinja b/deployer/_templates/deployment/cloudbuild_local.yaml.jinja new file mode 100644 index 0000000..27adb1c --- /dev/null +++ b/deployer/_templates/deployment/cloudbuild_local.yaml.jinja @@ -0,0 +1,36 @@ +# This config file is meant to be used from a local dev machine to submit a vertex base image build to Cloud Build. +# This generic image will then be used in all the Vertex components of your pipeline. + +steps: + # Build base image + - name: 'gcr.io/cloud-builders/docker' + args: [ + 'build', + '-t', '${_GAR_IMAGE_PATH}', + '-f', '{{ dockerfile_path }}', + '--build-arg', 'PROJECT_ID=${PROJECT_ID}', + '--build-arg', 'GCP_REGION=${_GCP_REGION}', + '--build-arg', 'GAR_LOCATION=${_GAR_LOCATION}', + '--build-arg', 'GAR_PIPELINES_REPO_ID=${_GAR_PIPELINES_REPO_ID}', + '--build-arg', 'VERTEX_STAGING_BUCKET_NAME=${_VERTEX_STAGING_BUCKET_NAME}', + '--build-arg', 'VERTEX_SERVICE_ACCOUNT=${_VERTEX_SERVICE_ACCOUNT}', + '.', + ] + id: build-base-image + +substitutions: + _GAR_IMAGE_PATH: '${_GAR_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_GAR_DOCKER_REPO_ID}/${_GAR_VERTEX_BASE_IMAGE_NAME}:${_TAG}' + _GCP_REGION: '${GCP_REGION}' + _GAR_PIPELINES_REPO_ID: '${GAR_PIPELINES_REPO_ID}' + _VERTEX_STAGING_BUCKET_NAME: '${VERTEX_STAGING_BUCKET_NAME}' + _VERTEX_SERVICE_ACCOUNT: '${VERTEX_SERVICE_ACCOUNT}' + +options: + logging: CLOUD_LOGGING_ONLY + dynamic_substitutions: true + +images: +- '${_GAR_IMAGE_PATH}' + +tags: + - vertex-${_GAR_DOCKER_REPO_ID}-base-image-local-${_TAG} diff --git a/deployer/_templates/pipelines/pipeline_minimal.py.jinja b/deployer/_templates/pipelines/pipeline_minimal.py.jinja new file mode 100644 index 0000000..0c39a94 --- /dev/null +++ b/deployer/_templates/pipelines/pipeline_minimal.py.jinja @@ -0,0 +1,9 @@ +import kfp.dsl + +from {{ component_module }} import dummy_component + + +@kfp.dsl.pipeline(name="{{ pipeline_name }}") +def {{ pipeline_name }}(name: str): + """This pipeline prints hello {name}""" + dummy_component(name=name) diff --git a/deployer/_templates/requirements-vertex.txt.jinja b/deployer/_templates/requirements-vertex.txt.jinja new file mode 100644 index 0000000..9be2991 --- /dev/null +++ b/deployer/_templates/requirements-vertex.txt.jinja @@ -0,0 +1,8 @@ +# base +# We consider that your initial requirements are defined in a requirements.txt file +-r requirements.txt + +# deploy +kfp +google-cloud-aiplatform +vertex-deployer={{ deployer_version }} diff --git a/deployer/cli.py b/deployer/cli.py index 76db8a0..28463d4 100644 --- a/deployer/cli.py +++ b/deployer/cli.py @@ -12,21 +12,25 @@ from typing_extensions import Annotated from deployer import constants +from deployer.init_deployer import ( + _create_file_from_template, + build_default_folder_structure, + configure_deployer, + ensure_pyproject_toml, + show_commands, +) from deployer.settings import ( DeployerSettings, - find_pyproject_toml, load_deployer_settings, - update_pyproject_toml, ) from deployer.utils.config import ( ConfigType, - VertexPipelinesSettings, list_config_filepaths, load_config, load_vertex_settings, validate_or_log_settings, ) -from deployer.utils.console import ask_user_for_model_fields, console +from deployer.utils.console import console from deployer.utils.logging import LoguruLevel from deployer.utils.utils import ( dict_to_repr, @@ -223,17 +227,6 @@ def deploy( # noqa: C901 "Defaults to '{pipeline_name}-experiment'.", ), ] = None, - local_package_path: Annotated[ - Path, - typer.Option( - "--local-package-path", - "-lpp", - help="Local dir path where pipelines will be compiled.", - dir_okay=True, - file_okay=False, - resolve_path=True, - ), - ] = constants.DEFAULT_LOCAL_PACKAGE_PATH, skip_validation: Annotated[ bool, typer.Option( @@ -286,13 +279,13 @@ def deploy( # noqa: C901 pipeline_func=pipeline_func, gar_location=vertex_settings.GAR_LOCATION, gar_repo_id=vertex_settings.GAR_PIPELINES_REPO_ID, - local_package_path=local_package_path, + local_package_path=deployer_settings.local_package_path, ) if run or schedule: if config_name is not None: config_filepath = ( - Path(deployer_settings.config_root_path) / pipeline_name / config_name + Path(deployer_settings.configs_root_path) / pipeline_name / config_name ) parameter_values, input_artifacts = load_config(config_filepath) @@ -388,7 +381,7 @@ def check( * Checking that the pipeline can be compiled using `kfp.compiler.Compiler`. - * Checking that config files in `{config_root_path}/{pipeline_name}` are corresponding to the + * Checking that config files in `{configs_root_path}/{pipeline_name}` are corresponding to the pipeline parameters definition, using Pydantic. --- @@ -409,7 +402,8 @@ def check( if config_filepath is None: to_check = { - p: list_config_filepaths(deployer_settings.config_root_path, p) for p in pipeline_names + p: list_config_filepaths(deployer_settings.configs_root_path, p) + for p in pipeline_names } else: to_check = {p: [config_filepath] for p in pipeline_names} @@ -423,7 +417,7 @@ def check( "pipeline_name": p, "config_paths": config_filepaths, "pipelines_root_path": deployer_settings.pipelines_root_path, - "config_root_path": deployer_settings.config_root_path, + "configs_root_path": deployer_settings.configs_root_path, } for p, config_filepaths in to_check.items() } @@ -454,7 +448,7 @@ def list_pipelines( """List all pipelines.""" if with_configs: pipelines_dict = { - p.name: list_config_filepaths(ctx.obj["settings"].config_root_path, p.name) + p.name: list_config_filepaths(ctx.obj["settings"].configs_root_path, p.name) for p in ctx.obj["pipeline_names"].__members__.values() } else: @@ -473,7 +467,7 @@ def create_pipeline( config_type: Annotated[ ConfigType, typer.Option("--config-type", "-ct", help="The type of the config to create."), - ] = ConfigType.json, + ] = ConfigType.py, ): """Create files structure for a new pipeline.""" invalid_pipelines = [p for p in pipeline_names if not re.match(r"^[a-zA-Z0-9_]+$", p)] @@ -485,7 +479,7 @@ def create_pipeline( deployer_settings: DeployerSettings = ctx.obj["settings"] - for path in [deployer_settings.pipelines_root_path, deployer_settings.config_root_path]: + for path in [deployer_settings.pipelines_root_path, deployer_settings.configs_root_path]: if not Path(path).is_dir(): raise FileNotFoundError( f"Path '{path}' does not exist." @@ -505,94 +499,96 @@ def create_pipeline( for pipeline_name in pipeline_names: pipeline_filepath = deployer_settings.pipelines_root_path / f"{pipeline_name}.py" - pipeline_filepath.touch(exist_ok=False) - pipeline_filepath.write_text( - constants.PIPELINE_MINIMAL_TEMPLATE.format(pipeline_name=pipeline_name) + _create_file_from_template( + path=pipeline_filepath, + template_path=constants.PIPELINE_MINIMAL_TEMPLATE, + pipeline_name=pipeline_name, + component_module=str(deployer_settings.vertex_folder_path / "components").replace( + "/", "." + ), ) try: - config_dirpath = Path(deployer_settings.config_root_path) / pipeline_name + config_dirpath = Path(deployer_settings.configs_root_path) / pipeline_name config_dirpath.mkdir(exist_ok=True) for config_name in ["test", "dev", "prod"]: config_filepath = config_dirpath / f"{config_name}.{config_type}" - config_filepath.touch(exist_ok=False) - if config_type == ConfigType.py: - config_filepath.write_text(constants.PYTHON_CONFIG_TEMPLATE) + config_template = constants.CONFIG_TEMPLATE_MAPPING[config_type] + _create_file_from_template( + path=config_filepath, + template_path=config_template, + ) + except Exception as e: pipeline_filepath.unlink() raise e console.print( - f"Pipeline '{pipeline_name}' created at '{pipeline_filepath}'" - f" with config files: {[str(p) for p in config_dirpath.glob('*')]}. :sparkles:", + f"\n Pipeline '{pipeline_name}' created at '{pipeline_filepath}'" + f"\n with config files: {[str(p) for p in config_dirpath.glob('*')]}. :sparkles: \n", style="blue", ) @app.command(name="init") -def init_deployer(ctx: typer.Context): # noqa: C901 - deployer_settings: DeployerSettings = ctx.obj["settings"] - - console.print("Welcome to Vertex Deployer!", style="blue") - console.print("This command will help you getting fired up.", style="blue") - - if Confirm.ask("Do you want to configure the deployer?"): - pyproject_toml_filepath = find_pyproject_toml(Path.cwd().resolve()) - - if pyproject_toml_filepath is None: - console.print( - "No pyproject.toml file found. Creating one in current directory.", - style="yellow", - ) - pyproject_toml_filepath = Path("./pyproject.toml") - pyproject_toml_filepath.touch() - - set_fields = ask_user_for_model_fields(DeployerSettings) - - new_deployer_settings = DeployerSettings(**set_fields) - - update_pyproject_toml(pyproject_toml_filepath, new_deployer_settings) - console.print("Configuration saved in pyproject.toml :sparkles:", style="blue") - - if Confirm.ask("Do you want to build default folder structure"): - - def create_file_or_dir(path: Path, text: str = ""): - """Create a file (if text is provided) or a directory at path. Warn if path exists.""" - if path.exists(): - console.print( - f"Path '{path}' already exists. Skipping creation of path.", style="yellow" - ) - else: - if text: - path.touch() - path.write_text(text) +def init_deployer( + ctx: typer.Context, + default: bool = typer.Option( + False, + "--default", + "-d", + help="Instantly creates the full vertex structure and files without configuration prompts", + ), +): + """Initialize the deployer.""" + deployer_settings = ctx.obj["settings"] + console.print("Welcome to Vertex Deployer!", style="bold blue") + + if default or Confirm.ask( + "Do you want to instantly create the full vertex structure and templates\n" + "without configuration prompts? This will use the default settings." + ): + console.print("Performing quick initialization...", style="bold blue") + ensure_pyproject_toml() + deployer_settings = load_deployer_settings() + build_default_folder_structure(deployer_settings) + create_pipeline(ctx, pipeline_names=["dummy_pipeline"], config_type=ConfigType.py) + + console.print("Default initialization done :sparkles:\n", style="bold blue") + console.print("Here are some commands on how to use the deployer:", style="blue") + show_commands(deployer_settings) + else: + console.print("This command will help you getting fired up! :fire:\n", style="blue") + + if Confirm.ask("Do you want to configure the deployer?"): + deployer_settings = configure_deployer() + ctx.obj["settings"] = deployer_settings + + if Confirm.ask("Do you want to build default folder structure"): + build_default_folder_structure(deployer_settings) + + if Confirm.ask("Do you want to create a pipeline?"): + wrong_name = True + while wrong_name: + pipeline_name = Prompt.ask("What is the name of the pipeline?") + + try: + config_type = deployer_settings.create.config_type.name + create_pipeline(ctx, pipeline_names=[pipeline_name], config_type=config_type) + except typer.BadParameter as e: + console.print(e, style="red") + except FileExistsError: + console.print( + f"Pipeline '{pipeline_name}' already exists. Skipping creation.", + style="yellow", + ) else: - path.mkdir(parents=True) + wrong_name = False - create_file_or_dir(deployer_settings.pipelines_root_path) - create_file_or_dir(deployer_settings.config_root_path) - create_file_or_dir( - Path("./.env"), "=\n".join(VertexPipelinesSettings.model_json_schema()["required"]) - ) - - if Confirm.ask("Do you want to create a pipeline?"): - wrong_name = True - while wrong_name: - pipeline_name = Prompt.ask("What is the name of the pipeline?") - - try: - create_pipeline(ctx, pipeline_names=[pipeline_name]) - except typer.BadParameter as e: - console.print(e, style="red") - except FileExistsError: - console.print( - f"Pipeline '{pipeline_name}' already exists. Skipping creation.", - style="yellow", - ) - else: - wrong_name = False + console.print("All done :sparkles:\n", style="bold blue") - console.print("All done :sparkles:", style="blue") + if Confirm.ask("Do you want to see some instructions on how to use the deployer"): + show_commands(deployer_settings) @app.command(name="config") diff --git a/deployer/constants.py b/deployer/constants.py index 51f0c51..a157901 100644 --- a/deployer/constants.py +++ b/deployer/constants.py @@ -1,31 +1,34 @@ from pathlib import Path -DEFAULT_PIPELINE_ROOT_PATH = Path("vertex/pipelines") -DEFAULT_CONFIG_ROOT_PATH = Path("vertex/configs") +TEMPLATES_PATH = Path(__file__).parent / "_templates" + +DEFAULT_VERTEX_FOLDER_PATH = "vertex" DEFAULT_LOG_LEVEL = "INFO" DEFAULT_SCHEDULER_TIMEZONE = "Europe/Paris" -DEFAULT_LOCAL_PACKAGE_PATH = Path("vertex/pipelines/compiled_pipelines") DEFAULT_TAGS = None TEMP_LOCAL_PACKAGE_PATH = ".vertex-deployer-temp" +PIPELINE_MINIMAL_TEMPLATE = Path(TEMPLATES_PATH / "pipelines/pipeline_minimal.py.jinja") +PYTHON_CONFIG_TEMPLATE = Path(TEMPLATES_PATH / "configs/python_config.py.jinja") +JSON_CONFIG_TEMPLATE = Path(TEMPLATES_PATH / "configs/json_config.json.jinja") +TOML_CONFIG_TEMPLATE = Path(TEMPLATES_PATH / "configs/toml_config.toml.jinja") -PIPELINE_MINIMAL_TEMPLATE = """import kfp.dsl - - -@kfp.dsl.pipeline(name="{pipeline_name}") -def {pipeline_name}(): - pass - -""" +TEMPLATES_DEFAULT_STRUCTURE = { + "dummy_component": Path(TEMPLATES_PATH / "components/dummy_component.py.jinja"), + "deployer_env": Path(TEMPLATES_PATH / "deployer.env.jinja"), + "requirements_vertex": Path(TEMPLATES_PATH / "requirements-vertex.txt.jinja"), + "dockerfile": Path(TEMPLATES_PATH / "deployment/Dockerfile.jinja"), + "cloudbuild_local": Path(TEMPLATES_PATH / "deployment/cloudbuild_local.yaml.jinja"), + "build_base_image": Path(TEMPLATES_PATH / "deployment/build_base_image.sh.jinja"), +} -PYTHON_CONFIG_TEMPLATE = """from kfp.dsl import Artifact, Dataset, Input, Output, Metrics - -parameter_values = {} -input_artifacts = {} - -""" +CONFIG_TEMPLATE_MAPPING = { + "json": JSON_CONFIG_TEMPLATE, + "toml": TOML_CONFIG_TEMPLATE, + "py": PYTHON_CONFIG_TEMPLATE, +} PIPELINE_CHECKS_TABLE_COLUMNS = [ "Status", @@ -36,3 +39,28 @@ def {pipeline_name}(): "Config Error Type", "Config Error Message", ] + + +INSTRUCTIONS = ( + "\n" + "Now that your deployer is configured, make sure that you're also done with the setup!\n" + "You can find all the instructions in the README.md file.\n" + "\n" + "If your setup is complete you're ready to start building your pipelines! :tada:\n" + "Here are the commands you need to run to build your project:\n" + "\n" + "1. Build the base image:\n" + "$ bash {build_base_image_path}\n" + "\n" + "2. Check all the pipelines:\n" + "$ vertex-deployer check --all\n" + "\n" + "3. Deploy a pipeline and run it:\n" + "$ vertex-deployer deploy pipeline_name --run\n" + "If not set during configuration, you will need to provide the config path or name:\n" + "$ vertex-deployer deploy pipeline_name --cfp=path/to/your/config.type\n" + "\n" + "4. Schedule a pipeline:\n" + "you can add the following flags to the deploy command if not set in your config:\n" + "--schedule --cron=cron_expression --scheduler-timezone=IANA_time_zone\n" +) diff --git a/deployer/init_deployer.py b/deployer/init_deployer.py new file mode 100644 index 0000000..96eaa02 --- /dev/null +++ b/deployer/init_deployer.py @@ -0,0 +1,182 @@ +from pathlib import Path + +import jinja2 +from jinja2 import Environment, FileSystemLoader, meta +from rich.tree import Tree + +from deployer import constants +from deployer.__init__ import __version__ as deployer_version +from deployer.settings import ( + DeployerSettings, + find_pyproject_toml, + update_pyproject_toml, +) +from deployer.utils.console import ask_user_for_model_fields, console +from deployer.utils.exceptions import TemplateFileCreationError + + +def ensure_pyproject_toml(): + """Ensure that a pyproject.toml file exists in the current directory.""" + pyproject_toml_filepath = find_pyproject_toml(Path.cwd().resolve()) + + if pyproject_toml_filepath is None: + console.print( + "No pyproject.toml file found. Creating one in the current directory.", + style="yellow", + ) + pyproject_toml_filepath = Path("./pyproject.toml") + pyproject_toml_filepath.touch() + + return pyproject_toml_filepath + + +def configure_deployer(): + """Configure the deployer settings.""" + pyproject_toml_filepath = ensure_pyproject_toml() + + set_fields = ask_user_for_model_fields(DeployerSettings) + + tags = set_fields.get("deploy", {}).get("tags", None) + if tags: + set_fields["deploy"]["tags"] = [tags] + + new_deployer_settings = DeployerSettings(**set_fields) + + update_pyproject_toml(pyproject_toml_filepath, new_deployer_settings) + console.print("\n Configuration saved in pyproject.toml :sparkles:\n", style="bold blue") + + return new_deployer_settings + + +def _create_dir(path: Path): + """Create a directory at path. Warn if path exists.""" + if path.exists(): + console.print(f"Directory '{path}' already exists. Skipping creation.", style="yellow") + else: + path.mkdir(parents=True) + + +def _create_file_from_template(path: Path, template_path: Path, **kwargs): + """Create a file at path from a template file. + + Raises: + TemplateFileCreationError: Raised when the file cannot be created from the template. + """ + try: + env = Environment(loader=FileSystemLoader(str(template_path.parent)), autoescape=True) + template = env.get_template(template_path.name) + content = template.render(**kwargs) + if path.exists(): + console.print( + f"Path '{path}' already exists. Skipping creation of path.", style="yellow" + ) + else: + path.write_text(content) + except (FileNotFoundError, KeyError, jinja2.TemplateError) as e: + raise TemplateFileCreationError( + f"An error occurred while creating the file from template: {e}" + ) from e + except Exception as e: + console.print(f"An unexpected error occurred: {e}", style="red") + + +def _generate_templates_mapping( + templates_dict: dict, mapping_variables: dict, vertex_folder_path: Path +): + """Generate the mapping of a list of templates to create and their variables.""" + templates_mapping = {} + env = Environment(loader=FileSystemLoader(str(constants.TEMPLATES_PATH)), autoescape=True) + for template, template_path in templates_dict.items(): + template_name = str(template_path.relative_to(constants.TEMPLATES_PATH)) + template_source = env.loader.get_source(env, template_name)[0] + parsed_content = env.parse(template_source) + variables = meta.find_undeclared_variables(parsed_content) + if template in [ + "deployer_env", + "requirements_vertex", + ]: + new_file_path = Path(template_name.replace(".jinja", "")) + else: + new_file_path = vertex_folder_path / template_name.replace(".jinja", "") + if variables: + template_variables = {variable: mapping_variables[variable] for variable in variables} + templates_mapping[new_file_path] = (template_path, template_variables) + else: + templates_mapping[new_file_path] = (template_path, {}) + return templates_mapping + + +def build_default_folder_structure(deployer_settings: DeployerSettings): + """Create the default folder structure for the Vertex Pipelines project.""" + vertex_folder_path = deployer_settings.vertex_folder_path + dockerfile_path = vertex_folder_path / str( + constants.TEMPLATES_DEFAULT_STRUCTURE["dockerfile"].relative_to(constants.TEMPLATES_PATH) + ).replace(".jinja", "") + cloud_build_path = vertex_folder_path / str( + constants.TEMPLATES_DEFAULT_STRUCTURE["cloudbuild_local"].relative_to( + constants.TEMPLATES_PATH + ) + ).replace(".jinja", "") + + # Create the folder structure + for folder in ["configs", "components", "deployment", "lib", "pipelines"]: + _create_dir(vertex_folder_path / folder) + + mapping_variables = { + "cloud_build_path": cloud_build_path, + "dockerfile_path": dockerfile_path, + "vertex_folder_path": vertex_folder_path, + "deployer_version": deployer_version, + } + + templates_mapping = _generate_templates_mapping( + constants.TEMPLATES_DEFAULT_STRUCTURE, mapping_variables, vertex_folder_path + ) + + # Create the files + for new_file_path, (template_path, content) in templates_mapping.items(): + _create_file_from_template(new_file_path, template_path, **content) + + console.print("\n Complete folder structure created :sparkles: \n", style="bold blue") + + tree = generate_tree(vertex_folder_path) + console.print(tree, "\n") + + +def generate_tree(vertex_folder_path: Path): + """Generate a tree of the folder structure.""" + root = Tree("your_project_root", style="blue", guide_style="bold green") + vertex_node = root.add( + f":file_folder: [bold magenta]{vertex_folder_path.name}", guide_style="bold magenta" + ) + + folder_structure = { + "configs": ["your_pipeline"], + "components": ["your_component.py"], + "deployment": ["Dockerfile", "cloudbuild_local.yaml", "build_base_image.sh"], + "lib": ["your_lib.py"], + "pipelines": ["your_pipeline.py"], + } + + for folder, files in folder_structure.items(): + folder_node = vertex_node.add(f":file_folder: [yellow]{folder}", guide_style="yellow") + for file in files: + if folder == "configs": + folder_node.add(file).add("config.type") + else: + folder_node.add(file) + + root.add("deployer.env") + root.add("requirements-vertex.txt") + root.add("pyproject.toml") + return root + + +def show_commands(deployer_settings: DeployerSettings): + """Show the commands to run to build the project.""" + vertex_folder_path = deployer_settings.vertex_folder_path + build_base_image_path = vertex_folder_path / "deployment" / "build_base_image.sh" + + console.print( + constants.INSTRUCTIONS.format(build_base_image_path=build_base_image_path), style="blue" + ) diff --git a/deployer/pipeline_checks.py b/deployer/pipeline_checks.py index 31a9405..60d693d 100644 --- a/deployer/pipeline_checks.py +++ b/deployer/pipeline_checks.py @@ -57,7 +57,7 @@ class Pipeline(CustomBaseModel): pipeline_name: str config_paths: Annotated[List[Path], Field(validate_default=True)] = None pipelines_root_path: Path - config_root_path: Path + configs_root_path: Path configs: Optional[Dict[str, ConfigDynamicModel]] = None # Optional because populated after @model_validator(mode="before") @@ -66,7 +66,7 @@ def populate_config_names(cls, data: Any) -> Any: """Populate config names before validation""" if data.get("config_paths") is None: data["config_paths"] = list_config_filepaths( - str(data["config_root_path"]), data["pipeline_name"] + str(data["configs_root_path"]), data["pipeline_name"] ) return data diff --git a/deployer/pipeline_deployer.py b/deployer/pipeline_deployer.py index 6324971..7b591d6 100644 --- a/deployer/pipeline_deployer.py +++ b/deployer/pipeline_deployer.py @@ -11,7 +11,6 @@ from loguru import logger from requests import HTTPError -from deployer.constants import DEFAULT_LOCAL_PACKAGE_PATH from deployer.utils.exceptions import ( MissingGoogleArtifactRegistryHostError, TagNotFoundError, @@ -31,7 +30,7 @@ def __init__( service_account: Optional[str] = None, gar_location: Optional[str] = None, gar_repo_id: Optional[str] = None, - local_package_path: Path = DEFAULT_LOCAL_PACKAGE_PATH, + local_package_path: Optional[Path] = None, ) -> None: """I don't want to write a dostring here but ruff wants me to""" self.project_id = project_id diff --git a/deployer/settings.py b/deployer/settings.py index 5c9fe06..8f77515 100644 --- a/deployer/settings.py +++ b/deployer/settings.py @@ -30,7 +30,6 @@ class _DeployerDeploySettings(CustomBaseModel): config_name: Optional[str] = None enable_caching: Optional[bool] = None experiment_name: Optional[str] = None - local_package_path: Path = constants.DEFAULT_LOCAL_PACKAGE_PATH skip_validation: bool = True @@ -53,7 +52,7 @@ class _DeployerListSettings(CustomBaseModel): class _DeployerCreateSettings(CustomBaseModel): """Settings for Vertex Deployer `create` command.""" - config_type: ConfigType = ConfigType.json + config_type: ConfigType = ConfigType.py class _DeployerConfigSettings(CustomBaseModel): @@ -65,8 +64,7 @@ class _DeployerConfigSettings(CustomBaseModel): class DeployerSettings(CustomBaseModel): """Settings for Vertex Deployer.""" - pipelines_root_path: Path = constants.DEFAULT_PIPELINE_ROOT_PATH - config_root_path: Path = constants.DEFAULT_CONFIG_ROOT_PATH + vertex_folder_path: Path = constants.DEFAULT_VERTEX_FOLDER_PATH log_level: str = "INFO" deploy: _DeployerDeploySettings = _DeployerDeploySettings() check: _DeployerCheckSettings = _DeployerCheckSettings() @@ -74,6 +72,21 @@ class DeployerSettings(CustomBaseModel): create: _DeployerCreateSettings = _DeployerCreateSettings() config: _DeployerConfigSettings = _DeployerConfigSettings() + @property + def pipelines_root_path(self) -> Path: + """Construct the pipelines root path.""" + return self.vertex_folder_path / "pipelines" + + @property + def configs_root_path(self) -> Path: + """Construct the configs root path.""" + return self.vertex_folder_path / "configs" + + @property + def local_package_path(self) -> Path: + """Construct the local package path.""" + return self.vertex_folder_path / "pipelines" / "compiled_pipelines" + def find_pyproject_toml(path_project_root: Path) -> Optional[str]: """Find the pyproject.toml file.""" diff --git a/deployer/utils/config.py b/deployer/utils/config.py index cba1495..eb3392a 100644 --- a/deployer/utils/config.py +++ b/deployer/utils/config.py @@ -85,17 +85,17 @@ class ConfigType(str, Enum): # noqa: D101 toml = "toml" -def list_config_filepaths(config_root_path: Union[Path, str], pipeline_name: str) -> List[Path]: +def list_config_filepaths(configs_root_path: Path, pipeline_name: str) -> List[Path]: """List the config filepaths for a pipeline. Args: - config_root_path (Path): A `Path` object representing the root path of the configs. + configs_root_path (Path): A `Path` object representing the root path of the configs. pipeline_name (str): The name of the pipeline. Returns: List[Path]: A list of `Path` objects representing the config filepaths. """ - configs_dirpath = Path(config_root_path) / pipeline_name + configs_dirpath = Path(configs_root_path) / pipeline_name config_filepaths = [ x for config_type in ConfigType.__members__.values() diff --git a/deployer/utils/console.py b/deployer/utils/console.py index eeda8c9..7559cf2 100644 --- a/deployer/utils/console.py +++ b/deployer/utils/console.py @@ -19,6 +19,7 @@ def ask_user_for_model_fields(model: Type[BaseModel]) -> dict: dict: A dictionary of the set fields. """ set_fields = {} + for field_name, field_info in model.model_fields.items(): if isclass(field_info.annotation) and issubclass(field_info.annotation, BaseModel): answer = Confirm.ask(f"Do you want to configure command {field_name}?", default=False) @@ -36,9 +37,13 @@ def ask_user_for_model_fields(model: Type[BaseModel]) -> dict: if isclass(annotation) and annotation == bool: answer = Confirm.ask(field_name, default=default) else: - answer = Prompt.ask(field_name, default=default, choices=choices) + answer = Prompt.ask( + field_name, default=default if default is not None else "None", choices=choices + ) - if answer != field_info.default: + if answer != field_info.default and not ( + answer in [None, "None"] and field_info.default is None + ): set_fields[field_name] = answer return set_fields diff --git a/deployer/utils/exceptions.py b/deployer/utils/exceptions.py index a6e7278..bbbb6bf 100644 --- a/deployer/utils/exceptions.py +++ b/deployer/utils/exceptions.py @@ -16,3 +16,7 @@ class BadConfigError(ValueError): class InvalidPyProjectTOMLError(Exception): """Raised when the configuration is invalid.""" + + +class TemplateFileCreationError(Exception): + """Exception raised when a file cannot be created from a template.""" diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index 638ecd8..6db310a 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -20,7 +20,7 @@ $ vertex-deployer [OPTIONS] COMMAND [ARGS]... * `config`: Display the configuration from... * `create`: Create files structure for a new pipeline. * `deploy`: Compile, upload, run and schedule pipelines. -* `init` +* `init`: Initialize the deployer. * `list`: List all pipelines. ## `vertex-deployer check` @@ -34,7 +34,7 @@ Checking that a pipeline is valid includes: * Checking that the pipeline can be compiled using `kfp.compiler.Compiler`. -* Checking that config files in `{config_root_path}/{pipeline_name}` are corresponding to the +* Checking that config files in `{configs_root_path}/{pipeline_name}` are corresponding to the pipeline parameters definition, using Pydantic. --- @@ -91,7 +91,7 @@ $ vertex-deployer create [OPTIONS] PIPELINE_NAMES... **Options**: -* `-ct, --config-type [json|py|toml]`: The type of the config to create. [default: json] +* `-ct, --config-type [json|py|toml]`: The type of the config to create. [default: py] * `--help`: Show this message and exit. ## `vertex-deployer deploy` @@ -123,12 +123,13 @@ $ vertex-deployer deploy [OPTIONS] PIPELINE_NAMES... * `-cn, --config-name TEXT`: Name of the json/py file with parameter values and input artifacts to use when running the pipeline. It must be in the pipeline config dir. e.g. `config_dev.json` for `./vertex/configs/{pipeline-name}/config_dev.json`. * `-ec, --enable-caching / -nec, --no-cache`: Whether to turn on caching for the run.If this is not set, defaults to the compile time settings, which are True for alltasks by default, while users may specify different caching options for individualtasks. If this is set, the setting applies to all tasks in the pipeline.Overrides the compile time settings. Defaults to None. * `-en, --experiment-name TEXT`: The name of the experiment to run the pipeline in.Defaults to '{pipeline_name}-experiment'. -* `-lpp, --local-package-path DIRECTORY`: Local dir path where pipelines will be compiled. [default: vertex/pipelines/compiled_pipelines] * `-y, --skip-validation / -n, --no-skip`: Whether to continue without user validation of the settings. [default: skip-validation] * `--help`: Show this message and exit. ## `vertex-deployer init` +Initialize the deployer. + **Usage**: ```console @@ -137,6 +138,7 @@ $ vertex-deployer init [OPTIONS] **Options**: +* `-d, --default`: Instantly creates the full vertex structure and files without configuration prompts * `--help`: Show this message and exit. ## `vertex-deployer list` diff --git a/example/Makefile b/example/Makefile index 4325b21..028ea5b 100644 --- a/example/Makefile +++ b/example/Makefile @@ -2,7 +2,7 @@ .PHONY: build-base-image build-base-image: @gcloud builds submit --config ./vertex/deployment/cloudbuild_local.yaml \ - --substitutions=_GAR_LOCATION=${GAR_LOCATION},_GAR_DOCKER_REPO_ID=${GAR_DOCKER_REPO_ID},_GAR_VERTEX_BASE_IMAGE_NAME=${GAR_VERTEX_BASE_IMAGE_NAME},_TAG=${TAG} + --substitutions=_GCP_REGION=${GCP_REGION},_GAR_LOCATION=${GAR_LOCATION},_GAR_DOCKER_REPO_ID=${GAR_DOCKER_REPO_ID},_GAR_VERTEX_BASE_IMAGE_NAME=${GAR_VERTEX_BASE_IMAGE_NAME},_TAG=${TAG},_GAR_PIPELINES_REPO_ID=${GAR_PIPELINES_REPO_ID},_VERTEX_STAGING_BUCKET_NAME=${VERTEX_STAGING_BUCKET_NAME},_VERTEX_SERVICE_ACCOUNT=${VERTEX_SERVICE_ACCOUNT} # --8<-- [end:build-base-image] # --8<-- [start:deploy-pipeline] diff --git a/example/vertex/deployment/cloudbuild_local.yaml b/example/vertex/deployment/cloudbuild_local.yaml index 079d3ac..f8fcc40 100644 --- a/example/vertex/deployment/cloudbuild_local.yaml +++ b/example/vertex/deployment/cloudbuild_local.yaml @@ -20,6 +20,10 @@ steps: substitutions: _GAR_IMAGE_PATH: '${_GAR_LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_GAR_DOCKER_REPO_ID}/${_GAR_VERTEX_BASE_IMAGE_NAME}:${_TAG}' + _GCP_REGION: '${GCP_REGION}' + _GAR_PIPELINES_REPO_ID: '${GAR_PIPELINES_REPO_ID}' + _VERTEX_STAGING_BUCKET_NAME: '${VERTEX_STAGING_BUCKET_NAME}' + _VERTEX_SERVICE_ACCOUNT: '${VERTEX_SERVICE_ACCOUNT}' options: logging: CLOUD_LOGGING_ONLY diff --git a/pyproject.toml b/pyproject.toml index b8d10cb..2814384 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ pydantic = "^2.3" pyinstrument = { version = "^4.5", optional = true } toml = "^0.10" tomlkit = "^0.12" +jinja2 = "^3.1.3" [tool.poetry.group.dev.dependencies] pytest = "^8.0" @@ -54,6 +55,9 @@ build-backend = "poetry.core.masonry.api" [tool.ruff] target-version = "py38" line-length = 99 +exclude = [ + "deployer/_templates/*", +] [tool.ruff.lint] ignore = [ diff --git a/tests/integration_tests/test_configuration.py b/tests/integration_tests/test_configuration.py index 52f1ab9..ab074c8 100644 --- a/tests/integration_tests/test_configuration.py +++ b/tests/integration_tests/test_configuration.py @@ -66,7 +66,7 @@ def test_deployer_cli_and_settings_consistency(): configured_parameters = { k: v for k, v in get_model_recursive_signature(DeployerSettings).items() - if k not in ["pipelines_root_path", "config_root_path", "log_level"] + if k not in ["vertex_folder_path", "pipelines_root_path", "configs_root_path", "log_level"] } cli_parameters = get_typer_app_signature(app) diff --git a/tests/unit_tests/test_settings.py b/tests/unit_tests/test_settings.py index 444c46f..ae40527 100644 --- a/tests/unit_tests/test_settings.py +++ b/tests/unit_tests/test_settings.py @@ -39,11 +39,11 @@ def test_update_pyproject_toml_updates_file_with_default_settings_in_existing_se build-backend = "poetry.core.masonry.api" [tool.vertex_deployer] - pipelines_root_path = "pipelines" + vertex_folder_path = "vrtx" """, encoding="utf-8", ) - deployer_settings = DeployerSettings(pipelines_root_path="vertex/pipelines") + deployer_settings = DeployerSettings(vertex_folder_path="vertex") # When update_pyproject_toml(path_pyproject_toml, deployer_settings) @@ -53,9 +53,7 @@ def test_update_pyproject_toml_updates_file_with_default_settings_in_existing_se toml_document = tomlkit.loads(path_pyproject_toml.read_text()) assert toml_document["build-system"]["requires"] == ["poetry-core>=1.0.0"] assert toml_document["build-system"]["build-backend"] == "poetry.core.masonry.api" - assert ( - toml_document["tool"]["vertex_deployer"]["pipelines_root_path"] == "vertex/pipelines" - ) + assert toml_document["tool"]["vertex_deployer"]["vertex_folder_path"] == "vertex" def test_update_pyproject_toml_successfully_updates_file_with_non_default_settings( self, tmp_path @@ -64,8 +62,7 @@ def test_update_pyproject_toml_successfully_updates_file_with_non_default_settin path_pyproject_toml = tmp_path / "pyproject.toml" path_pyproject_toml.write_text("", encoding="utf-8") deployer_settings = DeployerSettings( - pipelines_root_path=tmp_path / "pipelines", - config_root_path=tmp_path / "config", + vertex_folder_path=tmp_path, log_level="DEBUG", deploy={ "env_file": tmp_path / ".env", @@ -81,7 +78,6 @@ def test_update_pyproject_toml_successfully_updates_file_with_non_default_settin "config_name": "config_name", "enable_caching": True, "experiment_name": "experiment_name", - "local_package_path": tmp_path / "package", }, check={ "all": True, @@ -106,8 +102,7 @@ def test_update_pyproject_toml_successfully_updates_file_with_non_default_settin assert path_pyproject_toml.exists() toml_document = tomlkit.loads(path_pyproject_toml.read_text()) deployer_section = toml_document["tool"]["vertex_deployer"] - assert deployer_section["pipelines_root_path"] == str(tmp_path / "pipelines") - assert deployer_section["config_root_path"] == str(tmp_path / "config") + assert deployer_section["vertex_folder_path"] == str(tmp_path) assert deployer_section["log_level"] == "DEBUG" assert deployer_section["deploy"]["env_file"] == str(tmp_path / ".env") assert deployer_section["deploy"]["compile"] is False @@ -122,7 +117,6 @@ def test_update_pyproject_toml_successfully_updates_file_with_non_default_settin assert deployer_section["deploy"]["config_name"] == "config_name" assert deployer_section["deploy"]["enable_caching"] is True assert deployer_section["deploy"]["experiment_name"] == "experiment_name" - assert deployer_section["deploy"]["local_package_path"] == str(tmp_path / "package") assert deployer_section["check"]["all"] is True assert deployer_section["check"]["config_filepath"] == str(tmp_path / "config.toml") assert deployer_section["check"]["raise_error"] is True From 8e1faf87f01a2cf060f05e6702165ad3943cc1f6 Mon Sep 17 00:00:00 2001 From: Jules Bertrand <33326907+julesbertrand@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:20:41 +0200 Subject: [PATCH 2/2] docs: improve readme badges (#180) --- README.md | 14 ++++++++------ docs/index.md | 12 ------------ 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 142c31b..d30a84d 100644 --- a/README.md +++ b/README.md @@ -16,16 +16,18 @@
-[![Python Version](https://img.shields.io/badge/Python-3.8_3.9_3.10-blue?logo=python)](#supported-python-versions) -[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) -[![Linting: ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) -[![Pre-commit](https://img.shields.io/badge/pre--commit-enabled-informational?logo=pre-commit&logoColor=white)](https://github.com/ornikar/vertex-eduscore/blob/develop/.pre-commit-config.yaml) -[![License](https://img.shields.io/github/license/artefactory/vertex-pipelines-deployer)](https://github.com/artefactory/vertex-pipelines-deployer/blob/main/LICENSE) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/vertex-deployer?logo=python) +![PyPI - Status](https://img.shields.io/pypi/v/vertex-deployer) +![PyPI - Downloads](https://img.shields.io/pypi/dm/vertex-deployer?color=blue) +![PyPI - License](https://img.shields.io/pypi/l/vertex-deployer) [![CI](https://github.com/artefactory/vertex-pipelines-deployer/actions/workflows/ci.yaml/badge.svg?branch=main&event=push)](https://github.com/artefactory/vertex-pipelines-deployer/actions/workflows/ci.yaml) [![Release](https://github.com/artefactory/vertex-pipelines-deployer/actions/workflows/release.yaml/badge.svg?branch=main&event=push)](https://github.com/artefactory/vertex-pipelines-deployer/actions/workflows/release.yaml) +[![Pre-commit](https://img.shields.io/badge/pre--commit-enabled-informational?logo=pre-commit&logoColor=white)](https://github.com/ornikar/vertex-eduscore/blob/develop/.pre-commit-config.yaml) +[![Linting: ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat)](https://pycqa.github.io/isort/) +
diff --git a/docs/index.md b/docs/index.md index fa051e5..f167285 100644 --- a/docs/index.md +++ b/docs/index.md @@ -14,18 +14,6 @@
- -[![Python Version](https://img.shields.io/badge/Python-3.8_3.9_3.10-blue?logo=python)](#supported-python-versions) -[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) -[![Linting: ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) -[![Pre-commit](https://img.shields.io/badge/pre--commit-enabled-informational?logo=pre-commit&logoColor=white)](https://github.com/ornikar/vertex-eduscore/blob/develop/.pre-commit-config.yaml) -[![License](https://img.shields.io/github/license/artefactory/vertex-pipelines-deployer)](https://github.com/artefactory/vertex-pipelines-deployer/blob/develop/LICENSE) - -[![CI](https://github.com/artefactory/vertex-pipelines-deployer/actions/workflows/ci.yaml/badge.svg?branch%3Adevelop&event%3Apush)](https://github.com/artefactory/vertex-pipelines-deployer/actions/workflows/ci.yaml/badge.svg?query=branch%3Adevelop) -[![Release](https://github.com/artefactory/vertex-pipelines-deployer/actions/workflows/release.yaml/badge.svg?branch%3Amain&event%3Apush)](https://github.com/artefactory/vertex-pipelines-deployer/actions/workflows/release.yaml/badge.svg?query=branch%3Amain) - - !!! info This project is looking for beta testers and contributors.