diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 5645b11..bb36185 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.1 +current_version = 0.2.0 [bumpversion:file:pyproject.toml] diff --git a/.copier-answers.yml b/.copier-answers.yml index b438428..8dacc71 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -7,7 +7,7 @@ description: Kedro plugin with AWS SageMaker Pipelines support docs_url: https://kedro-sagemaker.readthedocs.io/ full_name: Kedro SageMaker Pipelines plugin github_url: https://github.com/getindata/kedro-sagemaker -initial_version: 0.1.1 +initial_version: 0.2.0 keywords: - kedro - sagemaker diff --git a/.gitignore b/.gitignore index 198447e..498f135 100644 --- a/.gitignore +++ b/.gitignore @@ -164,13 +164,5 @@ credentials.json /mlruns/ /ga4/ -# terraform -terraform/terraform.tfstate.backup -terraform/.terraform.lock.hcl -terraform/.terraform/providers/registry.terraform.io/hashicorp/google-beta/4.21.0/darwin_amd64/terraform-provider-google-beta_v4.21.0_x5 -terraform/.terraform/providers/registry.terraform.io/hashicorp/google/4.21.0/darwin_amd64/terraform-provider-google_v4.21.0_x5 -terraform/terraform.tfstate - .idea -conf/azure/credentials.yml - +tests/mlruns diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 98cac3a..5bb94b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.12.0 hooks: - id: isort args: ["--profile", "black", "--line-length=79"] diff --git a/CHANGELOG.md b/CHANGELOG.md index faaa4cb..1c65e6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased] +## [0.2.0] - 2023-02-08 + +- Support for Mlflow with shared run across pipeline steps +- Fixed ability to overwrite docker image in `kedro sagemaker run` + ## [0.1.1] - 2022-12-30 - Pass missing environment to the internal entrypoint @@ -15,7 +20,9 @@ - Project seed prepared -[Unreleased]: https://github.com/getindata/kedro-sagemaker/compare/0.1.1...HEAD +[Unreleased]: https://github.com/getindata/kedro-sagemaker/compare/0.2.0...HEAD + +[0.2.0]: https://github.com/getindata/kedro-sagemaker/compare/0.1.1...0.2.0 [0.1.1]: https://github.com/getindata/kedro-sagemaker/compare/0.1.0...0.1.1 diff --git a/docs/images/pipeline_with_mlflow.gif b/docs/images/pipeline_with_mlflow.gif new file mode 100644 index 0000000..9878fe6 Binary files /dev/null and b/docs/images/pipeline_with_mlflow.gif differ diff --git a/docs/source/02_installation.md b/docs/source/02_installation.md index f68134f..6f7a56f 100644 --- a/docs/source/02_installation.md +++ b/docs/source/02_installation.md @@ -10,7 +10,7 @@ First, you need to install base Kedro package ```console -$ pip install ">=0.18.3,<0.19" +$ pip install "kedro>=0.18.3,<0.19" ``` ## Plugin installation diff --git a/docs/source/03_quickstart.rst b/docs/source/03_quickstart.rst index 9fb0ad6..19623cf 100644 --- a/docs/source/03_quickstart.rst +++ b/docs/source/03_quickstart.rst @@ -5,7 +5,7 @@ Before you start, make sure that you have the following: - AWS CLI installed - AWS SageMaker domain -- SageMaker Execution role ARN (in a form `arn:aws:iam:::role/service-role/AmazonSageMaker-ExecutionRole-`) +- SageMaker Execution role ARN (in a form `arn:aws:iam:::role/service-role/AmazonSageMaker-ExecutionRole-`). If you don't have one, follow the [official AWS docs](https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-roles.html#sagemaker-roles-create-execution-role). - S3 bucket that the above role has R/W access - Docker installed - Amazon Elastic Container Registry (`Amazon ECR `__) repository created that the above role has read access and you have write access @@ -100,6 +100,10 @@ Finally, you will see similar logs in your terminal: |Kedro SageMaker Pipelines execution| +Additionally, if you have (`kedro-mlflow `__) plugin installed, an additional node called `start-mlflow-run` will appear on execution graph. It's job is to log the SageMaker's Pipeline Execution ARN (so you can link runs with mlflow with runs in SageMaker) and make sure that all nodes use common Mlflow run. + +|Kedro SageMaker Pipeline with Mlflow| .. |Kedro SageMaker Pipelines execution| image:: ../images/sagemaker_running_pipeline.gif +.. |Kedro SageMaker Pipeline with Mlflow| image:: ../images/pipeline_with_mlflow.gif diff --git a/kedro_sagemaker/__init__.py b/kedro_sagemaker/__init__.py index 485f44a..d3ec452 100644 --- a/kedro_sagemaker/__init__.py +++ b/kedro_sagemaker/__init__.py @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.2.0" diff --git a/kedro_sagemaker/cli.py b/kedro_sagemaker/cli.py index 40f4118..f78af0a 100644 --- a/kedro_sagemaker/cli.py +++ b/kedro_sagemaker/cli.py @@ -12,6 +12,7 @@ from kedro_sagemaker.cli_functions import ( docker_autobuild, get_context_and_pipeline, + lookup_mlflow_run_id, parse_extra_params, write_file_and_confirm_overwrite, ) @@ -20,14 +21,17 @@ from kedro_sagemaker.constants import ( KEDRO_SAGEMAKER_ARGS, KEDRO_SAGEMAKER_DEBUG, + KEDRO_SAGEMAKER_EXECUTION_ARN, KEDRO_SAGEMAKER_S3_TEMP_DIR_NAME, KEDRO_SAGEMAKER_WORKING_DIRECTORY, + MLFLOW_TAG_EXECUTION_ARN, ) from kedro_sagemaker.docker import DOCKERFILE_TEMPLATE, DOCKERIGNORE_TEMPLATE from kedro_sagemaker.runner import SageMakerPipelinesRunner from kedro_sagemaker.utils import ( CliContext, KedroContextManager, + is_mlflow_enabled, parse_flat_parameters, ) @@ -196,9 +200,11 @@ def run( ) is_ok = client.run( - local, - wait_for_completion, - lambda p: click.echo(f"Pipeline ARN: {p.describe()['PipelineArn']}"), + is_local=local, + wait_for_completion=wait_for_completion, + on_pipeline_started=lambda p: click.echo( + f"Pipeline ARN: {p.describe()['PipelineArn']}" + ), ) if is_ok: @@ -341,5 +347,35 @@ def execute(ctx: CliContext, pipeline: str, node: str, params: str): with KedroContextManager( ctx.metadata.package_name, env=ctx.env, extra_params=parameters ) as mgr: + if is_mlflow_enabled(): + env_key, env_value = lookup_mlflow_run_id( + mgr.context, os.getenv(KEDRO_SAGEMAKER_EXECUTION_ARN) + ) + if env_value is not None: + click.echo(f"Mlflow run id: {env_value}") + os.environ[env_key] = env_value + runner = SageMakerPipelinesRunner() mgr.session.run(pipeline, node_names=[node], runner=runner) + + +@sagemaker_group.command(hidden=True) +@click.pass_obj +def mlflow_start(ctx: CliContext): + """ + Registers new mlflow run with Sagemaker Execution ARN inside the tags + """ + import mlflow + from kedro_mlflow.config.kedro_mlflow_config import KedroMlflowConfig + + with KedroContextManager(ctx.metadata.package_name, env=ctx.env) as mgr: + mlflow_conf: KedroMlflowConfig = mgr.context.mlflow + + run = mlflow.start_run( + experiment_id=mlflow.get_experiment_by_name( + mlflow_conf.tracking.experiment.name + ).experiment_id, + nested=False, + ) + mlflow.set_tag(MLFLOW_TAG_EXECUTION_ARN, os.environ[KEDRO_SAGEMAKER_EXECUTION_ARN]) + click.echo(f"Started run: {run.info.run_id}") diff --git a/kedro_sagemaker/cli_functions.py b/kedro_sagemaker/cli_functions.py index fb48247..256f683 100644 --- a/kedro_sagemaker/cli_functions.py +++ b/kedro_sagemaker/cli_functions.py @@ -1,4 +1,6 @@ +import importlib import json +import logging from contextlib import contextmanager from pathlib import Path from typing import Callable, Iterator, Optional, Tuple @@ -6,6 +8,7 @@ import click from sagemaker.workflow.pipeline import Pipeline as SageMakerPipeline +from kedro_sagemaker.constants import MLFLOW_TAG_EXECUTION_ARN from kedro_sagemaker.generator import KedroSageMakerGenerator from kedro_sagemaker.utils import ( CliContext, @@ -14,6 +17,8 @@ docker_push, ) +logger = logging.getLogger() + def parse_extra_params(params, silent=False): if params and (parameters := json.loads(params.strip("'"))): @@ -94,3 +99,26 @@ def write_file_and_confirm_overwrite( filepath.write_text(contents) elif on_denied_overwrite: on_denied_overwrite(filepath) + + +def lookup_mlflow_run_id(context, sagemaker_execution_arn: str): + import mlflow + from kedro_mlflow.config.kedro_mlflow_config import KedroMlflowConfig + + mlflow_conf: KedroMlflowConfig = context.mlflow + mlflow_runs = mlflow.search_runs( + experiment_names=[mlflow_conf.tracking.experiment.name], + filter_string=f'tags.`{MLFLOW_TAG_EXECUTION_ARN}` = "{sagemaker_execution_arn}"', + max_results=1, + output_format="list", + ) + importlib.reload(mlflow.tracking.request_header.registry) + + if len(mlflow_runs) == 0: + logger.warning( + "Unable to find parent mlflow run id for the current execution (%s)", + sagemaker_execution_arn, + ) + return mlflow.tracking._RUN_ID_ENV_VAR, None + + return mlflow.tracking._RUN_ID_ENV_VAR, mlflow_runs[0].info.run_id diff --git a/kedro_sagemaker/constants.py b/kedro_sagemaker/constants.py index be4075f..b43622d 100644 --- a/kedro_sagemaker/constants.py +++ b/kedro_sagemaker/constants.py @@ -5,6 +5,8 @@ KEDRO_SAGEMAKER_DEBUG = f"{KEDRO_SAGEMAKER}_DEBUG" KEDRO_SAGEMAKER_WORKING_DIRECTORY = f"{KEDRO_SAGEMAKER}_WD" KEDRO_SAGEMAKER_PARAMETERS = f"{KEDRO_SAGEMAKER}_PARAMETERS" +KEDRO_SAGEMAKER_EXECUTION_ARN = f"{KEDRO_SAGEMAKER}_EXECUTION_ARN" KEDRO_SAGEMAKER_PARAM_KEY_PREFIX = f"{KEDRO_SAGEMAKER}_PARAM_KEY_" KEDRO_SAGEMAKER_PARAM_VALUE_PREFIX = f"{KEDRO_SAGEMAKER}_PARAM_VALUE_" KEDRO_SAGEMAKER_S3_TEMP_DIR_NAME = "kedro-sagemaker-tmp" +MLFLOW_TAG_EXECUTION_ARN = "sagemaker_execution_arn" diff --git a/kedro_sagemaker/generator.py b/kedro_sagemaker/generator.py index c240586..2c26a0c 100644 --- a/kedro_sagemaker/generator.py +++ b/kedro_sagemaker/generator.py @@ -1,7 +1,7 @@ import json from itertools import chain +from types import SimpleNamespace from typing import Dict, Iterator, List, Optional, Tuple, Union -from uuid import uuid4 from kedro.framework.context import KedroContext from kedro.io import DataCatalog @@ -9,6 +9,7 @@ from kedro.pipeline.node import Node as KedroNode from sagemaker import Model, Processor from sagemaker.estimator import Estimator +from sagemaker.workflow.execution_variables import ExecutionVariables from sagemaker.workflow.model_step import ModelStep from sagemaker.workflow.parameters import ( ParameterBoolean, @@ -21,7 +22,7 @@ LocalPipelineSession, PipelineSession, ) -from sagemaker.workflow.steps import ProcessingStep, TrainingStep +from sagemaker.workflow.steps import ProcessingStep, StepTypeEnum, TrainingStep from kedro_sagemaker.config import ( KedroSageMakerPluginConfig, @@ -30,6 +31,7 @@ ) from kedro_sagemaker.constants import ( KEDRO_SAGEMAKER_ARGS, + KEDRO_SAGEMAKER_EXECUTION_ARN, KEDRO_SAGEMAKER_METRICS, KEDRO_SAGEMAKER_PARAM_KEY_PREFIX, KEDRO_SAGEMAKER_PARAM_VALUE_PREFIX, @@ -38,7 +40,7 @@ ) from kedro_sagemaker.datasets import SageMakerModelDataset from kedro_sagemaker.runner import KedroSageMakerRunnerConfig -from kedro_sagemaker.utils import flatten_dict +from kedro_sagemaker.utils import flatten_dict, is_mlflow_enabled SageMakerStepType = Union[ProcessingStep, TrainingStep, ModelStep] @@ -54,7 +56,7 @@ def __init__( execution_role_arn: Optional[str] = None, ): self.is_local = is_local - self.docker_image = docker_image + self.docker_image = docker_image or config.docker.image self.config = config self.kedro_context = kedro_context self.pipeline_name = pipeline_name @@ -142,10 +144,7 @@ def _get_resources_for_node(self, node: KedroNode): return defaults def generate(self) -> SageMakerPipeline: - run_id = uuid4().hex - runner_config = KedroSageMakerRunnerConfig( - bucket=self.config.aws.bucket, run_id=run_id - ) + runner_config = KedroSageMakerRunnerConfig(bucket=self.config.aws.bucket) sagemaker_session = ( LocalPipelineSession() if self.is_local else PipelineSession() @@ -216,7 +215,9 @@ def generate(self) -> SageMakerPipeline: ) steps[node.name] = step - steps = self._add_step_dependencies(pipeline, steps) + self._add_step_dependencies(pipeline, steps) + if is_mlflow_enabled(): + self._add_mlflow_support(steps, runner_config) smp = SageMakerPipeline( self._get_sagemaker_pipeline_name(), @@ -226,6 +227,31 @@ def generate(self) -> SageMakerPipeline: ) return smp + def _add_mlflow_support(self, steps, runner_config): + mlflow_start_run = self._create_processing_step( + node=SimpleNamespace(name="start-mlflow-run"), + node_resources=ResourceConfig(instance_type="ml.t3.medium"), + runner_config=runner_config, + sm_node_name="start-mlflow-run", + sm_param_envs={}, + entrypoint=[ + "kedro", + "sagemaker", + "-e", + self.kedro_context.env or "local", + "mlflow-start", + ], + ) + for step in steps.values(): + if step.depends_on is not None: + continue + if step.step_type not in (StepTypeEnum.TRAINING, StepTypeEnum.PROCESSING): + continue + + step.add_depends_on([mlflow_start_run]) + + steps["start-mlflow-run"] = mlflow_start_run + def _add_step_dependencies( self, pipeline, steps: Dict[str, SageMakerStepType] ) -> Dict[str, SageMakerStepType]: @@ -237,19 +263,26 @@ def _add_step_dependencies( return steps def _create_processing_step( - self, node, node_resources, runner_config, sm_node_name, sm_param_envs + self, + node, + node_resources, + runner_config, + sm_node_name, + sm_param_envs, + entrypoint=None, ): step = ProcessingStep( sm_node_name, processor=Processor( - entrypoint=self._get_kedro_command(node), + entrypoint=entrypoint or self._get_kedro_command(node), role=self._execution_role, - image_uri=self.config.docker.image, + image_uri=self.docker_image, instance_count=node_resources.instance_count, instance_type=node_resources.instance_type, max_runtime_in_seconds=node_resources.timeout_seconds, env={ KEDRO_SAGEMAKER_RUNNER_CONFIG: runner_config.json(), + KEDRO_SAGEMAKER_EXECUTION_ARN: ExecutionVariables.PIPELINE_EXECUTION_ARN, **sm_param_envs, }, ), @@ -283,7 +316,7 @@ def _create_model_register_steps( ) -> Union[Tuple[ModelStep], Tuple[ModelStep, ModelStep]]: model_output_name = sagemaker_model_outputs[0].replace(".", "__") model = Model( - image_uri=self.config.docker.image, + image_uri=self.docker_image, model_data=step.properties.ModelArtifacts.S3ModelArtifacts, role=self._execution_role, name=model_output_name, @@ -303,7 +336,7 @@ def _create_model_register_steps( response_types=["application/json"], domain="MACHINE_LEARNING", task="OTHER", - image_uri=self.config.docker.image, + image_uri=self.docker_image, ) # TODO - maybe model metrics from https://docs.aws.amazon.com/sagemaker/latest/dg/define-pipeline.html ? @@ -335,7 +368,7 @@ def _create_training_step( return TrainingStep( sm_node_name, estimator=Estimator( - image_uri=self.config.docker.image, + image_uri=self.docker_image, role=self._execution_role, instance_count=node_resources.instance_count, instance_type=node_resources.instance_type, @@ -348,6 +381,7 @@ def _create_training_step( KEDRO_SAGEMAKER_ARGS: self._get_kedro_command(node, as_string=True), KEDRO_SAGEMAKER_RUNNER_CONFIG: runner_config.json(), KEDRO_SAGEMAKER_WORKING_DIRECTORY: self.config.docker.working_directory, + KEDRO_SAGEMAKER_EXECUTION_ARN: ExecutionVariables.PIPELINE_EXECUTION_ARN, # "PYTHONPATH": "/home/kedro/src", # # TODO - this will not be needed if plugin is installed, I hope :D, **sm_param_envs, diff --git a/kedro_sagemaker/runner.py b/kedro_sagemaker/runner.py index b8d755c..94e6602 100644 --- a/kedro_sagemaker/runner.py +++ b/kedro_sagemaker/runner.py @@ -8,7 +8,10 @@ from pluggy import PluginManager from pydantic import BaseModel -from kedro_sagemaker.constants import KEDRO_SAGEMAKER_RUNNER_CONFIG +from kedro_sagemaker.constants import ( + KEDRO_SAGEMAKER_EXECUTION_ARN, + KEDRO_SAGEMAKER_RUNNER_CONFIG, +) from kedro_sagemaker.datasets import ( CloudpickleDataset, DistributedCloudpickleDataset, @@ -20,7 +23,6 @@ class KedroSageMakerRunnerConfig(BaseModel): bucket: str - run_id: str class SageMakerPipelinesRunner(SequentialRunner): @@ -34,6 +36,7 @@ def __init__(self, is_async: bool = False): self.runner_config = KedroSageMakerRunnerConfig.parse_raw( self.runner_config_raw ) + self.run_id = os.getenv(KEDRO_SAGEMAKER_EXECUTION_ARN, "local").split(":")[-1] def run( self, @@ -58,7 +61,7 @@ def create_default_data_set(self, ds_name: str) -> AbstractDataSet: dataset_cls = DistributedCloudpickleDataset return dataset_cls( - self.runner_config.bucket, - ds_name, - self.runner_config.run_id, + bucket=self.runner_config.bucket, + dataset_name=ds_name, + run_id=self.run_id, ) diff --git a/kedro_sagemaker/utils.py b/kedro_sagemaker/utils.py index 29f1966..7033776 100644 --- a/kedro_sagemaker/utils.py +++ b/kedro_sagemaker/utils.py @@ -171,3 +171,13 @@ def docker_push(image: str) -> int: if rv: logger.error("Docker push has failed.") return rv + + +def is_mlflow_enabled() -> bool: + try: + import kedro_mlflow # NOQA + import mlflow # NOQA + + return True + except ImportError: + return False diff --git a/poetry.lock b/poetry.lock index b675130..67620a6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -173,6 +173,27 @@ files = [ {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, ] +[[package]] +name = "alembic" +version = "1.9.2" +description = "A database migration tool for SQLAlchemy." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "alembic-1.9.2-py3-none-any.whl", hash = "sha256:e8a6ff9f3b1887e1fed68bfb8fb9a000d8f61c21bdcc85b67bb9f87fcbc4fce3"}, + {file = "alembic-1.9.2.tar.gz", hash = "sha256:6880dec4f28dd7bd999d2ed13fbe7c9d4337700a44d11a524c0ce0c59aaf0dbd"}, +] + +[package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.9\""} +importlib-resources = {version = "*", markers = "python_version < \"3.9\""} +Mako = "*" +SQLAlchemy = ">=1.3.0" + +[package.extras] +tz = ["python-dateutil"] + [[package]] name = "anyconfig" version = "0.10.1" @@ -653,6 +674,26 @@ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.1 [package.extras] toml = ["tomli"] +[[package]] +name = "databricks-cli" +version = "0.17.4" +description = "A command line interface for Databricks" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "databricks-cli-0.17.4.tar.gz", hash = "sha256:bc0c4dd082f033cb6d7978cacaca5261698efe3a4c70f52f98762c38db925ce0"}, + {file = "databricks_cli-0.17.4-py2-none-any.whl", hash = "sha256:bbd57bc21c88ac6d1f8f0b250db986e500490c4d3cb69664229384632eaeed81"}, +] + +[package.dependencies] +click = ">=7.0" +oauthlib = ">=3.1.0" +pyjwt = ">=1.7.0" +requests = ">=2.17.3" +six = ">=1.10.0" +tabulate = ">=0.7.7" + [[package]] name = "dill" version = "0.3.6" @@ -680,6 +721,28 @@ files = [ {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] +[[package]] +name = "docker" +version = "6.0.1" +description = "A Python library for the Docker Engine API." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "docker-6.0.1-py3-none-any.whl", hash = "sha256:dbcb3bd2fa80dca0788ed908218bf43972772009b881ed1e20dfc29a65e49782"}, + {file = "docker-6.0.1.tar.gz", hash = "sha256:896c4282e5c7af5c45e8b683b0b0c33932974fe6e50fc6906a0a83616ab3da97"}, +] + +[package.dependencies] +packaging = ">=14.0" +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" +websocket-client = ">=0.32.0" + +[package.extras] +ssh = ["paramiko (>=2.4.3)"] + [[package]] name = "docutils" version = "0.19" @@ -714,6 +777,18 @@ toml = ["toml"] vault = ["hvac"] yaml = ["ruamel.yaml"] +[[package]] +name = "entrypoints" +version = "0.4" +description = "Discover and load entry points from installed packages." +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "entrypoints-0.4-py3-none-any.whl", hash = "sha256:f174b5ff827504fd3cd97cc3f8649f3693f51538c7e4bdf3ef002c8429d42f9f"}, + {file = "entrypoints-0.4.tar.gz", hash = "sha256:b706eddaa9218a19ebcd67b56818f05bb27589b1ca9e8d797b74affad4ccacd4"}, +] + [[package]] name = "filelock" version = "3.8.2" @@ -730,6 +805,29 @@ files = [ docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] testing = ["covdefaults (>=2.2.2)", "coverage (>=6.5)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] +[[package]] +name = "flask" +version = "2.2.2" +description = "A simple framework for building complex web applications." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Flask-2.2.2-py3-none-any.whl", hash = "sha256:b9c46cc36662a7949f34b52d8ec7bb59c0d74ba08ba6cb9ce9adc1d8676d9526"}, + {file = "Flask-2.2.2.tar.gz", hash = "sha256:642c450d19c4ad482f96729bd2a8f6d32554aa1e231f4f6b4e7e5264b16cca2b"}, +] + +[package.dependencies] +click = ">=8.0" +importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""} +itsdangerous = ">=2.0" +Jinja2 = ">=3.0" +Werkzeug = ">=2.2.2" + +[package.extras] +async = ["asgiref (>=3.2)"] +dotenv = ["python-dotenv"] + [[package]] name = "frozenlist" version = "1.3.3" @@ -895,6 +993,101 @@ files = [ [package.dependencies] six = "*" +[[package]] +name = "greenlet" +version = "2.0.2" +description = "Lightweight in-process concurrent programming" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +files = [ + {file = "greenlet-2.0.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:bdfea8c661e80d3c1c99ad7c3ff74e6e87184895bbaca6ee8cc61209f8b9b85d"}, + {file = "greenlet-2.0.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9d14b83fab60d5e8abe587d51c75b252bcc21683f24699ada8fb275d7712f5a9"}, + {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, + {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, + {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, + {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, + {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, + {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, + {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470"}, + {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3a51c9751078733d88e013587b108f1b7a1fb106d402fb390740f002b6f6551a"}, + {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, + {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, + {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, + {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, + {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, + {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, + {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:eff4eb9b7eb3e4d0cae3d28c283dc16d9bed6b193c2e1ace3ed86ce48ea8df19"}, + {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5454276c07d27a740c5892f4907c86327b632127dd9abec42ee62e12427ff7e3"}, + {file = "greenlet-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5"}, + {file = "greenlet-2.0.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6"}, + {file = "greenlet-2.0.2-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43"}, + {file = "greenlet-2.0.2-cp35-cp35m-win32.whl", hash = "sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a"}, + {file = "greenlet-2.0.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4b58adb399c4d61d912c4c331984d60eb66565175cdf4a34792cd9600f21b394"}, + {file = "greenlet-2.0.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:703f18f3fda276b9a916f0934d2fb6d989bf0b4fb5a64825260eb9bfd52d78f0"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:32e5b64b148966d9cccc2c8d35a671409e45f195864560829f395a54226408d3"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dd11f291565a81d71dab10b7033395b7a3a5456e637cf997a6f33ebdf06f8db"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099"}, + {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75"}, + {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:3c9b12575734155d0c09d6c3e10dbd81665d5c18e1a7c6597df72fd05990c8cf"}, + {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b9ec052b06a0524f0e35bd8790686a1da006bd911dd1ef7d50b77bfbad74e292"}, + {file = "greenlet-2.0.2-cp36-cp36m-win32.whl", hash = "sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9"}, + {file = "greenlet-2.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9f35ec95538f50292f6d8f2c9c9f8a3c6540bbfec21c9e5b4b751e0a7c20864f"}, + {file = "greenlet-2.0.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2780572ec463d44c1d3ae850239508dbeb9fed38e294c68d19a24d925d9223ca"}, + {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937e9020b514ceedb9c830c55d5c9872abc90f4b5862f89c0887033ae33c6f73"}, + {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:36abbf031e1c0f79dd5d596bfaf8e921c41df2bdf54ee1eed921ce1f52999a86"}, + {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33"}, + {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, + {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, + {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, + {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acd2162a36d3de67ee896c43effcd5ee3de247eb00354db411feb025aa319857"}, + {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a"}, + {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, + {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, + {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, + {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be4ed120b52ae4d974aa40215fcdfde9194d63541c7ded40ee12eb4dda57b76b"}, + {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94c817e84245513926588caf1152e3b559ff794d505555211ca041f032abbb6b"}, + {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8"}, + {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9"}, + {file = "greenlet-2.0.2-cp39-cp39-win32.whl", hash = "sha256:ea9872c80c132f4663822dd2a08d404073a5a9b5ba6155bea72fb2a79d1093b5"}, + {file = "greenlet-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564"}, + {file = "greenlet-2.0.2.tar.gz", hash = "sha256:e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0"}, +] + +[package.extras] +docs = ["Sphinx", "docutils (<0.18)"] +test = ["objgraph", "psutil"] + +[[package]] +name = "gunicorn" +version = "20.1.0" +description = "WSGI HTTP Server for UNIX" +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "gunicorn-20.1.0-py3-none-any.whl", hash = "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e"}, + {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, +] + +[package.dependencies] +setuptools = ">=3.0" + +[package.extras] +eventlet = ["eventlet (>=0.24.1)"] +gevent = ["gevent (>=1.4.0)"] +setproctitle = ["setproctitle"] +tornado = ["tornado (>=0.2)"] + [[package]] name = "identify" version = "2.5.11" @@ -985,6 +1178,18 @@ files = [ {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] +[[package]] +name = "itsdangerous" +version = "2.1.2" +description = "Safely pass data to untrusted environments and back." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"}, + {file = "itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"}, +] + [[package]] name = "jinja2" version = "3.1.2" @@ -1116,6 +1321,47 @@ video-videodataset = ["opencv-python (>=4.5.5.64,<4.6.0.0)"] yaml = ["PyYAML (>=4.2,<7.0)", "pandas (>=1.3,<2.0)"] yaml-yamldataset = ["PyYAML (>=4.2,<7.0)", "pandas (>=1.3,<2.0)"] +[[package]] +name = "kedro-mlflow" +version = "0.11.7" +description = "A kedro-plugin to use mlflow in your kedro projects" +category = "dev" +optional = false +python-versions = ">=3.7, <3.11" +files = [ + {file = "kedro_mlflow-0.11.7.tar.gz", hash = "sha256:0dab6d7ba49ab4bb748de967b03736e47c694408dff746c38fd2931ae3b2aef8"}, +] + +[package.dependencies] +kedro = ">=0.18.1,<0.19.0" +mlflow = ">=1.0.0,<2.0.0" +pydantic = ">=1.0.0,<2.0.0" + +[package.extras] +dev = ["jupyter (>=1.0.0,<2.0.0)", "pre-commit (>=2.0.0,<3.0.0)"] +doc = ["myst-parser (>=0.17.2,<0.19.0)", "sphinx (>=4.5.0,<7.0.0)", "sphinx-click (>=3.1,<4.5)", "sphinx-markdown-tables (>=0.0.15,<0.1.0)", "sphinx_copybutton (>=0.5.0,<0.6.0)", "sphinx_rtd_theme (>=1.0,<1.2)"] +test = ["black (==22.12.0)", "flake8 (==5.0.4)", "isort (==5.11.4)", "pytest (>=5.4.0,<8.0.0)", "pytest-cov (>=2.8.0,<5.0.0)", "pytest-lazy-fixture (>=0.6.0,<1.0.0)", "pytest-mock (>=3.1.0,<4.0.0)", "scikit-learn (>=0.23.0,<1.3.0)"] + +[[package]] +name = "mako" +version = "1.2.4" +description = "A super-fast templating language that borrows the best ideas from the existing templating languages." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Mako-1.2.4-py3-none-any.whl", hash = "sha256:c97c79c018b9165ac9922ae4f32da095ffd3c4e6872b45eded42926deea46818"}, + {file = "Mako-1.2.4.tar.gz", hash = "sha256:d60a3903dc3bb01a18ad6a89cdbe2e4eadc69c0bc8ef1e3773ba53d44c3f7a34"}, +] + +[package.dependencies] +MarkupSafe = ">=0.9.2" + +[package.extras] +babel = ["Babel"] +lingua = ["lingua"] +testing = ["pytest"] + [[package]] name = "markupsafe" version = "2.1.1" @@ -1166,6 +1412,49 @@ files = [ {file = "MarkupSafe-2.1.1.tar.gz", hash = "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b"}, ] +[[package]] +name = "mlflow" +version = "1.27.0" +description = "MLflow: A Platform for ML Development and Productionization" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mlflow-1.27.0-py3-none-any.whl", hash = "sha256:d759f3eefad2ff509a0fbc10507224204c6f6bb8d7f437bbf0bb9961cf74ff95"}, + {file = "mlflow-1.27.0.tar.gz", hash = "sha256:6a1e34d6be266725e41d4547572a8425d86d6623e1c8888cf3f22b90019be0aa"}, +] + +[package.dependencies] +alembic = "*" +click = ">=7.0" +cloudpickle = "*" +databricks-cli = ">=0.8.7" +docker = ">=4.0.0" +entrypoints = "*" +Flask = "*" +gitpython = ">=2.1.0" +gunicorn = {version = "*", markers = "platform_system != \"Windows\""} +importlib-metadata = ">=3.7.0,<4.7.0 || >4.7.0" +numpy = "*" +packaging = "*" +pandas = "*" +prometheus-flask-exporter = "*" +protobuf = ">=3.12.0" +pytz = "*" +pyyaml = ">=5.1" +querystring-parser = "*" +requests = ">=2.17.3" +scipy = "*" +sqlalchemy = ">=1.4.0" +sqlparse = ">=0.3.1" +waitress = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +aliyun-oss = ["aliyunstoreplugin"] +extras = ["azureml-core (>=1.2.0)", "boto3", "google-cloud-storage (>=1.30.0)", "kubernetes", "mlserver (>=0.5.3)", "mlserver-mlflow (>=0.5.3)", "pyarrow", "pysftp", "scikit-learn", "virtualenv"] +pipelines = ["Jinja2 (>=3.0)", "ipython (>=7.0)", "markdown (>=3.3)", "pandas-profiling (>=3.1)", "pyarrow (>=7.0)", "scikit-learn (>=1.0)", "shap (>=0.40)"] +sqlserver = ["mlflow-dbstore"] + [[package]] name = "multidict" version = "6.0.4" @@ -1330,6 +1619,23 @@ files = [ {file = "numpy-1.24.1.tar.gz", hash = "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2"}, ] +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + [[package]] name = "packaging" version = "22.0" @@ -1536,6 +1842,37 @@ nodeenv = ">=0.11.1" pyyaml = ">=5.1" virtualenv = ">=20.10.0" +[[package]] +name = "prometheus-client" +version = "0.16.0" +description = "Python client for the Prometheus monitoring system." +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "prometheus_client-0.16.0-py3-none-any.whl", hash = "sha256:0836af6eb2c8f4fed712b2f279f6c0a8bbab29f9f4aa15276b91c7cb0d1616ab"}, + {file = "prometheus_client-0.16.0.tar.gz", hash = "sha256:a03e35b359f14dd1630898543e2120addfdeacd1a6069c1367ae90fd93ad3f48"}, +] + +[package.extras] +twisted = ["twisted"] + +[[package]] +name = "prometheus-flask-exporter" +version = "0.21.0" +description = "Prometheus metrics exporter for Flask" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "prometheus_flask_exporter-0.21.0-py3-none-any.whl", hash = "sha256:6dc4a010c299d1ed94b6151d91f129c4513fb8aa04310db00be4ccb0006de400"}, + {file = "prometheus_flask_exporter-0.21.0.tar.gz", hash = "sha256:ebbc016c1e3d16e7cd39fe651a6c52ac68779858b2d5d1be6ddbc9e66f7fc29f"}, +] + +[package.dependencies] +flask = "*" +prometheus-client = "*" + [[package]] name = "protobuf" version = "3.20.3" @@ -1675,6 +2012,24 @@ files = [ [package.extras] plugins = ["importlib-metadata"] +[[package]] +name = "pyjwt" +version = "2.6.0" +description = "JSON Web Token implementation in Python" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, + {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pytest" version = "6.2.5" @@ -1787,6 +2142,30 @@ files = [ {file = "pytz-2022.7.tar.gz", hash = "sha256:7ccfae7b4b2c067464a6733c6261673fdb8fd1be905460396b97a073e9fa683a"}, ] +[[package]] +name = "pywin32" +version = "305" +description = "Python for Window Extensions" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, + {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, + {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, + {file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"}, + {file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"}, + {file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"}, + {file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"}, + {file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"}, + {file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"}, + {file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"}, + {file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"}, + {file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"}, + {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, + {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, +] + [[package]] name = "pyyaml" version = "6.0" @@ -1837,6 +2216,21 @@ files = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +[[package]] +name = "querystring-parser" +version = "1.2.4" +description = "QueryString parser for Python/Django that correctly handles nested dictionaries" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "querystring_parser-1.2.4-py2.py3-none-any.whl", hash = "sha256:d2fa90765eaf0de96c8b087872991a10238e89ba015ae59fedfed6bd61c242a0"}, + {file = "querystring_parser-1.2.4.tar.gz", hash = "sha256:644fce1cffe0530453b43a83a38094dbe422ccba8c9b2f2a1c00280e14ca8a62"}, +] + +[package.dependencies] +six = "*" + [[package]] name = "recommonmark" version = "0.7.1" @@ -2000,6 +2394,45 @@ files = [ [package.dependencies] contextlib2 = ">=0.5.5" +[[package]] +name = "scipy" +version = "1.10.0" +description = "Fundamental algorithms for scientific computing in Python" +category = "dev" +optional = false +python-versions = "<3.12,>=3.8" +files = [ + {file = "scipy-1.10.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:b901b423c91281a974f6cd1c36f5c6c523e665b5a6d5e80fcb2334e14670eefd"}, + {file = "scipy-1.10.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:16ba05d3d1b9f2141004f3f36888e05894a525960b07f4c2bfc0456b955a00be"}, + {file = "scipy-1.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:151f066fe7d6653c3ffefd489497b8fa66d7316e3e0d0c0f7ff6acca1b802809"}, + {file = "scipy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2f9ea0a37aca111a407cb98aa4e8dfde6e5d9333bae06dfa5d938d14c80bb5c3"}, + {file = "scipy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:27e548276b5a88b51212b61f6dda49a24acf5d770dff940bd372b3f7ced8c6c2"}, + {file = "scipy-1.10.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:42ab8b9e7dc1ebe248e55f54eea5307b6ab15011a7883367af48dd781d1312e4"}, + {file = "scipy-1.10.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e096b062d2efdea57f972d232358cb068413dc54eec4f24158bcbb5cb8bddfd8"}, + {file = "scipy-1.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4df25a28bd22c990b22129d3c637fd5c3be4b7c94f975dca909d8bab3309b694"}, + {file = "scipy-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ad449db4e0820e4b42baccefc98ec772ad7818dcbc9e28b85aa05a536b0f1a2"}, + {file = "scipy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:6faf86ef7717891195ae0537e48da7524d30bc3b828b30c9b115d04ea42f076f"}, + {file = "scipy-1.10.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:4bd0e3278126bc882d10414436e58fa3f1eca0aa88b534fcbf80ed47e854f46c"}, + {file = "scipy-1.10.0-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:38bfbd18dcc69eeb589811e77fae552fa923067fdfbb2e171c9eac749885f210"}, + {file = "scipy-1.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ab2a58064836632e2cec31ca197d3695c86b066bc4818052b3f5381bfd2a728"}, + {file = "scipy-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cd7a30970c29d9768a7164f564d1fbf2842bfc77b7d114a99bc32703ce0bf48"}, + {file = "scipy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:9b878c671655864af59c108c20e4da1e796154bd78c0ed6bb02bc41c84625686"}, + {file = "scipy-1.10.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:3afcbddb4488ac950ce1147e7580178b333a29cd43524c689b2e3543a080a2c8"}, + {file = "scipy-1.10.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:6e4497e5142f325a5423ff5fda2fff5b5d953da028637ff7c704378c8c284ea7"}, + {file = "scipy-1.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:441cab2166607c82e6d7a8683779cb89ba0f475b983c7e4ab88f3668e268c143"}, + {file = "scipy-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0490dc499fe23e4be35b8b6dd1e60a4a34f0c4adb30ac671e6332446b3cbbb5a"}, + {file = "scipy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:954ff69d2d1bf666b794c1d7216e0a746c9d9289096a64ab3355a17c7c59db54"}, + {file = "scipy-1.10.0.tar.gz", hash = "sha256:c8b3cbc636a87a89b770c6afc999baa6bcbb01691b5ccbbc1b1791c7c0a07540"}, +] + +[package.dependencies] +numpy = ">=1.19.5,<1.27.0" + +[package.extras] +dev = ["click", "doit (>=0.36.0)", "flake8", "mypy", "pycodestyle", "pydevtool", "rich-click", "typing_extensions"] +doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-design (>=0.2.0)"] +test = ["asv", "gmpy2", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + [[package]] name = "setuptools" version = "65.6.3" @@ -2196,6 +2629,111 @@ files = [ lint = ["docutils-stubs", "flake8", "mypy"] test = ["pytest"] +[[package]] +name = "sqlalchemy" +version = "2.0.0" +description = "Database Abstraction Library" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "SQLAlchemy-2.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:807d4f83dcf0b7fd60b7af5f677e3d20151083c3454304813e450f6f6e4b4a5c"}, + {file = "SQLAlchemy-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:419228c073060face5e35388ddf00229f1be3664c91143f6e6897d67254589f7"}, + {file = "SQLAlchemy-2.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e052ae0c2a887472a74405e3afa5aa5c75cddc8a98a49bbf4a84a09dbc1cb896"}, + {file = "SQLAlchemy-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0bc643a0228179bfcbc8df81c8d197b843e48d97073f41f90ada8f6aad1614d"}, + {file = "SQLAlchemy-2.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:492dfab60c3df7105c97474a08408f15a506966340643eeaf40f59daa08a516e"}, + {file = "SQLAlchemy-2.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:20ef6ed15ecc17036523157e1f9900f0fa9163c29ce793d441b0bdd337057354"}, + {file = "SQLAlchemy-2.0.0-cp310-cp310-win32.whl", hash = "sha256:86bc43f80b3fdae55f2dc6a3b0a9fe6f5c69001763e4095998e467b068a037d2"}, + {file = "SQLAlchemy-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:6e6cb16419328100fc92ee676bcb09846034586461aeb96c89a072feb48c9a6d"}, + {file = "SQLAlchemy-2.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a0b4047d7d9405005637fbfd70122746c78f2dada934067bfdd439bc934cb5fb"}, + {file = "SQLAlchemy-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b850d709cddfe0fa03f0ce7d58389947813053a3cfd5c7cc2fa5a49b77b7f7b5"}, + {file = "SQLAlchemy-2.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:176ddfce8d720f90ffccfecfe66f41b1af8906bb74acc536068d067bdb0fd080"}, + {file = "SQLAlchemy-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef0794ed9ed2cc3c42475998baf3ead135ce3849e72993fd61c82722a1def8a5"}, + {file = "SQLAlchemy-2.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3758f3e12dd7a1448d8d2c5d4d36dc32a504a0ff6dded23b06d955c73f1b71b4"}, + {file = "SQLAlchemy-2.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7a2709d68ec901add77aa378253568905ba8112ae82ae8b3d3e85fd56b06f44d"}, + {file = "SQLAlchemy-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c658c985830d4d80598387b2eca5944507acc9d52af8ec867d4c9fa0d4e27fd7"}, + {file = "SQLAlchemy-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:92b828f195bb967f85bda508bed5b4fe24b4ef0cac9ac2d9e403584ba504a304"}, + {file = "SQLAlchemy-2.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7b2231470060cd55b870806fb654f2ba66f7fc822f56fe594fa1fbd95e646da5"}, + {file = "SQLAlchemy-2.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:228851937becdbaeefdc937a3b43e9711b0a094eccc745f00b993ecd860a913b"}, + {file = "SQLAlchemy-2.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b4428bf59a5f12549f92f4274c8b2667313f105e36a7822c47727ea5572e0f7"}, + {file = "SQLAlchemy-2.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d1b1004e00023b37cc2385da670db28cb3dd96b9f01aafc3f9c437c030bf73f8"}, + {file = "SQLAlchemy-2.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a0f6d402a12ce2dc9243553ae8088459e94540b2afc4b4c3fc3a0272b9aa2827"}, + {file = "SQLAlchemy-2.0.0-cp37-cp37m-win32.whl", hash = "sha256:c1cae76f84755527c269ceb49e3a79ff370101bfd93d3f9d298bd7951e1b5e41"}, + {file = "SQLAlchemy-2.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:83db15a39539c6acb92075215aa68b9757085717d222ef678b0040cdf192adbb"}, + {file = "SQLAlchemy-2.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7c6934dfa9ab53853b1d31723ea2b8ea494de73ad3f36ea42f5859b74cb3afc3"}, + {file = "SQLAlchemy-2.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:05923707b416b7034c0b14e59e14614cb1432647188ba46bcfd911998cdea48d"}, + {file = "SQLAlchemy-2.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33a05cc9533a580f94a69852c8dea26d7dec0bc8182bb8d68180a5103c0b0add"}, + {file = "SQLAlchemy-2.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6e4a17bbcb882fcff597d6ffdf113144383ea346bcae97079b96faaf7d460fb"}, + {file = "SQLAlchemy-2.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:54fa0308430ea13239557b6b38a41988ab9d0356420879b2e8b976f58c8b8229"}, + {file = "SQLAlchemy-2.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6c75b77de2fd99bd19a609c00e870325574000c441f7bdb0cd33d15961ed93bc"}, + {file = "SQLAlchemy-2.0.0-cp38-cp38-win32.whl", hash = "sha256:28f8371e07c66f7bd8d665c0532e68986e1616f0505bef05a9bcb384889f94f2"}, + {file = "SQLAlchemy-2.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:2051899b520a4332da0fe7098d155e0981044aed91567623c7aff4bd4addddc8"}, + {file = "SQLAlchemy-2.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0fb3b58ba21898b94255e86da5e3bfc15cf99e039babcaccaa2ce10b6322929e"}, + {file = "SQLAlchemy-2.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:77666361fdd70868a414762d0eead141183caf1e0cb6735484c0cad6d41ac869"}, + {file = "SQLAlchemy-2.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dc81c03d4bccc82c81e4e21da5cea2071eca2fcddb248b462b151911c4b47b8"}, + {file = "SQLAlchemy-2.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b17bc162c317973d87613eac869cc50c1fef7a8b9d657b7d7f764ab5d9fee72"}, + {file = "SQLAlchemy-2.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7809546951b4a5ad1f0b5d5c87b42218af8c0574f50e89d141dfff531c069389"}, + {file = "SQLAlchemy-2.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:606f55614af6777261e54cb5d541a5c555539c5abc5e0b40d299c9a3bd06fae5"}, + {file = "SQLAlchemy-2.0.0-cp39-cp39-win32.whl", hash = "sha256:47348dad936e0899e2910853df1af736a84b3bddbd5dfe6471a5a39e00b32f06"}, + {file = "SQLAlchemy-2.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:13929b9806b002e3018a2f4d6666466298f43043c53b037a27520d8e8dad238d"}, + {file = "SQLAlchemy-2.0.0-py3-none-any.whl", hash = "sha256:192210daec1062e93fcc732de0c602c4b58097257c56874baa6e491849e82ceb"}, + {file = "SQLAlchemy-2.0.0.tar.gz", hash = "sha256:92388d03220eda6d744277a4d2cbcbb557509c7f7582215f61f8a04ec264be59"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} +typing-extensions = ">=4.2.0" + +[package.extras] +aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx-oracle (>=7)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +pymysql = ["pymysql"] +sqlcipher = ["sqlcipher3-binary"] + +[[package]] +name = "sqlparse" +version = "0.4.3" +description = "A non-validating SQL parser." +category = "dev" +optional = false +python-versions = ">=3.5" +files = [ + {file = "sqlparse-0.4.3-py3-none-any.whl", hash = "sha256:0323c0ec29cd52bceabc1b4d9d579e311f3e4961b98d174201d5622a23b85e34"}, + {file = "sqlparse-0.4.3.tar.gz", hash = "sha256:69ca804846bb114d2ec380e4360a8a340db83f0ccf3afceeb1404df028f57268"}, +] + +[[package]] +name = "tabulate" +version = "0.9.0" +description = "Pretty-print tabular data" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, + {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, +] + +[package.extras] +widechars = ["wcwidth"] + [[package]] name = "tarsafe" version = "0.0.4" @@ -2332,6 +2870,57 @@ platformdirs = ">=2.4,<3" docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] +[[package]] +name = "waitress" +version = "2.1.2" +description = "Waitress WSGI server" +category = "dev" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "waitress-2.1.2-py3-none-any.whl", hash = "sha256:7500c9625927c8ec60f54377d590f67b30c8e70ef4b8894214ac6e4cad233d2a"}, + {file = "waitress-2.1.2.tar.gz", hash = "sha256:780a4082c5fbc0fde6a2fcfe5e26e6efc1e8f425730863c04085769781f51eba"}, +] + +[package.extras] +docs = ["Sphinx (>=1.8.1)", "docutils", "pylons-sphinx-themes (>=1.0.9)"] +testing = ["coverage (>=5.0)", "pytest", "pytest-cover"] + +[[package]] +name = "websocket-client" +version = "1.5.0" +description = "WebSocket client for Python with low level API options" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "websocket-client-1.5.0.tar.gz", hash = "sha256:561ca949e5bbb5d33409a37235db55c279235c78ee407802f1d2314fff8a8536"}, + {file = "websocket_client-1.5.0-py3-none-any.whl", hash = "sha256:fb5d81b95d350f3a54838ebcb4c68a5353bbd1412ae8f068b1e5280faeb13074"}, +] + +[package.extras] +docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] +optional = ["python-socks", "wsaccel"] +test = ["websockets"] + +[[package]] +name = "werkzeug" +version = "2.2.2" +description = "The comprehensive WSGI web application library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Werkzeug-2.2.2-py3-none-any.whl", hash = "sha256:f979ab81f58d7318e064e99c4506445d60135ac5cd2e177a2de0089bfd4c9bd5"}, + {file = "Werkzeug-2.2.2.tar.gz", hash = "sha256:7ea2d48322cc7c0f8b3a215ed73eabd7b5d75d0b50e31ab006286ccff9e00b8f"}, +] + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog"] + [[package]] name = "wheel" version = "0.38.4" @@ -2595,4 +3184,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.11" -content-hash = "74580fef5b6710a35ca497fe35f2480500b798465e06113e9b2fb3feb19f9182" +content-hash = "6c85923ce54d48162085ecb19116c3cf30eafd2426999b856e17e0f751425314" diff --git a/pyproject.toml b/pyproject.toml index a2d9973..5914d25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "kedro-sagemaker" -version = "0.1.1" +version = "0.2.0" description = "Kedro plugin with AWS SageMaker Pipelines support" readme = "README.md" authors = ['Marcin Zabłocki '] @@ -21,6 +21,7 @@ numpy = "^1.23.5" pandas = "^1.5.2" pre-commit = "^2.20.0" recommonmark = "^0.7.1" +kedro-mlflow = "^0.11.6" [build-system] diff --git a/sonar-project.properties b/sonar-project.properties index 234a64d..3665699 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -6,7 +6,7 @@ sonar.tests=tests/ sonar.python.coverage.reportPaths=coverage.xml sonar.python.version=3.9 -sonar.projectVersion=0.1.1 +sonar.projectVersion=0.2.0 sonar.projectDescription=Kedro plugin with AWS SageMaker Pipelines support sonar.links.homepage=https://kedro-sagemaker.readthedocs.io/ sonar.links.ci=https://github.com/getindata/kedro-sagemaker/actions diff --git a/tests/conf/base/credentials.yml b/tests/conf/base/credentials.yml new file mode 100644 index 0000000..e69de29 diff --git a/tests/conf/base/mlflow.yml b/tests/conf/base/mlflow.yml new file mode 100644 index 0000000..e69de29 diff --git a/tests/conf/base/parameters.yml b/tests/conf/base/parameters.yml new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py index 5fc5f56..eda3802 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,3 +95,9 @@ def dummy_pipeline() -> Pipeline: node(identity, inputs="i3", outputs="output_data", name="node3"), ] ) + + +@pytest.fixture() +def no_mlflow(): + with patch.dict("sys.modules", {"mlflow": None}): + yield diff --git a/tests/test_cli.py b/tests/test_cli.py index c3c12fa..909cbbb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,5 @@ import json +import logging import os from pathlib import Path from unittest.mock import MagicMock, Mock, patch @@ -7,16 +8,17 @@ import pytest import yaml from click.testing import CliRunner -from sagemaker.workflow.pipeline import Pipeline as SageMakerPipeline from kedro_sagemaker import cli from kedro_sagemaker.config import KedroSageMakerPluginConfig from kedro_sagemaker.constants import ( KEDRO_SAGEMAKER_ARGS, KEDRO_SAGEMAKER_DEBUG, + KEDRO_SAGEMAKER_EXECUTION_ARN, KEDRO_SAGEMAKER_PARAM_KEY_PREFIX, KEDRO_SAGEMAKER_PARAM_VALUE_PREFIX, KEDRO_SAGEMAKER_WORKING_DIRECTORY, + MLFLOW_TAG_EXECUTION_ARN, ) from kedro_sagemaker.generator import KedroSageMakerGenerator from tests.utils import assert_has_any_call_with_args @@ -99,7 +101,7 @@ def test_can_compile_the_pipeline( @patch("click.confirm") @patch("subprocess.run", return_value=Mock(returncode=0)) -@patch("kedro_sagemaker.client.SageMakerClient") +@patch("kedro_sagemaker.cli.SageMakerClient") @pytest.mark.parametrize( "wait_for_completion", (False, True), ids=("no wait", "wait for completion") ) @@ -112,6 +114,14 @@ def test_can_compile_the_pipeline( "auto_build", (False, True), ids=("no auto-build", "with auto-build") ) @pytest.mark.parametrize("yes", (False, True), ids=("without --yes", "with --yes")) +@pytest.mark.parametrize( + "image", (None, "custom-image"), ids=("with default image", "with custom image") +) +@pytest.mark.parametrize( + "execution_role", + (None, "arn::yo"), + ids=("with default execution role", "with custom execution_role"), +) def test_can_run_the_pipeline( sagemaker_client, subprocess_run, @@ -122,20 +132,16 @@ def test_can_run_the_pipeline( cli_context, dummy_pipeline, yes: bool, + image: str, + execution_role: str, tmp_path: Path, wait_for_completion: bool, ): - mock_image = f"docker_image:{uuid4().hex}" - started_pipeline = MagicMock() + expected_image = image or "docker:image" + expected_execution_role = execution_role or "arn::unit/tests/role/arn" with patch.object( KedroSageMakerGenerator, "get_kedro_pipeline", return_value=dummy_pipeline - ), patch.object(SageMakerPipeline, "upsert") as upsert, patch.object( - SageMakerPipeline, "start", return_value=started_pipeline - ) as start, patch( - "sagemaker.model.Model" - ), patch( - "sagemaker.workflow.model_step.ModelStep" - ): + ), patch("sagemaker.model.Model"), patch("sagemaker.workflow.model_step.ModelStep"): runner = CliRunner() result = runner.invoke( cli.run, @@ -143,23 +149,26 @@ def test_can_run_the_pipeline( + (["--auto-build"] if auto_build else []) + (["--yes"] if yes else []) + (["--wait-for-completion"] if wait_for_completion else []) - + ["-i", mock_image], + + ["-i", image] + + ["--execution-role", execution_role], obj=cli_context, catch_exceptions=False, ) assert result.exit_code == 0 - sagemaker_client.run.asset_called_once() - upsert.assert_called_once() - start.assert_called_once() + sagemaker_client.return_value.run.asset_called_once() + sm_pipeline = sagemaker_client.call_args.args[0] + execution_role = sagemaker_client.call_args.args[1] + assert execution_role == expected_execution_role + assert sm_pipeline.steps[0].processor.image_uri == expected_image assert_docker_build = lambda: assert_has_any_call_with_args( # noqa: E731 subprocess_run, - ["docker", "build", str(Path.cwd().absolute()), "-t", mock_image], + ["docker", "build", str(Path.cwd().absolute()), "-t", expected_image], ) assert_docker_push = lambda: assert_has_any_call_with_args( # noqa: E731 - subprocess_run, ["docker", "push", mock_image] + subprocess_run, ["docker", "push", expected_image] ) # noqa: E731 if auto_build: @@ -175,10 +184,117 @@ def test_can_run_the_pipeline( with pytest.raises(AssertionError): assert_docker_push() - if wait_for_completion: - started_pipeline.wait.assert_called_once() - else: - started_pipeline.wait.assert_not_called() + assert ( + sagemaker_client.return_value.run.call_args.kwargs["wait_for_completion"] + == wait_for_completion + ) + + +@patch("mlflow.start_run") +@patch("mlflow.set_tag") +@patch("mlflow.get_experiment_by_name") +def test_mlflow_start( + mlflow_get_experiment_by_name, + mlflow_set_tag, + mlflow_start_run, + cli_context, + patched_kedro_package, +): + mlflow_get_experiment_by_name.return_value.experiment_id = 42 + runner = CliRunner() + with patch.dict( + os.environ, + {KEDRO_SAGEMAKER_EXECUTION_ARN: "execution-arn"}, + ): + result = runner.invoke( + cli.mlflow_start, obj=cli_context, catch_exceptions=False + ) + assert result.exit_code == 0 + + mlflow_start_run.assert_called_with(experiment_id=42, nested=False) + mlflow_set_tag.assert_called_with(MLFLOW_TAG_EXECUTION_ARN, "execution-arn") + + +@patch("kedro_sagemaker.cli.SageMakerPipelinesRunner") +@patch("mlflow.search_runs") +@patch("kedro_sagemaker.utils.KedroSession.run") +def test_mlflow_run_id_injection_in_execute( + kedro_session_run, + mlflow_search_runs, + sagemaker_pipelines_runner, + cli_context, + patched_kedro_package, +): + runner = CliRunner() + mlflow_search_runs.return_value = [MagicMock(info=MagicMock(run_id="abcdef"))] + with patch.dict( + os.environ, + {KEDRO_SAGEMAKER_EXECUTION_ARN: "execution-arn"}, + ): + + result = runner.invoke( + cli.execute, ["-n", "node"], obj=cli_context, catch_exceptions=False + ) + assert result.exit_code == 0 + assert os.environ["MLFLOW_RUN_ID"] == "abcdef" + + assert kedro_session_run.called_with("__default__", node_names=["node"]) + + +@patch("kedro_sagemaker.cli.SageMakerPipelinesRunner") +@patch("mlflow.search_runs") +@patch("kedro_sagemaker.utils.KedroSession.run") +def test_warn_if_unable_to_lookup_mlflow_run_id_in_execute( + kedro_session_run, + mlflow_search_runs, + sagemaker_pipelines_runner, + cli_context, + patched_kedro_package, + caplog, +): + runner = CliRunner() + mlflow_search_runs.return_value = [] + execution_arn = "pipeline/execution/arn" + with patch.dict( + os.environ, + {KEDRO_SAGEMAKER_EXECUTION_ARN: execution_arn}, + ), caplog.at_level(logging.WARN): + + result = runner.invoke( + cli.execute, ["-n", "node"], obj=cli_context, catch_exceptions=False + ) + assert result.exit_code == 0 + assert "MLFLOW_RUN_ID" not in os.environ + + assert ( + f"Unable to find parent mlflow run id for the current execution ({execution_arn})" + in caplog.text + ) + assert kedro_session_run.called_with("__default__", node_names=["node"]) + + +@patch("kedro_sagemaker.cli.SageMakerPipelinesRunner") +@patch("kedro_sagemaker.utils.KedroSession.run") +def test_no_mlflow_run_id_injected_if_mlflow_support_not_enabled( + kedro_session_run, + sagemaker_pipelines_runner, + cli_context, + patched_kedro_package, + no_mlflow, +): + runner = CliRunner() + with patch.dict( + os.environ, + {KEDRO_SAGEMAKER_EXECUTION_ARN: "execution-arn"}, + ): + + result = runner.invoke( + cli.execute, ["-n", "node"], obj=cli_context, catch_exceptions=False + ) + assert result.exit_code == 0 + assert "MLFLOW_RUN_ID" not in os.environ + + assert kedro_session_run.called_with("__default__", node_names=["node"]) @pytest.mark.parametrize("kedro_sagemaker_debug", ("0", "1")) diff --git a/tests/test_generator.py b/tests/test_generator.py index 9a0280d..9dab5e4 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -5,6 +5,7 @@ from kedro.io import DataCatalog from kedro.pipeline import node, pipeline from sagemaker.workflow import pipeline_context +from sagemaker.workflow.execution_variables import ExecutionVariables from sagemaker.workflow.steps import StepTypeEnum from kedro_sagemaker.config import _CONFIG_TEMPLATE, ResourceConfig @@ -23,7 +24,7 @@ @patch("kedro.framework.project.pipelines", {"__default__": sample_pipeline}) @patch("kedro.framework.context.KedroContext") -def test_should_generate_pipeline_with_processing_steps(context_mock): +def test_should_generate_pipeline_with_processing_steps(context_mock, no_mlflow): # given config = _CONFIG_TEMPLATE.copy(deep=True) generator = KedroSageMakerGenerator( @@ -152,7 +153,10 @@ def test_should_create_processor_based_on_the_config(context_mock): json.loads(processor.env["KEDRO_SAGEMAKER_RUNNER_CONFIG"])["bucket"] == "__bucket_name__" ) - assert "run_id" in json.loads(processor.env["KEDRO_SAGEMAKER_RUNNER_CONFIG"]) + assert ( + processor.env["KEDRO_SAGEMAKER_EXECUTION_ARN"] + == ExecutionVariables.PIPELINE_EXECUTION_ARN + ) @patch("kedro.framework.project.pipelines", {"__default__": sample_pipeline}) @@ -182,7 +186,7 @@ def test_should_use_default_resources_spec_in_processing_step(context_mock): @patch("kedro_sagemaker.generator.Model") @patch("kedro_sagemaker.generator.ModelStep") def test_should_generate_training_steps_and_register_model( - model_step_mock, model_mock, context_mock + model_step_mock, model_mock, context_mock, no_mlflow ): # given config = _CONFIG_TEMPLATE.copy(deep=True) @@ -211,8 +215,41 @@ def test_should_generate_training_steps_and_register_model( @patch("kedro.framework.context.KedroContext") @patch("kedro_sagemaker.generator.Model") @patch("kedro_sagemaker.generator.ModelStep") -def test_should_generate_training_steps_and_skip_model_registration( +def test_should_generate_training_steps_and_register_model_with_mlflow( model_step_mock, model_mock, context_mock +): + # given + config = _CONFIG_TEMPLATE.copy(deep=True) + config.docker.image = "__image_uri__" + context_mock.catalog = DataCatalog({"i2": SageMakerModelDataset()}) + context_mock.env = "base" + generator = KedroSageMakerGenerator( + "__default__", context_mock, config, is_local=False + ) + + # when + pipeline = generator.generate() + + # then + steps = {step.name: step for step in pipeline.steps} + assert len(steps) == 4 + assert "start-mlflow-run" in steps + assert steps["start-mlflow-run"].processor.entrypoint == [ + "kedro", + "sagemaker", + "-e", + "base", + "mlflow-start", + ] + assert steps["node1"].depends_on[0].name == "start-mlflow-run" + + +@patch("kedro.framework.project.pipelines", {"__default__": sample_pipeline}) +@patch("kedro.framework.context.KedroContext") +@patch("kedro_sagemaker.generator.Model") +@patch("kedro_sagemaker.generator.ModelStep") +def test_should_generate_training_steps_and_skip_model_registration( + model_step_mock, model_mock, context_mock, no_mlflow ): # given config = _CONFIG_TEMPLATE.copy(deep=True) @@ -236,7 +273,7 @@ def test_should_generate_training_steps_and_skip_model_registration( @patch("kedro_sagemaker.generator.Model") @patch("kedro_sagemaker.generator.ModelStep") def test_should_create_estimator_based_on_the_config( - model_step_mock, model_mock, context_mock + model_step_mock, model_mock, context_mock, no_mlflow ): # given config = _CONFIG_TEMPLATE.copy(deep=True) @@ -267,18 +304,19 @@ def test_should_create_estimator_based_on_the_config( == f"kedro sagemaker -e {env} execute --pipeline=__default__ --node=node1" ) assert estimator.environment["KEDRO_SAGEMAKER_WD"] == "/home/kedro" + assert ( + estimator.environment["KEDRO_SAGEMAKER_EXECUTION_ARN"] + == ExecutionVariables.PIPELINE_EXECUTION_ARN + ) assert ( json.loads(estimator.environment["KEDRO_SAGEMAKER_RUNNER_CONFIG"])["bucket"] == "__bucket_name__" ) - assert "run_id" in json.loads( - estimator.environment["KEDRO_SAGEMAKER_RUNNER_CONFIG"] - ) @patch("kedro.framework.project.pipelines", {"__default__": sample_pipeline}) @patch("kedro.framework.context.KedroContext") -def test_should_mark_node_as_estimator_if_it_exposes_metrics(context_mock): +def test_should_mark_node_as_estimator_if_it_exposes_metrics(context_mock, no_mlflow): # given context_mock.env = uuid4().hex config = _CONFIG_TEMPLATE.copy(deep=True) diff --git a/tests/test_runner.py b/tests/test_runner.py index ec2da29..35152ae 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -1,6 +1,13 @@ +import os +from unittest.mock import patch + from kedro.io import DataCatalog, MemoryDataSet from kedro.pipeline import Pipeline +from kedro_sagemaker.constants import ( + KEDRO_SAGEMAKER_EXECUTION_ARN, + KEDRO_SAGEMAKER_RUNNER_CONFIG, +) from kedro_sagemaker.runner import SageMakerPipelinesRunner @@ -32,3 +39,19 @@ def test_runner_fills_missing_datasets( catalog, ) assert results["output_data"] == input_data, "Invalid output data" + + +def test_runner_creating_default_datasets_based_on_execution_arn(): + with patch.dict( + os.environ, + { + KEDRO_SAGEMAKER_EXECUTION_ARN: "execution-arn", + KEDRO_SAGEMAKER_RUNNER_CONFIG: '{"bucket": "s3-bucket"}', + }, + ): + runner = SageMakerPipelinesRunner() + dataset = runner.create_default_data_set("output_data") + assert ( + dataset._get_target_path() + == "s3://s3-bucket/kedro-sagemaker-tmp/execution-arn/output_data.bin" + )