From 2b87b60e7b3182878cbca754b0b360e0800f2ece Mon Sep 17 00:00:00 2001 From: Gilad_Shapira Date: Mon, 21 Aug 2023 15:51:24 +0300 Subject: [PATCH 1/8] add milvus --- mlsetup.py | 121 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 110 insertions(+), 11 deletions(-) diff --git a/mlsetup.py b/mlsetup.py index ddc06ce..039f918 100644 --- a/mlsetup.py +++ b/mlsetup.py @@ -7,8 +7,9 @@ import socket import subprocess import sys +import time import urllib.request -from typing import List +from typing import List,Tuple import click import dotenv @@ -16,6 +17,30 @@ logging.basicConfig(level=logging.DEBUG, format="%(levelname)s: %(message)s") is_dummy_mode = os.environ.get("DUMMY_MODE", "") default_env_file = "~/.mlrun.env" +#default jupyter extra env +k8s_jupyter_extra_env = [ + {'name': 'MLRUN_ARTIFACT_PATH', + 'value': 's3://mlrun/projects/{{run.project}}/artifacts'}, + {'name': 'MLRUN_FEATURE_STORE__DATA_PREFIXES__DEFAULT', + 'value': 's3://mlrun/projects/{project}/FeatureStore/{name}/{kind}'}, + {'name': 'MLRUN_FEATURE_STORE__DATA_PREFIXES__NOSQL', 'value': ''}, + {'name': 'MLRUN_FEATURE_STORE__DEFAULT_TARGETS', 'value': 'parquet'}, + {'name': 'S3_ENDPOINT_URL', + 'value': 'http://minio.mlrun.svc.cluster.local:9000'}, + {'name': 'AWS_SECRET_ACCESS_KEY', 'value': 'minio123'}, + {'name': 'AWS_ACCESS_KEY_ID', 'value': 'minio'}] +#Build a method that added new env for jupyter if needed, that becuase jupyter helm does not support key value env add like mlrun api +def edit_k8s_jupyter_extra_env(env:List[Tuple[str,int or str]]): + global k8s_jupyter_extra_env + for name,value in env: + k8s_jupyter_extra_env.append({"name":name,"value":value}) + k8s_jupyter_extra_env_set = [] + for i in range(len(k8s_jupyter_extra_env)): + name = k8s_jupyter_extra_env[i].get("name") + value = k8s_jupyter_extra_env[i].get("value") or k8s_jupyter_extra_env[i].get("valueFrom") + k8s_jupyter_extra_env_set.append(f"jupyterNotebook.extraEnv[{i}].name={name}") + k8s_jupyter_extra_env_set.append(f"jupyterNotebook.extraEnv[{i}].value={value}") + return k8s_jupyter_extra_env_set scaled_deplyoments = [ "mlrun-api-chief", "mlrun-db", @@ -38,10 +63,10 @@ docker_services = ["jupyter", "milvus", "mysql"] k8s_services = ["spark", "monitoring", "jupyter", "pipelines"] service_map = { - "s": "spark-operator", - "m": "kube-prometheus-stack", - "j": "jupyterNotebook", - "p": "pipelines", + "spark": "spark-operator", + "monitoring": "kube-prometheus-stack", + "jupyter": "jupyterNotebook", + "pipelines": "pipelines", } # auto detect if running inside GitHub Codespaces is_codespaces = "CODESPACES" in os.environ and "CODESPACE_NAME" in os.environ @@ -416,6 +441,11 @@ def remote(url, username, access_key, artifact_path, env_file, env_vars, verbose default="", help="deploy Jupyter container, can provide jupyter image as argument", ) +@click.option( + "--milvus", + is_flag=True, + help="install milvus standalone service", +) def kubernetes( name, namespace, @@ -431,6 +461,7 @@ def kubernetes( simulate, chart_ver, jupyter, + milvus, ): """Install MLRun service on Kubernetes""" config = K8sConfig(env_file, verbose, env_vars_opt=env_vars, simulate=simulate) @@ -448,6 +479,7 @@ def kubernetes( disable, chart_ver, jupyter, + milvus, ) @@ -839,7 +871,9 @@ def start( raise SystemExit(returncode) print() - print(f"MLRun API address: http://localhost:{port} (internal: http://mlrun-api:{port})") + print( + f"MLRun API address: http://localhost:{port} (internal: http://mlrun-api:{port})" + ) print( f"MLRun UI address: http://localhost:{os.environ.get('MLRUN_UI_PORT', '8060')}" ) @@ -849,12 +883,18 @@ def start( if jupyter: print("Jupyter address: http://localhost:8888") if "milvus" in options: - print("Milvus API address: http://localhost:19530 (internal: http://milvus:19530)") - print("Minio API address: http://localhost:9000 (internal: http://minio:9000)") + print( + "Milvus API address: http://localhost:19530 (internal: http://milvus:19530)" + ) + print( + "Minio API address: http://localhost:9000 (internal: http://minio:9000)" + ) if "mysql" in options: print(f"MySQL connection str:") print(f" From Containers: {mysql_connection_url}") - print(f" From host: {mysql_connection_url.replace('sqldb', 'localhost')}") + print( + f" From host: {mysql_connection_url.replace('sqldb', 'localhost')}" + ) def stop(self, force=None, cleanup=None): compose_file = self.get_env().get("MLRUN_CONF_COMPOSE_PATH", "") @@ -935,6 +975,7 @@ def start( disable=None, chart_ver=None, jupyter="", + milvus=None, **kwargs, ): logging.info("Start installing MLRun CE") @@ -981,7 +1022,19 @@ def start( # Install and update Helm charts helm_commands = [ - ["helm", "repo", "add", "mlrun-ce", "https://mlrun.github.io/ce"], + ["helm", "repo", "add", "mlrun-ce", "https://mlrun.github.io/ce"] + ] + if milvus: + helm_commands += [ + [ + "helm", + "repo", + "add", + "milvus", + "https://milvus-io.github.io/milvus-helm/", + ] + ] + helm_commands += [ ["helm", "repo", "list"], ["helm", "repo", "update"], ] @@ -1002,8 +1055,49 @@ def start( for setting, value in new_settings.items(): env_settings["MLRUN_CONF_K8S_" + setting] = value self.set_env(env_settings) + if milvus: + helm_milvus_install = [ + "helm", + "install", + "-n", + namespace, + "milvus", + "milvus/milvus", + "--set", + "cluster.enabled=false", + "--set", + "etcd.replicaCount=1", + "--set", + "minio.mode=standalone", + "--set", + "pulsar.enabled=false", + "--set", + "service.type=NodePort", + "--set", + "livenessProbe.enabled=false", + "--set", + "readinessProbe.enabled=False", + "--set", + "service.nodePort=30020" + ] + # change the default mlrun minio port, cause milvus minio using those ports + service_options += [ + "minio.service.port=9010", + "minio.consoleService.port=9011", + "minio.endpointPort==9010", + "minio.minioAPIPort=9010", + "minio.minioConsolePort=9011", + "mlrun-ce.minio.service.url=http://minio.mlrun.svc.cluster.local:9010", + ] + #Point Jupyter for the new minio port for local runs + service_options += edit_k8s_jupyter_extra_env(env=[("S3_ENDPOINT_URL","http://minio.mlrun.svc.cluster.local:9010")]) + logging.info("Running Milvus helm install...") + returncode, _, _ = self.do_popen(helm_milvus_install) + if returncode != 0: + raise SystemExit(returncode) # run helm to install mlrun + env_settings["MILVUS_INSTALL"] = "true" helm_run_cmd = [ "helm", "--namespace", @@ -1062,7 +1156,7 @@ def start( if self.verbose: helm_run_cmd += ["--debug"] helm_run_cmd += ["mlrun-ce/mlrun-ce"] - + print(helm_run_cmd) logging.info("Running helm install...") returncode, _, _ = self.do_popen(helm_run_cmd) if returncode != 0: @@ -1075,6 +1169,11 @@ def start( "configure your mlrun client environment to use the installed service:\n" f"mlrun config set -a {dbpath}" ) + milvus_path = f"http://{external_addr or 'localhost'}:{30020}" + print( + "\nMILVUS is exposed externally at:\n" + f"{milvus_path}" + ) self.set_env(env_settings) @staticmethod From 9fc1956575577049e62b0d3854da73a808bb504b Mon Sep 17 00:00:00 2001 From: Gilad_Shapira Date: Mon, 21 Aug 2023 15:53:23 +0300 Subject: [PATCH 2/8] add fix --- mlsetup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mlsetup.py b/mlsetup.py index 039f918..827709c 100644 --- a/mlsetup.py +++ b/mlsetup.py @@ -1156,7 +1156,6 @@ def start( if self.verbose: helm_run_cmd += ["--debug"] helm_run_cmd += ["mlrun-ce/mlrun-ce"] - print(helm_run_cmd) logging.info("Running helm install...") returncode, _, _ = self.do_popen(helm_run_cmd) if returncode != 0: From b32ade378cd559402fcf717f90595f4f4acb1349 Mon Sep 17 00:00:00 2001 From: Gilad_Shapira Date: Mon, 21 Aug 2023 16:01:17 +0300 Subject: [PATCH 3/8] add milvus_path_internal messae, and fix disable and allow services issue --- mlsetup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mlsetup.py b/mlsetup.py index 827709c..95ecb5f 100644 --- a/mlsetup.py +++ b/mlsetup.py @@ -1168,10 +1168,13 @@ def start( "configure your mlrun client environment to use the installed service:\n" f"mlrun config set -a {dbpath}" ) - milvus_path = f"http://{external_addr or 'localhost'}:{30020}" + milvus_path_external = f"http://{external_addr or 'localhost'}:{30020}" + milvus_path_internal = f"http://milvus:{19530}" print( "\nMILVUS is exposed externally at:\n" - f"{milvus_path}" + f"{milvus_path_external}" + "\nMILVUS is exposed internally at:\n" + f"{milvus_path_internal}" ) self.set_env(env_settings) From 43852c37d3a4402fe7c495ab6c503fe23d061078 Mon Sep 17 00:00:00 2001 From: Gilad_Shapira Date: Mon, 21 Aug 2023 16:02:45 +0300 Subject: [PATCH 4/8] black --- mlsetup.py | 54 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/mlsetup.py b/mlsetup.py index 95ecb5f..d8e68a3 100644 --- a/mlsetup.py +++ b/mlsetup.py @@ -7,9 +7,8 @@ import socket import subprocess import sys -import time import urllib.request -from typing import List,Tuple +from typing import List, Tuple import click import dotenv @@ -17,30 +16,40 @@ logging.basicConfig(level=logging.DEBUG, format="%(levelname)s: %(message)s") is_dummy_mode = os.environ.get("DUMMY_MODE", "") default_env_file = "~/.mlrun.env" -#default jupyter extra env +# default jupyter extra env k8s_jupyter_extra_env = [ - {'name': 'MLRUN_ARTIFACT_PATH', - 'value': 's3://mlrun/projects/{{run.project}}/artifacts'}, - {'name': 'MLRUN_FEATURE_STORE__DATA_PREFIXES__DEFAULT', - 'value': 's3://mlrun/projects/{project}/FeatureStore/{name}/{kind}'}, - {'name': 'MLRUN_FEATURE_STORE__DATA_PREFIXES__NOSQL', 'value': ''}, - {'name': 'MLRUN_FEATURE_STORE__DEFAULT_TARGETS', 'value': 'parquet'}, - {'name': 'S3_ENDPOINT_URL', - 'value': 'http://minio.mlrun.svc.cluster.local:9000'}, - {'name': 'AWS_SECRET_ACCESS_KEY', 'value': 'minio123'}, - {'name': 'AWS_ACCESS_KEY_ID', 'value': 'minio'}] -#Build a method that added new env for jupyter if needed, that becuase jupyter helm does not support key value env add like mlrun api -def edit_k8s_jupyter_extra_env(env:List[Tuple[str,int or str]]): + { + "name": "MLRUN_ARTIFACT_PATH", + "value": "s3://mlrun/projects/{{run.project}}/artifacts", + }, + { + "name": "MLRUN_FEATURE_STORE__DATA_PREFIXES__DEFAULT", + "value": "s3://mlrun/projects/{project}/FeatureStore/{name}/{kind}", + }, + {"name": "MLRUN_FEATURE_STORE__DATA_PREFIXES__NOSQL", "value": ""}, + {"name": "MLRUN_FEATURE_STORE__DEFAULT_TARGETS", "value": "parquet"}, + {"name": "S3_ENDPOINT_URL", "value": "http://minio.mlrun.svc.cluster.local:9000"}, + {"name": "AWS_SECRET_ACCESS_KEY", "value": "minio123"}, + {"name": "AWS_ACCESS_KEY_ID", "value": "minio"}, +] + + +# Build a method that added new env for jupyter if needed, that becuase jupyter helm does not support key value env add like mlrun api +def edit_k8s_jupyter_extra_env(env: List[Tuple[str, int or str]]): global k8s_jupyter_extra_env - for name,value in env: - k8s_jupyter_extra_env.append({"name":name,"value":value}) + for name, value in env: + k8s_jupyter_extra_env.append({"name": name, "value": value}) k8s_jupyter_extra_env_set = [] for i in range(len(k8s_jupyter_extra_env)): name = k8s_jupyter_extra_env[i].get("name") - value = k8s_jupyter_extra_env[i].get("value") or k8s_jupyter_extra_env[i].get("valueFrom") + value = k8s_jupyter_extra_env[i].get("value") or k8s_jupyter_extra_env[i].get( + "valueFrom" + ) k8s_jupyter_extra_env_set.append(f"jupyterNotebook.extraEnv[{i}].name={name}") k8s_jupyter_extra_env_set.append(f"jupyterNotebook.extraEnv[{i}].value={value}") return k8s_jupyter_extra_env_set + + scaled_deplyoments = [ "mlrun-api-chief", "mlrun-db", @@ -1078,7 +1087,7 @@ def start( "--set", "readinessProbe.enabled=False", "--set", - "service.nodePort=30020" + "service.nodePort=30020", ] # change the default mlrun minio port, cause milvus minio using those ports service_options += [ @@ -1088,10 +1097,11 @@ def start( "minio.minioAPIPort=9010", "minio.minioConsolePort=9011", "mlrun-ce.minio.service.url=http://minio.mlrun.svc.cluster.local:9010", - ] - #Point Jupyter for the new minio port for local runs - service_options += edit_k8s_jupyter_extra_env(env=[("S3_ENDPOINT_URL","http://minio.mlrun.svc.cluster.local:9010")]) + # Point Jupyter for the new minio port for local runs + service_options += edit_k8s_jupyter_extra_env( + env=[("S3_ENDPOINT_URL", "http://minio.mlrun.svc.cluster.local:9010")] + ) logging.info("Running Milvus helm install...") returncode, _, _ = self.do_popen(helm_milvus_install) if returncode != 0: From 26f38e97859bbabc4e51a5a4bdf4a3b298adc1b9 Mon Sep 17 00:00:00 2001 From: Gilad_Shapira Date: Mon, 21 Aug 2023 16:07:49 +0300 Subject: [PATCH 5/8] fix install milvus message --- mlsetup.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mlsetup.py b/mlsetup.py index d8e68a3..4a4ae9d 100644 --- a/mlsetup.py +++ b/mlsetup.py @@ -1178,14 +1178,15 @@ def start( "configure your mlrun client environment to use the installed service:\n" f"mlrun config set -a {dbpath}" ) - milvus_path_external = f"http://{external_addr or 'localhost'}:{30020}" - milvus_path_internal = f"http://milvus:{19530}" - print( - "\nMILVUS is exposed externally at:\n" - f"{milvus_path_external}" - "\nMILVUS is exposed internally at:\n" - f"{milvus_path_internal}" - ) + if milvus: + milvus_path_external = f"http://{external_addr or 'localhost'}:{30020}" + milvus_path_internal = f"http://milvus:{19530}" + print( + "\nMILVUS is exposed externally at:\n" + f"{milvus_path_external}" + "\nMILVUS is exposed internally at:\n" + f"{milvus_path_internal}" + ) self.set_env(env_settings) @staticmethod From c6cf6ea218e448e63e60373011790954c006960d Mon Sep 17 00:00:00 2001 From: Gilad_Shapira Date: Tue, 22 Aug 2023 22:44:40 +0300 Subject: [PATCH 6/8] fix --- mlsetup.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/mlsetup.py b/mlsetup.py index 4a4ae9d..54063a7 100644 --- a/mlsetup.py +++ b/mlsetup.py @@ -70,12 +70,13 @@ def edit_k8s_jupyter_extra_env(env: List[Tuple[str, int or str]]): "push_secret", ] docker_services = ["jupyter", "milvus", "mysql"] -k8s_services = ["spark", "monitoring", "jupyter", "pipelines"] +k8s_services = ["spark", "monitoring", "jupyter", "pipelines","milvus"] service_map = { "spark": "spark-operator", "monitoring": "kube-prometheus-stack", "jupyter": "jupyterNotebook", "pipelines": "pipelines", + "milvus": "milvus" } # auto detect if running inside GitHub Codespaces is_codespaces = "CODESPACES" in os.environ and "CODESPACE_NAME" in os.environ @@ -450,11 +451,7 @@ def remote(url, username, access_key, artifact_path, env_file, env_vars, verbose default="", help="deploy Jupyter container, can provide jupyter image as argument", ) -@click.option( - "--milvus", - is_flag=True, - help="install milvus standalone service", -) + def kubernetes( name, namespace, @@ -470,7 +467,6 @@ def kubernetes( simulate, chart_ver, jupyter, - milvus, ): """Install MLRun service on Kubernetes""" config = K8sConfig(env_file, verbose, env_vars_opt=env_vars, simulate=simulate) @@ -488,7 +484,6 @@ def kubernetes( disable, chart_ver, jupyter, - milvus, ) @@ -984,7 +979,6 @@ def start( disable=None, chart_ver=None, jupyter="", - milvus=None, **kwargs, ): logging.info("Start installing MLRun CE") @@ -1033,7 +1027,7 @@ def start( helm_commands = [ ["helm", "repo", "add", "mlrun-ce", "https://mlrun.github.io/ce"] ] - if milvus: + if "milvus" in options: helm_commands += [ [ "helm", @@ -1064,7 +1058,7 @@ def start( for setting, value in new_settings.items(): env_settings["MLRUN_CONF_K8S_" + setting] = value self.set_env(env_settings) - if milvus: + if "milvus" in options: helm_milvus_install = [ "helm", "install", @@ -1178,7 +1172,7 @@ def start( "configure your mlrun client environment to use the installed service:\n" f"mlrun config set -a {dbpath}" ) - if milvus: + if "milvus" in options: milvus_path_external = f"http://{external_addr or 'localhost'}:{30020}" milvus_path_internal = f"http://milvus:{19530}" print( From cfea009584b52a526a6c4b6c46d739e4de2ae6df Mon Sep 17 00:00:00 2001 From: Gilad_Shapira Date: Sun, 27 Aug 2023 17:38:29 +0300 Subject: [PATCH 7/8] fix --- mlsetup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlsetup.py b/mlsetup.py index 54063a7..0352eb8 100644 --- a/mlsetup.py +++ b/mlsetup.py @@ -1159,7 +1159,7 @@ def start( if self.verbose: helm_run_cmd += ["--debug"] - helm_run_cmd += ["mlrun-ce/mlrun-ce"] + helm_run_cmd += ["/Users/Gilad_Shapira/new-ce/new-mlrun-ce"] logging.info("Running helm install...") returncode, _, _ = self.do_popen(helm_run_cmd) if returncode != 0: From 3b2820bdaf3bba6e80fceead818ce7bc2f5e4dd8 Mon Sep 17 00:00:00 2001 From: Gilad_Shapira Date: Sun, 27 Aug 2023 18:57:38 +0300 Subject: [PATCH 8/8] fix --- mlsetup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mlsetup.py b/mlsetup.py index 0352eb8..b0505d2 100644 --- a/mlsetup.py +++ b/mlsetup.py @@ -9,7 +9,7 @@ import sys import urllib.request from typing import List, Tuple - +import time import click import dotenv @@ -1102,6 +1102,7 @@ def start( raise SystemExit(returncode) # run helm to install mlrun env_settings["MILVUS_INSTALL"] = "true" + time.sleep(100) helm_run_cmd = [ "helm", "--namespace", @@ -1159,7 +1160,7 @@ def start( if self.verbose: helm_run_cmd += ["--debug"] - helm_run_cmd += ["/Users/Gilad_Shapira/new-ce/new-mlrun-ce"] + helm_run_cmd += ["mlrun-ce/mlrun-ce"] logging.info("Running helm install...") returncode, _, _ = self.do_popen(helm_run_cmd) if returncode != 0: @@ -1724,4 +1725,4 @@ def get_latest_mlrun_tag(): if __name__ == "__main__": - main() + main() \ No newline at end of file