From 7e0a9ca5df8375258335abeb57d24be5c9bcfd62 Mon Sep 17 00:00:00 2001 From: khoaguin Date: Wed, 27 Sep 2023 15:50:39 +0700 Subject: [PATCH 1/5] remove `-f` when pruning volumes to ask for confirmation --- packages/hagrid/hagrid/cli.py | 14 ++++++++++---- packages/hagrid/hagrid/orchestra.py | 30 ++++++++++------------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 4d4636aebcc..769de28144a 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -3149,17 +3149,18 @@ def create_land_cmd(verb: GrammarVerb, kwargs: TypeDict[str, Any]) -> str: if host in ["docker"]: target = verb.get_named_term_grammar("node_name").input + prune_volumes: bool = kwargs.get("prune_vol", False) if target == "all": # subprocess.call("docker rm `docker ps -aq` --force", shell=True) # nosec - if "prune_vol" in kwargs: - return "docker rm `docker ps -aq` --force && docker volume prune -f" + if prune_volumes: + return "docker rm `docker ps -aq` --force && docker volume prune" else: return "docker rm `docker ps -aq` --force" version = check_docker_version() if version: - return create_land_docker_cmd(verb=verb) + return create_land_docker_cmd(verb=verb, prune_volumes=prune_volumes) elif host == "localhost" or is_valid_ip(host): parsed_kwargs = {} if DEPENDENCIES["ansible-playbook"]: @@ -3236,7 +3237,7 @@ def create_land_cmd(verb: GrammarVerb, kwargs: TypeDict[str, Any]) -> str: ) -def create_land_docker_cmd(verb: GrammarVerb) -> str: +def create_land_docker_cmd(verb: GrammarVerb, prune_volumes: bool = False) -> str: node_name = verb.get_named_term_type(name="node_name") snake_name = str(node_name.snake_input) containers = shell("docker ps --format '{{.Names}}' | " + f"grep {snake_name}") @@ -3256,6 +3257,11 @@ def create_land_docker_cmd(verb: GrammarVerb) -> str: cmd += ' --project-name "' + snake_name + '"' cmd += " down --remove-orphans" + if prune_volumes: + cmd += ( + f' && docker volume rm $(docker volume ls --filter name="{snake_name}" -q)' + ) + cmd = "cd " + path + env_var + cmd return cmd diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py index 540a992d8e8..9a9de5fd43b 100644 --- a/packages/hagrid/hagrid/orchestra.py +++ b/packages/hagrid/hagrid/orchestra.py @@ -561,11 +561,16 @@ def land( Orchestra.reset(name, deployment_type_enum=deployment_type_enum) @staticmethod - def shutdown(name: str, deployment_type_enum: DeploymentType) -> None: + def shutdown( + name: str, deployment_type_enum: DeploymentType, reset: bool = False + ) -> None: if deployment_type_enum != DeploymentType.PYTHON: snake_name = to_snake_case(name) - land_output = shell(f"hagrid land {snake_name} --force") + if reset: + land_output = shell(f"hagrid land {snake_name} --force --prune-vol") + else: + land_output = shell(f"hagrid land {snake_name} --force") if "Removed" in land_output: print(f" ✅ {snake_name} Container Removed") else: @@ -580,24 +585,9 @@ def reset(name: str, deployment_type_enum: DeploymentType) -> None: deployment_type_enum == DeploymentType.CONTAINER_STACK or deployment_type_enum == DeploymentType.SINGLE_CONTAINER ): - if container_exists(name=name): - Orchestra.shutdown(name=name, deployment_type_enum=deployment_type_enum) - - snake_name = to_snake_case(name) - - volumes = ["mongo-data", "credentials-data"] - - for volume in volumes: - volume_output = shell( - f"docker volume rm {snake_name}_{volume} --force || true" - ) - - if "Error" not in volume_output: - print(f" ✅ {snake_name}_{volume} Volume Removed") - else: - print( - f"❌ Unable to remove container volume: {snake_name} :{volume_output}" - ) + Orchestra.shutdown( + name=name, deployment_type_enum=deployment_type_enum, reset=True + ) else: raise NotImplementedError( f"Reset not implemented for the deployment type:{deployment_type_enum}" From 6b25f6f13f4ec0ed54f64bea7dc9c7a2205dda69 Mon Sep 17 00:00:00 2001 From: khoaguin Date: Thu, 28 Sep 2023 17:02:07 +0700 Subject: [PATCH 2/5] - Adding labels metadata to the volumes that belong to syft deployment - Removing the syft volumes according to labels when doing `hagrid land all --prune-vol` Co-authored-by: Madhava Jay --- packages/grid/docker-compose.yml | 17 +++++++++++++++++ packages/hagrid/hagrid/cli.py | 13 ++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 820f4bf417e..9a84b79028c 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -230,6 +230,7 @@ services: - "/etc/seaweedfs/start.sh" volumes: - seaweedfs-data:/data/blob + - seaweedfs-data-2:/data - ./seaweedfs/filer.toml:/etc/seaweedfs/filer.toml - ./seaweedfs/start.sh:/etc/seaweedfs/start.sh @@ -241,6 +242,7 @@ services: - MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD} volumes: - mongo-data:/data/db + - mongo-config-data:/data/configdb jaeger: profiles: @@ -262,6 +264,8 @@ services: # - "14250:14250" # - "14269:14269" # - "9411:9411" + volumes: + - jaeger-data:/tmp volumes: credentials-data: @@ -269,7 +273,20 @@ volumes: headscale-data: # app-redis-data: seaweedfs-data: + labels: + orgs.openmined.syft: "this is a syft seaweedfs volume" + seaweedfs-data-2: + labels: + orgs.openmined.syft: "this is a syft seaweedfs volume" mongo-data: + labels: + orgs.openmined.syft: "this is a syft mongo volume" + mongo-config-data: + labels: + orgs.openmined.syft: "this is a syft mongo volume" + jaeger-data: + labels: + orgs.openmined.syft: "this is a syft jaeger volume" networks: traefik-public: diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 769de28144a..36f8642bc98 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -3150,17 +3150,21 @@ def create_land_cmd(verb: GrammarVerb, kwargs: TypeDict[str, Any]) -> str: if host in ["docker"]: target = verb.get_named_term_grammar("node_name").input prune_volumes: bool = kwargs.get("prune_vol", False) - if target == "all": - # subprocess.call("docker rm `docker ps -aq` --force", shell=True) # nosec + if target == "all": + # land all syft nodes if prune_volumes: - return "docker rm `docker ps -aq` --force && docker volume prune" + land_cmd = "docker rm `docker ps -aq` --force " + land_cmd += "&& docker volume rm " + land_cmd += "$(docker volume ls --filter label=orgs.openmined.syft -q)" + return land_cmd else: return "docker rm `docker ps -aq` --force" version = check_docker_version() if version: return create_land_docker_cmd(verb=verb, prune_volumes=prune_volumes) + elif host == "localhost" or is_valid_ip(host): parsed_kwargs = {} if DEPENDENCIES["ansible-playbook"]: @@ -3238,6 +3242,9 @@ def create_land_cmd(verb: GrammarVerb, kwargs: TypeDict[str, Any]) -> str: def create_land_docker_cmd(verb: GrammarVerb, prune_volumes: bool = False) -> str: + """ + Create docker `land` command to remove containers when a node's name is specified + """ node_name = verb.get_named_term_type(name="node_name") snake_name = str(node_name.snake_input) containers = shell("docker ps --format '{{.Names}}' | " + f"grep {snake_name}") From fc719f1eda4ec9b0246a0e11c8ed25710b446e1f Mon Sep 17 00:00:00 2001 From: khoaguin Date: Fri, 29 Sep 2023 17:29:01 +0700 Subject: [PATCH 3/5] - add labels to syft's containers - only remove syft's containers when doing `hagrid land all` --- packages/grid/docker-compose.yml | 16 ++++++++++++++++ packages/hagrid/hagrid/cli.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 9a84b79028c..c02868028e9 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -26,6 +26,8 @@ services: - "${HTTP_PORT}:81" extra_hosts: - "host.docker.internal:host-gateway" + labels: + - "orgs.openmined.syft=this is a syft proxy container" # depends_on: # - "docker-host" @@ -51,6 +53,8 @@ services: depends_on: - proxy network_mode: service:proxy + labels: + - "orgs.openmined.syft=this is a syft tailscale container" frontend: restart: always @@ -72,6 +76,8 @@ services: - VITE_PUBLIC_API_BASE_URL=${VITE_PUBLIC_API_BASE_URL} extra_hosts: - "host.docker.internal:host-gateway" + labels: + - "orgs.openmined.syft=this is a syft frontend container" # redis: # restart: always @@ -131,6 +137,8 @@ services: - ${BACKEND_STORAGE_PATH}:/storage stdin_open: true tty: true + labels: + - "orgs.openmined.syft=this is a syft backend container" # backend_stream: # restart: always @@ -213,6 +221,8 @@ services: - RELEASE=${RELEASE:-production} - NETWORK_NAME=omnet - STACK_API_KEY=$STACK_API_KEY + labels: + - "orgs.openmined.syft=this is a syft headscale container" seaweedfs: profiles: @@ -233,6 +243,8 @@ services: - seaweedfs-data-2:/data - ./seaweedfs/filer.toml:/etc/seaweedfs/filer.toml - ./seaweedfs/start.sh:/etc/seaweedfs/start.sh + labels: + - "orgs.openmined.syft=this is a syft seaweedfs container" mongo: image: "${MONGO_IMAGE}:${MONGO_VERSION}" @@ -243,6 +255,8 @@ services: volumes: - mongo-data:/data/db - mongo-config-data:/data/configdb + labels: + - "orgs.openmined.syft=this is a syft mongo container" jaeger: profiles: @@ -266,6 +280,8 @@ services: # - "9411:9411" volumes: - jaeger-data:/tmp + labels: + - "orgs.openmined.syft=this is a syft jaeger container" volumes: credentials-data: diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 36f8642bc98..b51770a2e11 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -3154,7 +3154,7 @@ def create_land_cmd(verb: GrammarVerb, kwargs: TypeDict[str, Any]) -> str: if target == "all": # land all syft nodes if prune_volumes: - land_cmd = "docker rm `docker ps -aq` --force " + land_cmd = "docker rm `docker ps --filter label=orgs.openmined.syft -q` --force " land_cmd += "&& docker volume rm " land_cmd += "$(docker volume ls --filter label=orgs.openmined.syft -q)" return land_cmd From 309aab82a7e340ffeea8ae5c0f6b23177251329e Mon Sep 17 00:00:00 2001 From: khoaguin Date: Fri, 29 Sep 2023 20:52:48 +0700 Subject: [PATCH 4/5] fix `docker remove` command when not pruning volumes --- packages/hagrid/hagrid/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index b51770a2e11..3d7a5b92044 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -3159,7 +3159,7 @@ def create_land_cmd(verb: GrammarVerb, kwargs: TypeDict[str, Any]) -> str: land_cmd += "$(docker volume ls --filter label=orgs.openmined.syft -q)" return land_cmd else: - return "docker rm `docker ps -aq` --force" + return "docker rm `docker ps --filter label=orgs.openmined.syft -q` --force" version = check_docker_version() if version: From bfa198e6f56e4b95b3d0d44259095b8e6b5eab4f Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 2 Oct 2023 18:00:54 +0530 Subject: [PATCH 5/5] add label to the volumes in worker docker-compose.yml --- packages/grid/worker/docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/grid/worker/docker-compose.yml b/packages/grid/worker/docker-compose.yml index 84952f7eb93..155c33a1f6d 100644 --- a/packages/grid/worker/docker-compose.yml +++ b/packages/grid/worker/docker-compose.yml @@ -31,9 +31,15 @@ services: - credentials-data:/storage extra_hosts: - "host.docker.internal:host-gateway" + labels: + - "orgs.openmined.syft=this is a syft worker container" volumes: credentials-data: + labels: + - "orgs.openmined.syft=this is a syft worker container" networks: worker-public: + labels: + - "orgs.openmined.syft=this is a syft worker container"