Merge pull request #8118 from khoaguin/bugfix/hagrid-land-can-delete-…

…volumes-from-other-containers fix: `hagrind land all --prune-vol` can delete volumes from other containers
OpenMined · Oct 2, 2023 · 8ecf66c · 8ecf66c
2 parents 233b231 + bfa198e
commit 8ecf66c
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 27 deletions.
diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml
@@ -26,6 +26,8 @@ services:
       - "${HTTP_PORT}:81"
     extra_hosts:
       - "host.docker.internal:host-gateway"
+    labels:
+      - "orgs.openmined.syft=this is a syft proxy container"
 
     # depends_on:
     #   - "docker-host"
@@ -51,6 +53,8 @@ services:
     depends_on:
       - proxy
     network_mode: service:proxy
+    labels:
+      - "orgs.openmined.syft=this is a syft tailscale container"
 
   frontend:
     restart: always
@@ -72,6 +76,8 @@ services:
       - VITE_PUBLIC_API_BASE_URL=${VITE_PUBLIC_API_BASE_URL}
     extra_hosts:
       - "host.docker.internal:host-gateway"
+    labels:
+      - "orgs.openmined.syft=this is a syft frontend container"
 
   # redis:
   #   restart: always
@@ -131,6 +137,8 @@ services:
       - ${BACKEND_STORAGE_PATH}:/storage
     stdin_open: true
     tty: true
+    labels:
+      - "orgs.openmined.syft=this is a syft backend container"
 
   # backend_stream:
   #   restart: always
@@ -213,6 +221,8 @@ services:
       - RELEASE=${RELEASE:-production}
       - NETWORK_NAME=omnet
       - STACK_API_KEY=$STACK_API_KEY
+    labels:
+      - "orgs.openmined.syft=this is a syft headscale container"
 
   seaweedfs:
     profiles:
@@ -230,8 +240,11 @@ services:
       - "/etc/seaweedfs/start.sh"
     volumes:
       - seaweedfs-data:/data/blob
+      - seaweedfs-data-2:/data
       - ./seaweedfs/filer.toml:/etc/seaweedfs/filer.toml
       - ./seaweedfs/start.sh:/etc/seaweedfs/start.sh
+    labels:
+      - "orgs.openmined.syft=this is a syft seaweedfs container"
 
   mongo:
     image: "${MONGO_IMAGE}:${MONGO_VERSION}"
@@ -241,6 +254,9 @@ services:
       - MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD}
     volumes:
       - mongo-data:/data/db
+      - mongo-config-data:/data/configdb
+    labels:
+      - "orgs.openmined.syft=this is a syft mongo container"
 
   jaeger:
     profiles:
@@ -262,14 +278,31 @@ services:
       # - "14250:14250"
       # - "14269:14269"
       # - "9411:9411"
+    volumes:
+      - jaeger-data:/tmp
+    labels:
+      - "orgs.openmined.syft=this is a syft jaeger container"
 
 volumes:
   credentials-data:
   tailscale-data:
   headscale-data:
   # app-redis-data:
   seaweedfs-data:
+    labels:
+      orgs.openmined.syft: "this is a syft seaweedfs volume"
+  seaweedfs-data-2:
+    labels:
+      orgs.openmined.syft: "this is a syft seaweedfs volume"
   mongo-data:
+    labels:
+      orgs.openmined.syft: "this is a syft mongo volume"
+  mongo-config-data:
+    labels:
+      orgs.openmined.syft: "this is a syft mongo volume"
+  jaeger-data:
+    labels:
+      orgs.openmined.syft: "this is a syft jaeger volume"
 
 networks:
   traefik-public:

diff --git a/packages/grid/worker/docker-compose.yml b/packages/grid/worker/docker-compose.yml
@@ -31,9 +31,15 @@ services:
       - credentials-data:/storage
     extra_hosts:
       - "host.docker.internal:host-gateway"
+    labels:
+      - "orgs.openmined.syft=this is a syft worker container"
 
 volumes:
   credentials-data:
+    labels:
+      - "orgs.openmined.syft=this is a syft worker container"
 
 networks:
   worker-public:
+    labels:
+      - "orgs.openmined.syft=this is a syft worker container"
diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py
@@ -3149,17 +3149,22 @@ def create_land_cmd(verb: GrammarVerb, kwargs: TypeDict[str, Any]) -> str:
 
     if host in ["docker"]:
         target = verb.get_named_term_grammar("node_name").input
-        if target == "all":
-            # subprocess.call("docker rm `docker ps -aq` --force", shell=True) # nosec
+        prune_volumes: bool = kwargs.get("prune_vol", False)
 
-            if "prune_vol" in kwargs:
-                return "docker rm `docker ps -aq` --force && docker volume prune -f"
+        if target == "all":
+            # land all syft nodes
+            if prune_volumes:
+                land_cmd = "docker rm `docker ps --filter label=orgs.openmined.syft -q` --force "
+                land_cmd += "&& docker volume rm "
+                land_cmd += "$(docker volume ls --filter label=orgs.openmined.syft -q)"
+                return land_cmd
             else:
-                return "docker rm `docker ps -aq` --force"
+                return "docker rm `docker ps --filter label=orgs.openmined.syft -q` --force"
 
         version = check_docker_version()
         if version:
-            return create_land_docker_cmd(verb=verb)
+            return create_land_docker_cmd(verb=verb, prune_volumes=prune_volumes)
+
     elif host == "localhost" or is_valid_ip(host):
         parsed_kwargs = {}
         if DEPENDENCIES["ansible-playbook"]:
@@ -3236,7 +3241,10 @@ def create_land_cmd(verb: GrammarVerb, kwargs: TypeDict[str, Any]) -> str:
     )
 
 
-def create_land_docker_cmd(verb: GrammarVerb) -> str:
+def create_land_docker_cmd(verb: GrammarVerb, prune_volumes: bool = False) -> str:
+    """
+    Create docker `land` command to remove containers when a node's name is specified
+    """
     node_name = verb.get_named_term_type(name="node_name")
     snake_name = str(node_name.snake_input)
     containers = shell("docker ps --format '{{.Names}}' | " + f"grep {snake_name}")
@@ -3256,6 +3264,11 @@ def create_land_docker_cmd(verb: GrammarVerb) -> str:
     cmd += ' --project-name "' + snake_name + '"'
     cmd += " down --remove-orphans"
 
+    if prune_volumes:
+        cmd += (
+            f' && docker volume rm $(docker volume ls --filter name="{snake_name}" -q)'
+        )
+
     cmd = "cd " + path + env_var + cmd
     return cmd
 

diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py
@@ -561,11 +561,16 @@ def land(
             Orchestra.reset(name, deployment_type_enum=deployment_type_enum)
 
     @staticmethod
-    def shutdown(name: str, deployment_type_enum: DeploymentType) -> None:
+    def shutdown(
+        name: str, deployment_type_enum: DeploymentType, reset: bool = False
+    ) -> None:
         if deployment_type_enum != DeploymentType.PYTHON:
             snake_name = to_snake_case(name)
 
-            land_output = shell(f"hagrid land {snake_name} --force")
+            if reset:
+                land_output = shell(f"hagrid land {snake_name} --force --prune-vol")
+            else:
+                land_output = shell(f"hagrid land {snake_name} --force")
             if "Removed" in land_output:
                 print(f" ✅ {snake_name} Container Removed")
             else:
@@ -580,24 +585,9 @@ def reset(name: str, deployment_type_enum: DeploymentType) -> None:
             deployment_type_enum == DeploymentType.CONTAINER_STACK
             or deployment_type_enum == DeploymentType.SINGLE_CONTAINER
         ):
-            if container_exists(name=name):
-                Orchestra.shutdown(name=name, deployment_type_enum=deployment_type_enum)
-
-            snake_name = to_snake_case(name)
-
-            volumes = ["mongo-data", "credentials-data"]
-
-            for volume in volumes:
-                volume_output = shell(
-                    f"docker volume rm {snake_name}_{volume} --force || true"
-                )
-
-                if "Error" not in volume_output:
-                    print(f" ✅ {snake_name}_{volume} Volume Removed")
-                else:
-                    print(
-                        f"❌ Unable to remove container volume: {snake_name} :{volume_output}"
-                    )
+            Orchestra.shutdown(
+                name=name, deployment_type_enum=deployment_type_enum, reset=True
+            )
         else:
             raise NotImplementedError(
                 f"Reset not implemented for the deployment type:{deployment_type_enum}"