From 9b6800f434c4bfbd977a14439bb34c01f0e57d70 Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Wed, 23 Oct 2024 14:53:05 +0200 Subject: [PATCH 1/4] feat(#501): enables to copy file from cluster into local container - add --file-from flag to run command - copies file/directory from cluster using tar - writes tar into local container --- client/gefyra/api/run.py | 59 ++++++++++++++++++++++++++++-- client/gefyra/cli/run.py | 11 ++++++ client/gefyra/cli/utils.py | 37 +++++++++++++++++++ client/gefyra/cluster/utils.py | 65 ++++++++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 2 deletions(-) diff --git a/client/gefyra/api/run.py b/client/gefyra/api/run.py index d77ec989..73837b5b 100644 --- a/client/gefyra/api/run.py +++ b/client/gefyra/api/run.py @@ -2,8 +2,9 @@ import os import sys from threading import Thread, Event -from typing import Dict, List, Optional, TYPE_CHECKING -from gefyra.cluster.utils import retrieve_pod_and_container +from typing import Dict, List, Optional, TYPE_CHECKING, Tuple +from client.gefyra.cli.utils import FileFromArgument +from gefyra.cluster.utils import get_file_from_pod_container, retrieve_pod_and_container if TYPE_CHECKING: from docker.models.containers import Container @@ -43,6 +44,7 @@ def run( namespace: str = "", env: Optional[List] = None, env_from: str = "", + file_from: Optional[Tuple[FileFromArgument]] = None, pull: str = "missing", platform: str = "linux/amd64", ) -> bool: @@ -100,6 +102,7 @@ def run( except ApiException as e: logger.error(f"Cannot copy environment from Pod: {e.reason} ({e.status}).") return False + if env: env_overrides = generate_env_dict_from_strings(env) env_dict.update(env_overrides) @@ -131,6 +134,58 @@ def run( else: raise RuntimeError(e.explanation) + # + # 3. copy files from a K8s container + # + if file_from: + for file_from_argument in file_from: + try: + file_from_pod, file_from_container = retrieve_pod_and_container( + file_from_argument.workload, namespace=namespace, config=config + ) + except ApiException as e: + logger.error( + f"Cannot copy file/directory from pod: {e.reason} ({e.status})." + ) + continue + + logger.debug( + f"Copying file/directory from {file_from_pod}/{file_from_container}" + ) + + try: + tar_buffer = get_file_from_pod_container( + config, + file_from_pod, + namespace, + file_from_container, + file_from_argument.source, + ) + except (RuntimeError, ApiException) as e: + logger.info( + f"Cannot copy file/directory from pod:" + f" --file-from {str(file_from_argument.workload)}:" + f"{file_from_argument.source}:{file_from_argument.destination}" + ) + logger.debug(e) + del tar_buffer + continue + + try: + client = config.DOCKER + client.containers.get(name).put_archive( + file_from_argument.destination, + tar_buffer.read(), + ) + except Exception as e: + logger.info( + f"Cannot copy file/directory into container:" + f" --file-from {str(file_from_argument.workload)}:" + f"{file_from_argument.source}:{file_from_argument.destination}" + ) + logger.debug(e) + continue + logger.info( f"Container image '{', '.join(container.image.tags)}' started with name" f" '{container.name}' in Kubernetes namespace '{namespace}' (from {ns_source})" diff --git a/client/gefyra/cli/run.py b/client/gefyra/cli/run.py index 9585cc18..7871090f 100644 --- a/client/gefyra/cli/run.py +++ b/client/gefyra/cli/run.py @@ -4,6 +4,7 @@ OptionEatAll, check_connection_name, parse_env, + parse_file_from, parse_ip_port_map, parse_workload, ) @@ -40,6 +41,13 @@ type=str, callback=parse_workload, ) +@click.option( + "--file-from", + help="Copy the file from the container in the notation 'Pod/Container'", + type=str, + callback=parse_file_from, + multiple=True, +) @click.option( "-v", "--volume", @@ -102,6 +110,7 @@ def run( auto_remove, expose, env_from, + file_from, volume, env, namespace, @@ -116,12 +125,14 @@ def run( if command: command = ast.literal_eval(command)[0] + api.run( image=image, name=name, command=command, namespace=namespace, env_from=env_from, + file_from=file_from, env=env, ports=expose, auto_remove=auto_remove, diff --git a/client/gefyra/cli/utils.py b/client/gefyra/cli/utils.py index 8ecb5d2d..efb68e9f 100644 --- a/client/gefyra/cli/utils.py +++ b/client/gefyra/cli/utils.py @@ -2,6 +2,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import click from click import ClickException +from typing import NamedTuple def standard_error_handler(func): @@ -237,6 +238,42 @@ def parse_workload(ctx, param, workload: str) -> str: return workload +class FileFromArgument(NamedTuple): + workload: str + source: str + destination: str = "/" + + +def parse_file_from(ctx, param, file_from_arguments) -> Tuple[FileFromArgument]: + if not file_from_arguments: + return None + + MSG = ( + "Invalid argument format. Please provide the file " + "in the format 'type/name:source:destination' or " + "'type/name/container-name:source:destination'." + ) + + file_from = [] + for file_from_argument in file_from_arguments: + split_argument = file_from_argument.split(":") + if len(split_argument) != 3: + raise ValueError(MSG) + + workload, source, destination = split_argument + if "/" not in workload: + raise ValueError(MSG) + + file_from_argument_parsed = FileFromArgument( + workload=workload, + source=source, + destination=destination, + ) + file_from.append(file_from_argument_parsed) + + return tuple(file_from) + + def check_connection_name(ctx, param, selected: Optional[str] = None) -> str: from gefyra import api diff --git a/client/gefyra/cluster/utils.py b/client/gefyra/cluster/utils.py index 1f0d7787..e6beda1b 100644 --- a/client/gefyra/cluster/utils.py +++ b/client/gefyra/cluster/utils.py @@ -2,6 +2,7 @@ from time import sleep from typing import Tuple, TYPE_CHECKING from gefyra.api.utils import get_workload_type +import io if TYPE_CHECKING: from kubernetes.client.models import V1Pod @@ -52,6 +53,70 @@ def get_env_from_pod_container( ) +def get_file_from_pod_container( + config: ClientConfiguration, + pod_name: str, + namespace: str, + container_name: str, + source: str, +) -> io.BytesIO: + from kubernetes.client import ApiException + from kubernetes.stream import stream + + retries = 10 + counter = 0 + interval = 1 + while counter < retries: + try: + import os + + exec_command = [ + "/bin/sh", + "-c", + f"cd {os.path.dirname(source)} && tar cf - {os.path.basename(source)}", + ] + + resp = stream( + config.K8S_CORE_API.connect_get_namespaced_pod_exec, + pod_name, + namespace, + container=container_name, + command=exec_command, + stderr=True, + stdin=False, + stdout=True, + tty=False, + _preload_content=False, + ) + + tar_buffer = io.BytesIO() + while resp.is_open(): + resp.update(timeout=1) + if resp.peek_stdout(): + tar_buffer.write(resp.read_stdout().encode("utf-8")) + if resp.peek_stderr(): + logger.debug("Error:", resp.read_stderr()) + + resp.close() + + tar_buffer.seek(0) + return tar_buffer + except ApiException as e: + if "500 Internal Server Error" in e.reason: + sleep(interval) + counter += 1 + logger.debug( + f"Failed to copy file from pod {pod_name} in namespace {namespace} on" + f" try {counter}." + ) + else: + raise e + raise RuntimeError( + f"Failed to copy file from pod {pod_name} in namespace {namespace} after" + f" {retries} tries." + ) + + def get_container(pod, container_name: str): for container in pod.spec.containers: if container.name == container_name: From a61973016b7ba1ba816cf89bff54a7c5f2756b5d Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Wed, 23 Oct 2024 15:09:31 +0200 Subject: [PATCH 2/4] fix(#501): import and function complexity --- client/gefyra/api/run.py | 115 ++++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 51 deletions(-) diff --git a/client/gefyra/api/run.py b/client/gefyra/api/run.py index 73837b5b..01621b08 100644 --- a/client/gefyra/api/run.py +++ b/client/gefyra/api/run.py @@ -3,7 +3,7 @@ import sys from threading import Thread, Event from typing import Dict, List, Optional, TYPE_CHECKING, Tuple -from client.gefyra.cli.utils import FileFromArgument +from gefyra.cli.utils import FileFromArgument from gefyra.cluster.utils import get_file_from_pod_container, retrieve_pod_and_container if TYPE_CHECKING: @@ -31,6 +31,63 @@ def check_input(): stop_thread.set() +def copy_files_or_directories_from_cluster_to_local_container( + config, + namespace: str, + local_container_name: str, + file_from: Tuple[FileFromArgument], +): + from kubernetes.client import ApiException + + for file_from_argument in file_from: + try: + file_from_pod, file_from_container = retrieve_pod_and_container( + file_from_argument.workload, namespace=namespace, config=config + ) + except ApiException as e: + logger.error( + f"Cannot copy file/directory from pod: {e.reason} ({e.status})." + ) + continue + + logger.debug( + f"Copying file/directory from {file_from_pod}/{file_from_container}" + ) + + try: + tar_buffer = get_file_from_pod_container( + config, + file_from_pod, + namespace, + file_from_container, + file_from_argument.source, + ) + except (RuntimeError, ApiException) as e: + logger.info( + f"Cannot copy file/directory from pod:" + f" --file-from {str(file_from_argument.workload)}:" + f"{file_from_argument.source}:{file_from_argument.destination}" + ) + logger.debug(e) + del tar_buffer + continue + + try: + client = config.DOCKER + client.containers.get(local_container_name).put_archive( + file_from_argument.destination, + tar_buffer.read(), + ) + except Exception as e: + logger.info( + f"Cannot copy file/directory into container:" + f" --file-from {str(file_from_argument.workload)}:" + f"{file_from_argument.source}:{file_from_argument.destination}" + ) + logger.debug(e) + continue + + @stopwatch def run( image: str, @@ -134,57 +191,13 @@ def run( else: raise RuntimeError(e.explanation) - # - # 3. copy files from a K8s container - # if file_from: - for file_from_argument in file_from: - try: - file_from_pod, file_from_container = retrieve_pod_and_container( - file_from_argument.workload, namespace=namespace, config=config - ) - except ApiException as e: - logger.error( - f"Cannot copy file/directory from pod: {e.reason} ({e.status})." - ) - continue - - logger.debug( - f"Copying file/directory from {file_from_pod}/{file_from_container}" - ) - - try: - tar_buffer = get_file_from_pod_container( - config, - file_from_pod, - namespace, - file_from_container, - file_from_argument.source, - ) - except (RuntimeError, ApiException) as e: - logger.info( - f"Cannot copy file/directory from pod:" - f" --file-from {str(file_from_argument.workload)}:" - f"{file_from_argument.source}:{file_from_argument.destination}" - ) - logger.debug(e) - del tar_buffer - continue - - try: - client = config.DOCKER - client.containers.get(name).put_archive( - file_from_argument.destination, - tar_buffer.read(), - ) - except Exception as e: - logger.info( - f"Cannot copy file/directory into container:" - f" --file-from {str(file_from_argument.workload)}:" - f"{file_from_argument.source}:{file_from_argument.destination}" - ) - logger.debug(e) - continue + copy_files_or_directories_from_cluster_to_local_container( + config, + namespace=namespace, + local_container_name=name, + file_from=file_from, + ) logger.info( f"Container image '{', '.join(container.image.tags)}' started with name" From b8e6e88d3c027dca43f879c17fd39306eea4fb5c Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Wed, 23 Oct 2024 15:15:50 +0200 Subject: [PATCH 3/4] fix(#501): remove tar_buffer error - remove del line, not sure why this was an issue --- client/gefyra/api/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/client/gefyra/api/run.py b/client/gefyra/api/run.py index 01621b08..649bdb67 100644 --- a/client/gefyra/api/run.py +++ b/client/gefyra/api/run.py @@ -69,7 +69,6 @@ def copy_files_or_directories_from_cluster_to_local_container( f"{file_from_argument.source}:{file_from_argument.destination}" ) logger.debug(e) - del tar_buffer continue try: From e60e174772c90c093249830b7d22826483987c73 Mon Sep 17 00:00:00 2001 From: Christian Busch Date: Fri, 25 Oct 2024 14:26:13 +0200 Subject: [PATCH 4/4] feat(#501): make --file-from destination optional - implemented feedback from PR - add tests for parse_file_from --- .vscode/settings.json | 20 ++++---- client/gefyra/cli/utils.py | 31 +++++++----- client/tests/unit/test_file_from.py | 77 +++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 23 deletions(-) create mode 100644 client/tests/unit/test_file_from.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 1ee0dd7b..dc5cf55a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,11 +1,11 @@ { - "python.testing.pytestArgs": [ - "operator" - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "python.formatting.provider": "black", - "gefyra.up": { - "minikube": false - }, -} \ No newline at end of file + "python.testing.pytestArgs": ["tests/"], + "python.testing.cwd": "${workspaceFolder}/client/", + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.formatting.provider": "black", + "gefyra.up": { + "minikube": false + }, + "python.analysis.include": ["client"] +} diff --git a/client/gefyra/cli/utils.py b/client/gefyra/cli/utils.py index efb68e9f..360770e9 100644 --- a/client/gefyra/cli/utils.py +++ b/client/gefyra/cli/utils.py @@ -244,32 +244,37 @@ class FileFromArgument(NamedTuple): destination: str = "/" -def parse_file_from(ctx, param, file_from_arguments) -> Tuple[FileFromArgument]: +def parse_file_from( + ctx, param, file_from_arguments +) -> Optional[Tuple[FileFromArgument]]: if not file_from_arguments: return None - MSG = ( - "Invalid argument format. Please provide the file " - "in the format 'type/name:source:destination' or " - "'type/name/container-name:source:destination'." - ) - file_from = [] for file_from_argument in file_from_arguments: split_argument = file_from_argument.split(":") - if len(split_argument) != 3: - raise ValueError(MSG) + if len(split_argument) not in (2, 3) or "/" not in split_argument[0]: + raise ValueError( + ( + "Invalid argument format. Please provide the file " + "in the format 'type/name[/container-name]:src[:dest]'." + ) + ) - workload, source, destination = split_argument - if "/" not in workload: - raise ValueError(MSG) + if len(split_argument) == 2: + workload, source = split_argument + destination = source + else: + workload, source, destination = split_argument file_from_argument_parsed = FileFromArgument( workload=workload, source=source, destination=destination, ) - file_from.append(file_from_argument_parsed) + + if file_from_argument_parsed not in file_from: + file_from.append(file_from_argument_parsed) return tuple(file_from) diff --git a/client/tests/unit/test_file_from.py b/client/tests/unit/test_file_from.py new file mode 100644 index 00000000..239462ae --- /dev/null +++ b/client/tests/unit/test_file_from.py @@ -0,0 +1,77 @@ +import unittest +from gefyra.cli.utils import FileFromArgument, parse_file_from + + +class ParseFileFromTest(unittest.TestCase): + def test_invalid_input_string(self): + with self.assertRaises(ValueError): + parse_file_from(None, None, ["invalid"]) + + def test_invalid_input_format(self): + with self.assertRaises(ValueError): + parse_file_from( + None, + None, + ["deployment/hello-world:/home/test.txt:/home/test.txt:/home/test.txt"], + ) + + def test_workload_source_destination(self): + file_from_arguments = parse_file_from( + None, + None, + ["deployment/hello-world/container:/home/test.txt:/home/test.txt"], + ) + expected = ( + FileFromArgument( + workload="deployment/hello-world/container", + source="/home/test.txt", + destination="/home/test.txt", + ), + ) + assert file_from_arguments == expected + + def test_workload_source(self): + file_from_arguments = parse_file_from( + None, None, ["deployment/hello-world/container:/home/test.txt"] + ) + expected = ( + FileFromArgument( + workload="deployment/hello-world/container", + source="/home/test.txt", + destination="/home/test.txt", + ), + ) + assert file_from_arguments == expected + + def test_duplicate_input(self): + file_from_arguments = parse_file_from( + None, + None, + [ + "deployment/hello-world/container:/home/test.txt", + "deployment/hello-world/container:/home/test.txt", + ], + ) + expected = ( + FileFromArgument( + workload="deployment/hello-world/container", + source="/home/test.txt", + destination="/home/test.txt", + ), + ) + assert file_from_arguments == expected + + def test_folder(self): + file_from_arguments = parse_file_from( + None, + None, + ["deployment/hello-world/container:/home:/home"], + ) + expected = ( + FileFromArgument( + workload="deployment/hello-world/container", + source="/home", + destination="/home", + ), + ) + assert file_from_arguments == expected