From 894266139dd7967e0fbd2b7f90022e33accb2384 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Thu, 12 Oct 2023 22:12:43 +0100 Subject: [PATCH 1/7] Add podman_executable --- repo2podman/podman.py | 90 ++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 26 deletions(-) diff --git a/repo2podman/podman.py b/repo2podman/podman.py index 578ed17..b69af45 100644 --- a/repo2podman/podman.py +++ b/repo2podman/podman.py @@ -162,7 +162,7 @@ def __str__(self): return s -def exec_podman(args, *, capture, read_timeout=None, break_callback=None): +def exec_podman(args, *, capture, exe="podman", read_timeout=None, break_callback=None): """ Execute a podman command capture: @@ -177,7 +177,7 @@ def exec_podman(args, *, capture, read_timeout=None, break_callback=None): Note podman usually exits with code 125 if a podman error occurred to differentiate it from the exit code of the container. """ - cmd = ["podman"] + args + cmd = [exe] + args log_debug("Executing: {}".format(" ".join(cmd))) try: p = execute_cmd(cmd, capture=capture, break_callback=break_callback) @@ -194,13 +194,13 @@ def exec_podman(args, *, capture, read_timeout=None, break_callback=None): raise PodmanCommandError(e, lines) from None -def exec_podman_stream(args, *, read_timeout=None, break_callback=None): +def exec_podman_stream(args, *, exe="podman", read_timeout=None, break_callback=None): """ Execute a podman command and stream the output Passes on CalledProcessError if exit code is not 0 """ - cmd = ["podman"] + args + cmd = [exe] + args log_debug("Executing: {}".format(" ".join(cmd))) p = execute_cmd(cmd, capture="both", break_callback=break_callback) # This will stream the output and also pass any exceptions to the caller @@ -208,14 +208,16 @@ def exec_podman_stream(args, *, read_timeout=None, break_callback=None): class PodmanContainer(Container): - def __init__(self, cid): + def __init__(self, cid, podman_executable="podman"): self.id = cid + self._podman_executable = podman_executable self.reload() def reload(self): lines = exec_podman( ["inspect", "--type", "container", "--format", "json", self.id], capture="stdout", + exe=self._podman_executable, ) d = json.loads("".join(lines)) assert len(d) == 1 @@ -225,7 +227,9 @@ def reload(self): def _exited(self): status = "\n".join( exec_podman( - ["inspect", "--format={{.State.Status}}", self.id], capture="both" + ["inspect", "--format={{.State.Status}}", self.id], + capture="both", + exe=self._podman_executable, ) ) return status.strip() == "exited" @@ -244,6 +248,7 @@ def iter_logs(cid): try: for line in exec_podman_stream( log_command + ["--follow", cid], + exe=self._podman_executable, read_timeout=2, break_callback=self._exited, ): @@ -254,26 +259,38 @@ def iter_logs(cid): return iter_logs(self.id) - return "\n".join(exec_podman(log_command + [self.id], capture="both")).encode( - "utf-8" - ) + return "\n".join( + exec_podman( + log_command + [self.id], capture="both", exe=self._podman_executable + ) + ).encode("utf-8") def kill(self, *, signal="KILL"): - lines = exec_podman(["kill", "--signal", signal, self.id], capture="stdout") + lines = exec_podman( + ["kill", "--signal", signal, self.id], + capture="stdout", + exe=self._podman_executable, + ) log_info(lines) def remove(self): - lines = exec_podman(["rm", self.id], capture="stdout") + lines = exec_podman( + ["rm", self.id], capture="stdout", exe=self._podman_executable + ) log_info(lines) def stop(self, *, timeout=10): lines = exec_podman( - ["stop", "--timeout", str(timeout), self.id], capture="stdout" + ["stop", "--timeout", str(timeout), self.id], + capture="stdout", + exe=self._podman_executable, ) log_info(lines) def wait(self): - lines = exec_podman(["wait", self.id], capture="stdout") + lines = exec_podman( + ["wait", self.id], capture="stdout", exe=self._podman_executable + ) log_info(lines) @property @@ -290,12 +307,6 @@ class PodmanEngine(ContainerEngine): Podman container engine """ - def __init__(self, *, parent): - super().__init__(parent=parent) - - lines = exec_podman(["info"], capture="stdout") - log_debug(lines) - default_transport = Unicode( "docker://", help=""" @@ -304,6 +315,21 @@ def __init__(self, *, parent): config=True, ) + podman_executable = Unicode( + "podman", + help="""The podman executable to use for all commands. + For example, you could use an alternative podman/docker compatible command. + Defaults to `podman` on the PATH. + """, + config=True, + ) + + def __init__(self, *, parent): + super().__init__(parent=parent) + + lines = exec_podman(["info"], capture="stdout", exe=self.podman_executable) + log_debug(lines) + def build( self, *, @@ -388,12 +414,16 @@ def build( lines = execute_cmd(["ls", "-lRa", builddir], capture="stdout") log_debug(lines) - for line in exec_podman_stream(cmdargs + [builddir]): + for line in exec_podman_stream( + cmdargs + [builddir], exe=self.podman_executable + ): yield line else: builddir = path assert path - for line in exec_podman_stream(cmdargs + [builddir]): + for line in exec_podman_stream( + cmdargs + [builddir], exe=self.podman_executable + ): yield line def images(self): @@ -405,7 +435,11 @@ def remove_local(tags): if tag.startswith("localhost/"): yield tag[10:] - lines = exec_podman(["image", "list", "--format", "json"], capture="stdout") + lines = exec_podman( + ["image", "list", "--format", "json"], + capture="stdout", + exe=self.podman_executable, + ) lines = "".join(lines) if lines.strip(): @@ -426,7 +460,9 @@ def remove_local(tags): def inspect_image(self, image): lines = exec_podman( - ["inspect", "--type", "image", "--format", "json", image], capture="stdout" + ["inspect", "--type", "image", "--format", "json", image], + capture="stdout", + exe=self.podman_executable, ) d = json.loads("".join(lines)) assert len(d) == 1 @@ -448,7 +484,7 @@ def push(self, image_spec): args = ["push", image_spec, destination] def iter_out(): - for line in exec_podman_stream(args): + for line in exec_podman_stream(args, exe=self.podman_executable): yield line return iter_out() @@ -499,7 +535,7 @@ def run( raise ValueError("Additional kwargs not supported") cmdline = cmdargs + [image_spec] + command - lines = exec_podman(cmdline, capture="stdout") + lines = exec_podman(cmdline, capture="stdout", exe=self.podman_executable) # Note possible race condition: # If the container exits immediately and remove=True the next line may fail @@ -507,4 +543,6 @@ def run( # If image was pulled the progress logs will also be present # assert len(lines) == 1, lines - return PodmanContainer(lines[-1].strip()) + return PodmanContainer( + lines[-1].strip(), podman_executable=self.podman_executable + ) From 32edc8a86a8fc69a33ad7d9ed37f08cb9a3897ab Mon Sep 17 00:00:00 2001 From: Simon Li Date: Thu, 12 Oct 2023 22:13:10 +0100 Subject: [PATCH 2/7] Test podman custom executable --- tests/unit/test_podman.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/unit/test_podman.py b/tests/unit/test_podman.py index c3dc883..a5586a3 100644 --- a/tests/unit/test_podman.py +++ b/tests/unit/test_podman.py @@ -129,3 +129,36 @@ def test_run_detach_stream_exited(): c.remove() with pytest.raises(PodmanCommandError): c.reload() + + +def test_custom_executable(tmp_path): + # Use a wrapper script to log commands + log = tmp_path.joinpath("log") + exe = tmp_path.joinpath("custom_exe.sh") + with exe.open("w") as f: + f.write("#!/bin/sh\n") + f.write(f"echo $@ >> {log}\n") + f.write("exec podman $@\n") + exe.chmod(0o755) + + client = PodmanEngine(parent=None) + client.podman_executable = str(exe) + c = client.run(BUSYBOX, command=["id", "-un"]) + assert isinstance(c, PodmanContainer) + assert c._podman_executable == str(exe) + cid = c.id + + # If image was pulled the progress logs will also be present + out = c.logs().splitlines() + assert out[-1].strip() == b"root", out + + c.remove() + + with log.open() as f: + lines = f.read().splitlines() + assert lines == [ + f"run --detach --log-level=debug {BUSYBOX} id -un", + f"inspect --type container --format json {cid}", + f"logs {cid}", + f"rm {cid}", + ] From 5e5ece2c690a478231af59a75ce1aaab67babaf5 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Fri, 13 Oct 2023 23:43:26 +0100 Subject: [PATCH 3/7] parse image ls and inspect outputs as json or jsonl --- repo2podman/podman.py | 50 +++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/repo2podman/podman.py b/repo2podman/podman.py index b69af45..d786ad0 100644 --- a/repo2podman/podman.py +++ b/repo2podman/podman.py @@ -207,6 +207,24 @@ def exec_podman_stream(args, *, exe="podman", read_timeout=None, break_callback= yield from p +def _parse_json_or_jsonl(lines): + """ + Parse an array of lines as JSON or JSONL + """ + is_jsonl = True + for line in lines: + line = line.strip() + if not line or line[0] != "{" or line[-1] != "}": + is_jsonl = False + break + if is_jsonl: + return [json.loads(line) for line in lines] + lines = "".join(lines) + if lines.strip(): + return json.loads(lines) + return [] + + class PodmanContainer(Container): def __init__(self, cid, podman_executable="podman"): self.id = cid @@ -219,7 +237,7 @@ def reload(self): capture="stdout", exe=self._podman_executable, ) - d = json.loads("".join(lines)) + d = _parse_json_or_jsonl(lines) assert len(d) == 1 self.attrs = d[0] assert self.attrs["Id"].startswith(self.id) @@ -440,23 +458,19 @@ def remove_local(tags): capture="stdout", exe=self.podman_executable, ) - lines = "".join(lines) + # Podman returns an array, nerdctl returns JSONL + images = _parse_json_or_jsonl(lines) - if lines.strip(): - images = json.loads(lines) - try: - return [ - Image(tags=list(remove_local(image["names"]))) for image in images - ] - except KeyError: - # Podman 1.9.1+ - # Some images may not have a name - return [ - Image(tags=list(remove_local(image["Names"]))) - for image in images - if "Names" in image - ] - return [] + try: + return [Image(tags=list(remove_local(image["names"]))) for image in images] + except KeyError: + # Podman 1.9.1+ + # Some images may not have a name + return [ + Image(tags=list(remove_local(image["Names"]))) + for image in images + if "Names" in image + ] def inspect_image(self, image): lines = exec_podman( @@ -464,7 +478,7 @@ def inspect_image(self, image): capture="stdout", exe=self.podman_executable, ) - d = json.loads("".join(lines)) + d = _parse_json_or_jsonl(lines) assert len(d) == 1 tags = d[0]["RepoTags"] config = d[0]["Config"] From 5c932f8fd9b382e80df8714c28d0b8894dc7c155 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Fri, 13 Oct 2023 22:39:35 +0100 Subject: [PATCH 4/7] Remove --force-rm from build --- repo2podman/podman.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repo2podman/podman.py b/repo2podman/podman.py index d786ad0..3264b74 100644 --- a/repo2podman/podman.py +++ b/repo2podman/podman.py @@ -395,7 +395,8 @@ def build( except KeyError: pass - cmdargs.append("--force-rm") + # Disable for better compatibility with other CLIs + # cmdargs.append("--force-rm") cmdargs.append("--rm") From 81b4f8484cd16caf7f818b018836b8bd10099e65 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Fri, 13 Oct 2023 22:40:17 +0100 Subject: [PATCH 5/7] Configurable log-level --- repo2podman/podman.py | 6 ++++-- tests/unit/test_podman.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/repo2podman/podman.py b/repo2podman/podman.py index 3264b74..6c5043a 100644 --- a/repo2podman/podman.py +++ b/repo2podman/podman.py @@ -342,6 +342,8 @@ class PodmanEngine(ContainerEngine): config=True, ) + podman_loglevel = Unicode("", help="Podman log level", config=True) + def __init__(self, *, parent): super().__init__(parent=parent) @@ -541,8 +543,8 @@ def run( if remove: cmdargs.append("--rm") - # TODO: Make this configurable via a config traitlet - cmdargs.append("--log-level=debug") + if self.podman_loglevel: + cmdargs.append(f"--log-level={self.podman_loglevel}") command = command or [] diff --git a/tests/unit/test_podman.py b/tests/unit/test_podman.py index a5586a3..5e4053b 100644 --- a/tests/unit/test_podman.py +++ b/tests/unit/test_podman.py @@ -143,6 +143,7 @@ def test_custom_executable(tmp_path): client = PodmanEngine(parent=None) client.podman_executable = str(exe) + client.podman_loglevel = "debug" c = client.run(BUSYBOX, command=["id", "-un"]) assert isinstance(c, PodmanContainer) assert c._podman_executable == str(exe) From 03a88d218e4bfbcebac28acc3a49e2777bfb78cb Mon Sep 17 00:00:00 2001 From: Simon Li Date: Fri, 13 Oct 2023 23:11:49 +0100 Subject: [PATCH 6/7] Fix order of ports in `--publish` --- repo2podman/podman.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repo2podman/podman.py b/repo2podman/podman.py index 6c5043a..68ff3a4 100644 --- a/repo2podman/podman.py +++ b/repo2podman/podman.py @@ -525,10 +525,11 @@ def run( cmdargs.append("--publish-all") ports = ports or {} + # container-port/protocol:host-port for k, v in ports.items(): if k.endswith("/tcp"): k = k[:-4] - cmdargs.extend(["--publish", "{}:{}".format(k, v)]) + cmdargs.extend(["--publish", "{}:{}".format(v, k)]) cmdargs.append("--detach") From 3455155e18d34b650b70326dab598b5f9714f6ff Mon Sep 17 00:00:00 2001 From: Simon Li Date: Sat, 14 Oct 2023 21:59:17 +0100 Subject: [PATCH 7/7] Mention nerdctl and podman-remote in README --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a82f7c5..81027c0 100644 --- a/README.md +++ b/README.md @@ -15,4 +15,18 @@ Requires Podman 3+. Simply include `--engine podman` in the arguments to `repo2docker`: - repo2docker --engine podman repository/to/build + repo2docker --engine podman + +### Using a different Podman executable + +repo2podman uses the `podman` command line executable, so it should be possible to substitute any other docker/podman compatible command line tool. + +For example, `nerdctl`: + + repo2docker --engine podman --PodmanEngine.podman_executable=nerdctl + +`podman-remote`: + + export CONTAINER_HOST=ssh://@/home//podman.sock + export CONTAINER_SSHKEY=$HOME/.ssh/ + repo2docker --engine=podman --PodmanEngine.podman_executable=podman-remote