From 418d1a09b70ccb66919a7a29085ef9d7974b8720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Gr=C3=B3dek?= Date: Fri, 3 Nov 2023 16:47:32 +0100 Subject: [PATCH] Update documentation --- DESCRIPTION.rst | 3 +- docs/api/ds_pycontain.rst | 8 ++++ docs/code_documentation.md | 25 +---------- docs/index.rst | 5 +-- docs/licenses.rst | 1 + docs/project_overview.md | 90 +++++++++++++++++++++++++++++++++++++- 6 files changed, 103 insertions(+), 29 deletions(-) diff --git a/DESCRIPTION.rst b/DESCRIPTION.rst index 24143a2..10ee8ce 100644 --- a/DESCRIPTION.rst +++ b/DESCRIPTION.rst @@ -1,7 +1,7 @@ `deepsense.ai `_ **ds_pycontain** ------------------------------------------------------------- -**ds_pycontain** is a small python package to help with docker containers and images. +**ds_pycontain** is a small python package to help with docker containers and images and provide Python REPL running in a docker container. Example use case you might consider is to isolate python code execution generated by untrusted LLM by running it in a docker container. @@ -11,3 +11,4 @@ This package makes it a bit easier to: * Pull docker images from dockerhub (or similar). * Run docker container to execute a one-off command. * Run docker container to execute a long-running process and communicate with it. +* Run python commands in a container and get the result. \ No newline at end of file diff --git a/docs/api/ds_pycontain.rst b/docs/api/ds_pycontain.rst index a571553..a614b9e 100644 --- a/docs/api/ds_pycontain.rst +++ b/docs/api/ds_pycontain.rst @@ -12,6 +12,14 @@ ds_pycontain.docker_containers module :undoc-members: :show-inheritance: +ds_pycontain.python_dockerized_repl module +---------------------------------------------------- + +.. automodule:: ds_pycontain.python_dockerized_repl + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/code_documentation.md b/docs/code_documentation.md index 0a14709..32c4bbd 100644 --- a/docs/code_documentation.md +++ b/docs/code_documentation.md @@ -1,29 +1,6 @@ # Code documentation -**ds_pycontain** is a python package which provides an abstraction over the docker API. - -Supported functionality covers: -- Building docker images from Dockerfiles -- Pulling docker images from dockerhub (or similar) -- Running docker containers to execute a one-off command -- Running docker containers to execute a long-running process and communicate with it - - -```python - from ds_pycontain import DockerContainer, DockerImage, get_docker_client - - client = get_docker_client() - - # This will fetch the image from dockerhub if it is not already present - # with the "latest" tag. Then container is started and commands are run - with DockerContainer(DockerImage.from_tag("alpine")) as container: - ret_code, output = container.run("touch /animal.txt") - assert ret_code == 0 - - ret_code, output = container.run("ls /") - assert ret_code == 0 - assert cast(bytes, output).find(b"animal.txt") >= 0 -``` +This is the documentation for the code of the project API. ```{toctree} --- diff --git a/docs/index.rst b/docs/index.rst index 0b68844..650268e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,9 +5,7 @@ `deepsense.ai `_ - pycontain ==================================================================================================================== -Documentation for **ds_pycontain** python package to work with docker containers and images. - -Example use case you might consider is to isolate python code execution generated by untrusted LLM by running it in a docker container. +Documentation for **ds_pycontain** python package to work with docker containers and images, as well as providing python REPL running in a container. This package makes it a bit easier to: @@ -15,6 +13,7 @@ This package makes it a bit easier to: * Pull docker images from dockerhub (or similar). * Run docker container to execute a one-off command. * Run docker container to execute a long-running process and communicate with it. +* Run python code in a docker container and communicate with it. .. toctree:: diff --git a/docs/licenses.rst b/docs/licenses.rst index bc815bc..c2bc0a5 100644 --- a/docs/licenses.rst +++ b/docs/licenses.rst @@ -2,5 +2,6 @@ Licenses ===================== List of automatically detected licenses of all detected python packages with `pip-licenses`. +They are all projects that are used in the project development. .. include:: licenses_table.rst \ No newline at end of file diff --git a/docs/project_overview.md b/docs/project_overview.md index 6011cbb..99e1948 100644 --- a/docs/project_overview.md +++ b/docs/project_overview.md @@ -1,6 +1,94 @@ # Project overview +**ds_pycontain** is a python package which provides an abstraction over the docker API and provide Python REPL running in a docker container. + +Supported functionality covers: +- Building docker images from Dockerfiles +- Pulling docker images from dockerhub (or similar) +- Running docker containers to execute a one-off command +- Running docker containers to execute a long-running process and communicate with it +- Run python commands in a container and get the result. + +## Motivation + +Main motivation is to allow to orchestrate running unsafe code or commands in isolated environment. The docker API is quite complicated and not well documented or typed. This project aims to provide a higher level abstraction over the docker API. -Main motivation is to allow to orchestrate running unsafe code or commands in isolated environment. +What is also provided is **a python REPL running in a docker container**. + +This might be useful to improve security for execution of LLM models/agents generated code, which generally should not be trusted. + +## Example code snippets + +### Execute commands in container running in the background: + +Below is a short snippet showcasing how to run docker container in the background and execute commands in it. + +```python + from ds_pycontain import DockerContainer, DockerImage, get_docker_client + + client = get_docker_client() + + # This will fetch the image from dockerhub if it is not already present + # with the "latest" tag. Then container is started and commands are run + with DockerContainer(DockerImage.from_tag("alpine")) as container: + ret_code, output = container.run("touch /animal.txt") + assert ret_code == 0 + + ret_code, output = container.run("ls /") + assert ret_code == 0 + assert cast(bytes, output).find(b"animal.txt") >= 0 +``` + +### Docker images + +Images can be pulled from dockerhub or built from dockerfile. + +```python +from ds_pycontain import DockerImage + +# pull or use alpine:latest +image = DockerImage.from_tag("alpine") +# use provided tag to pull/use the image +image = DockerImage.from_tag("python", tag="3.9-slim") +# use this dockerfile to build a new local image +image = DockerImage.from_dockerfile("example/Dockerfile") +# you can provide a directory path which contains Dockerfile, set custom image name +image = DockerImage.from_dockerfile("path/to/dir_with_Dockerfile/", name="cow") +``` + +### Python REPL running in docker container + +Running Python code in docker container is rather easy with this package. + +```python + from ds_pycontain.python_dockerized_repl import PythonContainerREPL + + # To start python REPL in container it is easy, + # just be aware that it will take some time to start the container + # and ports might be allocated by OS, so use different port/retry + # if you get error. + repl = PythonContainerREPL(port=7121) + + # You can run python commands in the container + # and it will keep state between commands. + out1 = repl.exec("x = [1, 2, 3]") + assert out1 == "" + # Eval returns string representation of the python command + # as it would be in python REPL: + out2 = repl.eval("len(x)") + assert out2 == "3" + + # Exec returns captured standard output (stdout) + # so it won't return anything in this case: + out3 = repl.exec("len(x)") + assert out3 == "" + # but exec with print works: + out4 = repl.exec("print(len(x))") + assert out4 == "3\n" + + # You can also get error messages if code is wrong: + err = repl.exec("print(x") + assert "SyntaxError" in err +``` \ No newline at end of file