Vpn (#3)

* original version * on workflow_dispatch - wip * on workflow_dispatch - wip * env * start debug * add _AGENT_ROOT_DIR env with default value * start debug * debug mode with disabled telemetry * build-push-dev workflow * remove file * dummy-server-app works * cache gha * fix agent sources * pip sly * remove version * fix LABEL_VERSION * build on release (#2) * build on release * Rename release-build-push-dev to build-push-release * rename gha name * Update build-push-release.yml * Update build-push-release.yml * no label_version Co-authored-by: Pavel Bartsits <[email protected]>
supervisely · Jan 18, 2022 · 07111fa · 07111fa
1 parent 7885cdb
commit 07111fa
Show file tree

Hide file tree

Showing 9 changed files with 100 additions and 14 deletions.
diff --git a/.github/workflows/build-push-dev.yml b/.github/workflows/build-push-dev.yml
@@ -45,10 +45,12 @@ jobs:
             supervisely/agent:dev
             ${{ secrets.DOCKER_ENTERPRISE_REGISTRY }}/agent:dev
           build-args: |
-            LABEL_VERSION=dev
+            LABEL_VERSION=agent:dev
             LABEL_INFO=${{ env.LABEL_INFO }}
             LABEL_MODES=${{ env.LABEL_MODES }}
             LABEL_README=${{ env.LABEL_README }}
             LABEL_BUILT_AT=${{ env.LABEL_BUILT_AT }}
-          cache-from: type=registry,ref=supervisely/agent:cache
-          cache-to: type=registry,ref=supervisely/agent:cache,mode=max
+          # cache-from: type=registry,ref=supervisely/agent:cache
+          # cache-to: type=registry,ref=supervisely/agent:cache,mode=max
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
diff --git a/.github/workflows/build-push-release.yml b/.github/workflows/build-push-release.yml
@@ -0,0 +1,59 @@
+name: build-push-release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Write Tag to ENV variable
+        run: echo "LABEL_VERSION=${GITHUB_REF_NAME:1}" >> $GITHUB_ENV
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME_COMMUNITY }}
+          password: ${{ secrets.DOCKER_TOKEN_COMMUNITY }}
+      -
+        name: Login to Docker Supervisely Enterprise
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ secrets.DOCKER_ENTERPRISE_REGISTRY }}
+          username: ${{ secrets.DOCKER_USERNAME_ENTERPRISE }}
+          password: ${{ secrets.DOCKER_PASSWORD_ENTERPRISE }}
+      -
+        name: Get Docker Labels from python script
+        run: python .github/workflows/docker_labels.py
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          push: true
+          tags: |
+            supervisely/agent:dev
+            supervisely/agent:${{ env.LABEL_VERSION }}
+            ${{ secrets.DOCKER_ENTERPRISE_REGISTRY }}/agent:dev
+            ${{ secrets.DOCKER_ENTERPRISE_REGISTRY }}/agent:${{ env.LABEL_VERSION }}
+          build-args: |
+            LABEL_VERSION=agent:${{ env.LABEL_VERSION }}
+            LABEL_INFO=${{ env.LABEL_INFO }}
+            LABEL_MODES=${{ env.LABEL_MODES }}
+            LABEL_README=${{ env.LABEL_README }}
+            LABEL_BUILT_AT=${{ env.LABEL_BUILT_AT }}
+          # cache-from: type=registry,ref=supervisely/agent:cache
+          # cache-to: type=registry,ref=supervisely/agent:cache,mode=max
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
diff --git a/.github/workflows/docker_labels.py b/.github/workflows/docker_labels.py
@@ -8,7 +8,7 @@
 # version_dir = "my_app"
 # with open(os.path.join(version_dir, 'VERSION')) as f:
 #     label_version = f.read()
-label_version = "dev"
+#label_version = "dev"
 
 ## PLUGIN INFO
 info_path = "./plugin_info.json"
@@ -32,7 +32,7 @@
 
 ## Create ENV variables
 with open(env_file, "a") as workflow:
-    workflow.write(f"LABEL_VERSION={label_version}\n")
+    #workflow.write(f"LABEL_VERSION={label_version}\n")
     workflow.write(f"LABEL_INFO={label_info}\n")
     workflow.write(f"LABEL_MODES={label_modes}\n")
     workflow.write(f"LABEL_README={label_readme}\n")

diff --git a/Dockerfile b/Dockerfile
@@ -48,14 +48,16 @@ RUN apt-get update \
 
 RUN pip install docker --upgrade
 RUN pip install version-parser==1.0.1
-RUN pip install supervisely==6.4.3
+
 
 ############### copy code ###############
-ARG MODULE_PATH
-COPY $MODULE_PATH /workdir
 #COPY supervisely_lib /workdir/supervisely_lib
+RUN pip install supervisely==6.4.3
+
+COPY . /workdir
 
 #ENV PYTHONPATH /workdir:/workdir/src:/workdir/supervisely_lib/worker_proto:$PYTHONPATH
-WORKDIR /workdir/src
+WORKDIR /workdir/agent
+
+ENTRYPOINT ["sh", "-c", "python -u /workdir/agent/main.py"]
 
-ENTRYPOINT ["sh", "-c", "python -u /workdir/src/main.py"]
diff --git a/agent/.gitignore b/agent/.gitignore
@@ -0,0 +1,3 @@
+__pycache__
+*.pyc
+*.pyo
diff --git a/agent/main.py b/agent/main.py
@@ -22,7 +22,6 @@
 from worker.agent import Agent
 
 
-
 def parse_envs():
     args_req = {x: os.environ[x] for x in constants.get_required_settings()}
     args_opt = {x: constants.read_optional_setting(x) for x in constants.get_optional_defaults().keys()}

diff --git a/agent/worker/constants.py b/agent/worker/constants.py
@@ -63,6 +63,8 @@
     str(PullPolicy.NEVER): PullPolicy.NEVER
 }
 
+_DOCKER_NET = 'DOCKER_NET'
+
 _OPTIONAL_DEFAULTS = {
     _WITH_LOCAL_STORAGE: 'true',
     _UPLOAD_RESULT_IMAGES: 'true',
@@ -87,6 +89,7 @@
     _REQUESTS_CA_BUNDLE: None,
     _HOST_REQUESTS_CA_BUNDLE: None,
     _SHM_SIZE: "5G",
+    _DOCKER_NET: None,  # or string value 'supervisely-vpn'
     _AGENT_ROOT_DIR: '/sly_agent',
     _DISABLE_TELEMETRY: None
 }
@@ -110,7 +113,6 @@ def HOST_DIR():
 
 def AGENT_ROOT_DIR():
     return read_optional_setting(_AGENT_ROOT_DIR)
-    return '/sly_agent'
 
 
 def _agent_to_host_path(local_path):
@@ -350,6 +352,10 @@ def SHM_SIZE():
     return read_optional_setting(_SHM_SIZE)
 
 
+def DOCKER_NET():
+    return read_optional_setting(_DOCKER_NET)
+
+
 def DISABLE_TELEMETRY():
     return read_optional_setting(_DISABLE_TELEMETRY)
 

diff --git a/agent/worker/task_app.py b/agent/worker/task_app.py
@@ -218,7 +218,12 @@ def find_or_run_container(self):
         sly.docker_utils.docker_pull_if_needed(self._docker_api, self.docker_image_name, constants.PULL_POLICY(), self.logger)
         self.sync_pip_cache()
         if self._container is None:
-            self.spawn_container(add_labels=add_labels)
+            add_envs = None
+            if constants.DOCKER_NET() is not None:
+                add_envs = {
+                    'VIRTUAL_HOST': f'task-{self.info["task_id"]}.supervisely.local'
+                }
+            self.spawn_container(add_envs=add_envs, add_labels=add_labels)
             self.logger.info("Double check pip cache for old agents")
             self.install_pip_requirements(container_id=self._container.id)
             self.logger.info("pip second install for old agents is finished")
@@ -310,6 +315,9 @@ def main_step_envs(self):
             'LOGLEVEL': str(self.app_info.get('logLevel', 'INFO')),
             'PYTHONUNBUFFERED': 1
         }
+
+        if constants.DOCKER_NET() is not None:
+            envs['VIRTUAL_HOST'] = f'task-{self.info["task_id"]}.supervisely.local'
         return envs
 
     def process_logs(self):

diff --git a/agent/worker/task_dockerized.py b/agent/worker/task_dockerized.py
@@ -141,6 +141,12 @@ def spawn_container(self, add_envs=None, add_labels=None, entrypoint_func=None):
 
         self._container_lock.acquire()
         volumes = self._get_task_volumes()
+
+        if constants.DOCKER_NET() is not None:
+            if add_envs is None:
+                add_envs = {}
+            add_envs['VIRTUAL_HOST'] = f'task-{self.info["task_id"]}.supervisely.local'
+
         try:
             self._container = self._docker_api.containers.run(
                 self.docker_image_name,
@@ -172,7 +178,8 @@ def spawn_container(self, add_envs=None, add_labels=None, entrypoint_func=None):
                 cpu_period=constants.CPU_PERIOD(),
                 cpu_quota=constants.CPU_QUOTA(),
                 mem_limit=constants.MEM_LIMIT(),
-                memswap_limit=constants.MEM_LIMIT()
+                memswap_limit=constants.MEM_LIMIT(),
+                network=constants.DOCKER_NET()
             )
             self._container.reload()
             self.logger.debug('After spawning. Container status: {}'.format(str(self._container.status)))