skill deployments

skill-manager/requirements.txt

@@ -7,3 +7,4 @@ pydantic>=1.8.2
 pymongo>=3.12.1
 requests>=2.26.0
 docker>=5.0.3
+python-multipart>=0.0.5

skill-manager/skill_manager/models/skill_deployment.py

@@ -0,0 +1,8 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+class SkillDeployment(BaseModel):
+    skill_template_id: str = Field()
+    deployed: bool = Field()
+    url: Optional[str] = Field()

skill-manager/skill_manager/models/skill_template.py

@@ -28,8 +28,3 @@ class SkillTemplate(MongoModel):
     client_secret: Optional[str] = Field(
         None, description="The cleint secret of the skill stored in Keycloak."
     )
-
-class SkillDeployment(BaseModel):
-    skill_template_id: str = Field()
-    deployed: bool = Field()
-    url: Optional[str] = Field()

skill-manager/skill_manager/routers/skill_deployments.py

@@ -0,0 +1,79 @@
+from typing import List
+
+from celery.result import AsyncResult
+from fastapi import APIRouter, Depends, Path
+from square_auth.auth import Auth
+
+from skill_manager.core import tasks
+from skill_manager.core.docker_client import SkillManagerDockerClient
+from skill_manager.models.skill_deployment import SkillDeployment
+from skill_manager.models.skill_template import SkillTemplate
+from skill_manager.models.task import TaskStatus
+from skill_manager.routers.skill_templates import (get_skill_template_by_id,
+                                                   get_skill_templates)
+
+router = APIRouter(prefix="/deployments")
+
+auth = Auth()
+
+@router.get("/{id}", response_model=SkillDeployment)
+async def get_deployment_by_id(
+    skill_template_id=Path(..., alias="id"),
+    skill_manager_docker_client: SkillManagerDockerClient = Depends(
+        SkillManagerDockerClient
+    ),
+):
+    container = skill_manager_docker_client.get_skill_template_container_by_id(
+        skill_template_id
+    )
+    if container:
+        deployed = True
+        url = container.labels["url"]
+    else:
+        deployed = False
+        url = None
+
+    return SkillDeployment(
+        skill_template_id=skill_template_id,
+        deployed=deployed,
+        url=url,
+    )
+
+
+@router.get("", response_model=List[SkillDeployment])
+async def get_deployments(
+    skill_manager_docker_client: SkillManagerDockerClient = Depends(
+        SkillManagerDockerClient
+    ),
+):
+    skill_templates: List[SkillTemplate] = await get_skill_templates()
+    containers = skill_manager_docker_client.get_skill_template_containers()
+    container_skill_template_ids = [c.labels.get("skill-template-id", "") for c in containers]
+
+    skill_deployments = []
+    for skill_template in skill_templates:
+        if skill_template.id in container_skill_template_ids:
+            container_idx = container_skill_template_ids.index(skill_template.id)
+            url = containers[container_idx].labels["url"]
+            deployed = True
+        else:
+            url = None
+            deployed = False
+        skill_deployments.append(
+            SkillDeployment(
+                skill_template_id=str(skill_template.id),
+                deployed=deployed,
+                url=url,
+            )
+        )
+    return skill_deployments
+
+
+@router.post("/{id}", response_model=TaskStatus)
+async def deploy_skill_template(skill_template_id=Path(..., alias="id")):
+    skill_template = await get_skill_template_by_id(skill_template_id)
+    result: AsyncResult = tasks.build_and_deploy_skill_template_container.delay(
+        skill_template
+    )
+
+    return TaskStatus(task_id=result.id, status=result.status)

skill-manager/skill_manager/routers/skill_templates.py

@@ -1,15 +1,15 @@
 import logging
+import os
 from typing import Dict, List
 
 from bson import ObjectId
-from fastapi import APIRouter, Depends, Request, Path
+from fastapi import APIRouter, Depends, Path, UploadFile
 from square_auth.auth import Auth
+
 from skill_manager import mongo_client
 from skill_manager.core.keycloak_client import KeycloakClient
-
 from skill_manager.models.skill_template import SkillTemplate
 
-
 logger = logging.getLogger(__name__)
 
 router = APIRouter(prefix="/skill-templates")
@@ -48,7 +48,7 @@ async def create_skill_template(
         realm=realm, username=username, skill_name=skill_template.name
     )
     skill_template.client_id = client["clientId"]
-    
+
     # save skill template in mongo db
     skill_template_id = mongo_client.client.skill_manager.skill_templates.insert_one(
         skill_template.mongo()
@@ -85,3 +85,12 @@ async def delete_skill_template(skill_template_id=Path(..., alias="id")):
         return
     else:
         raise RuntimeError(delete_result.raw_result)
+
+
+@router.post("/{id}/upload-function")
+async def upload_function(file: UploadFile, skill_template_id=Path(..., alias="id")):
+    tmp = await file.read()
+    fn_dir = os.environ["FUNCTION_DUMP_DIR"]
+    os.makedirs(fn_dir, exist_ok=True)
+    with open(f"{fn_dir}/{skill_template_id}.pickle", "wb") as fh:
+        fh.write(tmp)

skill-manager/tests/test_skill_deployments_api.py

@@ -0,0 +1,124 @@
+import uuid
+from unittest.mock import MagicMock
+
+import pytest
+from docker import DockerClient
+from fastapi.testclient import TestClient
+from skill_manager.core.docker_client import SkillManagerDockerClient
+from skill_manager.main import app
+from skill_manager.models.skill_deployment import SkillDeployment
+from skill_manager.core.keycloak_client import KeycloakClient
+from skill_manager.routers.skill_templates import auth
+
+
+keycloak_client_mock = MagicMock()
+keycloak_client_mock.create_client.return_value = {
+    "clientId": "test-client-id",
+    "secret": "test-secret",
+}
+keycloak_client_override = lambda: keycloak_client_mock
+
+realm = "test-realm"
+username = "test-user"
+app.dependency_overrides[KeycloakClient] = keycloak_client_override
+app.dependency_overrides[auth] = lambda: dict(realm=realm, username=username)
+
+@pytest.mark.parametrize(
+    "skill_template_running", [False, True], ids=["not-runnning", "running"]
+)
+def test_get_deployment_by_id(
+    mongo_db, docker_client: DockerClient, skill_template_running
+):
+
+    skill_template_id = str(uuid.uuid1())
+    sm_docker_client_mock = MagicMock(spec=SkillManagerDockerClient)
+    if skill_template_running:
+        test_image = "containous/whoami"
+        test_url = "http://test.test"
+        container = docker_client.containers.run(
+            test_image,
+            detach=True,
+            remove=True,
+            labels={"skill-template-id": f"{skill_template_id}", "url": test_url},
+        )
+    else:
+        test_url = None
+        container = None
+    sm_docker_client_mock.get_skill_template_container_by_id.return_value = container
+    app.dependency_overrides[SkillManagerDockerClient] = lambda: sm_docker_client_mock
+
+    with TestClient(app) as test_client:
+        response = test_client.get("/api/deployments/{id}".format(id=skill_template_id))
+        assert response.status_code == 200
+        actual_skill_deployment = SkillDeployment.parse_obj(response.json())
+        expected_skill_deployment = SkillDeployment(
+            skill_template_id=skill_template_id,
+            deployed=skill_template_running,
+            url=test_url,
+        )
+        assert actual_skill_deployment == expected_skill_deployment
+
+    if container:
+        container.stop()
+
+
+@pytest.mark.parametrize("num_containers", [0, 3], ids=["zero-runnning", "n-running"])
+def test_get_deployments(
+    mongo_db,
+    docker_client: DockerClient,
+    num_containers,
+    token_factory,
+    skill_template_factory,
+):
+
+    username = "test-user"
+    token = token_factory(preferred_username=username)
+
+    sm_docker_client_mock = MagicMock(spec=SkillManagerDockerClient)
+    containers = []
+    test_image = "containous/whoami"
+    test_url = "http://test.test"
+    for _ in range(num_containers):
+        # create the skill template in mongodb
+        with TestClient(app) as test_client:
+            skill_template_name = f"test-create-skill-template"
+            test_skill_template = skill_template_factory(
+                name=skill_template_name, user_id=username
+            )
+            response = test_client.post(
+                "/api/skill-templates",
+                data=test_skill_template.json(),
+                headers=dict(Authorization="Bearer " + token),
+            )
+            skill_template_id = response.json()["id"]
+
+        # deploy a container 
+        container = docker_client.containers.run(
+            test_image,
+            detach=True,
+            remove=True,
+            labels={
+                "type": "skill-template",
+                "skill-template-id": f"{skill_template_id}",
+                "url": test_url,
+            },
+        )
+        containers.append(container) 
+    sm_docker_client_mock.get_skill_template_containers.return_value = containers
+    app.dependency_overrides[SkillManagerDockerClient] = lambda: sm_docker_client_mock
+
+    for _ in range(4):
+        container = docker_client.containers.run(
+            test_image,
+            detach=True, 
+            remove=True,
+        )
+        containers.append(container)
+
+    with TestClient(app) as test_client:
+        response = test_client.get("/api/deployments")
+        assert response.status_code == 200, response.content
+        assert len(response.json()) == num_containers, response.json()
+
+    for container in containers:
+        container.stop()

skill-manager/tests/test_skill_templates_api.py

@@ -1,13 +1,11 @@
+import uuid
 from unittest.mock import MagicMock
 
-import pytest
 from bson import ObjectId
 from fastapi.testclient import TestClient
-
 from skill_manager import mongo_client
 from skill_manager.core.keycloak_client import KeycloakClient
 from skill_manager.main import app
-from skill_manager.models.skill_template import SkillTemplate
 from skill_manager.routers.skill_templates import auth
 
 keycloak_client_mock = MagicMock()
@@ -22,22 +20,6 @@
 app.dependency_overrides[KeycloakClient] = keycloak_client_override
 app.dependency_overrides[auth] = lambda: dict(realm=realm, username=username)
 
-@pytest.fixture
-def skill_template_factory():
-    def create_skill_template(
-        name="test-skill-template",
-        user_id="test-user",
-        url="http://test-skill-template.test",
-        **kwargs
-    ):
-        skill_template = SkillTemplate(name=name, user_id=user_id, url=url, **kwargs)
-        if not skill_template.id:
-            del skill_template.id
-
-        return skill_template
-
-    return create_skill_template
-
 
 def test_get_skill_by_id(mongo_db, token_factory, skill_template_factory):
     token = token_factory(preferred_username=username)
@@ -53,9 +35,11 @@ def test_get_skill_by_id(mongo_db, token_factory, skill_template_factory):
             headers=dict(Authorization="Bearer " + token),
         )
         assert response.status_code == 201, response.content
-        
+
         skill_template_id = response.json()["id"]
-        response = test_client.get("/api/skill-templates/{id}".format(id=skill_template_id))
+        response = test_client.get(
+            "/api/skill-templates/{id}".format(id=skill_template_id)
+        )
         assert response.status_code == 200, response.content
 
         response = response.json()
@@ -84,6 +68,7 @@ def test_get_skill(mongo_db, token_factory, skill_template_factory):
         response = response.json()
         assert any(r["name"] == skill_template_name for r in response)
 
+
 def test_create_skill(mongo_db, token_factory, skill_template_factory):
 
     token = token_factory(preferred_username=username)
@@ -108,7 +93,6 @@ def test_create_skill(mongo_db, token_factory, skill_template_factory):
             )
         )
         assert mongo_skill_template["name"] == skill_template_name
-
 
 
 def test_update_skill(mongo_db, token_factory, skill_template_factory):
@@ -126,7 +110,7 @@ def test_update_skill(mongo_db, token_factory, skill_template_factory):
             headers=dict(Authorization="Bearer " + token),
         )
         assert response.status_code == 201, response.content
-        
+
         skill_template_id = response.json()["id"]
         updated_skill_template_name = "test-update-skill-template-updated"
         response = test_client.put(
@@ -159,19 +143,44 @@ def test_delete_skill(mongo_db, token_factory, skill_template_factory):
             headers=dict(Authorization="Bearer " + token),
         )
         assert response.status_code == 201, response.content
-        
+
         skill_template_id = response.json()["id"]
 
         response = test_client.delete(
             "/api/skill-templates/{id}".format(id=skill_template_id),
             data=test_skill_template.json(),
             headers=dict(Authorization="Bearer " + token),
         )
-        
+
         # assert it has been deleted from mongodb
         mongo_skill_template = (
             mongo_client.client.skill_manager.skill_templates.find_one(
                 {"_id": ObjectId(skill_template_id)}
             )
         )
         assert mongo_skill_template is None
+
+
+def test_upload_function(mongo_db, monkeypatch, tmp_path_factory):
+
+    skill_template_id = str(uuid.uuid1())
+    filename = f"{skill_template_id}.pickle"
+    source_dir = tmp_path_factory.mktemp("source")
+
+    source_path = source_dir / filename
+    source_path.write_text(f"{skill_template_id}")
+
+    target_dir = tmp_path_factory.mktemp("target")
+    monkeypatch.setenv("FUNCTION_DUMP_DIR", str(target_dir))
+
+    with TestClient(app) as test_client:
+        with open(source_path, "rb") as file:
+            test_client.post(
+                f"/api/skill-templates/{skill_template_id}/upload-function",
+                files={"file": file},
+            )
+
+        assert (
+            open(target_dir / f"{skill_template_id}.pickle", "r").read()
+            == skill_template_id
+        )