feat: add tests for secrets helper (#40)

feat: remove default env on deletion
feat: add local dev setup Makefile

.pre-commit-config.yaml

@@ -4,6 +4,7 @@ repos:
     hooks:
       - id: isort
         name: isort (python)
+        args: ["--profile", "black"]
 
   - repo: https://github.com/psf/black
     rev: 23.7.0

Makefile

@@ -0,0 +1,25 @@
+
+.PHONY: local-dev
+local-dev:
+	cp .env.sample .env
+	echo "\nDEPLOYMENTS_MOUNT_DIR='$(PWD)/deployments' # DO NOT EDIT THIS - FIX FOR RESOURCE SHARING BUG" >> .env
+	echo "\nNGINX_PROXY_CONF_LOCATION='$(PWD)/nginx-confs' # DO NOT EDIT THIS - FIX FOR RESOURCE SHARING BUG" >> .env
+	echo "\nDEPLOYMENT_HOST='localhost' # DO NOT EDIT THIS" >> .env
+	bash setup-vault.sh docker-compose-local.yml
+	echo "\nVAULT_BASE_URL='http://localhost:8200'" >> .env
+	mkdir deployments nginx-confs
+	docker-compose -f docker-compose-local.yml up -d nginx
+
+.PHONY: sarthi
+sarthi:
+	python app.py
+
+.PHONY: test
+test:
+	python -m pytest -vvv tests
+
+.PHONY: reset
+reset:
+	rm -f .env keys.txt parsed-key.txt ansi-keys.txt
+	docker compose -f docker-compose-local.yml down -v
+	rm -rf deployments nginx-confs

app.py

@@ -10,7 +10,7 @@
 
 load_dotenv()
 
-if os.environ.get("ENV").lower() == "local":
+if (os.environ.get("ENV") or "local").lower() == "local":
     logging.basicConfig(level=logging.NOTSET)
 
 

docker-compose-local.yml

@@ -0,0 +1,63 @@
+version: "3"
+
+services:
+  nginx:
+    image: nginx:latest
+    restart: always
+    container_name: sarthi_nginx
+    ports:
+      - "80:80"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ./nginx-confs:/etc/nginx/conf.d
+
+  sarthi:
+    build: .
+    restart: always
+    volumes:
+      # hack to bypass file resource sharing error
+      # not tested and no support for windows server 💩
+      - ./deployments:${DEPLOYMENTS_MOUNT_DIR:-/deployments}
+      - ./nginx-confs:${NGINX_PROXY_CONF_LOCATION:-/nginx-confs}
+      - /var/run/docker.sock:/var/run/docker.sock
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      DEPLOYMENTS_MOUNT_DIR: ${DEPLOYMENTS_MOUNT_DIR}
+      NGINX_PROXY_CONF_LOCATION: ${NGINX_PROXY_CONF_LOCATION}
+      ENV: ${ENV:-local}
+      DOMAIN_NAME: ${DOMAIN_NAME:-localhost}
+      VAULT_TOKEN: ${VAULT_TOKEN}
+      VAULT_BASE_URL: ${VAULT_BASE_URL:-http://vault:8200}
+      SECRET_TEXT: ${SECRET_TEXT}
+    depends_on:
+      - vault
+
+  vault:
+    image: vault:1.12.3
+    restart: always
+    ports:
+      - "8200:8200"
+    volumes:
+      - ./vault/vault.json:/vault/config/vault.json
+      - vault-secrets-dev:/vault/file
+    environment:
+      VAULT_ADDR: http://0.0.0.0:8200
+      VAULT_API_ADDR: http://0.0.0.0:8200
+      VAULT_ADDRESS: http://0.0.0.0:8200
+    cap_add:
+      - IPC_LOCK
+    command: vault server -config=/vault/config/vault.json
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "wget --spider http://127.0.0.1:8200/v1/sys/health || exit 1",
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+volumes:
+  vault-secrets-dev:

docker-compose.yml

@@ -22,11 +22,10 @@ services:
     build: .
     restart: always
     volumes:
-      # hack to bypass file resource sharing error to make my development easy on mac
-      # docker mount dir not meant to be used on linux
+      # hack to bypass file resource sharing error
       # not tested and no support for windows server 💩
-      - ./deployments:${DEPLOYMENTS_MOUNT_DIR}
-      - ./nginx-confs:${NGINX_PROXY_CONF_LOCATION}
+      - ./deployments:${DEPLOYMENTS_MOUNT_DIR:-/deployments}
+      - ./nginx-confs:${NGINX_PROXY_CONF_LOCATION:-/nginx-confs}
       - /var/run/docker.sock:/var/run/docker.sock
     extra_hosts:
       - "host.docker.internal:host-gateway"

server/deployer.py

@@ -118,3 +118,4 @@ def delete_preview_environment(self):
             self._nginx_helper.remove_outer_proxy()
             self._nginx_helper.reload_nginx()
             self._delete_deployment_files()
+            self._secrets_helper.cleanup_deployment_variables()

server/utils.py

@@ -295,12 +295,15 @@ def remove_outer_proxy(self):
 
 class SecretsHelper:
     def __init__(self, project_name, branch_name, project_path):
+        vault_url = os.environ.get("VAULT_BASE_URL")
+        vault_token = os.environ.get("VAULT_TOKEN")
         self._project_path = project_path
         self._secrets_namespace = f"{project_name}/{branch_name}"
-        self._secret_url = (
-            f"{os.environ.get('VAULT_BASE_URL')}/v1/kv/data/{self._secrets_namespace}"
+        self._secret_url = f"{vault_url}/v1/kv/data/{self._secrets_namespace}"
+        self._secret_metadata_url = (
+            f"{vault_url}/v1/kv/metadata/{self._secrets_namespace}"
         )
-        self._headers = {"X-Vault-Token": os.environ.get("VAULT_TOKEN")}
+        self._headers = {"X-Vault-Token": vault_token}
 
     def _create_env_placeholder(self):
         sample_envs = {"key": "secret-value"}
@@ -335,6 +338,17 @@ def inject_env_variables(self, project_path):
             for key, value in secret_data["data"]["data"].items():
                 file.write(f'{key}="{value}"\n')
 
+    def cleanup_deployment_variables(self):
+        response = requests.delete(url=self._secret_metadata_url, headers=self._headers)
+        logger.debug(
+            f"Tried Removing Deployment variables from Vault {response.status_code}"
+        )
+        try:
+            response.raise_for_status()
+        except requests.HTTPError as e:
+            logger.debug(f"Error removing deployment secrets {e}")
+        return response
+
 
 def get_random_stub(project_name: str) -> str:
     return hashlib.md5(project_name.encode()).hexdigest()[:16]

setup-vault.sh

@@ -45,25 +45,27 @@ else
   sed 's/\x1B\[[0-9;]*[JKmsu]//g' < ansi-keys.txt  > keys.txt
 fi
 
-sed -n 's/Unseal Key [1-1]\+: \(.*\)/\1/p' keys.txt > parsed-key.txt
+awk '/Unseal Key [1-1]:/ {print $NF}' keys.txt > parsed-key.txt
 key=$(cat parsed-key.txt)
 docker-compose -f "$COMPOSE_FILE" exec -T "$SERVICE_NAME" vault operator unseal "$key" < /dev/null
 
-sed -n 's/Unseal Key [2-2]\+: \(.*\)/\1/p' keys.txt > parsed-key.txt
+
+
+awk '/Unseal Key [2-2]:/ {print $NF}' keys.txt > parsed-key.txt
 key=$(cat parsed-key.txt)
 docker-compose -f "$COMPOSE_FILE" exec -T "$SERVICE_NAME" vault operator unseal "$key" < /dev/null
 
-sed -n 's/Unseal Key [3-3]\+: \(.*\)/\1/p' keys.txt > parsed-key.txt
+awk '/Unseal Key [3-3]:/ {print $NF}' keys.txt > parsed-key.txt
 key=$(cat parsed-key.txt)
 docker-compose -f "$COMPOSE_FILE" exec -T "$SERVICE_NAME" vault operator unseal "$key" < /dev/null
 
-root_token=$(sed -n 's/Initial Root Token: \(.*\)/\1/p' keys.txt | tr -dc '[:print:]')
+root_token=$(awk '/Initial Root Token:/ {print $NF}' keys.txt | tr -dc '[:print:]')
 
 if [[ $vault_status == *"Initialized     true"* ]]; then
     echo "Vault is initialized already. Skipping creating a KV engine"
 else
   echo -e "\nVAULT_TOKEN=random_token" >> .env
-  sed -i "s/VAULT_TOKEN=.*/VAULT_TOKEN=$root_token/" ".env"
+  awk -v root_token="$root_token" '/^VAULT_TOKEN=/ {gsub(/=.*/, "=" root_token)} 1' ".env" > temp && mv temp ".env"
   docker-compose -f "$COMPOSE_FILE" exec -e VAULT_TOKEN=$root_token -T "$SERVICE_NAME" vault secrets enable -path=kv kv-v2
 fi
 

tests/conftest.py

@@ -1,6 +1,6 @@
 import pytest
 
-from server.utils import ComposeHelper, DeploymentConfig, NginxHelper
+from server.utils import ComposeHelper, DeploymentConfig, NginxHelper, SecretsHelper
 
 
 @pytest.fixture
@@ -86,3 +86,13 @@ def nginx_helper(deployment_config):
     outer_conf_base_path = "/path/to/outer/conf"
     deployment_project_path = "/path/to/deployment/project"
     return NginxHelper(deployment_config, outer_conf_base_path, deployment_project_path)
+
+
+@pytest.fixture
+def secrets_helper_instance(mocker):
+    mock_os = mocker.patch("server.utils.os")
+    mock_os.environ = {
+        "VAULT_TOKEN": "hvs.randomToken",
+        "VAULT_BASE_URL": "http://vault:8200",
+    }
+    return SecretsHelper("project_name", "branch_name", "/path/to/project")

tests/test_utils.py

@@ -1,6 +1,8 @@
 import pathlib
+from unittest.mock import MagicMock, patch
 
 import pytest
+import requests
 
 from server.utils import ComposeHelper
 
@@ -244,3 +246,142 @@ def test_deployment_config_repr(deployment_config):
         "'https://github.com/tushar5526/test-project-name.git', 'docker-compose.yml', 'POST')"
     )
     assert repr(deployment_config) == expected_repr
+
+
+@patch("server.utils.os")
+@patch("server.utils.requests")
+def test_create_env_placeholder_with_sample_env_file(
+    mock_requests, mock_os, secrets_helper_instance
+):
+    # Mocking necessary dependencies
+    mock_os.path.exists.return_value = True
+    mock_dotenv_values = MagicMock(return_value={"key": "secret-value"})
+    with patch("server.utils.dotenv_values", mock_dotenv_values):
+        mock_response = MagicMock()
+        mock_response.raise_for_status.return_value = None
+        mock_requests.post.return_value = mock_response
+
+        # Calling the method under test
+        secrets_helper_instance._create_env_placeholder()
+
+        # Assertions
+        mock_dotenv_values.assert_called()
+        mock_requests.post.assert_called_once_with(
+            url="http://vault:8200/v1/kv/data/project_name/branch_name",
+            headers={"X-Vault-Token": "hvs.randomToken"},
+            data='{"data": {"key": "secret-value"}}',
+        )
+
+
+@patch("server.utils.os")
+@patch("server.utils.requests")
+def test_create_env_placeholder_with_sample_env_file_missing(
+    mock_requests, mock_os, secrets_helper_instance
+):
+    # Mocking necessary dependencies
+    mock_os.path.join.return_value = "/path/to/project/.env.sample"
+    mock_os.path.exists.return_value = False
+
+    # Calling the method under test
+    secrets_helper_instance._create_env_placeholder()
+
+    # Assertions
+    mock_os.path.exists.assert_called_with("/path/to/project/.env.sample")
+    mock_requests.post.assert_called_once_with(
+        url="http://vault:8200/v1/kv/data/project_name/branch_name",
+        headers={"X-Vault-Token": "hvs.randomToken"},
+        data='{"data": {"key": "secret-value"}}',
+    )
+
+
+@patch("server.utils.os")
+@patch("server.utils.requests")
+def test_inject_env_variables_with_secrets_found(
+    mock_requests, mock_os, secrets_helper_instance
+):
+    # Mocking necessary dependencies
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.json.return_value = {"data": {"data": {"key": "secret-value"}}}
+    mock_requests.get.return_value = mock_response
+    mock_open = MagicMock()
+    mock_os.path.join.return_value = "/path/to/project/.env"
+    with patch("builtins.open", mock_open):
+        # Calling the method under test
+        secrets_helper_instance.inject_env_variables("/path/to/project")
+
+        # Assertions
+        mock_requests.get.assert_called_once_with(
+            url="http://vault:8200/v1/kv/data/project_name/branch_name",
+            headers={"X-Vault-Token": "hvs.randomToken"},
+        )
+        mock_open.assert_called_once_with("/path/to/project/.env", "w")
+        # TODO: Add check for what is written
+        # mock_open().write.assert_called_once_with('key="secret-value"\n')
+
+
+@patch("server.utils.os")
+@patch("server.utils.requests")
+def test_inject_env_variables_with_no_secrets(
+    mock_requests, mock_os, secrets_helper_instance
+):
+    # Mocking necessary dependencies
+    mock_response = MagicMock()
+    mock_response.status_code = 404
+    mock_requests.get.return_value = mock_response
+    mock_create_env_placeholder = MagicMock()
+    with patch.object(
+        secrets_helper_instance, "_create_env_placeholder", mock_create_env_placeholder
+    ):
+        # Calling the method under test
+        secrets_helper_instance.inject_env_variables("/path/to/project")
+
+        # Assertions
+        mock_requests.get.assert_called_once_with(
+            url="http://vault:8200/v1/kv/data/project_name/branch_name",
+            headers={"X-Vault-Token": "hvs.randomToken"},
+        )
+        mock_create_env_placeholder.assert_called_once_with()
+
+
+@patch("server.utils.requests.delete", autospec=True)
+def test_cleanup_deployment_variables_success(
+    mock_requests_delete, secrets_helper_instance
+):
+    # Mocking necessary dependencies
+    mock_response = MagicMock()
+    mock_response.status_code = 204
+    mock_requests_delete.return_value = mock_response
+
+    # Calling the method under test
+    result = secrets_helper_instance.cleanup_deployment_variables()
+
+    # Assertions
+    mock_requests_delete.assert_called_once_with(
+        url="http://vault:8200/v1/kv/metadata/project_name/branch_name",
+        headers={"X-Vault-Token": "hvs.randomToken"},
+    )
+    assert result.status_code == 204
+
+
+@patch("server.utils.requests.delete", autospec=True)
+def test_cleanup_deployment_variables_failure(
+    mock_requests_delete, secrets_helper_instance
+):
+    # Mocking necessary dependencies
+    mock_response = MagicMock()
+    mock_response.status_code = 500
+    mock_response.raise_for_status.side_effect = requests.HTTPError(
+        "Internal Server Error"
+    )
+    mock_requests_delete.return_value = mock_response
+
+    # Calling the method under test
+    result = secrets_helper_instance.cleanup_deployment_variables()
+
+    # Assertions
+    mock_requests_delete.assert_called_once_with(
+        url="http://vault:8200/v1/kv/metadata/project_name/branch_name",
+        headers={"X-Vault-Token": "hvs.randomToken"},
+    )
+    assert result.status_code == 500