diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df79866..28326d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,7 @@ repos: hooks: - id: isort name: isort (python) + args: ["--profile", "black"] - repo: https://github.com/psf/black rev: 23.7.0 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..43e7d3b --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ + +.PHONY: local-dev +local-dev: + cp .env.sample .env + echo "\nDEPLOYMENTS_MOUNT_DIR='$(PWD)/deployments' # DO NOT EDIT THIS - FIX FOR RESOURCE SHARING BUG" >> .env + echo "\nNGINX_PROXY_CONF_LOCATION='$(PWD)/nginx-confs' # DO NOT EDIT THIS - FIX FOR RESOURCE SHARING BUG" >> .env + echo "\nDEPLOYMENT_HOST='localhost' # DO NOT EDIT THIS" >> .env + bash setup-vault.sh docker-compose-local.yml + echo "\nVAULT_BASE_URL='http://localhost:8200'" >> .env + mkdir deployments nginx-confs + docker-compose -f docker-compose-local.yml up -d nginx + +.PHONY: sarthi +sarthi: + python app.py + +.PHONY: test +test: + python -m pytest -vvv tests + +.PHONY: reset +reset: + rm -f .env keys.txt parsed-key.txt ansi-keys.txt + docker compose -f docker-compose-local.yml down -v + rm -rf deployments nginx-confs diff --git a/app.py b/app.py index bb386c6..7473d6d 100644 --- a/app.py +++ b/app.py @@ -10,7 +10,7 @@ load_dotenv() -if os.environ.get("ENV").lower() == "local": +if (os.environ.get("ENV") or "local").lower() == "local": logging.basicConfig(level=logging.NOTSET) diff --git a/docker-compose-local.yml b/docker-compose-local.yml new file mode 100644 index 0000000..7305be8 --- /dev/null +++ b/docker-compose-local.yml @@ -0,0 +1,63 @@ +version: "3" + +services: + nginx: + image: nginx:latest + restart: always + container_name: sarthi_nginx + ports: + - "80:80" + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ./nginx-confs:/etc/nginx/conf.d + + sarthi: + build: . + restart: always + volumes: + # hack to bypass file resource sharing error + # not tested and no support for windows server 💩 + - ./deployments:${DEPLOYMENTS_MOUNT_DIR:-/deployments} + - ./nginx-confs:${NGINX_PROXY_CONF_LOCATION:-/nginx-confs} + - /var/run/docker.sock:/var/run/docker.sock + extra_hosts: + - "host.docker.internal:host-gateway" + environment: + DEPLOYMENTS_MOUNT_DIR: ${DEPLOYMENTS_MOUNT_DIR} + NGINX_PROXY_CONF_LOCATION: ${NGINX_PROXY_CONF_LOCATION} + ENV: ${ENV:-local} + DOMAIN_NAME: ${DOMAIN_NAME:-localhost} + VAULT_TOKEN: ${VAULT_TOKEN} + VAULT_BASE_URL: ${VAULT_BASE_URL:-http://vault:8200} + SECRET_TEXT: ${SECRET_TEXT} + depends_on: + - vault + + vault: + image: vault:1.12.3 + restart: always + ports: + - "8200:8200" + volumes: + - ./vault/vault.json:/vault/config/vault.json + - vault-secrets-dev:/vault/file + environment: + VAULT_ADDR: http://0.0.0.0:8200 + VAULT_API_ADDR: http://0.0.0.0:8200 + VAULT_ADDRESS: http://0.0.0.0:8200 + cap_add: + - IPC_LOCK + command: vault server -config=/vault/config/vault.json + healthcheck: + test: + [ + "CMD-SHELL", + "wget --spider http://127.0.0.1:8200/v1/sys/health || exit 1", + ] + interval: 10s + timeout: 5s + retries: 3 + +volumes: + vault-secrets-dev: diff --git a/docker-compose.yml b/docker-compose.yml index 4e71b79..9a41e58 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,11 +22,10 @@ services: build: . restart: always volumes: - # hack to bypass file resource sharing error to make my development easy on mac - # docker mount dir not meant to be used on linux + # hack to bypass file resource sharing error # not tested and no support for windows server 💩 - - ./deployments:${DEPLOYMENTS_MOUNT_DIR} - - ./nginx-confs:${NGINX_PROXY_CONF_LOCATION} + - ./deployments:${DEPLOYMENTS_MOUNT_DIR:-/deployments} + - ./nginx-confs:${NGINX_PROXY_CONF_LOCATION:-/nginx-confs} - /var/run/docker.sock:/var/run/docker.sock extra_hosts: - "host.docker.internal:host-gateway" diff --git a/server/deployer.py b/server/deployer.py index a15ccaf..ebfbfe9 100644 --- a/server/deployer.py +++ b/server/deployer.py @@ -118,3 +118,4 @@ def delete_preview_environment(self): self._nginx_helper.remove_outer_proxy() self._nginx_helper.reload_nginx() self._delete_deployment_files() + self._secrets_helper.cleanup_deployment_variables() diff --git a/server/utils.py b/server/utils.py index 687b8f9..08fad61 100644 --- a/server/utils.py +++ b/server/utils.py @@ -295,12 +295,15 @@ def remove_outer_proxy(self): class SecretsHelper: def __init__(self, project_name, branch_name, project_path): + vault_url = os.environ.get("VAULT_BASE_URL") + vault_token = os.environ.get("VAULT_TOKEN") self._project_path = project_path self._secrets_namespace = f"{project_name}/{branch_name}" - self._secret_url = ( - f"{os.environ.get('VAULT_BASE_URL')}/v1/kv/data/{self._secrets_namespace}" + self._secret_url = f"{vault_url}/v1/kv/data/{self._secrets_namespace}" + self._secret_metadata_url = ( + f"{vault_url}/v1/kv/metadata/{self._secrets_namespace}" ) - self._headers = {"X-Vault-Token": os.environ.get("VAULT_TOKEN")} + self._headers = {"X-Vault-Token": vault_token} def _create_env_placeholder(self): sample_envs = {"key": "secret-value"} @@ -335,6 +338,17 @@ def inject_env_variables(self, project_path): for key, value in secret_data["data"]["data"].items(): file.write(f'{key}="{value}"\n') + def cleanup_deployment_variables(self): + response = requests.delete(url=self._secret_metadata_url, headers=self._headers) + logger.debug( + f"Tried Removing Deployment variables from Vault {response.status_code}" + ) + try: + response.raise_for_status() + except requests.HTTPError as e: + logger.debug(f"Error removing deployment secrets {e}") + return response + def get_random_stub(project_name: str) -> str: return hashlib.md5(project_name.encode()).hexdigest()[:16] diff --git a/setup-vault.sh b/setup-vault.sh index b2b1868..c445e5a 100755 --- a/setup-vault.sh +++ b/setup-vault.sh @@ -45,25 +45,27 @@ else sed 's/\x1B\[[0-9;]*[JKmsu]//g' < ansi-keys.txt > keys.txt fi -sed -n 's/Unseal Key [1-1]\+: \(.*\)/\1/p' keys.txt > parsed-key.txt +awk '/Unseal Key [1-1]:/ {print $NF}' keys.txt > parsed-key.txt key=$(cat parsed-key.txt) docker-compose -f "$COMPOSE_FILE" exec -T "$SERVICE_NAME" vault operator unseal "$key" < /dev/null -sed -n 's/Unseal Key [2-2]\+: \(.*\)/\1/p' keys.txt > parsed-key.txt + + +awk '/Unseal Key [2-2]:/ {print $NF}' keys.txt > parsed-key.txt key=$(cat parsed-key.txt) docker-compose -f "$COMPOSE_FILE" exec -T "$SERVICE_NAME" vault operator unseal "$key" < /dev/null -sed -n 's/Unseal Key [3-3]\+: \(.*\)/\1/p' keys.txt > parsed-key.txt +awk '/Unseal Key [3-3]:/ {print $NF}' keys.txt > parsed-key.txt key=$(cat parsed-key.txt) docker-compose -f "$COMPOSE_FILE" exec -T "$SERVICE_NAME" vault operator unseal "$key" < /dev/null -root_token=$(sed -n 's/Initial Root Token: \(.*\)/\1/p' keys.txt | tr -dc '[:print:]') +root_token=$(awk '/Initial Root Token:/ {print $NF}' keys.txt | tr -dc '[:print:]') if [[ $vault_status == *"Initialized true"* ]]; then echo "Vault is initialized already. Skipping creating a KV engine" else echo -e "\nVAULT_TOKEN=random_token" >> .env - sed -i "s/VAULT_TOKEN=.*/VAULT_TOKEN=$root_token/" ".env" + awk -v root_token="$root_token" '/^VAULT_TOKEN=/ {gsub(/=.*/, "=" root_token)} 1' ".env" > temp && mv temp ".env" docker-compose -f "$COMPOSE_FILE" exec -e VAULT_TOKEN=$root_token -T "$SERVICE_NAME" vault secrets enable -path=kv kv-v2 fi diff --git a/tests/conftest.py b/tests/conftest.py index 09be9c9..1609533 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ import pytest -from server.utils import ComposeHelper, DeploymentConfig, NginxHelper +from server.utils import ComposeHelper, DeploymentConfig, NginxHelper, SecretsHelper @pytest.fixture @@ -86,3 +86,13 @@ def nginx_helper(deployment_config): outer_conf_base_path = "/path/to/outer/conf" deployment_project_path = "/path/to/deployment/project" return NginxHelper(deployment_config, outer_conf_base_path, deployment_project_path) + + +@pytest.fixture +def secrets_helper_instance(mocker): + mock_os = mocker.patch("server.utils.os") + mock_os.environ = { + "VAULT_TOKEN": "hvs.randomToken", + "VAULT_BASE_URL": "http://vault:8200", + } + return SecretsHelper("project_name", "branch_name", "/path/to/project") diff --git a/tests/test_utils.py b/tests/test_utils.py index 4eac20f..1911027 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,8 @@ import pathlib +from unittest.mock import MagicMock, patch import pytest +import requests from server.utils import ComposeHelper @@ -244,3 +246,142 @@ def test_deployment_config_repr(deployment_config): "'https://github.com/tushar5526/test-project-name.git', 'docker-compose.yml', 'POST')" ) assert repr(deployment_config) == expected_repr + + +@patch("server.utils.os") +@patch("server.utils.requests") +def test_create_env_placeholder_with_sample_env_file( + mock_requests, mock_os, secrets_helper_instance +): + # Mocking necessary dependencies + mock_os.path.exists.return_value = True + mock_dotenv_values = MagicMock(return_value={"key": "secret-value"}) + with patch("server.utils.dotenv_values", mock_dotenv_values): + mock_response = MagicMock() + mock_response.raise_for_status.return_value = None + mock_requests.post.return_value = mock_response + + # Calling the method under test + secrets_helper_instance._create_env_placeholder() + + # Assertions + mock_dotenv_values.assert_called() + mock_requests.post.assert_called_once_with( + url="http://vault:8200/v1/kv/data/project_name/branch_name", + headers={"X-Vault-Token": "hvs.randomToken"}, + data='{"data": {"key": "secret-value"}}', + ) + + +@patch("server.utils.os") +@patch("server.utils.requests") +def test_create_env_placeholder_with_sample_env_file_missing( + mock_requests, mock_os, secrets_helper_instance +): + # Mocking necessary dependencies + mock_os.path.join.return_value = "/path/to/project/.env.sample" + mock_os.path.exists.return_value = False + + # Calling the method under test + secrets_helper_instance._create_env_placeholder() + + # Assertions + mock_os.path.exists.assert_called_with("/path/to/project/.env.sample") + mock_requests.post.assert_called_once_with( + url="http://vault:8200/v1/kv/data/project_name/branch_name", + headers={"X-Vault-Token": "hvs.randomToken"}, + data='{"data": {"key": "secret-value"}}', + ) + + +@patch("server.utils.os") +@patch("server.utils.requests") +def test_inject_env_variables_with_secrets_found( + mock_requests, mock_os, secrets_helper_instance +): + # Mocking necessary dependencies + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"data": {"data": {"key": "secret-value"}}} + mock_requests.get.return_value = mock_response + mock_open = MagicMock() + mock_os.path.join.return_value = "/path/to/project/.env" + with patch("builtins.open", mock_open): + # Calling the method under test + secrets_helper_instance.inject_env_variables("/path/to/project") + + # Assertions + mock_requests.get.assert_called_once_with( + url="http://vault:8200/v1/kv/data/project_name/branch_name", + headers={"X-Vault-Token": "hvs.randomToken"}, + ) + mock_open.assert_called_once_with("/path/to/project/.env", "w") + # TODO: Add check for what is written + # mock_open().write.assert_called_once_with('key="secret-value"\n') + + +@patch("server.utils.os") +@patch("server.utils.requests") +def test_inject_env_variables_with_no_secrets( + mock_requests, mock_os, secrets_helper_instance +): + # Mocking necessary dependencies + mock_response = MagicMock() + mock_response.status_code = 404 + mock_requests.get.return_value = mock_response + mock_create_env_placeholder = MagicMock() + with patch.object( + secrets_helper_instance, "_create_env_placeholder", mock_create_env_placeholder + ): + # Calling the method under test + secrets_helper_instance.inject_env_variables("/path/to/project") + + # Assertions + mock_requests.get.assert_called_once_with( + url="http://vault:8200/v1/kv/data/project_name/branch_name", + headers={"X-Vault-Token": "hvs.randomToken"}, + ) + mock_create_env_placeholder.assert_called_once_with() + + +@patch("server.utils.requests.delete", autospec=True) +def test_cleanup_deployment_variables_success( + mock_requests_delete, secrets_helper_instance +): + # Mocking necessary dependencies + mock_response = MagicMock() + mock_response.status_code = 204 + mock_requests_delete.return_value = mock_response + + # Calling the method under test + result = secrets_helper_instance.cleanup_deployment_variables() + + # Assertions + mock_requests_delete.assert_called_once_with( + url="http://vault:8200/v1/kv/metadata/project_name/branch_name", + headers={"X-Vault-Token": "hvs.randomToken"}, + ) + assert result.status_code == 204 + + +@patch("server.utils.requests.delete", autospec=True) +def test_cleanup_deployment_variables_failure( + mock_requests_delete, secrets_helper_instance +): + # Mocking necessary dependencies + mock_response = MagicMock() + mock_response.status_code = 500 + mock_response.raise_for_status.side_effect = requests.HTTPError( + "Internal Server Error" + ) + mock_requests_delete.return_value = mock_response + + # Calling the method under test + result = secrets_helper_instance.cleanup_deployment_variables() + + # Assertions + mock_requests_delete.assert_called_once_with( + url="http://vault:8200/v1/kv/metadata/project_name/branch_name", + headers={"X-Vault-Token": "hvs.randomToken"}, + ) + assert result.status_code == 500