From f96a6a0335bffa92a959a9b33c21e22d92cc26d8 Mon Sep 17 00:00:00 2001 From: gauravjisri Date: Sun, 26 May 2024 00:46:37 +0530 Subject: [PATCH 1/7] Added Docker Compose modification script - Added a Python script named modify_compose.py to automate Docker Compose file modifications. - The script utilizes the PyYAML library to parse and modify YAML data. - Demonstrated how to use the script to modify a Docker Compose file, specifying a simple port mapping change. --- docker-compose.yml | 9 +- modify_compose.py | 22 ++++ processed-docker-compose.yml | 195 +++++++++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 5 deletions(-) create mode 100644 modify_compose.py create mode 100644 processed-docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml index de0f0b5..9aa4879 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,10 +4,9 @@ services: nginx: image: nginx:1.26.0 restart: always - # container name is used internally in code - don't change it container_name: sarthi_nginx ports: - - "80:80" + - "${PORT_NGINX:-80}:80" # Modified: Port number made variable extra_hosts: - "host.docker.internal:host-gateway" volumes: @@ -23,8 +22,6 @@ services: build: . restart: always volumes: - # hack to bypass file resource sharing error - # not tested and no support for windows server 💩 - ./deployments:${DEPLOYMENTS_MOUNT_DIR:-/deployments} - ./nginx-confs:${NGINX_PROXY_CONF_LOCATION:-/nginx-confs} - /var/run/docker.sock:/var/run/docker.sock @@ -45,7 +42,7 @@ services: image: grafana/loki:2.9.5 restart: always ports: - - 127.0.0.1:3100:3100 + - "${PORT_LOKI:-3100}:3100" # Modified: Port number made variable volumes: - ./logging-config/loki:/etc/loki command: -config.file=/etc/loki/loki-config.yaml @@ -63,6 +60,8 @@ services: grafana: image: grafana/grafana:10.1.10-ubuntu restart: always + ports: + - "${PORT_GRAFANA:-3000}:3000" # Modified: Port number made variable volumes: - grafana:/var/lib/grafana - ./logging-config/grafana/dashboards:/etc/grafana/provisioning/dashboards diff --git a/modify_compose.py b/modify_compose.py new file mode 100644 index 0000000..f885a78 --- /dev/null +++ b/modify_compose.py @@ -0,0 +1,22 @@ +import yaml + +def modify_compose(compose_file_path): + # Load the Docker Compose file + with open(compose_file_path, 'r') as file: + compose_data = yaml.safe_load(file) + + # Perform modifications here + # Example : + # compose_data['services']['web']['ports'] = ["8000:80"] + # For example, you can add or update services, adjust configurations, etc. + + # Save the modified Docker Compose file + modified_compose_path = compose_file_path.replace('.yml', '-modified.yml') + with open(modified_compose_path, 'w') as file: + yaml.dump(compose_data, file) + + print(f"Modified Docker Compose file saved to: {modified_compose_path}") + +if __name__ == "__main__": + compose_file_path = "docker-compose.yml" # Specify the path to your Docker Compose file + modify_compose(compose_file_path) diff --git a/processed-docker-compose.yml b/processed-docker-compose.yml new file mode 100644 index 0000000..c2bb260 --- /dev/null +++ b/processed-docker-compose.yml @@ -0,0 +1,195 @@ +name: sarthi +services: + grafana: + depends_on: + loki: + condition: service_started + image: grafana/grafana:10.1.10-ubuntu + networks: + default: null + ports: + - mode: ingress + target: 3000 + published: "3000" + protocol: tcp + restart: always + volumes: + - type: volume + source: grafana + target: /var/lib/grafana + volume: {} + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/grafana/dashboards + target: /etc/grafana/provisioning/dashboards + bind: + create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/grafana/datasources + target: /etc/grafana/provisioning/datasources + bind: + create_host_path: true + loki: + command: + - -config.file=/etc/loki/loki-config.yaml + depends_on: + promtail: + condition: service_started + image: grafana/loki:2.9.5 + networks: + default: null + ports: + - mode: ingress + target: 3100 + published: "3100" + protocol: tcp + restart: always + volumes: + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/loki + target: /etc/loki + bind: + create_host_path: true + nginx: + container_name: sarthi_nginx + depends_on: + grafana: + condition: service_started + portainer: + condition: service_started + sarthi: + condition: service_started + vault: + condition: service_started + extra_hosts: + host.docker.internal: host-gateway + image: nginx:1.26.0 + networks: + default: null + ports: + - mode: ingress + target: 80 + published: "80" + protocol: tcp + restart: always + volumes: + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/nginx-confs + target: /etc/nginx/conf.d + bind: + create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/sarthi.conf + target: /etc/nginx/conf.d/sarthi.conf + bind: + create_host_path: true + portainer: + image: portainer/portainer-ce:2.20.2 + networks: + default: null + restart: unless-stopped + volumes: + - type: volume + source: portainer_data + target: /data + volume: {} + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + bind: + create_host_path: true + promtail: + command: + - -config.file=/etc/promtail/promtail-config.yaml + image: grafana/promtail:main-8978ecf + networks: + default: null + restart: always + volumes: + - type: bind + source: /var/log + target: /var/log + bind: + create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/promtail + target: /etc/promtail + bind: + create_host_path: true + sarthi: + build: + context: /Users/gauravjisrivastava/Desktop/sarthi + dockerfile: Dockerfile + depends_on: + vault: + condition: service_started + environment: + DEPLOYMENTS_MOUNT_DIR: your_value + DOMAIN_NAME: localhost + ENV: local + NGINX_PROXY_CONF_LOCATION: your_value + SECRET_TEXT: your_value + VAULT_BASE_URL: http://vault:8200 + VAULT_TOKEN: your_value + extra_hosts: + host.docker.internal: host-gateway + networks: + default: null + restart: always + volumes: + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/deployments + target: your_value + bind: + create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/nginx-confs + target: your_value + bind: + create_host_path: true + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + bind: + create_host_path: true + vault: + cap_add: + - IPC_LOCK + command: + - vault + - server + - -config=/vault/config/vault.json + environment: + VAULT_ADDR: http://0.0.0.0:8200 + VAULT_ADDRESS: http://0.0.0.0:8200 + VAULT_API_ADDR: http://0.0.0.0:8200 + healthcheck: + test: + - CMD-SHELL + - wget --spider http://127.0.0.1:8200/v1/sys/health || exit 1 + timeout: 5s + interval: 10s + retries: 3 + image: hashicorp/vault:1.16 + networks: + default: null + restart: always + volumes: + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/vault/vault.json + target: /vault/config/vault.json + bind: + create_host_path: true + - type: volume + source: vault-secrets + target: /vault/file + volume: {} +networks: + default: + name: sarthi_default +volumes: + grafana: + name: sarthi_grafana + portainer_data: + name: sarthi_portainer_data + vault-secrets: + name: sarthi_vault-secrets From 283171177eeb2d9a3919c0b0bd529a484e828676 Mon Sep 17 00:00:00 2001 From: Gaurav Ji Srivastava <103504986+GJS2162@users.noreply.github.com> Date: Sun, 26 May 2024 00:50:05 +0530 Subject: [PATCH 2/7] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 269c024..bbc18e7 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,9 @@ The following services are exposed: - Check the vault under the `project/feature-branch` namespace and find secrets there. - There is a default namespace reserved for developers to specify default secrets for all the PS. Secrets defined under `project/default-dev-secrets` are used if `project/feature-branch` secret path is empty. - If the default namespace is not configured as well, Sarthi automatically tries to find `sample.env`, `env.sample`, `.env.sample` and similar sample env files in the root directory and loads those sample environment variables to both `default-dev-secrets` and `project/feature-branch` + +### Modifying docker-compose.yml +1. There is a python script for modifying the docker compose file. ### Tips 💡 From 8d8a1f55e674b16b2ef8a4da3fd5b699c1873509 Mon Sep 17 00:00:00 2001 From: gauravjisri Date: Thu, 30 May 2024 16:38:19 +0530 Subject: [PATCH 3/7] Fix formatting issues in .pre-commit-config.yaml with prettier --- .pre-commit-config.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 28326d9..c7a41df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,3 +32,13 @@ repos: hooks: - id: prettier exclude: ^(frontend/|CHANGELOG.md) + + # Add the local repo for the custom hook + - repo: local + hooks: + - id: modify_compose # Unique identifier for the hook + name: Modify Docker Compose file # Descriptive name for the hook + entry: python3 modify_compose.py # Command to run your Python script + files: docker-compose.yml # File(s) to which the hook applies + language: python # Add this line to specify the language + additional_dependencies: [pyyaml] From b053024e9fa18fff9799114a13368f2f4696a83b Mon Sep 17 00:00:00 2001 From: gauravjisri Date: Thu, 30 May 2024 17:12:23 +0530 Subject: [PATCH 4/7] Fix formatting issues --- README.md | 3 ++- docker-compose.yml | 6 +++--- modify_compose.py | 12 ++++++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index bbc18e7..012ad59 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,9 @@ The following services are exposed: - Check the vault under the `project/feature-branch` namespace and find secrets there. - There is a default namespace reserved for developers to specify default secrets for all the PS. Secrets defined under `project/default-dev-secrets` are used if `project/feature-branch` secret path is empty. - If the default namespace is not configured as well, Sarthi automatically tries to find `sample.env`, `env.sample`, `.env.sample` and similar sample env files in the root directory and loads those sample environment variables to both `default-dev-secrets` and `project/feature-branch` - + ### Modifying docker-compose.yml + 1. There is a python script for modifying the docker compose file. ### Tips 💡 diff --git a/docker-compose.yml b/docker-compose.yml index 9aa4879..e8a35be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: restart: always container_name: sarthi_nginx ports: - - "${PORT_NGINX:-80}:80" # Modified: Port number made variable + - "${PORT_NGINX:-80}:80" # Modified: Port number made variable extra_hosts: - "host.docker.internal:host-gateway" volumes: @@ -42,7 +42,7 @@ services: image: grafana/loki:2.9.5 restart: always ports: - - "${PORT_LOKI:-3100}:3100" # Modified: Port number made variable + - "${PORT_LOKI:-3100}:3100" # Modified: Port number made variable volumes: - ./logging-config/loki:/etc/loki command: -config.file=/etc/loki/loki-config.yaml @@ -61,7 +61,7 @@ services: image: grafana/grafana:10.1.10-ubuntu restart: always ports: - - "${PORT_GRAFANA:-3000}:3000" # Modified: Port number made variable + - "${PORT_GRAFANA:-3000}:3000" # Modified: Port number made variable volumes: - grafana:/var/lib/grafana - ./logging-config/grafana/dashboards:/etc/grafana/provisioning/dashboards diff --git a/modify_compose.py b/modify_compose.py index f885a78..90d7e12 100644 --- a/modify_compose.py +++ b/modify_compose.py @@ -1,8 +1,9 @@ import yaml + def modify_compose(compose_file_path): # Load the Docker Compose file - with open(compose_file_path, 'r') as file: + with open(compose_file_path, "r") as file: compose_data = yaml.safe_load(file) # Perform modifications here @@ -11,12 +12,15 @@ def modify_compose(compose_file_path): # For example, you can add or update services, adjust configurations, etc. # Save the modified Docker Compose file - modified_compose_path = compose_file_path.replace('.yml', '-modified.yml') - with open(modified_compose_path, 'w') as file: + modified_compose_path = compose_file_path.replace(".yml", "-modified.yml") + with open(modified_compose_path, "w") as file: yaml.dump(compose_data, file) print(f"Modified Docker Compose file saved to: {modified_compose_path}") + if __name__ == "__main__": - compose_file_path = "docker-compose.yml" # Specify the path to your Docker Compose file + compose_file_path = ( + "docker-compose.yml" # Specify the path to your Docker Compose file + ) modify_compose(compose_file_path) From b98adfc5fb51eabe6027e48a200f155550ab2deb Mon Sep 17 00:00:00 2001 From: gauravjisri Date: Thu, 30 May 2024 17:29:52 +0530 Subject: [PATCH 5/7] fix processed-docker-compose.yml --- processed-docker-compose.yml | 184 +++++++++++++++++------------------ 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/processed-docker-compose.yml b/processed-docker-compose.yml index c2bb260..7b34f01 100644 --- a/processed-docker-compose.yml +++ b/processed-docker-compose.yml @@ -8,29 +8,29 @@ services: networks: default: null ports: - - mode: ingress - target: 3000 - published: "3000" - protocol: tcp + - mode: ingress + target: 3000 + published: "3000" + protocol: tcp restart: always volumes: - - type: volume - source: grafana - target: /var/lib/grafana - volume: {} - - type: bind - source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/grafana/dashboards - target: /etc/grafana/provisioning/dashboards - bind: - create_host_path: true - - type: bind - source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/grafana/datasources - target: /etc/grafana/provisioning/datasources - bind: - create_host_path: true + - type: volume + source: grafana + target: /var/lib/grafana + volume: {} + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/grafana/dashboards + target: /etc/grafana/provisioning/dashboards + bind: + create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/grafana/datasources + target: /etc/grafana/provisioning/datasources + bind: + create_host_path: true loki: command: - - -config.file=/etc/loki/loki-config.yaml + - -config.file=/etc/loki/loki-config.yaml depends_on: promtail: condition: service_started @@ -38,17 +38,17 @@ services: networks: default: null ports: - - mode: ingress - target: 3100 - published: "3100" - protocol: tcp + - mode: ingress + target: 3100 + published: "3100" + protocol: tcp restart: always volumes: - - type: bind - source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/loki - target: /etc/loki - bind: - create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/loki + target: /etc/loki + bind: + create_host_path: true nginx: container_name: sarthi_nginx depends_on: @@ -66,55 +66,55 @@ services: networks: default: null ports: - - mode: ingress - target: 80 - published: "80" - protocol: tcp + - mode: ingress + target: 80 + published: "80" + protocol: tcp restart: always volumes: - - type: bind - source: /Users/gauravjisrivastava/Desktop/sarthi/nginx-confs - target: /etc/nginx/conf.d - bind: - create_host_path: true - - type: bind - source: /Users/gauravjisrivastava/Desktop/sarthi/sarthi.conf - target: /etc/nginx/conf.d/sarthi.conf - bind: - create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/nginx-confs + target: /etc/nginx/conf.d + bind: + create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/sarthi.conf + target: /etc/nginx/conf.d/sarthi.conf + bind: + create_host_path: true portainer: image: portainer/portainer-ce:2.20.2 networks: default: null restart: unless-stopped volumes: - - type: volume - source: portainer_data - target: /data - volume: {} - - type: bind - source: /var/run/docker.sock - target: /var/run/docker.sock - bind: - create_host_path: true + - type: volume + source: portainer_data + target: /data + volume: {} + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + bind: + create_host_path: true promtail: command: - - -config.file=/etc/promtail/promtail-config.yaml + - -config.file=/etc/promtail/promtail-config.yaml image: grafana/promtail:main-8978ecf networks: default: null restart: always volumes: - - type: bind - source: /var/log - target: /var/log - bind: - create_host_path: true - - type: bind - source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/promtail - target: /etc/promtail - bind: - create_host_path: true + - type: bind + source: /var/log + target: /var/log + bind: + create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/logging-config/promtail + target: /etc/promtail + bind: + create_host_path: true sarthi: build: context: /Users/gauravjisrivastava/Desktop/sarthi @@ -136,36 +136,36 @@ services: default: null restart: always volumes: - - type: bind - source: /Users/gauravjisrivastava/Desktop/sarthi/deployments - target: your_value - bind: - create_host_path: true - - type: bind - source: /Users/gauravjisrivastava/Desktop/sarthi/nginx-confs - target: your_value - bind: - create_host_path: true - - type: bind - source: /var/run/docker.sock - target: /var/run/docker.sock - bind: - create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/deployments + target: your_value + bind: + create_host_path: true + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/nginx-confs + target: your_value + bind: + create_host_path: true + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + bind: + create_host_path: true vault: cap_add: - - IPC_LOCK + - IPC_LOCK command: - - vault - - server - - -config=/vault/config/vault.json + - vault + - server + - -config=/vault/config/vault.json environment: VAULT_ADDR: http://0.0.0.0:8200 VAULT_ADDRESS: http://0.0.0.0:8200 VAULT_API_ADDR: http://0.0.0.0:8200 healthcheck: test: - - CMD-SHELL - - wget --spider http://127.0.0.1:8200/v1/sys/health || exit 1 + - CMD-SHELL + - wget --spider http://127.0.0.1:8200/v1/sys/health || exit 1 timeout: 5s interval: 10s retries: 3 @@ -174,15 +174,15 @@ services: default: null restart: always volumes: - - type: bind - source: /Users/gauravjisrivastava/Desktop/sarthi/vault/vault.json - target: /vault/config/vault.json - bind: - create_host_path: true - - type: volume - source: vault-secrets - target: /vault/file - volume: {} + - type: bind + source: /Users/gauravjisrivastava/Desktop/sarthi/vault/vault.json + target: /vault/config/vault.json + bind: + create_host_path: true + - type: volume + source: vault-secrets + target: /vault/file + volume: {} networks: default: name: sarthi_default From be35c7c2b4f4faf4fd815395e632979783276221 Mon Sep 17 00:00:00 2001 From: gauravjisri Date: Thu, 30 May 2024 18:05:30 +0530 Subject: [PATCH 6/7] fixed --- docker-compose-modified.yml | 95 +++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 docker-compose-modified.yml diff --git a/docker-compose-modified.yml b/docker-compose-modified.yml new file mode 100644 index 0000000..b572f9b --- /dev/null +++ b/docker-compose-modified.yml @@ -0,0 +1,95 @@ +services: + grafana: + depends_on: + - loki + image: grafana/grafana:10.1.10-ubuntu + ports: + - ${PORT_GRAFANA:-3000}:3000 + restart: always + volumes: + - grafana:/var/lib/grafana + - ./logging-config/grafana/dashboards:/etc/grafana/provisioning/dashboards + - ./logging-config/grafana/datasources:/etc/grafana/provisioning/datasources + loki: + command: -config.file=/etc/loki/loki-config.yaml + depends_on: + - promtail + image: grafana/loki:2.9.5 + ports: + - ${PORT_LOKI:-3100}:3100 + restart: always + volumes: + - ./logging-config/loki:/etc/loki + nginx: + container_name: sarthi_nginx + depends_on: + - portainer + - vault + - sarthi + - grafana + extra_hosts: + - host.docker.internal:host-gateway + image: nginx:1.26.0 + ports: + - ${PORT_NGINX:-80}:80 + restart: always + volumes: + - ./nginx-confs:/etc/nginx/conf.d + - ./sarthi.conf:/etc/nginx/conf.d/sarthi.conf + portainer: + image: portainer/portainer-ce:2.20.2 + restart: unless-stopped + volumes: + - portainer_data:/data + - /var/run/docker.sock:/var/run/docker.sock + promtail: + command: -config.file=/etc/promtail/promtail-config.yaml + image: grafana/promtail:main-8978ecf + restart: always + volumes: + - /var/log:/var/log + - ./logging-config/promtail:/etc/promtail + sarthi: + build: . + depends_on: + - vault + environment: + DEPLOYMENTS_MOUNT_DIR: ${DEPLOYMENTS_MOUNT_DIR} + DOMAIN_NAME: ${DOMAIN_NAME:-localhost} + ENV: ${ENV:-local} + NGINX_PROXY_CONF_LOCATION: ${NGINX_PROXY_CONF_LOCATION} + SECRET_TEXT: ${SECRET_TEXT} + VAULT_BASE_URL: ${VAULT_BASE_URL:-http://vault:8200} + VAULT_TOKEN: ${VAULT_TOKEN} + extra_hosts: + - host.docker.internal:host-gateway + restart: always + volumes: + - ./deployments:${DEPLOYMENTS_MOUNT_DIR:-/deployments} + - ./nginx-confs:${NGINX_PROXY_CONF_LOCATION:-/nginx-confs} + - /var/run/docker.sock:/var/run/docker.sock + vault: + cap_add: + - IPC_LOCK + command: vault server -config=/vault/config/vault.json + environment: + VAULT_ADDR: http://0.0.0.0:8200 + VAULT_ADDRESS: http://0.0.0.0:8200 + VAULT_API_ADDR: http://0.0.0.0:8200 + healthcheck: + interval: 10s + retries: 3 + test: + - CMD-SHELL + - wget --spider http://127.0.0.1:8200/v1/sys/health || exit 1 + timeout: 5s + image: hashicorp/vault:1.16 + restart: always + volumes: + - ./vault/vault.json:/vault/config/vault.json + - vault-secrets:/vault/file +version: "3" +volumes: + grafana: null + portainer_data: null + vault-secrets: null From 11ac865f7a93504b145271460a07ac1f0e820f9c Mon Sep 17 00:00:00 2001 From: gauravjisri Date: Thu, 30 May 2024 18:12:34 +0530 Subject: [PATCH 7/7] fixed by removing modify_compose --- .pre-commit-config.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7a41df..28326d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,13 +32,3 @@ repos: hooks: - id: prettier exclude: ^(frontend/|CHANGELOG.md) - - # Add the local repo for the custom hook - - repo: local - hooks: - - id: modify_compose # Unique identifier for the hook - name: Modify Docker Compose file # Descriptive name for the hook - entry: python3 modify_compose.py # Command to run your Python script - files: docker-compose.yml # File(s) to which the hook applies - language: python # Add this line to specify the language - additional_dependencies: [pyyaml]