From 34236ab7d348b9f4c83c979d0aab1f42c6da7c7f Mon Sep 17 00:00:00 2001 From: Kozin Date: Thu, 19 Oct 2023 13:33:56 +0300 Subject: [PATCH 1/6] Refactored the project building and deploying procedure. Added frontend --- .github/workflows/main.yml | 42 +++++++++++++------------- .gitignore | 3 ++ infra_bt/docker-compose.production.yml | 21 ++++++------- infra_bt/docker-compose.yml | 37 +++++++++-------------- infra_bt/nginx.conf | 18 ++++------- 5 files changed, 54 insertions(+), 67 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e8899b8..785b57e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,26 +37,25 @@ jobs: push: true tags: ${{ secrets.DOCKER_USERNAME }}/volunteers_backend:latest -# build_frontend_and_push_to_docker_hub: -# name: Push frontend Docker image to DockerHub -# runs-on: ubuntu-latest -# steps: -# - name: Check out the repo -# uses: actions/checkout@v3 -# - name: Set up Docker Buildx -# uses: docker/setup-buildx-action@v2 -# - name: Login to Docker -# uses: docker/login-action@v2 -# with: -# username: ${{ secrets.DOCKER_USERNAME }} -# password: ${{ secrets.DOCKER_PASSWORD }} -# - name: Push to DockerHub -# uses: docker/build-push-action@v4 -# with: -# context: git@github.com:volunteers-for-city-projects/volunteers-frontend.git -# file: /infra_bt/Dockerfile -# push: true -# tags: ${{ secrets.DOCKER_USERNAME }}/volunteers_frontend:latest + build_frontend_and_push_to_docker_hub: + name: Push frontend Docker image to DockerHub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Push to DockerHub + uses: docker/build-push-action@v4 + with: + context: git@github.com:volunteers-for-city-projects/volunteers-frontend.git#deployDocker + push: true + tags: ${{ secrets.DOCKER_USERNAME }}/volunteers_frontend:latest deploy: runs-on: ubuntu-latest @@ -77,7 +76,8 @@ jobs: key: ${{ secrets.SSH_KEY }} passphrase: ${{ secrets.SSH_PASSPHRASE }} source: "infra_bt/docker-compose.production.yml" - target: "volunteers-for-city-projects" + target: "better-together/" + strip_components: 1 - name: Executing remote ssh commands to deploy uses: appleboy/ssh-action@master with: diff --git a/.gitignore b/.gitignore index aa219ec..189e472 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,6 @@ cython_debug/ # static **/collected_static/ media/ + +# pre-commit +.pre-commit-config.yaml diff --git a/infra_bt/docker-compose.production.yml b/infra_bt/docker-compose.production.yml index 5dfe601..97e6a7c 100644 --- a/infra_bt/docker-compose.production.yml +++ b/infra_bt/docker-compose.production.yml @@ -2,8 +2,8 @@ version: '3' volumes: pg_data: - static: - media: + static_data: + media_data: services: @@ -27,17 +27,16 @@ services: image: 1yunker/volunteers_backend env_file: .env volumes: - - static:/backend_static - - media:/media + - static_data:/backend_static depends_on: db: condition: service_healthy - # frontend: - # image: 1yunker/volunteers_frontend - # command: cp -r /app/result_build/. /static/ - # volumes: - # - static:/static + frontend: + image: 1yunker/volunteers_frontend + command: cp -r /app/build/. /static_files/ + volumes: + - static_data:/static_files gateway: image: nginx:1.25.2-alpine3.18-slim @@ -45,9 +44,7 @@ services: - 8000:80 volumes: - ./nginx.conf:/etc/nginx/conf.d/default.conf - # - ../docs/:/usr/share/nginx/html/api/docs/ - - static:/static - - media:/media + - static_data:/usr/share/nginx/html depends_on: backend: condition: service_started diff --git a/infra_bt/docker-compose.yml b/infra_bt/docker-compose.yml index 9796e0d..9e9e40f 100644 --- a/infra_bt/docker-compose.yml +++ b/infra_bt/docker-compose.yml @@ -2,8 +2,8 @@ version: '3.3' volumes: pg_data: - static: - media: + static_data: + media_data: name: volunteers @@ -32,14 +32,13 @@ services: build: ../backend/ env_file: .env volumes: - - static:/backend_static - - media:/media + - static_data:/backend_static command: - /bin/sh - -c - | python manage.py collectstatic -c --noinput - cp -r /app/collected_static/. /backend_static/static/ + cp -r /app/collected_static/. /backend_static/static python manage.py migrate gunicorn --bind 0.0.0.0:8000 backend.wsgi restart: unless-stopped @@ -47,19 +46,16 @@ services: db: condition: service_healthy - # frontend: - # build: - # context: git@github.com:volunteers-for-city-projects/volunteers-frontend.git - # dockerfile: Dockerfile - # command: - # - /bin/sh - # - -c - # - | - # npm run build result_build - # cp -r /app/result_build/ static_files/ - # volumes: - # - ../frontend:/app/ - # - static:/static_files + frontend: + build: + context: https://github.com/volunteers-for-city-projects/volunteers-frontend.git#feature/deployDocker + command: + - /bin/sh + - -c + - | + cp -r /app/. /static_files/ + volumes: + - static_data:/static_files nginx: image: nginx:1.25.2-alpine3.18-slim @@ -67,10 +63,7 @@ services: - "8000:80" volumes: - ./nginx.conf:/etc/nginx/conf.d/default.conf - # - ../frontend/build:/usr/share/nginx/html/ - # - ../docs/:/usr/share/nginx/html/api/docs/ - - static:/static - - media:/media + - static_data:/usr/share/nginx/html depends_on: backend: condition: service_started diff --git a/infra_bt/nginx.conf b/infra_bt/nginx.conf index d3d9065..326e8a2 100644 --- a/infra_bt/nginx.conf +++ b/infra_bt/nginx.conf @@ -1,6 +1,8 @@ server { listen 80; + root /usr/share/nginx/html; + location /api/ { proxy_set_header Host $http_host; proxy_pass http://backend:8000/api/; @@ -9,24 +11,16 @@ server { proxy_set_header Host $http_host; proxy_pass http://backend:8000/swagger/; } + location /redoc/ { + proxy_set_header Host $http_host; + proxy_pass http://backend:8000/redoc/; + } location /admin/ { proxy_set_header Host $http_host; proxy_pass http://backend:8000/admin/; } - location /static/ { - root /static; - } - location /media/ { - root /; - } - - # location / { - # proxy_set_header Host $http_host; - # proxy_pass http://backend:8000/; - # } location / { - root /usr/share/nginx/html; index index.html index.htm; try_files $uri /index.html; proxy_set_header Host $host; From 203f755eaaf6424fdbb5c07c02e381055bf7edad Mon Sep 17 00:00:00 2001 From: Kozin Date: Fri, 20 Oct 2023 01:55:06 +0300 Subject: [PATCH 2/6] Fixed missing isort package --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb83e7b..c1c3e6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip --no-cache-dir - pip install flake8 ruff --no-cache-dir + pip install flake8 ruff isort --no-cache-dir if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Git Clone Action uses: actions/checkout@v2 From 71357e3679b79cb2ff81ffc748cbf2c7e37c6754 Mon Sep 17 00:00:00 2001 From: Kozin Date: Fri, 20 Oct 2023 01:56:52 +0300 Subject: [PATCH 3/6] Added settings for media_data volume --- backend/backend/settings.py | 5 +++-- infra_bt/docker-compose.production.yml | 2 ++ infra_bt/docker-compose.yml | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 50e8779..3bc3ccf 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -1,7 +1,7 @@ import os +from pathlib import Path from dotenv import load_dotenv -from pathlib import Path # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -46,6 +46,7 @@ 'projects.apps.ProjectsConfig', 'users.apps.UsersConfig', 'corsheaders', + 'check.apps.CheckConfig', ] MIDDLEWARE = [ @@ -146,7 +147,7 @@ STATIC_ROOT = Path(BASE_DIR, 'collected_static') MEDIA_URL = 'media/' -MEDIA_ROOT = '/media' +MEDIA_ROOT = Path(BASE_DIR, 'media') # Default primary key field type # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field diff --git a/infra_bt/docker-compose.production.yml b/infra_bt/docker-compose.production.yml index 97e6a7c..f02a6ff 100644 --- a/infra_bt/docker-compose.production.yml +++ b/infra_bt/docker-compose.production.yml @@ -28,6 +28,7 @@ services: env_file: .env volumes: - static_data:/backend_static + - media_data:/app/media depends_on: db: condition: service_healthy @@ -45,6 +46,7 @@ services: volumes: - ./nginx.conf:/etc/nginx/conf.d/default.conf - static_data:/usr/share/nginx/html + - media_data:/usr/share/nginx/html/media depends_on: backend: condition: service_started diff --git a/infra_bt/docker-compose.yml b/infra_bt/docker-compose.yml index 9e9e40f..c01310f 100644 --- a/infra_bt/docker-compose.yml +++ b/infra_bt/docker-compose.yml @@ -33,6 +33,7 @@ services: env_file: .env volumes: - static_data:/backend_static + - media_data:/app/media command: - /bin/sh - -c @@ -64,6 +65,7 @@ services: volumes: - ./nginx.conf:/etc/nginx/conf.d/default.conf - static_data:/usr/share/nginx/html + - media_data:/usr/share/nginx/html/media/ depends_on: backend: condition: service_started From d7a62de41d5488937a2c1deb17807c759c7dc07f Mon Sep 17 00:00:00 2001 From: Kozin Date: Fri, 20 Oct 2023 02:41:32 +0300 Subject: [PATCH 4/6] Changed the nginx container deployment method --- .github/workflows/main.yml | 22 ++++++++++++++++++++++ infra_bt/.dockerignore | 5 +++++ infra_bt/Dockerfile | 3 +++ infra_bt/docker-compose.production.yml | 3 +-- 4 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 infra_bt/.dockerignore create mode 100644 infra_bt/Dockerfile diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 785b57e..a4f8294 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,6 +57,28 @@ jobs: push: true tags: ${{ secrets.DOCKER_USERNAME }}/volunteers_frontend:latest + build_gateway_and_push_to_docker_hub: + name: Push gateway Docker image to DockerHub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Push to DockerHub + uses: docker/build-push-action@v4 + with: + build-args: + NGINX_VERSION=${{ vars.NGINX_VERSION_BUILD }} + context: ./infra_bt/ + push: true + tags: ${{ secrets.DOCKER_USERNAME }}/volunteers_gateway:latest + deploy: runs-on: ubuntu-latest needs: diff --git a/infra_bt/.dockerignore b/infra_bt/.dockerignore new file mode 100644 index 0000000..7a2f7ba --- /dev/null +++ b/infra_bt/.dockerignore @@ -0,0 +1,5 @@ +.env +.env.example +docker-compose.production.yml +docker-compose.yml +Dockerfile diff --git a/infra_bt/Dockerfile b/infra_bt/Dockerfile new file mode 100644 index 0000000..00833b1 --- /dev/null +++ b/infra_bt/Dockerfile @@ -0,0 +1,3 @@ +ARG NGINX_VERSION_BUILD +FROM nginx:$NGINX_VERSION_BUILD +COPY ./nginx.conf /etc/nginx/conf.d/default.conf diff --git a/infra_bt/docker-compose.production.yml b/infra_bt/docker-compose.production.yml index f02a6ff..72935f1 100644 --- a/infra_bt/docker-compose.production.yml +++ b/infra_bt/docker-compose.production.yml @@ -40,11 +40,10 @@ services: - static_data:/static_files gateway: - image: nginx:1.25.2-alpine3.18-slim + image: 1yunker/volunteers_gateway ports: - 8000:80 volumes: - - ./nginx.conf:/etc/nginx/conf.d/default.conf - static_data:/usr/share/nginx/html - media_data:/usr/share/nginx/html/media depends_on: From 3fce5d277b1c8ac102823b4e400372912158319a Mon Sep 17 00:00:00 2001 From: Kozin Date: Fri, 20 Oct 2023 18:40:42 +0300 Subject: [PATCH 5/6] Corrections --- .github/workflows/ci.yml | 20 +++++++++++++++----- .github/workflows/main.yml | 9 +++------ backend/backend/settings.py | 1 - infra_bt/docker-compose.yml | 2 +- infra_bt/nginx.conf | 5 +++-- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1c3e6b..5e2de86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,25 +18,35 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | python -m pip install --upgrade pip --no-cache-dir pip install flake8 ruff isort --no-cache-dir if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Git Clone Action - uses: actions/checkout@v2 - with: - repository: ${{ github.repository }} - path: backend/ + - name: Run flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 backend --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings flake8 backend --count --exit-zero --max-complexity=10 --max-line-length=79 --statistics --config=setup.cfg + - name: Run ruff run: | python -m ruff check . + - name: Run isort run: | isort -c . + + - name: Send report + uses: appleboy/telegram-action@master + with: + fail_ci_if_error: true + to: ${{ secrets.A_TG_TO }} + token: ${{ secrets.A_TG_TOKEN }} + message: | + В репозитории {{ github.repository }} проверка кода прошла + успешно. + Данные: {{ fail_ci_if_error }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4f8294..044bd7f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,10 +41,6 @@ jobs: name: Push frontend Docker image to DockerHub runs-on: ubuntu-latest steps: - - name: Check out the repo - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - name: Login to Docker uses: docker/login-action@v2 with: @@ -53,7 +49,7 @@ jobs: - name: Push to DockerHub uses: docker/build-push-action@v4 with: - context: git@github.com:volunteers-for-city-projects/volunteers-frontend.git#deployDocker + context: https://github.com/volunteers-for-city-projects/volunteers-frontend.git#develop push: true tags: ${{ secrets.DOCKER_USERNAME }}/volunteers_frontend:latest @@ -84,7 +80,8 @@ jobs: needs: # Дождёмся билда всех образов - build_backend_and_push_to_docker_hub - # - build_frontend_and_push_to_docker_hub + - build_frontend_and_push_to_docker_hub + - build_gateway_and_push_to_docker_hub steps: - name: Checkout repo uses: actions/checkout@v3 diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 951ef48..f20fb89 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -50,7 +50,6 @@ 'projects.apps.ProjectsConfig', 'users.apps.UsersConfig', 'corsheaders', - 'check.apps.CheckConfig', ] MIDDLEWARE = [ diff --git a/infra_bt/docker-compose.yml b/infra_bt/docker-compose.yml index c01310f..fa537b3 100644 --- a/infra_bt/docker-compose.yml +++ b/infra_bt/docker-compose.yml @@ -49,7 +49,7 @@ services: frontend: build: - context: https://github.com/volunteers-for-city-projects/volunteers-frontend.git#feature/deployDocker + context: https://github.com/volunteers-for-city-projects/volunteers-frontend.git#develop command: - /bin/sh - -c diff --git a/infra_bt/nginx.conf b/infra_bt/nginx.conf index 326e8a2..95b2940 100644 --- a/infra_bt/nginx.conf +++ b/infra_bt/nginx.conf @@ -1,5 +1,6 @@ server { listen 80; + server_tokens off; root /usr/share/nginx/html; @@ -21,8 +22,8 @@ server { } location / { - index index.html index.htm; - try_files $uri /index.html; + limit_except GET POST; + try_files $uri $uri/ =404; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; From 92b75f358d83a553203fff6daac7db561cca84ee Mon Sep 17 00:00:00 2001 From: Kozin Date: Sat, 21 Oct 2023 14:21:20 +0300 Subject: [PATCH 6/6] Fix --- .github/workflows/ci.yml | 10 ---------- infra_bt/nginx.conf | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e2de86..8adbe1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,13 +40,3 @@ jobs: run: | isort -c . - - name: Send report - uses: appleboy/telegram-action@master - with: - fail_ci_if_error: true - to: ${{ secrets.A_TG_TO }} - token: ${{ secrets.A_TG_TOKEN }} - message: | - В репозитории {{ github.repository }} проверка кода прошла - успешно. - Данные: {{ fail_ci_if_error }} diff --git a/infra_bt/nginx.conf b/infra_bt/nginx.conf index 95b2940..f29f83e 100644 --- a/infra_bt/nginx.conf +++ b/infra_bt/nginx.conf @@ -1,6 +1,7 @@ server { listen 80; server_tokens off; + client_max_body_size 20M; root /usr/share/nginx/html; @@ -22,7 +23,6 @@ server { } location / { - limit_except GET POST; try_files $uri $uri/ =404; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr;