From b616f4a98dd84dbaead02539d3f1a2172fd6809a Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 24 May 2024 13:01:08 +0100 Subject: [PATCH 1/2] build: update all just commands, working just start tunnel --- Justfile | 123 +++++++-------- contrib/josm/docker-compose.yml | 6 +- contrib/just/README.md | 13 ++ .../dotenv/Justfile} | 31 ++-- contrib/just/start/Justfile | 140 ++++++++++++++++++ .../docker-compose.yml => just/stop/Justfile} | 36 +++-- contrib/just/test/Justfile | 37 +++++ contrib/tunnel/docker-compose.yml | 61 ++++++++ docker-compose.yml | 2 +- 9 files changed, 344 insertions(+), 105 deletions(-) create mode 100644 contrib/just/README.md rename contrib/{tunnel/odk/docker-compose.yml => just/dotenv/Justfile} (64%) create mode 100644 contrib/just/start/Justfile rename contrib/{tunnel/fmtm/docker-compose.yml => just/stop/Justfile} (64%) create mode 100644 contrib/just/test/Justfile create mode 100644 contrib/tunnel/docker-compose.yml diff --git a/Justfile b/Justfile index 6e287a87a2..f38977bbd6 100644 --- a/Justfile +++ b/Justfile @@ -1,4 +1,4 @@ -# Copyright (c) 2020, 2021 Humanitarian OpenStreetMap Team +# Copyright (c) 2024 Humanitarian OpenStreetMap Team # # This file is part of FMTM. # @@ -16,55 +16,29 @@ # along with FMTM. If not, see . # -# Builds +mod start 'contrib/just/start/Justfile' +mod stop 'contrib/just/stop/Justfile' +mod build 'contrib/just/build/Justfile' +mod test 'contrib/just/test/Justfile' +mod dotenv 'contrib/just/dotenv/Justfile' -build-backend: - docker compose build api +# Run the help script +default: + @just --unstable help -build-frontend: - docker compose build ui +# View available commands +help: + @just --unstable --list --justfile {{justfile()}} -build: build-backend build-frontend +# Run database migrations for backend +migrate: + docker compose up -d migrations -# Run - -run: - docker compose up -d - -run-without-central: - docker compose --profile no-odk up -d - -run-with-josm: - docker compose \ - -f docker-compose.yml \ - -f contrib/josm/docker-compose.yml \ - up -d - -run-with-tunnels: - docker compose \ - -f docker-compose.yml \ - -f contrib/tunnel/fmtm/docker-compose.yml \ - -f contrib/tunnel/odk/docker-compose.yml \ - up -d - -stop: - docker compose down - -clean-db: +# Delete local database, S3, and ODK Central data +clean: docker compose down -v -# Tests - -test-backend: - docker compose run --rm api pytest - -test-frontend: - docker compose run --rm ui-test - -test: test-backend test-frontend - -# Maintenance - +# Run pre-commit hooks lint: TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \ --volume $PWD:$PWD --workdir $PWD \ @@ -72,6 +46,7 @@ lint: 'git config --global --add safe.directory $PWD \ && pre-commit run --all-files' +# Increment version bump: TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \ --volume $PWD:$PWD --workdir $PWD \ @@ -82,33 +57,35 @@ bump: && cd src/backend \ && cz bump --check-consistency' -# Docs - -docs-rebuild: docs-clean docs-doxygen docs-uml - -docs-clean: - @rm -rf docs/{apidocs,html,docbook,man} docs/packages.png docs/classes.png - -docs-doxygen: - cd docs && doxygen +# Run docs website locally +docs: + @echo + @echo "\033[0;33m ############################################### \033[0m" + @echo + @echo + @echo "\033[0;34m Access the docs site on: http://localhost:55425 \033[0m" + @echo + @echo + @echo "\033[0;33m ############################################### \033[0m" + @echo -docs-uml: - cd docs && pyreverse -o png ../src/backend/app - -docs-pdf: - # Strip any unicode out of the markdown file before converting to PDF - # FIXME - MDS := \ - docs/dev/Backend.md \ - docs/dev/Database-Tips.md \ - docs/dev/Release-Cycle.md \ - docs/dev/Frontend.md \ - docs/dev/Production.md \ - docs/dev/Version-Control.md \ - docs/dev/Setup.md \ - docs/dev/Troubleshooting.md \ - PDFS := $(MDS:.md=.pdf) - @echo "Converting $PDFS to a PDF" - @new=$(notdir $(basename $PDFS)); \ - iconv -f utf-8 -t US $PDFS -c | \ - pandoc $PDFS -f markdown -t pdf -s -o /tmp/$$new.pdf \ No newline at end of file + TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \ + --volume $PWD:$PWD --workdir $PWD --publish 55425:3000 \ + --entrypoint='sh -c' api \ + 'git config --global --add safe.directory $PWD \ + && mkdocs serve --dev-addr 0.0.0.0:3000' + +# Mount an S3 bucket on your filesystem +mount-s3: + #!/usr/bin/env sh + fstab_entry="fmtm-data /mnt/fmtm/local fuse.s3fs _netdev,allow_other,\ + use_path_request_style,passwd_file=/home/$(whoami)/s3-creds/fmtm-local,\ + url=http://s3.fmtm.localhost:7050 0 0" + + if ! grep -q "$fstab_entry" /etc/fstab; then + echo "Mounting local FMTM S3 permanently in /etc/fstab" + echo "$fstab_entry" | sudo tee -a /etc/fstab > /dev/null + echo + else + echo "Local FMTM S3 is already mounted" + fi diff --git a/contrib/josm/docker-compose.yml b/contrib/josm/docker-compose.yml index a105874522..9fbe243caf 100644 --- a/contrib/josm/docker-compose.yml +++ b/contrib/josm/docker-compose.yml @@ -22,7 +22,7 @@ services: josm: image: "ghcr.io/hotosm/fmtm/josm:latest" build: - context: . + context: contrib/josm container_name: josm environment: - DISPLAY=josm-novnc:0.0 @@ -32,13 +32,13 @@ services: - fmtm-net - x11 ports: - - 8111:80 + - "8111:80" restart: "unless-stopped" josm-novnc: image: "ghcr.io/hotosm/fmtm/josm-novnc:latest" build: - context: novnc + context: contrib/josm/novnc container_name: josm_novnc environment: - DISPLAY_WIDTH=1280 diff --git a/contrib/just/README.md b/contrib/just/README.md new file mode 100644 index 0000000000..9045bbe80c --- /dev/null +++ b/contrib/just/README.md @@ -0,0 +1,13 @@ +# Just Submodules + +- This directory contains submodules for the parent Justfile. +- It allows for submodules to be used like: + +```bash +just build frontend +just build backend + +# Instead of (only top level) +just build-backend +just build-frontend +``` diff --git a/contrib/tunnel/odk/docker-compose.yml b/contrib/just/dotenv/Justfile similarity index 64% rename from contrib/tunnel/odk/docker-compose.yml rename to contrib/just/dotenv/Justfile index 77b7734197..5d55a4f94e 100644 --- a/contrib/tunnel/odk/docker-compose.yml +++ b/contrib/just/dotenv/Justfile @@ -1,4 +1,5 @@ -# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# Copyright (c) 2024 Humanitarian OpenStreetMap Team +# # This file is part of FMTM. # # FMTM is free software: you can redistribute it and/or modify @@ -15,17 +16,19 @@ # along with FMTM. If not, see . # -networks: - fmtm-net: - name: fmtm-${GIT_BRANCH:-local} +# Update a variable in the .env file +[no-cd] +update key value: + #!/usr/bin/env sh + + var_name={{key}} + var_value={{value}} + var_pattern="^${var_name}=" + new_var="${var_name}=${var_value}" + + if grep -Eq "${var_pattern}" .env; then + sed -i "s|${var_pattern}.*|${new_var}|" .env + else + echo "${new_var}" >> .env + fi -services: - central-tunnel: - image: "docker.io/cloudflare/cloudflared:latest" - depends_on: - central: - condition: service_healthy - networks: - - fmtm-net - restart: "unless-stopped" - command: tunnel --url http://central:8383 diff --git a/contrib/just/start/Justfile b/contrib/just/start/Justfile new file mode 100644 index 0000000000..262e7aeb2c --- /dev/null +++ b/contrib/just/start/Justfile @@ -0,0 +1,140 @@ +# Copyright (c) 2024 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +# Start FMTM +[no-cd] +default: + docker compose up -d + +# Start backend API only +[no-cd] +backend: + docker compose up -d api + +# Start frontend UI only +[no-cd] +frontend: + docker compose up -d ui + +# Start FMTM without ODK Central +[no-cd] +without-central: + docker compose --profile no-odk up -d + +# Start FMTM with JOSM +[no-cd] +josm: + docker compose \ + -f docker-compose.yml \ + -f contrib/josm/docker-compose.yml \ + up -d + + @echo + @echo "\033[0;33m ############################################### \033[0m" + @echo + @echo " Access the JOSM Remote API: http://localhost:8111" + @echo " Access the JOSM GUI in browser: http://localhost:8112" + @echo + @echo "\033[0;33m ############################################### \033[0m" + @echo + +# Start FMTM with tunnels +[no-cd] +tunnel: + #!/usr/bin/env sh + + docker compose \ + -f docker-compose.yml \ + -f contrib/tunnel/docker-compose.yml \ + up -d --wait + + # Workaround to until PR merged: + # https://github.com/cloudflare/cloudflared/pull/1135 + # Wait until services ready without HEALTHCHECK + sleep 5 + + fmtm_url=$(just --unstable start _get-tunnel-url 'frontend') + api_url=$(just --unstable start _get-tunnel-url 'api') + odk_url=$(just --unstable start _get-tunnel-url 'central') + s3_url=$(just --unstable start _get-tunnel-url 's3') + + just --unstable dotenv update "EXTRA_CORS_ORIGINS" "${fmtm_url}" + just --unstable dotenv update "S3_ENDPOINT" "${s3_url}" + + # Restart the API and UI with environment variables set + API_URL_OVERRIDE="${api_url}" docker compose \ + -f docker-compose.yml \ + -f contrib/tunnel/docker-compose.yml \ + up -d api ui + + # Restart ODK Central with domain override (for form download urls) + CENTRAL_DOMAIN_OVERRIDE="$(echo "${odk_url}" | sed 's|^https://||')" \ + docker compose \ + -f docker-compose.yml \ + -f contrib/tunnel/docker-compose.yml \ + up -d central + + just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url" + +# View the URLs for created tunnels +[no-cd] +view-tunnel-urls: + #!/usr/bin/env sh + + fmtm_url=$(just --unstable start _get-tunnel-url 'frontend') + api_url=$(just --unstable start _get-tunnel-url 'api') + odk_url=$(just --unstable start _get-tunnel-url 'central') + s3_url=$(just --unstable start _get-tunnel-url 's3') + + just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url" + +[no-cd] +_get-tunnel-url service_name: + #!/usr/bin/env sh + + service_url=$(docker compose \ + -f docker-compose.yml \ + -f contrib/tunnel/docker-compose.yml \ + logs {{service_name}}-tunnel | \ + grep 'Your quick Tunnel' -A 1 | tail -n 1 | \ + sed -n 's/.*| *\(https:\/\/[^ ]*\).*/\1/p') + + echo "$service_url" + +[no-cd] +_print-tunnel-urls fmtm_url api_url odk_url s3_url: + @echo + @echo "\033[0;33m ############################################### \033[0m" + @echo + @echo "\033[0;34m FMTM URL: \033[0m" + @echo " {{fmtm_url}}" + @echo + @echo "\033[0;34m API URL: \033[0m" + @echo " {{api_url}}" + @echo + @echo "\033[0;34m ODK Central URL: \033[0m" + @echo " {{odk_url}}" + @echo + @echo "\033[0;34m S3 URL: \033[0m" + @echo " {{s3_url}}" + @echo + @echo "\033[0;33m ############################################### \033[0m" + @echo + + + diff --git a/contrib/tunnel/fmtm/docker-compose.yml b/contrib/just/stop/Justfile similarity index 64% rename from contrib/tunnel/fmtm/docker-compose.yml rename to contrib/just/stop/Justfile index bdd0aa7eff..8237b78a8b 100644 --- a/contrib/tunnel/fmtm/docker-compose.yml +++ b/contrib/just/stop/Justfile @@ -1,4 +1,5 @@ -# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# Copyright (c) 2024 Humanitarian OpenStreetMap Team +# # This file is part of FMTM. # # FMTM is free software: you can redistribute it and/or modify @@ -15,17 +16,24 @@ # along with FMTM. If not, see . # -networks: - fmtm-net: - name: fmtm-${GIT_BRANCH:-local} +# Stop FMTM +[no-cd] +default: + docker compose down + +# Stop FMTM & JOSM +[no-cd] +josm: + docker compose \ + -f docker-compose.yml \ + -f contrib/josm/docker-compose.yml \ + down + +# Stop FMTM & tunnels +[no-cd] +tunnel: + docker compose \ + -f docker-compose.yml \ + -f contrib/tunnel/docker-compose.yml \ + down -services: - fmtm-tunnel: - image: "docker.io/cloudflare/cloudflared:latest" - depends_on: - proxy: - condition: service_healthy - networks: - - fmtm-net - restart: "unless-stopped" - command: tunnel --url http://proxy:80 diff --git a/contrib/just/test/Justfile b/contrib/just/test/Justfile new file mode 100644 index 0000000000..62b6610f25 --- /dev/null +++ b/contrib/just/test/Justfile @@ -0,0 +1,37 @@ +# Copyright (c) 2024 Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +# Test backend & frontend +[no-cd] +all: backend frontend + +# Test backend with pytest +[no-cd] +backend: + docker compose run --rm api pytest + +# Test frontend with Playwright +[no-cd] +frontend: + docker compose run --rm ui-test + +# Check coverage for backend tests +[no-cd] +coverage: + docker compose run --rm --entrypoint='sh -c' api \ + 'coverage run -m pytest && coverage report -m' diff --git a/contrib/tunnel/docker-compose.yml b/contrib/tunnel/docker-compose.yml new file mode 100644 index 0000000000..5ae8742926 --- /dev/null +++ b/contrib/tunnel/docker-compose.yml @@ -0,0 +1,61 @@ +# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +networks: + fmtm-net: + name: fmtm-${GIT_BRANCH:-local} + +services: + frontend-tunnel: + image: "docker.io/cloudflare/cloudflared:2024.5.0" + depends_on: + proxy: + condition: service_healthy + networks: + - fmtm-net + restart: "unless-stopped" + command: tunnel --url http://proxy:80 + + api-tunnel: + image: "docker.io/cloudflare/cloudflared:2024.5.0" + depends_on: + api: + condition: service_healthy + networks: + - fmtm-net + restart: "unless-stopped" + command: tunnel --url http://api:8000 + + central-tunnel: + image: "docker.io/cloudflare/cloudflared:2024.5.0" + depends_on: + central: + condition: service_healthy + networks: + - fmtm-net + restart: "unless-stopped" + command: tunnel --url http://central:8383 + + s3-tunnel: + image: "docker.io/cloudflare/cloudflared:2024.5.0" + depends_on: + s3: + condition: service_healthy + networks: + - fmtm-net + restart: "unless-stopped" + command: tunnel --url http://s3:9000 diff --git a/docker-compose.yml b/docker-compose.yml index 6f24d22d1f..e1347aa100 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,7 +119,7 @@ services: - ./src/frontend:/app - /app/node_modules/ environment: - - VITE_API_URL=http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050} + - VITE_API_URL=${API_URL_OVERRIDE:-http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}} ports: - "7051:7051" networks: From 29ad888f3c19d2b5ba65055fba53c0525604a663 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 24 May 2024 13:03:29 +0100 Subject: [PATCH 2/2] docs: update backend info using just commands --- docs/dev/Backend.md | 125 +++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 77 deletions(-) diff --git a/docs/dev/Backend.md b/docs/dev/Backend.md index 5adca39bbc..79e0a018bd 100644 --- a/docs/dev/Backend.md +++ b/docs/dev/Backend.md @@ -55,6 +55,9 @@ To run the local development setup without ODK Central (use external server): ```bash dc --profile no-odk up -d + +# Or via Just +just start without-central ``` ## 2. Start the API without Docker @@ -93,6 +96,9 @@ The API should now be accessible at: ```bash docker compose up -d migrations + +# Or via Just +just migrate ``` ### Type Checking @@ -166,6 +172,9 @@ To run the backend tests locally, run: ```bash docker compose run --rm api pytest + +# Or via Just +just test backend ``` To assess coverage of tests, run: @@ -173,6 +182,9 @@ To assess coverage of tests, run: ```bash docker compose run --rm --entrypoint='sh -c' api \ 'coverage run -m pytest && coverage report -m' + +# Or via Just +just test coverage ``` To assess performance of endpoints: @@ -208,13 +220,6 @@ Creating a new release during development may not always be feasible. The s3fs tool allows you to mount an S3 bucket on your filesystem, to browse like any other directory. -Install: - -```bash -sudo apt update -sudo apt install s3fs -``` - Create a credentials file: ```bash @@ -223,6 +228,21 @@ echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > ${HOME}/.passwd-s3fs chmod 600 ${HOME}/.passwd-s3fs ``` +#### Mount local S3 using Just + +```bash +just mount-s3 +``` + +#### Mount S3 manually + +Install s3fs: + +```bash +sudo apt update +sudo apt install s3fs +``` + Mount your bucket: > If you wish for this to be permanent, see below. @@ -248,88 +268,39 @@ url=http://s3.fmtm.localhost:7050 0 0` ### Running JOSM in the dev stack -- Run JOSM with FMTM: +- Run JOSM with FMTM via Just: ```bash -docker compose \ - -f docker-compose.yml \ - -f contrib/josm/docker-compose.yml \ - up -d +just start josm ``` This adds JOSM to the docker compose stack for local development. -Access the JOSM Remote API: -Access the JOSM GUI in browser: You can now call the JOSM API from FMTM and changes will be reflected in the GUI. -### Debugging local FMTM on mobile +### Debugging local services on mobile - It's difficult to debug services running on localhost from your mobile phone. - An easy way to do this is by tunneling: Cloudflare provides a great free solution for this (an alternative is Ngrok). -- To run the tunnel to the FMTM API: - - ```bash - docker compose \ - -f docker-compose.yml \ - -f contrib/tunnel/fmtm/docker-compose.yml \ - up -d - ``` - -- View the website to access FMTM remotely (e.g. via mobile): - - ```bash - docker compose \ - -f docker-compose.yml \ - -f contrib/tunnel/fmtm/docker-compose.yml \ - logs fmtm-tunnel - ``` - -- Now the final step is to add the provided tunnel URL to the allowed CORS - origins on API startup: - - ```bash - EXTRA_CORS_ORIGINS=https://the-url-you-were-given.trycloudflare.com \ - docker compose restart api - ``` - -### Using local ODK Central on mobile - -- Sometimes you wish to use a project in your local ODK Central, via ODK Collect - on your mobile. -- To run the tunnel to the ODK Central API: - - ```bash - docker compose \ - -f docker-compose.yml \ - -f contrib/tunnel/odk/docker-compose.yml \ - up -d - ``` - -- View the website to access ODK Central remotely (e.g. via mobile): - - ```bash - docker compose \ - -f docker-compose.yml \ - -f contrib/tunnel/odk/docker-compose.yml \ - logs central-tunnel - ``` - -1. Requirement: Restart ODK Central using the domain override - (required for form download URLs): - - ```bash - CENTRAL_DOMAIN_OVERRIDE=the-domain-without-protocol.trycloudflare.com \ - docker compose restart central - ``` +- We may also wish to debug our local ODK Central instance forms on our mobile ODK + Collect. +- To handle both of these instances set up tunnels for all services with: + +```bash +just start tunnel +``` + +To complete this setup, two additional steps must be complete: + +- **Requirement 1**: For login to work, use the temporary login. -2. Requirement: During project creation, set the ODK Central server URL to the - provided tunnel URL for the ODK Central API. +- **Requirement 2**: During project creation, set the ODK Central server URL + to the provided tunnel URL for the ODK Central API. - > The credentials for the local ODK Central instance are: - > Username: - > Password: Password1234 + > The credentials for the local ODK Central instance are: + > Username: + > Password: Password1234 -- Now when you access the project via a QRCode on mobile, the connection to ODK - Central should work. +Now when you access the project via a QRCode on mobile, the connection to ODK +Central should work.