build: update Just and Justfile modules for better dev experience (#1537

) * build: update all just commands, working just start tunnel * docs: update backend info using just commands
hotosm · May 24, 2024 · 5bf93cb · 5bf93cb
1 parent dfb5e7b
commit 5bf93cb
Show file tree

Hide file tree

Showing 10 changed files with 392 additions and 182 deletions.
diff --git a/Justfile b/Justfile
@@ -1,4 +1,4 @@
-# Copyright (c) 2020, 2021 Humanitarian OpenStreetMap Team
+# Copyright (c) 2024 Humanitarian OpenStreetMap Team
 #
 # This file is part of FMTM.
 #
@@ -16,62 +16,37 @@
 #     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
 #
 
-# Builds
+mod start 'contrib/just/start/Justfile'
+mod stop 'contrib/just/stop/Justfile'
+mod build 'contrib/just/build/Justfile'
+mod test 'contrib/just/test/Justfile'
+mod dotenv 'contrib/just/dotenv/Justfile'
 
-build-backend:
-  docker compose build api
+# Run the help script
+default:
+  @just --unstable help
 
-build-frontend:
-  docker compose build ui
+# View available commands
+help:
+  @just --unstable --list --justfile {{justfile()}}
 
-build: build-backend build-frontend
+# Run database migrations for backend
+migrate:
+  docker compose up -d migrations
 
-# Run
-
-run:
-  docker compose up -d
-
-run-without-central:
-  docker compose --profile no-odk up -d
-
-run-with-josm:
-  docker compose \
-    -f docker-compose.yml \
-    -f contrib/josm/docker-compose.yml \
-    up -d
-
-run-with-tunnels:
-  docker compose \
-    -f docker-compose.yml \
-    -f contrib/tunnel/fmtm/docker-compose.yml \
-    -f contrib/tunnel/odk/docker-compose.yml \
-    up -d
-
-stop:
-  docker compose down
-
-clean-db:
+# Delete local database, S3, and ODK Central data
+clean:
   docker compose down -v
 
-# Tests
-
-test-backend:
-  docker compose run --rm api pytest
-
-test-frontend:
-  docker compose run --rm ui-test
-
-test: test-backend test-frontend
-
-# Maintenance
-
+# Run pre-commit hooks
 lint:
   TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \
     --volume $PWD:$PWD --workdir $PWD \
     --entrypoint='sh -c' api \
     'git config --global --add safe.directory $PWD \
     && pre-commit run --all-files'
 
+# Increment version
 bump:
   TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \
     --volume $PWD:$PWD --workdir $PWD \
@@ -82,33 +57,35 @@ bump:
     && cd src/backend \
     && cz bump --check-consistency'
 
-# Docs
-
-docs-rebuild: docs-clean docs-doxygen docs-uml
-
-docs-clean:
-	@rm -rf docs/{apidocs,html,docbook,man} docs/packages.png docs/classes.png
-
-docs-doxygen:
-	cd docs && doxygen
+# Run docs website locally
+docs:
+  @echo
+  @echo "\033[0;33m ############################################### \033[0m"
+  @echo
+  @echo
+  @echo "\033[0;34m Access the docs site on: http://localhost:55425 \033[0m"
+  @echo
+  @echo
+  @echo "\033[0;33m ############################################### \033[0m"
+  @echo
 
-docs-uml:
-	cd docs && pyreverse -o png ../src/backend/app
-
-docs-pdf:
-  # Strip any unicode out of the markdown file before converting to PDF
-  # FIXME
-  MDS := \
-    docs/dev/Backend.md \
-    docs/dev/Database-Tips.md \
-    docs/dev/Release-Cycle.md \
-    docs/dev/Frontend.md \
-    docs/dev/Production.md \
-    docs/dev/Version-Control.md \
-    docs/dev/Setup.md \
-    docs/dev/Troubleshooting.md \
-  PDFS := $(MDS:.md=.pdf)
-	@echo "Converting $PDFS to a PDF"
-	@new=$(notdir $(basename $PDFS)); \
-	iconv -f utf-8 -t US $PDFS -c | \
-	pandoc $PDFS -f markdown -t pdf -s -o /tmp/$$new.pdf
+  TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \
+    --volume $PWD:$PWD --workdir $PWD --publish 55425:3000 \
+    --entrypoint='sh -c' api \
+    'git config --global --add safe.directory $PWD \
+    && mkdocs serve --dev-addr 0.0.0.0:3000'
+
+# Mount an S3 bucket on your filesystem
+mount-s3:
+  #!/usr/bin/env sh
+  fstab_entry="fmtm-data /mnt/fmtm/local fuse.s3fs _netdev,allow_other,\
+  use_path_request_style,passwd_file=/home/$(whoami)/s3-creds/fmtm-local,\
+  url=http://s3.fmtm.localhost:7050 0 0"
+
+  if ! grep -q "$fstab_entry" /etc/fstab; then
+      echo "Mounting local FMTM S3 permanently in /etc/fstab"
+      echo "$fstab_entry" | sudo tee -a /etc/fstab > /dev/null
+      echo
+  else
+      echo "Local FMTM S3 is already mounted"
+  fi
diff --git a/contrib/josm/docker-compose.yml b/contrib/josm/docker-compose.yml
@@ -22,7 +22,7 @@ services:
   josm:
     image: "ghcr.io/hotosm/fmtm/josm:latest"
     build:
-      context: .
+      context: contrib/josm
     container_name: josm
     environment:
       - DISPLAY=josm-novnc:0.0
@@ -32,13 +32,13 @@ services:
       - fmtm-net
       - x11
     ports:
-      - 8111:80
+      - "8111:80"
     restart: "unless-stopped"
 
   josm-novnc:
     image: "ghcr.io/hotosm/fmtm/josm-novnc:latest"
     build:
-      context: novnc
+      context: contrib/josm/novnc
     container_name: josm_novnc
     environment:
       - DISPLAY_WIDTH=1280

diff --git a/contrib/just/README.md b/contrib/just/README.md
@@ -0,0 +1,13 @@
+# Just Submodules
+
+- This directory contains submodules for the parent Justfile.
+- It allows for submodules to be used like:
+
+```bash
+just build frontend
+just build backend
+
+# Instead of (only top level)
+just build-backend
+just build-frontend
+```
diff --git a/contrib/tunnel/odk/docker-compose.yml → contrib/just/dotenv/Justfile b/contrib/tunnel/odk/docker-compose.yml → contrib/just/dotenv/Justfile
@@ -1,4 +1,5 @@
-# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# Copyright (c) 2024 Humanitarian OpenStreetMap Team
+#
 # This file is part of FMTM.
 #
 #     FMTM is free software: you can redistribute it and/or modify
@@ -15,17 +16,19 @@
 #     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
 #
 
-networks:
-  fmtm-net:
-    name: fmtm-${GIT_BRANCH:-local}
+# Update a variable in the .env file
+[no-cd]
+update key value:
+  #!/usr/bin/env sh
+
+  var_name={{key}}
+  var_value={{value}}
+  var_pattern="^${var_name}="
+  new_var="${var_name}=${var_value}"
+
+  if grep -Eq "${var_pattern}" .env; then
+    sed -i "s|${var_pattern}.*|${new_var}|" .env
+  else
+    echo "${new_var}" >> .env
+  fi
 
-services:
-  central-tunnel:
-    image: "docker.io/cloudflare/cloudflared:latest"
-    depends_on:
-      central:
-        condition: service_healthy
-    networks:
-      - fmtm-net
-    restart: "unless-stopped"
-    command: tunnel --url http://central:8383
diff --git a/contrib/just/start/Justfile b/contrib/just/start/Justfile
@@ -0,0 +1,140 @@
+# Copyright (c) 2024 Humanitarian OpenStreetMap Team
+#
+# This file is part of FMTM.
+#
+#     FMTM is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     FMTM is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
+#
+
+# Start FMTM
+[no-cd]
+default:
+  docker compose up -d
+
+# Start backend API only
+[no-cd]
+backend:
+  docker compose up -d api
+
+# Start frontend UI only
+[no-cd]
+frontend:
+  docker compose up -d ui
+
+# Start FMTM without ODK Central
+[no-cd]
+without-central:
+  docker compose --profile no-odk up -d
+
+# Start FMTM with JOSM
+[no-cd]
+josm:
+  docker compose \
+    -f docker-compose.yml \
+    -f contrib/josm/docker-compose.yml \
+    up -d
+
+  @echo
+  @echo "\033[0;33m ############################################### \033[0m"
+  @echo
+  @echo " Access the JOSM Remote API: http://localhost:8111"
+  @echo " Access the JOSM GUI in browser: http://localhost:8112"
+  @echo
+  @echo "\033[0;33m ############################################### \033[0m"
+  @echo
+
+# Start FMTM with tunnels
+[no-cd]
+tunnel:
+  #!/usr/bin/env sh
+
+  docker compose \
+    -f docker-compose.yml \
+    -f contrib/tunnel/docker-compose.yml \
+    up -d --wait
+
+  # Workaround to until PR merged:
+  # https://github.com/cloudflare/cloudflared/pull/1135
+  # Wait until services ready without HEALTHCHECK
+  sleep 5
+
+  fmtm_url=$(just --unstable start _get-tunnel-url 'frontend')
+  api_url=$(just --unstable start _get-tunnel-url 'api')
+  odk_url=$(just --unstable start _get-tunnel-url 'central')
+  s3_url=$(just --unstable start _get-tunnel-url 's3')
+
+  just --unstable dotenv update "EXTRA_CORS_ORIGINS" "${fmtm_url}"
+  just --unstable dotenv update "S3_ENDPOINT" "${s3_url}"
+
+  # Restart the API and UI with environment variables set
+  API_URL_OVERRIDE="${api_url}" docker compose \
+    -f docker-compose.yml \
+    -f contrib/tunnel/docker-compose.yml \
+    up -d api ui
+
+  # Restart ODK Central with domain override (for form download urls)
+  CENTRAL_DOMAIN_OVERRIDE="$(echo "${odk_url}" | sed 's|^https://||')" \
+  docker compose \
+    -f docker-compose.yml \
+    -f contrib/tunnel/docker-compose.yml \
+    up -d central
+
+  just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url"
+
+# View the URLs for created tunnels
+[no-cd]
+view-tunnel-urls:
+  #!/usr/bin/env sh
+
+  fmtm_url=$(just --unstable start _get-tunnel-url 'frontend')
+  api_url=$(just --unstable start _get-tunnel-url 'api')
+  odk_url=$(just --unstable start _get-tunnel-url 'central')
+  s3_url=$(just --unstable start _get-tunnel-url 's3')
+
+  just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url"
+
+[no-cd]
+_get-tunnel-url service_name:
+  #!/usr/bin/env sh
+
+  service_url=$(docker compose \
+    -f docker-compose.yml \
+    -f contrib/tunnel/docker-compose.yml \
+    logs {{service_name}}-tunnel | \
+    grep 'Your quick Tunnel' -A 1 | tail -n 1 | \
+    sed -n 's/.*| *\(https:\/\/[^ ]*\).*/\1/p')
+
+  echo "$service_url"
+
+[no-cd]
+_print-tunnel-urls fmtm_url api_url odk_url s3_url:
+  @echo
+  @echo "\033[0;33m ############################################### \033[0m"
+  @echo
+  @echo "\033[0;34m FMTM URL: \033[0m"
+  @echo " {{fmtm_url}}"
+  @echo
+  @echo "\033[0;34m API URL: \033[0m"
+  @echo " {{api_url}}"
+  @echo
+  @echo "\033[0;34m ODK Central URL: \033[0m"
+  @echo " {{odk_url}}"
+  @echo
+  @echo "\033[0;34m S3 URL: \033[0m"
+  @echo " {{s3_url}}"
+  @echo
+  @echo "\033[0;33m ############################################### \033[0m"
+  @echo
+
+
+