Merge branch 'development' of github.com:hotosm/fmtm into fix-create-…

…project-reload

.github/workflows/build_odk_imgs.yml

@@ -21,6 +21,7 @@ jobs:
         "ghcr.io/${{ github.repository }}/odkcentral:latest"
       extra_build_args: |
         ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }}
+      multi_arch: true
 
   build-odkcentral-ui:
     uses: hotosm/gh-workflows/.github/workflows/[email protected]
@@ -31,3 +32,4 @@ jobs:
         "ghcr.io/${{ github.repository }}/odkcentral-ui:latest"
       extra_build_args: |
         ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }}
+      multi_arch: true

.github/workflows/build_proxy_imgs.yml

@@ -18,6 +18,7 @@ jobs:
         "ghcr.io/${{ github.repository }}/proxy:certs-init-main"
       extra_build_args: |
         NGINX_TAG=${{ vars.NGINX_TAG }}
+      multi_arch: true
 
   build-cert-init-dev:
     uses: hotosm/gh-workflows/.github/workflows/[email protected]
@@ -29,6 +30,7 @@ jobs:
         "ghcr.io/${{ github.repository }}/proxy:certs-init-staging"
       extra_build_args: |
         NGINX_TAG=${{ vars.NGINX_TAG }}
+      multi_arch: true
 
   build-proxy-main:
     uses: hotosm/gh-workflows/.github/workflows/[email protected]
@@ -39,6 +41,7 @@ jobs:
         "ghcr.io/${{ github.repository }}/proxy:main"
       extra_build_args: |
         NGINX_TAG=${{ vars.NGINX_TAG }}
+      multi_arch: true
 
   build-proxy-main-plus-script:
     uses: hotosm/gh-workflows/.github/workflows/[email protected]
@@ -49,6 +52,7 @@ jobs:
         "ghcr.io/${{ github.repository }}/proxy:main-plus-script"
       extra_build_args: |
         NGINX_TAG=${{ vars.NGINX_TAG }}
+      multi_arch: true
 
   build-proxy-dev:
     uses: hotosm/gh-workflows/.github/workflows/[email protected]
@@ -60,3 +64,4 @@ jobs:
         "ghcr.io/${{ github.repository }}/proxy:staging"
       extra_build_args: |
         NGINX_TAG=${{ vars.NGINX_TAG }}
+      multi_arch: true

.github/workflows/tag_build.yml

@@ -14,4 +14,5 @@ jobs:
       context: src/backend
       build_target: prod
       image_name: ghcr.io/${{ github.repository }}/backend
+      multi_arch: true
 # Frontend is not built as build variables are required

INSTALL.md

@@ -274,6 +274,10 @@ colima start
 
 Then continue with the FMTM [installation](#software-requirements).
 
+> Note: only tagged backend images are multi-architecture, supporting
+> MacOS. The regular images for fast continuous deployment are not:
+> `backend:development`, `backend:staging`, `backend:main`.
+
 ### A Note on Docker Desktop
 
 While in theory FMTM should run using Docker-Desktop, it has not

docker-compose.development.yml

@@ -125,7 +125,7 @@ services:
       - SYSADMIN_EMAIL=${ODK_CENTRAL_USER}
       - SYSADMIN_PASSWD=${ODK_CENTRAL_PASSWD}
       - HTTPS_PORT=443
-      - DB_HOST=central-db
+      - DB_HOST=${CENTRAL_DB_HOST:-central-db}
       - DB_USER=${CENTRAL_DB_USER}
       - DB_PASSWORD=${CENTRAL_DB_PASSWORD}
       - DB_NAME=${CENTRAL_DB_NAME}

docker-compose.staging.yml

@@ -18,13 +18,26 @@
 version: "3"
 
 volumes:
-  fmtm_data:
+  fmtm_frontend:
+    name: fmtm-frontend-${GIT_BRANCH}
   fmtm_db_data:
+    name: fmtm-db-data-${GIT_BRANCH}
+  fmtm_data:
+    name: fmtm-s3-data-${GIT_BRANCH}
   fmtm_logs:
+    name: fmtm-logs-${GIT_BRANCH}
   fmtm_images:
+    name: fmtm-images-${GIT_BRANCH}
   fmtm_tiles:
-  central_db_data:
+    name: fmtm-tiles-${GIT_BRANCH}
   certs:
+    name: fmtm-certs-${GIT_BRANCH}
+  certbot_data:
+    name: fmtm-certbot-data-${GIT_BRANCH}
+  central_db_data:
+    name: fmtm-central-db-data-${GIT_BRANCH}
+  central_frontend:
+    name: fmtm-central-frontend-${GIT_BRANCH}
 
 networks:
   fmtm-net:
@@ -48,6 +61,10 @@ services:
     extends:
       file: docker-compose.development.yml
       service: central
+  central-ui:
+    extends:
+      file: docker-compose.development.yml
+      service: central-ui
   s3:
     extends:
       file: docker-compose.development.yml

docs/dev/Production.md

@@ -10,72 +10,24 @@ your own cloud server.
 - Get a cloud server (tested with Ubuntu 22.04).
 - Set up a domain name, and point the DNS to your cloud server.
 - SSH into your server. Set up a user with sudo called
-  fmtm. [this](https://www.digitalocean.com/community/tutorials/initial-server-setup-with-ubuntu-22-04)
+  svcfmtm. [this](https://www.digitalocean.com/community/tutorials/initial-server-setup-with-ubuntu-22-04)
   is a good guide for basic server setup including creation of a
   user.
 
-### Install some stuff it'll need
-
-#### Docker
-
-- Install
-  Docker. [Here](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-22-04)
-  is a good tutorial for that; do steps 1 and 2. At the time of
-  writing that consisted of:
-
-      sudo apt update
-      sudo apt install apt-transport-https ca-certificates curl software-properties-common
-      curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-      echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-      sudo apt update
-      sudo apt install docker-ce
-      sudo usermod -aG docker ${USER}
-      su - ${USER}
-
-- Now install Docker Compose (as per [this
-  tutorial](https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-22-04)). At
-  the time of writing (the latest version of Docker Compose may
-  change, so the version number might be out of date, but the rest
-  shouldn't change) this consisted of:
-
-      mkdir -p ~/.docker/cli-plugins/
-      curl -SL https://github.com/docker/compose/releases/download/v2.12.2/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose
-      sudo chmod +x ~/.docker/cli-plugins/docker-compose
-
-### Grab the FMTM code
-
-Clone the Git repo for the fmtm with `git clone https://github.com/hotosm/fmtm.git`. Step into the resulting directory
-with `cd fmtm`.
-
-### Set up the environment and utilities to launch
-
-Create the env file interactively with:
-
-```bash
-bash scripts/gen-env.sh
-```
-
-OR
+### Run the install script
 
 ```bash
-cp .env.example .env
+curl -L https://get.fmtm.dev -o install.sh
+bash install.sh
+# Alternative URL: https://fmtm.hotosm.org/install.sh
 
-# Then edit values manually
+# Then follow the prompts
 ```
 
-Main variables of note to update:
-
-```dotenv
-ODK_CENTRAL_USER=`<CHANGEME>`
-ODK_CENTRAL_PASSWD=`<CHANGEME>`
-
-CERT_EMAIL=`<EMAIL_ADDRESS_TO_GENERATE_CERT_FOR>`
-OSM_CLIENT_ID=`<CHANGEME>`
-OSM_CLIENT_SECRET=`<CHANGEME>`
+#### Additional Environment Variables
 
-S3_ACCESS_KEY=`<CHANGEME>`
-S3_SECRET_KEY=`<CHANGEME>`
-```
+Variables are set in `.env`.
+Some can be updated manually, as required.
 
 #### EXTRA_CORS_ORIGINS
 
@@ -113,7 +65,7 @@ stack, and variables should be set accordingly.
 If you run FMTM with ODK and Minio (S3) included, then the
 domains will default to:
 
-```
+```dotenv
 ${FMTM_DOMAIN} --> Frontend
 api.${FMTM_DOMAIN} --> Backend
 odk.${FMTM_DOMAIN} --> ODK Central
@@ -122,26 +74,17 @@ s3.${FMTM_DOMAIN} --> S3 / Minio
 
 These defaults can be overriden with respective environment variables:
 
-```
+```dotenv
 FMTM_API_DOMAIN
 FMTM_ODK_DOMAIN
 FMTM_S3_DOMAIN
 ```
 
-### Start the Compose Stack
-
-Run the production docker-compose config:
-`docker compose -f docker-compose.main.yml up -d`
-
-> Note: The images should be built already on Github.
-
-With any luck, this will launch the docker container where the project
-runs, and you can access the working website from the domain name!
-
 ### Connecting to a remote database
 
 - A database may be located on a headless Linux server in the cloud.
-- To access the database via GUI tool such as PGAdmin, it is possible using port tunneling.
+- To access the database via GUI tool such as PGAdmin,
+  it is possible using port tunneling.
 
 ```bash
 ssh [email protected] -N -f -L {local_port}:localhost:{remote_port}
@@ -156,18 +99,81 @@ This will map port 5432 on the remote machine to port 5430 on your local machine
 
 ```bash
 GIT_BRANCH=development
-backup_filename="fmtm-db-backup-$(date +'%Y-%m-%d').sql.gz"
+backup_filename="fmtm-db-${GIT_BRANCH}-$(date +'%Y-%m-%d').sql.gz"
 echo $backup_filename
 
-docker exec -i -e PGPASSWORD=PASSWORD_HERE fmtm-db-${GIT_BRANCH} pg_dump --verbose --format c -U fmtm fmtm | gzip -9 > "$backup_filename"
+docker exec -i -e PGPASSWORD=PASSWORD_HERE \
+fmtm-db-${GIT_BRANCH} \
+pg_dump --verbose --format c -U fmtm fmtm \
+| gzip -9 > "$backup_filename"
+
+# For ODK
+docker exec -i -e PGPASSWORD=PASSWORD_HERE \
+fmtm-central-db-${GIT_BRANCH} \
+pg_dump --verbose --format c -U odk odk | \
+gzip -9 > "$backup_filename"
 ```
 
+> Note: if you are dumping to import into a pre-existing
+> database, you should also include the --clean flag.
+>
+> This will drop the existing tables prior to import,
+> and should prevent conflicts.
+
 ## Manual Database Restores
 
+The restore should be as easy as:
+
 ```bash
 # On a different machine (else change the container name)
 GIT_BRANCH=development
-backup_filename=fmtm-db-backup-XXXX-XX-XX-sql.gz
+backup_filename=fmtm-db-${GIT_BRANCH}-XXXX-XX-XX-sql.gz
+
+cat "$backup_filename" | gunzip | \
+docker exec -i -e PGPASSWORD=NEW_PASSWORD_HERE \
+fmtm-db-${GIT_BRANCH} \
+pg_restore --verbose -U fmtm -d fmtm
+
+# For ODK
+cat "$backup_filename" | gunzip | \
+docker exec -i -e PGPASSWORD=NEW_PASSWORD_HERE \
+fmtm-central-db-${GIT_BRANCH} \
+pg_restore --verbose -U odk -d odk
+```
+
+However, in some cases you may have existing data
+in the database (i.e. if you started the docker
+compose stack & the API ran the migrations!).
+
+In this case you can import into a fresh db, before
+attaching to the FMTM containers:
+
+```bash
+export GIT_BRANCH=development
+
+# Shut down the running database & delete the data
+docker compose -f docker-compose.$GIT_BRANCH.yml down -v
+
+# First, ensure you have a suitable .env with database vars
+# Start the databases only
+docker compose -f docker-compose.$GIT_BRANCH.yml up -d fmtm-db central-db
+
+# (Optional) restore odk central from the backup
+backup_filename=fmtm-central-db-${GIT_BRANCH}-XXXX-XX-XX-sql.gz
+
+cat "$backup_filename" | gunzip | \
+docker exec -i \
+fmtm-central-db-${GIT_BRANCH} \
+pg_restore --verbose -U odk -d odk
+
+# Restore fmtm from the backup
+backup_filename=fmtm-db-${GIT_BRANCH}-XXXX-XX-XX-sql.gz
+
+cat "$backup_filename" | gunzip | \
+docker exec -i \
+fmtm-db-${GIT_BRANCH} \
+pg_restore --verbose -U fmtm -d fmtm
 
-cat "$backup_filename" | gunzip | docker exec -i -e PGPASSWORD=NEW_PASSWORD_HERE fmtm-db-${GIT_BRANCH} pg_restore --verbose -U fmtm -d fmtm
+# Run the entire docker compose stack
+docker compose -f docker-compose.$GIT_BRANCH.yml up -d
 ```

src/backend/app/projects/project_crud.py

@@ -102,6 +102,7 @@ def get_projects(
             .limit(limit)
             .all()
         )
+        project_count = db.query(db_models.DbProject).filter(and_(*filters)).count()
 
     else:
         db_projects = (
@@ -111,9 +112,10 @@ def get_projects(
             .limit(limit)
             .all()
         )
+        project_count = db.query(db_models.DbProject).count()
     if db_objects:
-        return db_projects
-    return convert_to_app_projects(db_projects)
+        return project_count, db_projects
+    return project_count, convert_to_app_projects(db_projects)
 
 
 def get_project_summaries(
@@ -139,8 +141,10 @@ def get_project_summaries(
     #         .filter(
     #         db_models.DbProject.author_id == user_id).offset(skip).limit(limit).all()
 
-    db_projects = get_projects(db, user_id, skip, limit, True, hashtags, search)
-    return convert_to_project_summaries(db_projects)
+    project_count, db_projects = get_projects(
+        db, user_id, skip, limit, True, hashtags, search
+    )
+    return project_count, convert_to_project_summaries(db_projects)
 
 
 def get_project(db: Session, project_id: int):
@@ -2854,3 +2858,22 @@ def get_tasks_count(db: Session, project_id: int):
     )
     task_count = len(db_task.tasks)
     return task_count
+
+
+def get_pagintaion(page: int, count: int, results_per_page: int, total: int):
+    total_pages = (count + results_per_page - 1) // results_per_page
+    hasNext = (page * results_per_page) < count
+    hasPrev = page > 1
+
+    pagination = project_schemas.PaginationInfo(
+        hasNext=hasNext,
+        hasPrev=hasPrev,
+        nextNum=page + 1 if hasNext else None,
+        page=page,
+        pages=total_pages,
+        prevNum=page - 1 if hasPrev else None,
+        perPage=results_per_page,
+        total=total,
+    )
+
+    return pagination