From d0c7c2d0446e5335435ea4fd90005aa37fef042d Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 13 Feb 2024 10:47:01 +0000 Subject: [PATCH 01/12] build: remove odk vars from frontend build dev compose --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6db8f404f5..06f8edd7ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -122,9 +122,6 @@ services: - /app/node_modules/ environment: - VITE_API_URL=http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050} - - VITE_ODK_CENTRAL_URL=${ODK_CENTRAL_URL} - - VITE_ODK_CENTRAL_USER=${ODK_CENTRAL_USER} - - VITE_ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD} ports: - "7051:7051" networks: From 391685d9b3c69aa0eca864b697da32b4085cab29 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 13 Feb 2024 10:48:40 +0000 Subject: [PATCH 02/12] build: remove user password field from init schema --- src/backend/migrations/init/fmtm_base_schema.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index c94ded36af..681e12c227 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -531,8 +531,7 @@ CREATE TABLE public.users ( tasks_invalidated integer NOT NULL, projects_mapped integer[], date_registered timestamp without time zone, - last_validation_date timestamp without time zone, - password character varying + last_validation_date timestamp without time zone ); ALTER TABLE public.users OWNER TO fmtm; CREATE SEQUENCE public.users_id_seq From 80127d8c7fa95622d6764ea4e2bbac946a8d9d24 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 13 Feb 2024 11:30:44 +0000 Subject: [PATCH 03/12] build: fix only continue migration if sql script succeeds --- src/backend/migrate-entrypoint.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/migrate-entrypoint.sh b/src/backend/migrate-entrypoint.sh index f7cee00b89..820ad4e9cf 100644 --- a/src/backend/migrate-entrypoint.sh +++ b/src/backend/migrate-entrypoint.sh @@ -128,7 +128,7 @@ backup_db() { --host "$FMTM_DB_HOST" --username "$FMTM_DB_USER" "$FMTM_DB_NAME" echo "gzipping file --> ${db_backup_file}.gz" - gzip "$db_backup_file" + gzip --force "$db_backup_file" db_backup_file="${db_backup_file}.gz" BUCKET_NAME="fmtm-db-backups" @@ -145,9 +145,9 @@ execute_migrations() { for script_name in "${scripts_to_execute[@]}"; do script_file="/opt/migrations/$script_name" pretty_echo "Executing migration: $script_name" - psql "$db_url" -a -f "$script_file" - - # Add an entry in the migrations table to indicate completion + # Apply migration & if succeeds, add an entry + # in the migrations table to indicate completion + psql "$db_url" -v ON_ERROR_STOP=1 -a -f "$script_file" && \ psql "$db_url" < Date: Tue, 13 Feb 2024 14:58:25 +0000 Subject: [PATCH 04/12] build: update migration entrypoint envsubst + error handling --- src/backend/migrate-entrypoint.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/backend/migrate-entrypoint.sh b/src/backend/migrate-entrypoint.sh index 820ad4e9cf..0e5adaba78 100644 --- a/src/backend/migrate-entrypoint.sh +++ b/src/backend/migrate-entrypoint.sh @@ -145,10 +145,11 @@ execute_migrations() { for script_name in "${scripts_to_execute[@]}"; do script_file="/opt/migrations/$script_name" pretty_echo "Executing migration: $script_name" - # Apply migration & if succeeds, add an entry - # in the migrations table to indicate completion - psql "$db_url" -v ON_ERROR_STOP=1 -a -f "$script_file" && \ - psql "$db_url" < Date: Tue, 13 Feb 2024 14:58:37 +0000 Subject: [PATCH 05/12] build: migration to create fmtm public beta org + svcfmtm --- .../migrations/008-init-public-beta.sql | 73 +++++++++++++++++++ .../revert/008-init-public-beta.sql | 25 +++++++ 2 files changed, 98 insertions(+) create mode 100644 src/backend/migrations/008-init-public-beta.sql create mode 100644 src/backend/migrations/revert/008-init-public-beta.sql diff --git a/src/backend/migrations/008-init-public-beta.sql b/src/backend/migrations/008-init-public-beta.sql new file mode 100644 index 0000000000..b77e8b17ae --- /dev/null +++ b/src/backend/migrations/008-init-public-beta.sql @@ -0,0 +1,73 @@ +-- ## Migration to: +-- * Add FMTM Public Beta org by default. +-- * Add svcfmtm admin user. + +-- Start a transaction +BEGIN; + +-- Insert FMTM Public Beta organisation +INSERT INTO public.organisations ( + name, + slug, + logo, + description, + url, + type, + approved, + odk_central_url, + odk_central_user, + odk_central_password +) +VALUES ( + 'FMTM Public Beta', + 'fmtm-public-beta', + 'https://avatars.githubusercontent.com/u/458752?s=280&v=4', + 'HOTOSM Public Beta for FMTM.', + 'https://hotosm.org', + 'FREE', + true, + '${ODK_CENTRAL_URL}', + '${ODK_CENTRAL_USER}', + '${ODK_CENTRAL_PASSWD}' +) +ON CONFLICT ("name") DO NOTHING; + +-- Insert svcfmtm admin user +INSERT INTO public.users ( + username, + role, + name, + email_address, + is_email_verified, + mapping_level, + tasks_mapped, + tasks_validated, + tasks_invalidated +) +VALUES ( + 'svcfmtm', + 'ADMIN', + 'Admin', + 'admin@hotosm.org', + true, + 'ADVANCED', + 0, + 0, + 0 +) +ON CONFLICT ("username") DO NOTHING; + +-- Set svcfmtm user as org admin +WITH org_cte AS ( + SELECT id FROM public.organisations + WHERE name = 'FMTM Public Beta' +), user_cte AS ( + SELECT id FROM public.users + WHERE username = 'svcfmtm' +) +INSERT INTO public.organisation_managers (organisation_id, user_id) +SELECT (SELECT id FROM org_cte), (SELECT id FROM user_cte) +ON CONFLICT DO NOTHING; + +-- Commit the transaction +COMMIT; diff --git a/src/backend/migrations/revert/008-init-public-beta.sql b/src/backend/migrations/revert/008-init-public-beta.sql new file mode 100644 index 0000000000..f1581db692 --- /dev/null +++ b/src/backend/migrations/revert/008-init-public-beta.sql @@ -0,0 +1,25 @@ +-- Start a transaction +BEGIN; + +-- Delete record in org managers +WITH org_cte AS ( + SELECT id FROM public.organisations + WHERE name = 'FMTM Public Beta' +), user_cte AS ( + SELECT id FROM public.users + WHERE username = 'svcfmtm' +) +DELETE FROM public.organisation_managers +WHERE organisation_id = (SELECT id FROM org_cte) + AND user_id = (SELECT id FROM user_cte); + +-- Delete svcfmtm admin user +DELETE FROM public.users +WHERE username = 'svcfmtm'; + +-- Delete FMTM Public Beta organisation +DELETE FROM public.organisations +WHERE name = 'FMTM Public Beta'; + +-- Commit the transaction +COMMIT; From b19a51a374e64bdd543999b92929c41511dff873 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 13 Feb 2024 14:58:54 +0000 Subject: [PATCH 06/12] build: add gettext for envsubst command in dockerfile --- src/backend/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index 9035aedafe..77f41b724c 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -96,6 +96,7 @@ RUN set -ex \ -y --no-install-recommends \ "nano" \ "curl" \ + "gettext-base" \ "libpcre3" \ "mime-support" \ "postgresql-client" \ From 2c708afcfe61d10164cd2857706b725712134ae8 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 13 Feb 2024 15:03:16 +0000 Subject: [PATCH 07/12] build: use odk user email for svcfmtm user email --- src/backend/migrations/008-init-public-beta.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/migrations/008-init-public-beta.sql b/src/backend/migrations/008-init-public-beta.sql index b77e8b17ae..01052f099e 100644 --- a/src/backend/migrations/008-init-public-beta.sql +++ b/src/backend/migrations/008-init-public-beta.sql @@ -48,7 +48,7 @@ VALUES ( 'svcfmtm', 'ADMIN', 'Admin', - 'admin@hotosm.org', + '${ODK_CENTRAL_USER}', true, 'ADVANCED', 0, From 65f7db0f271b58b766b2a833895cf593fcfc6ac9 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 13 Feb 2024 18:16:03 +0000 Subject: [PATCH 08/12] fix: replace org init with startup code for pass encrypt --- src/backend/app/config.py | 2 +- src/backend/app/main.py | 6 +- .../app/organisations/organisation_crud.py | 90 ++++++++++++++++++- .../migrations/008-init-public-beta.sql | 73 --------------- .../revert/008-init-public-beta.sql | 25 ------ 5 files changed, 94 insertions(+), 102 deletions(-) delete mode 100644 src/backend/migrations/008-init-public-beta.sql delete mode 100644 src/backend/migrations/revert/008-init-public-beta.sql diff --git a/src/backend/app/config.py b/src/backend/app/config.py index af84897407..749b518694 100644 --- a/src/backend/app/config.py +++ b/src/backend/app/config.py @@ -176,7 +176,7 @@ def get_cipher_suite(): return Fernet(settings.ENCRYPTION_KEY) -def encrypt_value(password: str) -> str: +def encrypt_value(password: Union[str, HttpUrlStr]) -> str: """Encrypt value before going to the DB.""" cipher_suite = get_cipher_suite() encrypted_password = cipher_suite.encrypt(password.encode("utf-8")) diff --git a/src/backend/app/main.py b/src/backend/app/main.py index 3940b4f6c6..d206619f43 100644 --- a/src/backend/app/main.py +++ b/src/backend/app/main.py @@ -36,6 +36,7 @@ from app.db.database import get_db from app.models.enums import HTTPStatus from app.organisations import organisation_routes +from app.organisations.organisation_crud import init_admin_org from app.projects import project_routes from app.projects.project_crud import read_xlsforms from app.submissions import submission_routes @@ -54,8 +55,11 @@ async def lifespan(app: FastAPI): """FastAPI startup/shutdown event.""" log.debug("Starting up FastAPI server.") + db_conn = next(get_db()) + log.debug("Initialising admin org and user in DB.") + await init_admin_org(db_conn) log.debug("Reading XLSForms from DB.") - await read_xlsforms(next(get_db()), xlsforms_path) + await read_xlsforms(db_conn, xlsforms_path) yield diff --git a/src/backend/app/organisations/organisation_crud.py b/src/backend/app/organisations/organisation_crud.py index b5de5ec413..f02515c3b9 100644 --- a/src/backend/app/organisations/organisation_crud.py +++ b/src/backend/app/organisations/organisation_crud.py @@ -22,11 +22,11 @@ from fastapi import File, HTTPException, Response, UploadFile from loguru import logger as log -from sqlalchemy import update +from sqlalchemy import text, update from sqlalchemy.orm import Session from app.auth.osm import AuthUser -from app.config import settings +from app.config import encrypt_value, settings from app.db import db_models from app.models.enums import HTTPStatus, UserRole from app.organisations.organisation_deps import ( @@ -37,6 +37,92 @@ from app.users.user_crud import get_user +async def init_admin_org(db: Session): + """Init admin org and user at application startup.""" + sql = text( + """ + -- Start a transaction + BEGIN; + + -- Insert FMTM Public Beta organisation + INSERT INTO public.organisations ( + name, + slug, + logo, + description, + url, + type, + approved, + odk_central_url, + odk_central_user, + odk_central_password + ) + VALUES ( + 'FMTM Public Beta', + 'fmtm-public-beta', + 'https://avatars.githubusercontent.com/u/458752?s=280&v=4', + 'HOTOSM Public Beta for FMTM.', + 'https://hotosm.org', + 'FREE', + true, + :odk_url, + :odk_user, + :odk_pass + ) + ON CONFLICT ("name") DO NOTHING; + + -- Insert svcfmtm admin user + INSERT INTO public.users ( + username, + role, + name, + email_address, + is_email_verified, + mapping_level, + tasks_mapped, + tasks_validated, + tasks_invalidated + ) + VALUES ( + 'svcfmtm', + 'ADMIN', + 'Admin', + :odk_user, + true, + 'ADVANCED', + 0, + 0, + 0 + ) + ON CONFLICT ("username") DO NOTHING; + + -- Set svcfmtm user as org admin + WITH org_cte AS ( + SELECT id FROM public.organisations + WHERE name = 'FMTM Public Beta' + ), user_cte AS ( + SELECT id FROM public.users + WHERE username = 'svcfmtm' + ) + INSERT INTO public.organisation_managers (organisation_id, user_id) + SELECT (SELECT id FROM org_cte), (SELECT id FROM user_cte) + ON CONFLICT DO NOTHING; + + -- Commit the transaction + COMMIT; + """ + ) + + db.execute( + sql, + { + "odk_url": settings.ODK_CENTRAL_URL, + "odk_user": settings.ODK_CENTRAL_USER, + "odk_pass": encrypt_value(settings.ODK_CENTRAL_PASSWD), + }, + ) + + async def get_organisations( db: Session, current_user: AuthUser, diff --git a/src/backend/migrations/008-init-public-beta.sql b/src/backend/migrations/008-init-public-beta.sql deleted file mode 100644 index 01052f099e..0000000000 --- a/src/backend/migrations/008-init-public-beta.sql +++ /dev/null @@ -1,73 +0,0 @@ --- ## Migration to: --- * Add FMTM Public Beta org by default. --- * Add svcfmtm admin user. - --- Start a transaction -BEGIN; - --- Insert FMTM Public Beta organisation -INSERT INTO public.organisations ( - name, - slug, - logo, - description, - url, - type, - approved, - odk_central_url, - odk_central_user, - odk_central_password -) -VALUES ( - 'FMTM Public Beta', - 'fmtm-public-beta', - 'https://avatars.githubusercontent.com/u/458752?s=280&v=4', - 'HOTOSM Public Beta for FMTM.', - 'https://hotosm.org', - 'FREE', - true, - '${ODK_CENTRAL_URL}', - '${ODK_CENTRAL_USER}', - '${ODK_CENTRAL_PASSWD}' -) -ON CONFLICT ("name") DO NOTHING; - --- Insert svcfmtm admin user -INSERT INTO public.users ( - username, - role, - name, - email_address, - is_email_verified, - mapping_level, - tasks_mapped, - tasks_validated, - tasks_invalidated -) -VALUES ( - 'svcfmtm', - 'ADMIN', - 'Admin', - '${ODK_CENTRAL_USER}', - true, - 'ADVANCED', - 0, - 0, - 0 -) -ON CONFLICT ("username") DO NOTHING; - --- Set svcfmtm user as org admin -WITH org_cte AS ( - SELECT id FROM public.organisations - WHERE name = 'FMTM Public Beta' -), user_cte AS ( - SELECT id FROM public.users - WHERE username = 'svcfmtm' -) -INSERT INTO public.organisation_managers (organisation_id, user_id) -SELECT (SELECT id FROM org_cte), (SELECT id FROM user_cte) -ON CONFLICT DO NOTHING; - --- Commit the transaction -COMMIT; diff --git a/src/backend/migrations/revert/008-init-public-beta.sql b/src/backend/migrations/revert/008-init-public-beta.sql deleted file mode 100644 index f1581db692..0000000000 --- a/src/backend/migrations/revert/008-init-public-beta.sql +++ /dev/null @@ -1,25 +0,0 @@ --- Start a transaction -BEGIN; - --- Delete record in org managers -WITH org_cte AS ( - SELECT id FROM public.organisations - WHERE name = 'FMTM Public Beta' -), user_cte AS ( - SELECT id FROM public.users - WHERE username = 'svcfmtm' -) -DELETE FROM public.organisation_managers -WHERE organisation_id = (SELECT id FROM org_cte) - AND user_id = (SELECT id FROM user_cte); - --- Delete svcfmtm admin user -DELETE FROM public.users -WHERE username = 'svcfmtm'; - --- Delete FMTM Public Beta organisation -DELETE FROM public.organisations -WHERE name = 'FMTM Public Beta'; - --- Commit the transaction -COMMIT; From 109059e3e44395afb595ef1ac940ed48368d5af7 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 13 Feb 2024 18:28:30 +0000 Subject: [PATCH 09/12] fix: specify user id for svcfmtm admin init --- src/backend/app/organisations/organisation_crud.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/backend/app/organisations/organisation_crud.py b/src/backend/app/organisations/organisation_crud.py index f02515c3b9..3536b97457 100644 --- a/src/backend/app/organisations/organisation_crud.py +++ b/src/backend/app/organisations/organisation_crud.py @@ -73,6 +73,7 @@ async def init_admin_org(db: Session): -- Insert svcfmtm admin user INSERT INTO public.users ( + id, username, role, name, @@ -84,7 +85,8 @@ async def init_admin_org(db: Session): tasks_invalidated ) VALUES ( - 'svcfmtm', + :user_id, + :username, 'ADMIN', 'Admin', :odk_user, @@ -100,12 +102,9 @@ async def init_admin_org(db: Session): WITH org_cte AS ( SELECT id FROM public.organisations WHERE name = 'FMTM Public Beta' - ), user_cte AS ( - SELECT id FROM public.users - WHERE username = 'svcfmtm' ) INSERT INTO public.organisation_managers (organisation_id, user_id) - SELECT (SELECT id FROM org_cte), (SELECT id FROM user_cte) + SELECT (SELECT id FROM org_cte), :user_id ON CONFLICT DO NOTHING; -- Commit the transaction @@ -116,6 +115,8 @@ async def init_admin_org(db: Session): db.execute( sql, { + "user_id": 20386219, + "username": "svcfmtm", "odk_url": settings.ODK_CENTRAL_URL, "odk_user": settings.ODK_CENTRAL_USER, "odk_pass": encrypt_value(settings.ODK_CENTRAL_PASSWD), From c40ca4702569c5082de79edd72d5428a7bc16de2 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 13 Feb 2024 18:49:41 +0000 Subject: [PATCH 10/12] build: fix remove sequential user id (create manually) --- src/backend/app/db/db_models.py | 2 +- src/backend/migrations/init/fmtm_base_schema.sql | 9 --------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index 7580d867cf..da43df754d 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -93,7 +93,7 @@ class DbUser(Base): __tablename__ = "users" - id = cast(int, Column(BigInteger, primary_key=True, index=True)) + id = cast(int, Column(BigInteger, primary_key=True)) username = cast(str, Column(String, unique=True)) profile_img = cast(str, Column(String)) role = cast(UserRole, Column(Enum(UserRole), default=UserRole.MAPPER)) diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index 681e12c227..5f2904ffe2 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -534,14 +534,6 @@ CREATE TABLE public.users ( last_validation_date timestamp without time zone ); ALTER TABLE public.users OWNER TO fmtm; -CREATE SEQUENCE public.users_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; -ALTER TABLE public.users_id_seq OWNER TO fmtm; -ALTER SEQUENCE public.users_id_seq OWNED BY public.users.id; CREATE TABLE public.xlsforms ( id integer NOT NULL, @@ -577,7 +569,6 @@ ALTER TABLE ONLY public.task_invalidation_history ALTER COLUMN id SET DEFAULT ne ALTER TABLE ONLY public.task_mapping_issues ALTER COLUMN id SET DEFAULT nextval('public.task_mapping_issues_id_seq'::regclass); ALTER TABLE ONLY public.tasks ALTER COLUMN id SET DEFAULT nextval('public.tasks_id_seq'::regclass); ALTER TABLE ONLY public.teams ALTER COLUMN id SET DEFAULT nextval('public.teams_id_seq'::regclass); -ALTER TABLE ONLY public.users ALTER COLUMN id SET DEFAULT nextval('public.users_id_seq'::regclass); ALTER TABLE ONLY public.xlsforms ALTER COLUMN id SET DEFAULT nextval('public.xlsforms_id_seq'::regclass); From 589539606b6944900bd60733787aa4a3be4e8a75 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 13 Feb 2024 18:49:59 +0000 Subject: [PATCH 11/12] build: remove default user creation from migration script --- src/backend/migrate-entrypoint.sh | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/src/backend/migrate-entrypoint.sh b/src/backend/migrate-entrypoint.sh index 0e5adaba78..24c4d2f178 100644 --- a/src/backend/migrate-entrypoint.sh +++ b/src/backend/migrate-entrypoint.sh @@ -163,20 +163,6 @@ SQL done } -create_svc_user() { - pretty_echo "Creating default svcfmtm user." - psql "$db_url" < Date: Wed, 14 Feb 2024 10:33:05 +0000 Subject: [PATCH 12/12] build: fix project-roles migration if enum exists --- src/backend/migrations/003-project-roles.sql | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/backend/migrations/003-project-roles.sql b/src/backend/migrations/003-project-roles.sql index 6c4bf50c69..4cf0882b69 100644 --- a/src/backend/migrations/003-project-roles.sql +++ b/src/backend/migrations/003-project-roles.sql @@ -1,20 +1,27 @@ -- ## Migration to: --- * Add public.projectrol enum. +-- * Add public.projectrole enum. -- * Update the user_roles table to use the enum -- Start a transaction BEGIN; -CREATE TYPE public.projectrole as ENUM ( +-- Create projectrole enum if it doesn't exist +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'projectrole') THEN + CREATE TYPE public.projectrole AS ENUM ( 'MAPPER', 'VALIDATOR', 'FIELD_MANAGER', 'ASSOCIATE_PROJECT_MANAGER', 'PROJECT_MANAGER' -); + ); + END IF; +END $$; +ALTER TYPE public.projectrole OWNER TO fmtm; + ALTER TABLE public.user_roles ALTER COLUMN "role" TYPE VARCHAR(24); ALTER TABLE public.user_roles ALTER COLUMN "role" TYPE public.projectrole USING role::public.projectrole; -ALTER TYPE public.projectrole OWNER TO fmtm; -- Commit the transaction COMMIT;