From 14201b1174568d1be911612fdee558e054166835 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 26 Jan 2024 11:41:37 +0000 Subject: [PATCH 01/20] feat(frontend): add qrcode-generator dependency --- src/frontend/package.json | 1 + src/frontend/pnpm-lock.yaml | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/frontend/package.json b/src/frontend/package.json index b5e7f6cdca..5e143edeec 100755 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -74,6 +74,7 @@ "lucide-react": "^0.276.0", "mini-css-extract-plugin": "^2.7.5", "ol-ext": "^4.0.11", + "qrcode-generator": "^1.4.4", "react": "^17.0.2", "react-dom": "^17.0.2", "react-lazy-load-image-component": "^1.5.6", diff --git a/src/frontend/pnpm-lock.yaml b/src/frontend/pnpm-lock.yaml index 61d0b03818..f7db10e250 100644 --- a/src/frontend/pnpm-lock.yaml +++ b/src/frontend/pnpm-lock.yaml @@ -98,6 +98,9 @@ dependencies: ol-ext: specifier: ^4.0.11 version: 4.0.11(ol@8.1.0) + qrcode-generator: + specifier: ^1.4.4 + version: 1.4.4 react: specifier: ^17.0.2 version: 17.0.2 @@ -6488,6 +6491,10 @@ packages: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} + /qrcode-generator@1.4.4: + resolution: {integrity: sha512-HM7yY8O2ilqhmULxGMpcHSF1EhJJ9yBj8gvDEuZ6M+KGJ0YY2hKpnXvRD+hZPLrDVck3ExIGhmPtSdcjC+guuw==} + dev: false + /querystringify@2.2.0: resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} dev: true From 7885e847cf88bea7bffa5214597087d854d36f3c Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 26 Jan 2024 11:42:09 +0000 Subject: [PATCH 02/20] fix: use existing task-list for frontend qrcodes --- src/frontend/src/api/Files.js | 32 ++++++++++--------- src/frontend/src/api/Project.js | 5 +-- .../src/components/QrcodeComponent.jsx | 9 ++---- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/frontend/src/api/Files.js b/src/frontend/src/api/Files.js index bdf97aa805..3f00305b9f 100755 --- a/src/frontend/src/api/Files.js +++ b/src/frontend/src/api/Files.js @@ -1,35 +1,37 @@ -import React, { useEffect, useState } from 'react'; +import { useEffect, useState } from 'react'; import CoreModules from '@/shared/CoreModules'; -export const ProjectFilesById = (url, taskId) => { +export const ProjectFilesById = (qrcode_base64, taskId) => { const [loading, setLoading] = useState(true); const [qrcode, setQrcode] = useState(''); - const source = CoreModules.axios.CancelToken.source(); useEffect(() => { - const fetchProjectFileById = async (url) => { + const fetchProjectFileById = async (qrcode_base64) => { try { setLoading(true); - const fileJson = await CoreModules.axios.get(url, { - cancelToken: source.token, - }); - const resp = fileJson.data; - const taskIndex = resp.findIndex((task) => task.id == taskId); - const getQrcodeByIndex = resp[taskIndex].qr_code_base64; - setQrcode(getQrcodeByIndex); + // TODO code to generate QR code + + // const json = JSON.stringify({ + // token: xxx, + // var1: xxx, + // }); + // Note btoa base64 encodes the JSON string + // code.addData(btoa(json)); + // code.make(); + // Note cellSize=3 + // return code.createImgTag(3, 0); + + setQrcode(qrcode_base64); setLoading(false); } catch (error) { setLoading(false); } }; - fetchProjectFileById(url); + fetchProjectFileById(qrcode_base64); const cleanUp = () => { setLoading(false); setQrcode(''); - if (source) { - source.cancel('component unmounted'); - } }; return cleanUp; diff --git a/src/frontend/src/api/Project.js b/src/frontend/src/api/Project.js index 625d531075..13f703d435 100755 --- a/src/frontend/src/api/Project.js +++ b/src/frontend/src/api/Project.js @@ -21,12 +21,13 @@ export const ProjectById = (existingProjectList, projectId) => { return { id: data.id, project_task_name: data.project_task_name, - task_status: task_priority_str[data.task_status], outline_geojson: data.outline_geojson, outline_centroid: data.outline_centroid, - task_history: data.task_history, + task_status: task_priority_str[data.task_status], locked_by_uid: data.locked_by_uid, locked_by_username: data.locked_by_username, + task_history: data.task_history, + qrcode: data.qr_code_base64, }; }); // added centroid from another api to projecttaskboundries diff --git a/src/frontend/src/components/QrcodeComponent.jsx b/src/frontend/src/components/QrcodeComponent.jsx index 3cbc603598..ef363e975e 100755 --- a/src/frontend/src/components/QrcodeComponent.jsx +++ b/src/frontend/src/components/QrcodeComponent.jsx @@ -15,18 +15,15 @@ const TasksComponent = ({ type, task, defaultTheme }) => { const currentProjectId = environment.decode(params.id); const projectIndex = projectData.findIndex((project) => project.id == currentProjectId); const token = CoreModules.useAppSelector((state) => state.login.loginToken); - const currentStatus = { + const selectedTask = { ...projectData?.[projectIndex]?.taskBoundries?.filter((indTask, i) => { return indTask.id == task; })?.[0], }; const checkIfTaskAssignedOrNot = - currentStatus?.locked_by_username === token?.username || currentStatus?.locked_by_username === null; + selectedTask?.locked_by_username === token?.username || selectedTask?.locked_by_username === null; - const { loading, qrcode } = ProjectFilesById( - `${import.meta.env.VITE_API_URL}/tasks/task-list?project_id=${environment.decode(params.id)}`, - task, - ); + const { qrLoading, qrcode } = ProjectFilesById(selectedTask.qrcode, task); const socialStyles = { copyContainer: { From f241413b5cec7aa714ccee7e36569eaa93351633 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 26 Jan 2024 12:34:42 +0000 Subject: [PATCH 03/20] build: add migration to remove qrcode table, add odk_token --- src/backend/migrations/005-remove-qrcode.sql | 18 ++++++++++ .../migrations/init/fmtm_base_schema.sql | 27 +-------------- .../migrations/revert/005-remove-qrcode.sql | 33 +++++++++++++++++++ 3 files changed, 52 insertions(+), 26 deletions(-) create mode 100644 src/backend/migrations/005-remove-qrcode.sql create mode 100644 src/backend/migrations/revert/005-remove-qrcode.sql diff --git a/src/backend/migrations/005-remove-qrcode.sql b/src/backend/migrations/005-remove-qrcode.sql new file mode 100644 index 0000000000..8951ba94af --- /dev/null +++ b/src/backend/migrations/005-remove-qrcode.sql @@ -0,0 +1,18 @@ +-- ## Migration to: +-- * Remove public.qr_code table. +-- * Remove public.tasks.odk_token field. +-- * Add public.tasks.odk_token field. + +-- Start a transaction +BEGIN; + +-- Drop qr_code table +DROP TABLE IF EXISTS public.qr_code CASCADE; + +-- Update field in projects table +ALTER TABLE IF EXISTS public.tasks + DROP COLUMN IF EXISTS qr_code_id, + ADD COLUMN IF NOT EXISTS odk_token VARCHAR; + +-- Commit the transaction +COMMIT; diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index 4e4447a772..f7c07219f6 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -386,23 +386,6 @@ ALTER TABLE public.projects_id_seq OWNER TO fmtm; ALTER SEQUENCE public.projects_id_seq OWNED BY public.projects.id; -CREATE TABLE public.qr_code ( - id integer NOT NULL, - filename character varying, - image bytea -); -ALTER TABLE public.qr_code OWNER TO fmtm; -CREATE SEQUENCE public.qr_code_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; -ALTER TABLE public.qr_code_id_seq OWNER TO fmtm; -ALTER SEQUENCE public.qr_code_id_seq OWNED BY public.qr_code.id; - - CREATE TABLE public.task_history ( id integer NOT NULL, project_id integer, @@ -481,7 +464,7 @@ CREATE TABLE public.tasks ( locked_by bigint, mapped_by bigint, validated_by bigint, - qr_code_id integer + odk_token character varying ); ALTER TABLE public.tasks OWNER TO fmtm; CREATE SEQUENCE public.tasks_id_seq @@ -589,7 +572,6 @@ ALTER TABLE ONLY public.mbtiles_path ALTER COLUMN id SET DEFAULT nextval('public ALTER TABLE ONLY public.organisations ALTER COLUMN id SET DEFAULT nextval('public.organisations_id_seq'::regclass); ALTER TABLE ONLY public.project_chat ALTER COLUMN id SET DEFAULT nextval('public.project_chat_id_seq'::regclass); ALTER TABLE ONLY public.projects ALTER COLUMN id SET DEFAULT nextval('public.projects_id_seq'::regclass); -ALTER TABLE ONLY public.qr_code ALTER COLUMN id SET DEFAULT nextval('public.qr_code_id_seq'::regclass); ALTER TABLE ONLY public.task_history ALTER COLUMN id SET DEFAULT nextval('public.task_history_id_seq'::regclass); ALTER TABLE ONLY public.task_invalidation_history ALTER COLUMN id SET DEFAULT nextval('public.task_invalidation_history_id_seq'::regclass); ALTER TABLE ONLY public.task_mapping_issues ALTER COLUMN id SET DEFAULT nextval('public.task_mapping_issues_id_seq'::regclass); @@ -649,9 +631,6 @@ ALTER TABLE ONLY public.project_teams ALTER TABLE ONLY public.projects ADD CONSTRAINT projects_pkey PRIMARY KEY (id); -ALTER TABLE ONLY public.qr_code - ADD CONSTRAINT qr_code_pkey PRIMARY KEY (id); - ALTER TABLE ONLY public.splitpolygons ADD CONSTRAINT splitpolygons_pkey PRIMARY KEY (polyid); @@ -708,7 +687,6 @@ CREATE INDEX ix_task_mapping_issues_task_history_id ON public.task_mapping_issue CREATE INDEX ix_tasks_locked_by ON public.tasks USING btree (locked_by); CREATE INDEX ix_tasks_mapped_by ON public.tasks USING btree (mapped_by); CREATE INDEX ix_tasks_project_id ON public.tasks USING btree (project_id); -CREATE INDEX ix_tasks_qr_code_id ON public.tasks USING btree (qr_code_id); CREATE INDEX ix_tasks_validated_by ON public.tasks USING btree (validated_by); CREATE INDEX ix_users_id ON public.users USING btree (id); CREATE INDEX textsearch_idx ON public.project_info USING btree (text_searchable); @@ -806,9 +784,6 @@ ALTER TABLE ONLY public.task_mapping_issues ALTER TABLE ONLY public.tasks ADD CONSTRAINT tasks_project_id_fkey FOREIGN KEY (project_id) REFERENCES public.projects(id); -ALTER TABLE ONLY public.tasks - ADD CONSTRAINT tasks_qr_code_id_fkey FOREIGN KEY (qr_code_id) REFERENCES public.qr_code(id); - ALTER TABLE ONLY public.user_licenses ADD CONSTRAINT user_licenses_license_fkey FOREIGN KEY (license) REFERENCES public.licenses(id); diff --git a/src/backend/migrations/revert/005-remove-qrcode.sql b/src/backend/migrations/revert/005-remove-qrcode.sql new file mode 100644 index 0000000000..7421b0c764 --- /dev/null +++ b/src/backend/migrations/revert/005-remove-qrcode.sql @@ -0,0 +1,33 @@ +-- Start a transaction +BEGIN; + +-- Add qr_code table +CREATE TABLE IF NOT EXISTS public.qr_code ( + id integer NOT NULL, + filename character varying, + image bytea +); +ALTER TABLE public.qr_code OWNER TO fmtm; +CREATE SEQUENCE public.qr_code_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; +ALTER TABLE public.qr_code_id_seq OWNER TO fmtm; +ALTER SEQUENCE public.qr_code_id_seq OWNED BY public.qr_code.id; +ALTER TABLE ONLY public.qr_code ALTER COLUMN id SET DEFAULT nextval('public.qr_code_id_seq'::regclass); +ALTER TABLE ONLY public.qr_code + ADD CONSTRAINT qr_code_pkey PRIMARY KEY (id); + +-- Update field in projects table +ALTER TABLE IF EXISTS public.tasks + DROP COLUMN IF EXISTS odk_token, + ADD COLUMN IF NOT EXISTS qr_code_id integer; +CREATE INDEX ix_tasks_qr_code_id ON public.tasks USING btree (qr_code_id); +ALTER TABLE ONLY public.tasks + ADD CONSTRAINT tasks_qr_code_id_fkey FOREIGN KEY (qr_code_id) REFERENCES public.qr_code(id); + +-- Commit the transaction +COMMIT; From 2e8720962fdbec190a77a5267dd1d833d15f8cd5 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 26 Jan 2024 15:33:05 +0000 Subject: [PATCH 04/20] build: add cryptography to dependencies for Fernet --- src/backend/pdm.lock | 45 +++++++++++++++++++++++++++++++++++++- src/backend/pyproject.toml | 2 +- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index eff69c98b7..d39f3bec49 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "debug", "dev", "docs", "test"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:d73a7c181c5594d5f391a5c715559194fb5c6c4a923e5a4d15e3c1a57b073e27" +content_hash = "sha256:128629bb2e26f4cf44df3189858f84b1e8a0aa6e74364a313855338c19be04be" [[package]] name = "annotated-types" @@ -415,6 +415,49 @@ files = [ {file = "coverage_badge-1.1.0-py2.py3-none-any.whl", hash = "sha256:e365d56e5202e923d1b237f82defd628a02d1d645a147f867ac85c58c81d7997"}, ] +[[package]] +name = "cryptography" +version = "42.0.1" +requires_python = ">=3.7" +summary = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +dependencies = [ + "cffi>=1.12; platform_python_implementation != \"PyPy\"", +] +files = [ + {file = "cryptography-42.0.1-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:265bdc693570b895eb641410b8fc9e8ddbce723a669236162b9d9cfb70bd8d77"}, + {file = "cryptography-42.0.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:160fa08dfa6dca9cb8ad9bd84e080c0db6414ba5ad9a7470bc60fb154f60111e"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:727387886c9c8de927c360a396c5edcb9340d9e960cda145fca75bdafdabd24c"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d84673c012aa698555d4710dcfe5f8a0ad76ea9dde8ef803128cc669640a2e0"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e6edc3a568667daf7d349d7e820783426ee4f1c0feab86c29bd1d6fe2755e009"}, + {file = "cryptography-42.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:d50718dd574a49d3ef3f7ef7ece66ef281b527951eb2267ce570425459f6a404"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9544492e8024f29919eac2117edd8c950165e74eb551a22c53f6fdf6ba5f4cb8"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:ab6b302d51fbb1dd339abc6f139a480de14d49d50f65fdc7dff782aa8631d035"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2fe16624637d6e3e765530bc55caa786ff2cbca67371d306e5d0a72e7c3d0407"}, + {file = "cryptography-42.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ed1b2130f5456a09a134cc505a17fc2830a1a48ed53efd37dcc904a23d7b82fa"}, + {file = "cryptography-42.0.1-cp37-abi3-win32.whl", hash = "sha256:e5edf189431b4d51f5c6fb4a95084a75cef6b4646c934eb6e32304fc720e1453"}, + {file = "cryptography-42.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:6bfd823b336fdcd8e06285ae8883d3d2624d3bdef312a0e2ef905f332f8e9302"}, + {file = "cryptography-42.0.1-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:351db02c1938c8e6b1fee8a78d6b15c5ccceca7a36b5ce48390479143da3b411"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:430100abed6d3652208ae1dd410c8396213baee2e01a003a4449357db7dc9e14"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dff7a32880a51321f5de7869ac9dde6b1fca00fc1fef89d60e93f215468e824"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:b512f33c6ab195852595187af5440d01bb5f8dd57cb7a91e1e009a17f1b7ebca"}, + {file = "cryptography-42.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:95d900d19a370ae36087cc728e6e7be9c964ffd8cbcb517fd1efb9c9284a6abc"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:6ac8924085ed8287545cba89dc472fc224c10cc634cdf2c3e2866fe868108e77"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:cb2861a9364fa27d24832c718150fdbf9ce6781d7dc246a516435f57cfa31fe7"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25ec6e9e81de5d39f111a4114193dbd39167cc4bbd31c30471cebedc2a92c323"}, + {file = "cryptography-42.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9d61fcdf37647765086030d81872488e4cb3fafe1d2dda1d487875c3709c0a49"}, + {file = "cryptography-42.0.1-cp39-abi3-win32.whl", hash = "sha256:16b9260d04a0bfc8952b00335ff54f471309d3eb9d7e8dbfe9b0bd9e26e67881"}, + {file = "cryptography-42.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:7911586fc69d06cd0ab3f874a169433db1bc2f0e40988661408ac06c4527a986"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d3594947d2507d4ef7a180a7f49a6db41f75fb874c2fd0e94f36b89bfd678bf2"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8d7efb6bf427d2add2f40b6e1e8e476c17508fa8907234775214b153e69c2e11"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:126e0ba3cc754b200a2fb88f67d66de0d9b9e94070c5bc548318c8dab6383cb6"}, + {file = "cryptography-42.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:802d6f83233cf9696b59b09eb067e6b4d5ae40942feeb8e13b213c8fad47f1aa"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0b7cacc142260ada944de070ce810c3e2a438963ee3deb45aa26fd2cee94c9a4"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:32ea63ceeae870f1a62e87f9727359174089f7b4b01e4999750827bf10e15d60"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d3902c779a92151f134f68e555dd0b17c658e13429f270d8a847399b99235a3f"}, + {file = "cryptography-42.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:50aecd93676bcca78379604ed664c45da82bc1241ffb6f97f6b7392ed5bc6f04"}, + {file = "cryptography-42.0.1.tar.gz", hash = "sha256:fd33f53809bb363cf126bebe7a99d97735988d9b0131a2be59fbf83e1259a5b7"}, +] + [[package]] name = "debugpy" version = "1.8.0" diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 30d43901e0..fef9e410b1 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -37,7 +37,6 @@ dependencies = [ "geojson==3.1.0", "shapely==2.0.2", "pyxform==1.12.2", - "segno==1.6.0", "sentry-sdk==1.38.0", "py-cpuinfo==9.0.0", "loguru==0.7.2", @@ -45,6 +44,7 @@ dependencies = [ "pyproj==3.6.1", "asgiref==3.7.2", "sozipfile==0.3.2", + "cryptography>=42.0.1", "osm-login-python==1.0.1", "osm-fieldwork==0.4.1", "osm-rawdata==0.1.7", From 596f642ad808a0fbb17b12d6a885849c1bdbca6b Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Fri, 26 Jan 2024 16:22:48 +0000 Subject: [PATCH 05/20] fix(frontend): incorrect import path for SelectFormValidation --- src/frontend/src/components/createnewproject/SelectForm.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/src/components/createnewproject/SelectForm.tsx b/src/frontend/src/components/createnewproject/SelectForm.tsx index 4d54fc193c..9991243f53 100644 --- a/src/frontend/src/components/createnewproject/SelectForm.tsx +++ b/src/frontend/src/components/createnewproject/SelectForm.tsx @@ -9,7 +9,7 @@ import { CreateProjectActions } from '@/store/slices/CreateProjectSlice'; import useForm from '@/hooks/useForm'; import { useAppSelector } from '@/types/reduxTypes'; import FileInputComponent from '@/components/common/FileInputComponent'; -import SelectFormValidation from '@/components/createproject/validation/SelectFormValidation'; +import SelectFormValidation from '@/components/createnewproject/validation/SelectFormValidation'; import { FormCategoryService, ValidateCustomForm } from '@/api/CreateProjectService'; import NewDefineAreaMap from '@/views/NewDefineAreaMap'; From 5d1e72528ae06aad82220ed271c9efbab216b223 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 28 Jan 2024 16:21:33 +0000 Subject: [PATCH 06/20] feat: add ENCRYPTION_KEY var, with encrypt/decrypt db val methods --- .env.example | 1 + src/backend/app/config.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/.env.example b/.env.example index 38ecf4a28e..ca7462a34c 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,7 @@ ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD:-"testuserpassword"} DEBUG=${DEBUG:-False} LOG_LEVEL=${LOG_LEVEL:-INFO} EXTRA_CORS_ORIGINS=${EXTRA_CORS_ORIGINS} +ENCRYPTION_KEY=${ENCRYPTION_KEY:-"pIxxYIXe4oAVHI36lTveyc97FKK2O_l2VHeiuqU-K_4="} FMTM_DOMAIN=${FMTM_DOMAIN:-"fmtm.localhost"} FMTM_DEV_PORT=${FMTM_DEV_PORT:-7050} CERT_EMAIL=${CERT_EMAIL} diff --git a/src/backend/app/config.py b/src/backend/app/config.py index 9c9f615e2d..cd422d964f 100644 --- a/src/backend/app/config.py +++ b/src/backend/app/config.py @@ -17,9 +17,11 @@ # """Config file for Pydantic and FastAPI, using environment variables.""" +import base64 from functools import lru_cache from typing import Any, Optional, Union +from cryptography.fernet import Fernet from pydantic import PostgresDsn, ValidationInfo, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -30,6 +32,7 @@ class Settings(BaseSettings): APP_NAME: str = "FMTM" DEBUG: bool = False LOG_LEVEL: str = "INFO" + ENCRYPTION_KEY: str = "" FMTM_DOMAIN: str FMTM_DEV_PORT: Optional[str] = "7050" @@ -161,4 +164,25 @@ def get_settings(): return _settings +@lru_cache +def get_cipher_suite(): + """Cache cypher suite.""" + return Fernet(settings.ENCRYPTION_KEY) + + +def encrypt_value(password: str) -> str: + """Encrypt value before going to the DB.""" + cipher_suite = get_cipher_suite() + encrypted_password = cipher_suite.encrypt(password.encode("utf-8")) + return base64.b64encode(encrypted_password).decode("utf-8") + + +def decrypt_value(db_password: str) -> str: + """Decrypt the database value.""" + cipher_suite = get_cipher_suite() + encrypted_password = base64.b64decode(db_password.encode("utf-8")) + decrypted_password = cipher_suite.decrypt(encrypted_password) + return decrypted_password.decode("utf-8") + + settings = get_settings() From 364bb298a4890070007253ea9f2c07b29a03a3ff Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 28 Jan 2024 16:24:33 +0000 Subject: [PATCH 07/20] refactor: remove qr code from db, add odk_token field for tasks --- src/backend/app/db/db_models.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index dccea2e90c..87ebfd9cc6 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -345,16 +345,6 @@ class DbTaskHistory(Base): ) -class DbQrCode(Base): - """QR Code.""" - - __tablename__ = "qr_code" - - id = Column(Integer, primary_key=True) - filename = Column(String) - image = Column(LargeBinary) - - class DbTask(Base): """Describes an individual mapping Task.""" @@ -380,13 +370,9 @@ class DbTask(Base): validated_by = Column( BigInteger, ForeignKey("users.id", name="fk_users_validator"), index=True ) + odk_token = Column(String, nullable=True) # Mapped objects - qr_code_id = Column(Integer, ForeignKey("qr_code.id"), index=True) - qr_code = relationship( - DbQrCode, cascade="all, delete, delete-orphan", single_parent=True - ) - task_history = relationship( DbTaskHistory, cascade="all", order_by=desc(DbTaskHistory.action_date) ) From 4627e82fb6ec302047aee2d7049a0cf13d19074f Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 28 Jan 2024 16:26:21 +0000 Subject: [PATCH 08/20] feat(backend): remove qrcode from tasks, replace with odk_token only --- src/backend/app/central/central_crud.py | 65 --- src/backend/app/central/central_schemas.py | 1 - src/backend/app/projects/project_crud.py | 485 ++++++++---------- .../app/submissions/submission_routes.py | 4 +- src/backend/app/tasks/tasks_crud.py | 23 +- src/backend/app/tasks/tasks_routes.py | 9 - src/backend/app/tasks/tasks_schemas.py | 31 +- 7 files changed, 233 insertions(+), 385 deletions(-) diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 5b873905ee..c0be1abd13 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -17,10 +17,7 @@ # """Logic for interaction with ODK Central & data.""" -import base64 -import json import os -import zlib from xml.etree import ElementTree # import osm_fieldwork @@ -161,36 +158,6 @@ async def delete_odk_project( return "Could not delete project from central odk" -def create_odk_app_user( - project_id: int, name: str, odk_credentials: project_schemas.ODKCentral = None -): - """Create an app user specific to a project on ODK Central. - - If odk credentials of the project are provided, use them to create an app user. - """ - if odk_credentials: - url = odk_credentials.odk_central_url - user = odk_credentials.odk_central_user - pw = odk_credentials.odk_central_password - - else: - log.debug("ODKCentral connection variables not set in function") - log.debug("Attempting extraction from environment variables") - url = settings.ODK_CENTRAL_URL - user = settings.ODK_CENTRAL_USER - pw = settings.ODK_CENTRAL_PASSWD - - odk_app_user = OdkAppUser(url, user, pw) - - log.debug( - "ODKCentral: attempting user creation: name: " f"{name} | project: {project_id}" - ) - result = odk_app_user.create(project_id, name) - - log.debug(f"ODKCentral response: {result.json()}") - return result - - def delete_odk_app_user( project_id: int, name: str, odk_central: project_schemas.ODKCentral = None ): @@ -537,38 +504,6 @@ def generate_updated_xform( return outfile -async def encode_qrcode_json( - project_id: int, token: str, name: str, odk_central_url: str = None -): - """Assemble the ODK Collect JSON and base64 encode. - - The base64 encoded string is used to generate a QR code later. - """ - if not odk_central_url: - log.debug("ODKCentral connection variables not set in function") - log.debug("Attempting extraction from environment variables") - odk_central_url = settings.ODK_CENTRAL_URL - - # QR code text json in the format acceptable by odk collect - qr_code_setting = { - "general": { - "server_url": f"{odk_central_url}/v1/key/{token}/projects/{project_id}", - "form_update_mode": "match_exactly", - "basemap_source": "osm", - "autosend": "wifi_and_cellular", - "metadata_username": "svcfmtm", - }, - "project": {"name": f"{name}"}, - "admin": {}, - } - - # Base64 encoded - qr_data = base64.b64encode( - zlib.compress(json.dumps(qr_code_setting).encode("utf-8")) - ) - return qr_data - - def upload_media( project_id: int, xform_id: str, diff --git a/src/backend/app/central/central_schemas.py b/src/backend/app/central/central_schemas.py index c9aa2a676c..a5d1e073b3 100644 --- a/src/backend/app/central/central_schemas.py +++ b/src/backend/app/central/central_schemas.py @@ -32,7 +32,6 @@ class Central(CentralBase): """ODK Central return, with extras.""" geometry_geojson: str - # qr_code_binary: bytes class CentralOut(CentralBase): diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 6125a975e2..b9e83b344a 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -17,7 +17,6 @@ # """Logic for FMTM project routes.""" -import io import json import os import time @@ -31,7 +30,6 @@ import geoalchemy2 import geojson import requests -import segno import shapely.wkb as wkblib import sozipfile.sozipfile as zipfile import sqlalchemy @@ -60,17 +58,15 @@ from sqlalchemy.orm import Session from app.central import central_crud -from app.config import settings +from app.config import encrypt_value, settings from app.db import db_models from app.db.database import get_db -from app.db.postgis_utils import geojson_to_flatgeobuf, geometry_to_geojson, timestamp +from app.db.postgis_utils import geojson_to_flatgeobuf, geometry_to_geojson from app.projects import project_schemas from app.s3 import add_obj_to_bucket, get_obj_from_bucket from app.tasks import tasks_crud from app.users import user_crud -QR_CODES_DIR = "QR_codes/" -TASK_GEOJSON_DIR = "geojson/" TILESDIR = "/opt/tiles" @@ -784,174 +780,200 @@ def remove_z_dimension(coord): return True -async def update_project_with_zip( - db: Session, - project_id: int, - project_name_prefix: str, - task_type_prefix: str, - uploaded_zip: UploadFile, -): - """Update a project from a zip file. - - TODO ensure that logged in user is user who created this project, - return 403 (forbidden) if not authorized. - """ - # ensure file upload is zip - if uploaded_zip.content_type not in [ - "application/zip", - "application/zip-compressed", - "application/x-zip-compressed", - ]: - raise HTTPException( - status_code=415, - detail=f"File must be a zip. Uploaded file was {uploaded_zip.content_type}", - ) - - with zipfile.ZipFile(io.BytesIO(uploaded_zip.file.read()), "r") as zip: - # verify valid zip file - bad_file = zip.testzip() - if bad_file: - raise HTTPException( - status_code=400, detail=f"Zip contained a bad file: {bad_file}" - ) - - # verify zip includes top level files & directories - listed_files = zip.namelist() - - if QR_CODES_DIR not in listed_files: - raise HTTPException( - status_code=400, - detail=f"Zip must contain directory named {QR_CODES_DIR}", - ) - - if TASK_GEOJSON_DIR not in listed_files: - raise HTTPException( - status_code=400, - detail=f"Zip must contain directory named {TASK_GEOJSON_DIR}", - ) - - outline_filename = f"{project_name_prefix}.geojson" - if outline_filename not in listed_files: - raise HTTPException( - status_code=400, - detail=( - f"Zip must contain file named '{outline_filename}' " - "that contains a FeatureCollection outlining the project" - ), - ) - - task_outlines_filename = f"{project_name_prefix}_polygons.geojson" - if task_outlines_filename not in listed_files: - raise HTTPException( - status_code=400, - detail=( - f"Zip must contain file named '{task_outlines_filename}' " - "that contains a FeatureCollection where each Feature " - "outlines a task" - ), - ) - - # verify project exists in db - db_project = await get_project_by_id(db, project_id) - if not db_project: - raise HTTPException( - status_code=428, detail=f"Project with id {project_id} does not exist" - ) - - # add prefixes - db_project.project_name_prefix = project_name_prefix - db_project.task_type_prefix = task_type_prefix - - # generate outline from file and add to project - outline_shape = await get_outline_from_geojson_file_in_zip( - zip, outline_filename, f"Could not generate Shape from {outline_filename}" - ) - await update_project_location_info(db_project, outline_shape.wkt) - - # get all task outlines from file - project_tasks_feature_collection = await get_json_from_zip( - zip, - task_outlines_filename, - f"Could not generate FeatureCollection from {task_outlines_filename}", - ) - - # generate task for each feature - try: - task_count = 0 - db_project.total_tasks = len(project_tasks_feature_collection["features"]) - for feature in project_tasks_feature_collection["features"]: - task_name = feature["properties"]["task"] - - # generate and save qr code in db - qr_filename = ( - f"{project_name_prefix}_{task_type_prefix}__{task_name}.png" - ) - db_qr = await get_dbqrcode_from_file( - zip, - QR_CODES_DIR + qr_filename, - ( - f"QRCode for task {task_name} does not exist. " - f"File should be in {qr_filename}" - ), - ) - db.add(db_qr) - - # save outline - task_outline_shape = await get_shape_from_json_str( - feature, - f"Could not create task outline for {task_name} using {feature}", - ) - - # extract task geojson - task_geojson_filename = ( - f"{project_name_prefix}_{task_type_prefix}__{task_name}.geojson" - ) - task_geojson = await get_json_from_zip( - zip, - TASK_GEOJSON_DIR + task_geojson_filename, - f"Geojson for task {task_name} does not exist", - ) - - # generate qr code id first - db.flush() - # save task in db - task = db_models.DbTask( - project_id=project_id, - project_task_index=feature["properties"]["fid"], - project_task_name=task_name, - qr_code=db_qr, - qr_code_id=db_qr.id, - outline=task_outline_shape.wkt, - # geometry_geojson=json.dumps(task_geojson), - initial_feature_count=len(task_geojson["features"]), - ) - db.add(task) - - # for error messages - task_count = task_count + 1 - db_project.last_updated = timestamp() - - db.commit() - # should now include outline, geometry and tasks - db.refresh(db_project) - - return db_project - - # Exception was raised by app logic and has an error message, - # just pass it along - except HTTPException as e: - log.error(e) - raise e from None - - # Unexpected exception - except Exception as e: - raise HTTPException( - status_code=500, - detail=( - f"{task_count} tasks were created before the " - f"following error was thrown: {e}, on feature: {feature}" - ), - ) from e +# TODO delete me (does not handle ODK project too) +# async def update_project_with_zip( +# db: Session, +# project_id: int, +# project_name_prefix: str, +# task_type_prefix: str, +# uploaded_zip: UploadFile, +# ): +# """Update a project from a zip file. + +# TODO ensure that logged in user is user who created this project, +# return 403 (forbidden) if not authorized. +# """ +# QR_CODES_DIR = "QR_codes/" +# TASK_GEOJSON_DIR = "geojson/" + +# # ensure file upload is zip +# if uploaded_zip.content_type not in [ +# "application/zip", +# "application/zip-compressed", +# "application/x-zip-compressed", +# ]: +# raise HTTPException( +# status_code=415, +# detail=( +# "File must be a zip. Uploaded file was " +# f"{uploaded_zip.content_type}", +# )) + +# with zipfile.ZipFile(io.BytesIO(uploaded_zip.file.read()), "r") as zip: +# # verify valid zip file +# bad_file = zip.testzip() +# if bad_file: +# raise HTTPException( +# status_code=400, detail=f"Zip contained a bad file: {bad_file}" +# ) + +# # verify zip includes top level files & directories +# listed_files = zip.namelist() + +# if QR_CODES_DIR not in listed_files: +# raise HTTPException( +# status_code=400, +# detail=f"Zip must contain directory named {QR_CODES_DIR}", +# ) + +# if TASK_GEOJSON_DIR not in listed_files: +# raise HTTPException( +# status_code=400, +# detail=f"Zip must contain directory named {TASK_GEOJSON_DIR}", +# ) + +# outline_filename = f"{project_name_prefix}.geojson" +# if outline_filename not in listed_files: +# raise HTTPException( +# status_code=400, +# detail=( +# f"Zip must contain file named '{outline_filename}' " +# "that contains a FeatureCollection outlining the project" +# ), +# ) + +# task_outlines_filename = f"{project_name_prefix}_polygons.geojson" +# if task_outlines_filename not in listed_files: +# raise HTTPException( +# status_code=400, +# detail=( +# f"Zip must contain file named '{task_outlines_filename}' " +# "that contains a FeatureCollection where each Feature " +# "outlines a task" +# ), +# ) + +# # verify project exists in db +# db_project = await get_project_by_id(db, project_id) +# if not db_project: +# raise HTTPException( +# status_code=428, detail=f"Project with id {project_id} does not exist" +# ) + +# # add prefixes +# db_project.project_name_prefix = project_name_prefix +# db_project.task_type_prefix = task_type_prefix + +# # generate outline from file and add to project +# outline_shape = await get_outline_from_geojson_file_in_zip( +# zip, outline_filename, f"Could not generate Shape from {outline_filename}" +# ) +# await update_project_location_info(db_project, outline_shape.wkt) + +# # get all task outlines from file +# project_tasks_feature_collection = await get_json_from_zip( +# zip, +# task_outlines_filename, +# f"Could not generate FeatureCollection from {task_outlines_filename}", +# ) + +# # TODO move me if required +# async def get_dbqrcode_from_file(zip, qr_filename: str, error_detail: str): +# """Get qr code from database during import.""" +# try: +# with zip.open(qr_filename) as qr_file: +# binary_qrcode = qr_file.read() +# if binary_qrcode: +# return db_models.DbQrCode( +# filename=qr_filename, +# image=binary_qrcode, +# ) +# else: +# raise HTTPException( +# status_code=400, detail=f"{qr_filename} is an empty file" +# ) from None +# except Exception as e: +# log.exception(e) +# raise HTTPException( +# status_code=400, detail=f"{error_detail} ----- Error: {e}" +# ) from e + +# # generate task for each feature +# try: +# task_count = 0 +# db_project.total_tasks = len(project_tasks_feature_collection["features"]) +# for feature in project_tasks_feature_collection["features"]: +# task_name = feature["properties"]["task"] + +# # TODO remove qr code entry to db +# # TODO replace with entry to tasks.odk_token +# # generate and save qr code in db +# db_qr = await get_dbqrcode_from_file( +# zip, +# QR_CODES_DIR + qr_filename, +# ( +# f"QRCode for task {task_name} does not exist. " +# f"File should be in {qr_filename}" +# ), +# ) +# db.add(db_qr) + +# # save outline +# task_outline_shape = await get_shape_from_json_str( +# feature, +# f"Could not create task outline for {task_name} using {feature}", +# ) + +# # extract task geojson +# task_geojson_filename = ( +# f"{project_name_prefix}_{task_type_prefix}__{task_name}.geojson" +# ) +# task_geojson = await get_json_from_zip( +# zip, +# TASK_GEOJSON_DIR + task_geojson_filename, +# f"Geojson for task {task_name} does not exist", +# ) + +# # generate qr code id first +# db.flush() +# # save task in db +# task = db_models.DbTask( +# project_id=project_id, +# project_task_index=feature["properties"]["fid"], +# project_task_name=task_name, +# qr_code=db_qr, +# qr_code_id=db_qr.id, +# outline=task_outline_shape.wkt, +# # geometry_geojson=json.dumps(task_geojson), +# initial_feature_count=len(task_geojson["features"]), +# ) +# db.add(task) + +# # for error messages +# task_count = task_count + 1 +# db_project.last_updated = timestamp() + +# db.commit() +# # should now include outline, geometry and tasks +# db.refresh(db_project) + +# return db_project + +# # Exception was raised by app logic and has an error message, +# # just pass it along +# except HTTPException as e: +# log.error(e) +# raise e from None + +# # Unexpected exception +# except Exception as e: +# raise HTTPException( +# status_code=500, +# detail=( +# f"{task_count} tasks were created before the " +# f"following error was thrown: {e}, on feature: {feature}" +# ), +# ) from e # --------------------------- @@ -1150,40 +1172,42 @@ def generate_task_files( odk_id = project.odkid project_name = project.project_name_prefix category = project.xform_title - name = f"{project_name}_{category}_{task_id}" + appuser_name = f"{project_name}_{category}_{task_id}" # Create an app user for the task - project_log.info(f"Creating odkcentral app user for task {task_id}") - appuser = central_crud.create_odk_app_user(odk_id, name, odk_credentials) + project_log.info( + f"Creating odkcentral app user ({appuser_name}) " + f"for FMTM task ({task_id}) in FMTM project ({project_id})" + ) + appuser = OdkAppUser( + odk_credentials.odk_central_url, + odk_credentials.odk_central_user, + odk_credentials.odk_central_password, + ) + appuser_json = appuser.create(odk_id, appuser_name) # If app user could not be created, raise an exception. - if not appuser: - project_log.error("Couldn't create appuser for project") + if not appuser_json: + project_log.error(f"Couldn't create appuser {appuser_name} for project") + return False + if not (appuser_token := appuser_json.get("token")): + project_log.error(f"Couldn't get token for appuser {appuser_name}") return False - - # prefix should be sent instead of name - project_log.info(f"Creating qr code for task {task_id}") - create_qr_sync = async_to_sync(create_qrcode) - qr_code = create_qr_sync( - db, - odk_id, - appuser.json()["token"], - project_name, - odk_credentials.odk_central_url, - ) get_task_sync = async_to_sync(tasks_crud.get_task) task = get_task_sync(db, task_id) - task.qr_code_id = qr_code["qr_code_id"] + task.odk_token = encrypt_value( + f"{odk_credentials.odk_central_url}/key/{appuser_token}/projects/{odk_id}" + ) db.commit() db.refresh(task) # This file will store xml contents of an xls form. - xform = f"/tmp/{name}.xml" - extracts = f"/tmp/{name}.geojson" # This file will store osm extracts + xform = f"/tmp/{appuser_name}.xml" + extracts = f"/tmp/{appuser_name}.geojson" # This file will store osm extracts # xform_id_format - xform_id = f"{name}".split("_")[2] + xform_id = f"{appuser_name}".split("_")[2] # Get the features for this task. # Postgis query to filter task inside this task outline and of this project @@ -1249,23 +1273,8 @@ def generate_task_files( project_log.info(f"Updating role for app user in task {task_id}") # Update the user role for the created xform. try: - # Pass odk credentials - if odk_credentials: - url = odk_credentials.odk_central_url - user = odk_credentials.odk_central_user - pw = odk_credentials.odk_central_password - - else: - log.debug("ODKCentral connection variables not set in function") - log.debug("Attempting extraction from environment variables") - url = settings.ODK_CENTRAL_URL - user = settings.ODK_CENTRAL_USER - pw = settings.ODK_CENTRAL_PASSWD - - odk_app = OdkAppUser(url, user, pw) - - odk_app.updateRole( - projectId=odk_id, xform=xform_id, actorId=appuser.json()["id"] + appuser.updateRole( + projectId=odk_id, xform=xform_id, actorId=appuser_json.get("id") ) except Exception as e: log.exception(e) @@ -1484,39 +1493,6 @@ def wrap_generate_task_files(task): raise e -async def create_qrcode( - db: Session, - odk_id: int, - token: str, - project_name: str, - odk_central_url: str = None, -): - """Create a QR code for a task.""" - # Make QR code for an app_user. - log.debug(f"Generating base64 encoded QR settings for token: {token}") - qrcode_data = await central_crud.encode_qrcode_json( - odk_id, token, project_name, odk_central_url - ) - - log.debug("Generating QR code from base64 settings") - qrcode = segno.make(qrcode_data, micro=False) - - log.debug("Saving to buffer and decoding") - buffer = io.BytesIO() - qrcode.save(buffer, kind="png", scale=5) - qrcode_binary = buffer.getvalue() - - log.debug(f"Writing QR code to database for token {token}") - qrdb = db_models.DbQrCode(image=qrcode_binary) - db.add(qrdb) - db.commit() - codes = table("qr_code", column("id")) - sql = select(sqlalchemy.func.count(codes.c.id)) - result = db.execute(sql) - rows = result.fetchone()[0] - return {"data": qrcode, "id": rows + 1, "qr_code_id": qrdb.id} - - async def get_project_geometry(db: Session, project_id: int): """Retrieves the geometry of a project. @@ -1654,27 +1630,6 @@ async def get_shape_from_json_str(feature: str, error_detail: str): ) from e -async def get_dbqrcode_from_file(zip, qr_filename: str, error_detail: str): - """Get qr code from database during import.""" - try: - with zip.open(qr_filename) as qr_file: - binary_qrcode = qr_file.read() - if binary_qrcode: - return db_models.DbQrCode( - filename=qr_filename, - image=binary_qrcode, - ) - else: - raise HTTPException( - status_code=400, detail=f"{qr_filename} is an empty file" - ) from None - except Exception as e: - log.exception(e) - raise HTTPException( - status_code=400, detail=f"{error_detail} ----- Error: {e}" - ) from e - - # -------------------- # ---- CONVERTERS ---- # -------------------- diff --git a/src/backend/app/submissions/submission_routes.py b/src/backend/app/submissions/submission_routes.py index c7ad5a7f55..4a2ab2c9d7 100644 --- a/src/backend/app/submissions/submission_routes.py +++ b/src/backend/app/submissions/submission_routes.py @@ -389,7 +389,7 @@ async def submission_table( pagination = await project_crud.get_pagination(page, count, results_per_page, count) response = submission_schemas.PaginatedSubmissions( results=data, - pagination=submission_schemas.PaginationInfo(**pagination.dict()), + pagination=submission_schemas.PaginationInfo(**pagination.model_dump()), ) return response @@ -424,6 +424,6 @@ async def task_submissions( pagination = await project_crud.get_pagination(page, count, limit, count) response = submission_schemas.PaginatedSubmissions( results=data, - pagination=submission_schemas.PaginationInfo(**pagination.dict()), + pagination=submission_schemas.PaginationInfo(**pagination.model_dump()), ) return response diff --git a/src/backend/app/tasks/tasks_crud.py b/src/backend/app/tasks/tasks_crud.py index 3345389235..654a0d33f0 100644 --- a/src/backend/app/tasks/tasks_crud.py +++ b/src/backend/app/tasks/tasks_crud.py @@ -17,7 +17,6 @@ # """Logic for FMTM tasks.""" -import base64 from datetime import datetime, timedelta from typing import List, Optional @@ -92,7 +91,7 @@ async def get_tasks( return db_tasks -async def get_task(db: Session, task_id: int): +async def get_task(db: Session, task_id: int) -> db_models.DbTask: """Get details for a specific task ID.""" log.debug(f"Getting task with ID '{task_id}' from database") return db.query(db_models.DbTask).filter(db_models.DbTask.id == task_id).first() @@ -222,24 +221,6 @@ async def create_task_history_for_status_change( # TODO: write tests for these -async def get_qr_codes_for_task( - db: Session, - task_id: int, -): - """Get the ODK Collect QR code for a task area.""" - task = await get_task(db=db, task_id=task_id) - if task: - if task.qr_code: - log.debug(f"QR code found for task ID {task.id}. Converting to base64") - qr_code = base64.b64encode(task.qr_code.image) - else: - log.debug(f"QR code not found for task ID {task.id}.") - qr_code = None - return {"id": task_id, "qr_code": qr_code} - else: - raise HTTPException(status_code=400, detail="Task does not exist") - - async def update_task_files( db: Session, project_id: int, @@ -335,7 +316,7 @@ async def edit_task_boundary(db: Session, task_id: int, boundary: str): async def update_task_history( - tasks: List[tasks_schemas.TaskBase], db: Session = Depends(database.get_db) + tasks: List[tasks_schemas.Task], db: Session = Depends(database.get_db) ): """Update task history with username and user profile image.""" diff --git a/src/backend/app/tasks/tasks_routes.py b/src/backend/app/tasks/tasks_routes.py index bf85659b4c..4a0df6a699 100644 --- a/src/backend/app/tasks/tasks_routes.py +++ b/src/backend/app/tasks/tasks_routes.py @@ -136,15 +136,6 @@ async def update_task_status( return updated_task -@router.post("/task-qr-code/{task_id}") -async def get_qr_code_list( - task_id: int, - db: Session = Depends(database.get_db), -): - """Get the associated ODK Collect QR code for a task.""" - return await tasks_crud.get_qr_codes_for_task(db=db, task_id=task_id) - - @router.post("/edit-task-boundary") async def edit_task_boundary( task_id: int, diff --git a/src/backend/app/tasks/tasks_schemas.py b/src/backend/app/tasks/tasks_schemas.py index 2899e490aa..3389c01bb0 100644 --- a/src/backend/app/tasks/tasks_schemas.py +++ b/src/backend/app/tasks/tasks_schemas.py @@ -17,8 +17,6 @@ # """Pydantic schemas for FMTM task areas.""" - -import base64 from datetime import datetime from typing import Any, List, Optional @@ -28,6 +26,7 @@ from pydantic.functional_serializers import field_serializer from pydantic.functional_validators import field_validator +from app.config import decrypt_value from app.db.postgis_utils import geometry_to_geojson, get_centroid from app.models.enums import TaskStatus @@ -56,7 +55,7 @@ class TaskHistoryCount(BaseModel): mapped: int -class TaskBase(BaseModel): +class Task(BaseModel): """Core fields for a Task.""" model_config = ConfigDict( @@ -67,7 +66,6 @@ class TaskBase(BaseModel): # Excluded lock_holder: Any = Field(exclude=True) outline: Any = Field(exclude=True) - qr_code: Any = Field(exclude=True) id: int project_id: int @@ -80,6 +78,7 @@ class TaskBase(BaseModel): locked_by_uid: Optional[int] = None locked_by_username: Optional[str] = None task_history: Optional[List[TaskHistoryBase]] = None + odk_token: Optional[str] = None @field_validator("outline_geojson", mode="before") @classmethod @@ -123,26 +122,14 @@ def get_locked_by_username(self, value: str) -> str: return self.lock_holder.username return None - -class Task(TaskBase): - """Task details plus base64 QR codes.""" - - qr_code_base64: Optional[str] = None - - @field_validator("qr_code_base64", mode="before") - @classmethod - def get_qrcode_base64(cls, value: Any, info: ValidationInfo) -> str: - """Get base64 encoded qrcode.""" - if qr_code := info.data.get("qr_code"): - log.debug( - f"QR code found for task ID {info.data.get('id')}. " - "Converting to base64" - ) - return base64.b64encode(qr_code.image) - else: - log.warning(f"No QR code found for task ID {info.data.get('id')}") + @field_serializer("odk_token") + def decrypt_password(self, value: str) -> str: + """Decrypt the ODK Token extracted from the db.""" + if not value: return "" + return decrypt_value(value) + class ReadTask(Task): """Task details plus updated task history.""" From 77dbe10d428170b9e5b91bafc0d897a8bdcee1c6 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 28 Jan 2024 16:26:50 +0000 Subject: [PATCH 09/20] feat(frontend): dyanamic qrcode generation on click --- src/frontend/src/api/Files.js | 47 ++++++++++--------- src/frontend/src/api/Project.js | 2 +- .../src/components/QrcodeComponent.jsx | 14 +++--- 3 files changed, 34 insertions(+), 29 deletions(-) diff --git a/src/frontend/src/api/Files.js b/src/frontend/src/api/Files.js index 3f00305b9f..f6cfb1b5c4 100755 --- a/src/frontend/src/api/Files.js +++ b/src/frontend/src/api/Files.js @@ -1,33 +1,38 @@ import { useEffect, useState } from 'react'; -import CoreModules from '@/shared/CoreModules'; +import qrcodeGenerator from 'qrcode-generator'; -export const ProjectFilesById = (qrcode_base64, taskId) => { +export const ProjectFilesById = (odkToken, projectName, osmUser, taskId) => { const [loading, setLoading] = useState(true); const [qrcode, setQrcode] = useState(''); useEffect(() => { - const fetchProjectFileById = async (qrcode_base64) => { - try { - setLoading(true); - // TODO code to generate QR code + const fetchProjectFileById = async (odkToken, projectName, osmUser, taskId) => { + setLoading(true); - // const json = JSON.stringify({ - // token: xxx, - // var1: xxx, - // }); - // Note btoa base64 encodes the JSON string - // code.addData(btoa(json)); - // code.make(); - // Note cellSize=3 - // return code.createImgTag(3, 0); + const odkCollectJson = JSON.stringify({ + general: { + server_url: odkToken, + form_update_mode: 'manual', + basemap_source: 'osm', + autosend: 'wifi_and_cellular', + metadata_username: osmUser, + metadata_email: taskId, + }, + project: { name: projectName }, + admin: {}, + }); - setQrcode(qrcode_base64); - setLoading(false); - } catch (error) { - setLoading(false); - } + // Note: error correction level = "L" + const code = qrcodeGenerator(0, 'L'); + // Note: btoa base64 encodes the JSON string + code.addData(btoa(odkCollectJson)); + code.make(); + + // Note: cell size = 3, margin = 5 + setQrcode(code.createDataURL(3, 5)); + setLoading(false); }; - fetchProjectFileById(qrcode_base64); + fetchProjectFileById(odkToken, projectName, osmUser, taskId); const cleanUp = () => { setLoading(false); diff --git a/src/frontend/src/api/Project.js b/src/frontend/src/api/Project.js index 13f703d435..1755b3f2b7 100755 --- a/src/frontend/src/api/Project.js +++ b/src/frontend/src/api/Project.js @@ -27,7 +27,7 @@ export const ProjectById = (existingProjectList, projectId) => { locked_by_uid: data.locked_by_uid, locked_by_username: data.locked_by_username, task_history: data.task_history, - qrcode: data.qr_code_base64, + odk_token: data.odk_token, }; }); // added centroid from another api to projecttaskboundries diff --git a/src/frontend/src/components/QrcodeComponent.jsx b/src/frontend/src/components/QrcodeComponent.jsx index ef363e975e..a54a9eb6d6 100755 --- a/src/frontend/src/components/QrcodeComponent.jsx +++ b/src/frontend/src/components/QrcodeComponent.jsx @@ -11,19 +11,20 @@ const TasksComponent = ({ type, task, defaultTheme }) => { const dispatch = CoreModules.useAppDispatch(); const [open, setOpen] = useState(false); const params = CoreModules.useParams(); - const projectData = CoreModules.useAppSelector((state) => state.project.projectTaskBoundries); + const projectName = CoreModules.useAppSelector((state) => state.project.projectInfo.title); + const projectTaskData = CoreModules.useAppSelector((state) => state.project.projectTaskBoundries); const currentProjectId = environment.decode(params.id); - const projectIndex = projectData.findIndex((project) => project.id == currentProjectId); + const projectIndex = projectTaskData.findIndex((project) => project.id == currentProjectId); const token = CoreModules.useAppSelector((state) => state.login.loginToken); const selectedTask = { - ...projectData?.[projectIndex]?.taskBoundries?.filter((indTask, i) => { + ...projectTaskData?.[projectIndex]?.taskBoundries?.filter((indTask, i) => { return indTask.id == task; })?.[0], }; const checkIfTaskAssignedOrNot = selectedTask?.locked_by_username === token?.username || selectedTask?.locked_by_username === null; - const { qrLoading, qrcode } = ProjectFilesById(selectedTask.qrcode, task); + const { qrLoading, qrcode } = ProjectFilesById(selectedTask.odk_token, projectName, token?.username, task); const socialStyles = { copyContainer: { @@ -63,7 +64,7 @@ const TasksComponent = ({ type, task, defaultTheme }) => { ) : ( - qrcode + qrcode )} @@ -71,9 +72,8 @@ const TasksComponent = ({ type, task, defaultTheme }) => { { - const linkSource = `data:image/png;base64,${qrcode}`; const downloadLink = document.createElement('a'); - downloadLink.href = linkSource; + downloadLink.href = qrcode; downloadLink.download = `Task_${task}`; downloadLink.click(); }} From 69337a4267e0e41c64e6f2f78a79648d6526417c Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 28 Jan 2024 16:27:26 +0000 Subject: [PATCH 10/20] fix: case when project_log.json does not exist --- src/backend/app/projects/project_routes.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 3c37290728..e93eba1509 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -216,10 +216,12 @@ async def read_project(project_id: int, db: Session = Depends(database.get_db)): async def delete_project( project: db_models.DbProject = Depends(project_deps.get_project_by_id), db: Session = Depends(database.get_db), - user_data: AuthUser = Depends(login_required), + current_user: AuthUser = Depends(login_required), ): """Delete a project from both ODK Central and the local database.""" - log.info(f"User {user_data.username} attempting deletion of project {project.id}") + log.info( + f"User {current_user.username} attempting deletion of project {project.id}" + ) # Odk crendentials odk_credentials = project_schemas.ODKCentral( odk_central_url=project.odk_central_url, @@ -549,6 +551,7 @@ async def generate_files( xls_form_config_file: Optional[UploadFile] = File(None), data_extracts: Optional[UploadFile] = File(None), db: Session = Depends(database.get_db), + # current_user: AuthUser = Depends(login_required), ): """Generate additional content to initialise the project. @@ -556,7 +559,7 @@ async def generate_files( Accepts a project ID, category, custom form flag, and an uploaded file as inputs. The generated files are associated with the project ID and stored in the database. - This api generates qr_code, forms. This api also creates an app user for + This api generates odk appuser tokens, forms. This api also creates an app user for each task and provides the required roles. Some of the other functionality of this api includes converting a xls file provided by the user to the xform, generates osm data extracts and uploads @@ -727,7 +730,9 @@ async def generate_log( .first() ).extract_completed_count - with open("/opt/logs/create_project.json", "r") as log_file: + project_log_file = Path("/opt/logs/create_project.json") + project_log_file.touch(exist_ok=True) + with open(project_log_file, "r") as log_file: logs = [json.loads(line) for line in log_file] filtered_logs = [ From a4328f3140231ecaa86003de45da0bb58f6836c5 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 28 Jan 2024 16:29:06 +0000 Subject: [PATCH 11/20] feat: add default odk credentials to organisation models --- src/backend/app/db/db_models.py | 5 +++ .../app/organisations/organisation_crud.py | 2 +- .../app/organisations/organisation_schemas.py | 40 ++++++++++++++++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/backend/app/db/db_models.py b/src/backend/app/db/db_models.py index 87ebfd9cc6..d7a2dc2831 100644 --- a/src/backend/app/db/db_models.py +++ b/src/backend/app/db/db_models.py @@ -149,6 +149,11 @@ class DbOrganisation(Base): type = Column(Enum(OrganisationType), default=OrganisationType.FREE, nullable=False) approved = Column(Boolean, default=False) + ## Odk central server + odk_central_url = Column(String) + odk_central_user = Column(String) + odk_central_password = Column(String) + managers = relationship( DbUser, secondary=organisation_managers, diff --git a/src/backend/app/organisations/organisation_crud.py b/src/backend/app/organisations/organisation_crud.py index 87d7a2f5e8..36e6ca649d 100644 --- a/src/backend/app/organisations/organisation_crud.py +++ b/src/backend/app/organisations/organisation_crud.py @@ -106,7 +106,7 @@ async def create_organisation( try: # Create new organisation without logo set - db_organisation = db_models.DbOrganisation(**org_model.dict()) + db_organisation = db_models.DbOrganisation(**org_model.model_dump()) db.add(db_organisation) db.commit() diff --git a/src/backend/app/organisations/organisation_schemas.py b/src/backend/app/organisations/organisation_schemas.py index 0a2b9aabc8..832021e8e0 100644 --- a/src/backend/app/organisations/organisation_schemas.py +++ b/src/backend/app/organisations/organisation_schemas.py @@ -22,8 +22,10 @@ from fastapi import Form from pydantic import BaseModel, Field, HttpUrl, computed_field +from pydantic.functional_serializers import field_serializer from pydantic.functional_validators import field_validator +from app.config import decrypt_value, encrypt_value from app.models.enums import OrganisationType # class OrganisationBase(BaseModel): @@ -37,7 +39,16 @@ class OrganisationIn(BaseModel): description: Optional[str] = Field( Form(None, description="Organisation description") ) - url: Optional[HttpUrl] = Field(Form(None, description="Organisation website URL")) + url: Optional[HttpUrl] = Field(Form(None, description=("Organisation website URL"))) + odk_central_url: Optional[str] = Field( + Form(None, description="Organisation default ODK URL") + ) + odk_central_user: Optional[str] = Field( + Form(None, description="Organisation default ODK User") + ) + odk_central_password: Optional[str] = Field( + Form(None, description="Organisation default ODK Password") + ) @field_validator("url", mode="after") @classmethod @@ -61,6 +72,14 @@ def slug(self) -> str: slug = sub(r"[-\s]+", "-", slug) return slug + @field_validator("odk_central_password", mode="before") + @classmethod + def encrypt_odk_password(cls, value: str) -> str: + """Encrypt the ODK Central password before db insertion.""" + if not value: + return "" + return encrypt_value(value) + class OrganisationEdit(OrganisationIn): """Organisation to edit via user input.""" @@ -79,3 +98,22 @@ class OrganisationOut(BaseModel): slug: Optional[str] url: Optional[str] type: OrganisationType + + +class OrganisationOutWithCreds(BaseModel): + """Organisation plus decrypted ODK Central password. + + WARNING Do not display this to the user. + WARNING contains decrypted credentials. + """ + + odk_central_url: Optional[str] = None + odk_central_user: Optional[str] = None + odk_central_password: Optional[str] = None + + @field_serializer("odk_central_password") + def decrypt_password(self, value: str) -> str: + """Decrypt the database password value.""" + if not value: + return "" + return decrypt_value(value) From 0d549ddf524bbcb7204ba54950ec696e2c3e95cc Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 28 Jan 2024 16:29:39 +0000 Subject: [PATCH 12/20] feat: encrypt project odk credentials by default --- src/backend/app/projects/project_schemas.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index aee4740fbe..a4d6f88ae7 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -25,7 +25,9 @@ from geojson_pydantic import Feature as GeojsonFeature from pydantic import BaseModel from pydantic.functional_serializers import field_serializer +from pydantic.functional_validators import field_validator +from app.config import decrypt_value, encrypt_value from app.db import db_models from app.models.enums import ProjectPriority, ProjectStatus, TaskSplitType from app.tasks import tasks_schemas @@ -39,6 +41,13 @@ class ODKCentral(BaseModel): odk_central_user: str odk_central_password: str + @field_serializer("odk_central_password") + def decrypt_password(self, value: str) -> str: + """Decrypt the database password value.""" + if not value: + return "" + return decrypt_value(value) + class ProjectInfo(BaseModel): """Basic project info.""" @@ -70,6 +79,14 @@ class ProjectUpload(BaseModel): task_num_buildings: Optional[int] = None data_extract_type: Optional[str] = None + @field_validator("odk_central_password", mode="before") + @classmethod + def encrypt_odk_password(cls, value: str) -> str: + """Encrypt the ODK Central password before db insertion.""" + if not value: + return "" + return encrypt_value(value) + # city: str # country: str From e95cc2c9ab132c5477c6e4b710ed215b1d7df37b Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 28 Jan 2024 16:31:58 +0000 Subject: [PATCH 13/20] refactor: move field_validator for odk password to base model --- src/backend/app/projects/project_schemas.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index a4d6f88ae7..05d950b012 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -48,6 +48,14 @@ def decrypt_password(self, value: str) -> str: return "" return decrypt_value(value) + @field_validator("odk_central_password", mode="before") + @classmethod + def encrypt_odk_password(cls, value: str) -> str: + """Encrypt the ODK Central password before db insertion.""" + if not value: + return "" + return encrypt_value(value) + class ProjectInfo(BaseModel): """Basic project info.""" @@ -79,14 +87,6 @@ class ProjectUpload(BaseModel): task_num_buildings: Optional[int] = None data_extract_type: Optional[str] = None - @field_validator("odk_central_password", mode="before") - @classmethod - def encrypt_odk_password(cls, value: str) -> str: - """Encrypt the ODK Central password before db insertion.""" - if not value: - return "" - return encrypt_value(value) - # city: str # country: str From d479bbe4fa4f1ab8733e6a223a3125139ca4f84e Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Sun, 28 Jan 2024 16:34:35 +0000 Subject: [PATCH 14/20] build: small migrations script to convert existing qrcodes --- scripts/qrcode_to_odktoken.py | 148 ++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 scripts/qrcode_to_odktoken.py diff --git a/scripts/qrcode_to_odktoken.py b/scripts/qrcode_to_odktoken.py new file mode 100644 index 0000000000..d81d2f6048 --- /dev/null +++ b/scripts/qrcode_to_odktoken.py @@ -0,0 +1,148 @@ +"""Convert a QR Code image in Postgres to a Fernet encrypted odk_token URL.""" + +from pathlib import Path +from io import BytesIO + +import argparse +import base64 +import zlib +import json +from segno import make as make_qr +# apt install libzbar-dev +from pyzbar.pyzbar import decode as decode_qr +# pip install pillow +from PIL import Image +from sqlalchemy import ForeignKey, Column, Integer, String, LargeBinary +from sqlalchemy.orm import relationship +from sqlalchemy.orm.attributes import InstrumentedAttribute + +from dotenv import load_dotenv + +load_dotenv(Path(__file__).parent.parent / ".env.example") + +from app.config import encrypt_value, decrypt_value # noqa: E402 +from app.db.database import Base, get_db +from app.db.db_models import DbProject, DbTask + + +class DbQrCode(Base): + """QR Code.""" + + __tablename__ = "qr_code" + + id = Column(Integer, primary_key=True) + filename = Column(String) + image = Column(LargeBinary) + + +class TaskPlusQR(DbTask): + """Task plus QR code foreign key.""" + qr_code_id = Column(Integer, ForeignKey('qr_code.id'), index=True) + qr_code = relationship( + DbQrCode, cascade="all", single_parent=True + ) + if not isinstance(DbTask.odk_token, InstrumentedAttribute): + odk_token = Column(String, nullable=True) + + +def odktoken_to_qr(): + """Extract odk_token field from db and convert to QR codes.""" + + db = next(get_db()) + projects = db.query(DbProject).all() + + for project in projects: + project_name = project.project_name_prefix + tasks = project.tasks + + for task in tasks: + odk_token = task.odk_token + if not odk_token: + continue + + decrypted_odk_token = decrypt_value(odk_token) + qr_code_setting = { + "general": { + "server_url": decrypted_odk_token, + "form_update_mode": "match_exactly", + "basemap_source": "osm", + "autosend": "wifi_and_cellular", + "metadata_username": "svcfmtm", + }, + "project": {"name": f"{project_name}"}, + "admin": {}, + } + + # Base64/zlib encoded + qrcode_data = base64.b64encode( + zlib.compress(json.dumps(qr_code_setting).encode("utf-8")) + ) + qrcode = make_qr(qrcode_data, micro=False) + buffer = BytesIO() + qrcode.save(buffer, kind="png", scale=5) + qrcode_binary = buffer.getvalue() + qrdb = DbQrCode(image=qrcode_binary) + db.add(qrdb) + print(f"Added qrcode for task {task.id} to db") + db.commit() + + +def qr_to_odktoken(): + """Extract QR codes from db and convert to odk_token field.""" + + db = next(get_db()) + tasks = db.query(TaskPlusQR).all() + + for task in tasks: + if task.qr_code: + qr_img = Image.open(BytesIO(task.qr_code.image)) + qr_data = decode_qr(qr_img)[0].data + + # Base64/zlib decoded + decoded_qr = zlib.decompress(base64.b64decode(qr_data)) + odk_token = json.loads(decoded_qr.decode("utf-8")).get("general", {}).get("server_url") + + task.odk_token = encrypt_value(odk_token) + print(f"Added odk token for task {task.id}") + db.commit() + + +def encrypt_odk_creds(): + """Encrypt project odk password in the db.""" + + db = next(get_db()) + projects = db.query(DbProject).all() + + for project in projects: + project.odk_central_password = encrypt_value(project.odk_central_password) + print(f"Encrypted odk password for project {project.id}") + db.commit() + + +def decrypt_odk_creds(): + """Decrypt project odk password in the db.""" + + db = next(get_db()) + projects = db.query(DbProject).all() + + for project in projects: + project.odk_central_password = decrypt_value(project.odk_central_password) + print(f"Encrypted odk password for project {project.id}") + db.commit() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Apply or revert changes to QR codes and odk tokens.") + parser.add_argument("--apply", action="store_true", help="Apply changes (convert QR codes to odk tokens).") + parser.add_argument("--revert", action="store_true", help="Revert changes (convert odk tokens to QR codes).") + + args = parser.parse_args() + + if args.apply: + qr_to_odktoken() + encrypt_odk_creds() + elif args.revert: + odktoken_to_qr() + decrypt_odk_creds() + else: + print("Please provide either --apply or --revert flag.") From b97314020a83777edbd41c0e13fcb051cd6a05d9 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 30 Jan 2024 10:58:44 +0000 Subject: [PATCH 15/20] build: update osm-fieldwork --> 0.4.2, fmtm-splitter --> 1.0.0 --- src/backend/pdm.lock | 25 +++++++------------------ src/backend/pyproject.toml | 4 ++-- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/backend/pdm.lock b/src/backend/pdm.lock index d39f3bec49..e872692d74 100644 --- a/src/backend/pdm.lock +++ b/src/backend/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "debug", "dev", "docs", "test"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:128629bb2e26f4cf44df3189858f84b1e8a0aa6e74364a313855338c19be04be" +content_hash = "sha256:73bb79db4e82351bb07d7b0faf51b90fc523e0d3f316dca54e14dffee0bc077e" [[package]] name = "annotated-types" @@ -611,7 +611,7 @@ files = [ [[package]] name = "fmtm-splitter" -version = "1.0.0rc0" +version = "1.0.0" requires_python = ">=3.10" summary = "A utility for splitting an AOI into multiple tasks." dependencies = [ @@ -622,8 +622,8 @@ dependencies = [ "shapely>=1.8.1", ] files = [ - {file = "fmtm-splitter-1.0.0rc0.tar.gz", hash = "sha256:56efe64a1076ef8188afdd423f5895c66602309ee894bf49599bd3ca7e5506ac"}, - {file = "fmtm_splitter-1.0.0rc0-py3-none-any.whl", hash = "sha256:9647a85e99308141df036546380e273ccf9e4317e21298d0131f988c6b61d622"}, + {file = "fmtm-splitter-1.0.0.tar.gz", hash = "sha256:e6c823b9341f0f58413ee892c2ebb7b91377cddcafb4e6a9edbb4382aee1dd2b"}, + {file = "fmtm_splitter-1.0.0-py3-none-any.whl", hash = "sha256:cb6b391b32caddcca489aa24bdd1e2bb9c4245f345c0b3d42fdd517694ac9bfc"}, ] [[package]] @@ -1341,7 +1341,7 @@ files = [ [[package]] name = "osm-fieldwork" -version = "0.4.1" +version = "0.4.2" requires_python = ">=3.10" summary = "Processing field data from OpenDataKit to OpenStreetMap format." dependencies = [ @@ -1356,10 +1356,8 @@ dependencies = [ "pandas>=1.5.0", "pmtiles>=3.2.0", "progress>=1.6", - "psycopg2>=2.9.1", "py-cpuinfo>=9.0.0", "pySmartDL>=1.3.4", - "pymbtiles>=0.5.0", "requests>=2.26.0", "segno>=1.5.2", "shapely>=1.8.5", @@ -1367,8 +1365,8 @@ dependencies = [ "xmltodict>=0.13.0", ] files = [ - {file = "osm-fieldwork-0.4.1.tar.gz", hash = "sha256:e3f3381b7024d816ffeb15082083accfbdbff573fa1a485e9976f68b2356f1b8"}, - {file = "osm_fieldwork-0.4.1-py3-none-any.whl", hash = "sha256:d0328fb1ea03649a052c96a5cd253218d96909ba8353f6c7fd92cbbfe1566924"}, + {file = "osm-fieldwork-0.4.2.tar.gz", hash = "sha256:9ae6cb4d90b5dd8a10045a5a3bd512a073c814ad3b16fb245edd023312aed17d"}, + {file = "osm_fieldwork-0.4.2-py3-none-any.whl", hash = "sha256:4e7a596c50bfaef7f91b002d44ed5e3d1b377e50bf0a2a509d915a60d7ddcab1"}, ] [[package]] @@ -1835,15 +1833,6 @@ files = [ {file = "pyinstrument-4.6.1.tar.gz", hash = "sha256:f4731b27121350f5a983d358d2272fe3df2f538aed058f57217eef7801a89288"}, ] -[[package]] -name = "pymbtiles" -version = "0.5.0" -summary = "MapBox Mbtiles Utilities" -files = [ - {file = "pymbtiles-0.5.0-py3-none-any.whl", hash = "sha256:91c1c2fa3e25f581d563a60e705105f7277b0dbb9ff727c8c28cb66f0f891c84"}, - {file = "pymbtiles-0.5.0.tar.gz", hash = "sha256:b4eb2c470d2eb3d94627cdc8a8ae448b8899af2dd696f9a5eca706ddf8293b58"}, -] - [[package]] name = "pymdown-extensions" version = "10.7" diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index fef9e410b1..5bf1107bfb 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -46,9 +46,9 @@ dependencies = [ "sozipfile==0.3.2", "cryptography>=42.0.1", "osm-login-python==1.0.1", - "osm-fieldwork==0.4.1", + "osm-fieldwork==0.4.2", "osm-rawdata==0.1.7", - "fmtm-splitter==1.0.0rc0", + "fmtm-splitter==1.0.0", ] requires-python = ">=3.10" readme = "../../README.md" From dbd505b94fdbb67aecc500234f1aabf8189ff159 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 30 Jan 2024 11:01:18 +0000 Subject: [PATCH 16/20] build: move qrcode_to_odktoken script to migrations dir --- .../backend/migrations}/qrcode_to_odktoken.py | 62 +++++++++++-------- 1 file changed, 36 insertions(+), 26 deletions(-) rename {scripts => src/backend/migrations}/qrcode_to_odktoken.py (79%) diff --git a/scripts/qrcode_to_odktoken.py b/src/backend/migrations/qrcode_to_odktoken.py similarity index 79% rename from scripts/qrcode_to_odktoken.py rename to src/backend/migrations/qrcode_to_odktoken.py index d81d2f6048..5ef8284cc2 100644 --- a/scripts/qrcode_to_odktoken.py +++ b/src/backend/migrations/qrcode_to_odktoken.py @@ -1,28 +1,29 @@ """Convert a QR Code image in Postgres to a Fernet encrypted odk_token URL.""" -from pathlib import Path -from io import BytesIO - import argparse import base64 -import zlib import json -from segno import make as make_qr -# apt install libzbar-dev -from pyzbar.pyzbar import decode as decode_qr +import zlib +from io import BytesIO +from pathlib import Path + +from dotenv import load_dotenv + # pip install pillow from PIL import Image -from sqlalchemy import ForeignKey, Column, Integer, String, LargeBinary + +# apt install libzbar-dev +from pyzbar.pyzbar import decode as decode_qr +from segno import make as make_qr +from sqlalchemy import Column, ForeignKey, Integer, LargeBinary, String from sqlalchemy.orm import relationship from sqlalchemy.orm.attributes import InstrumentedAttribute -from dotenv import load_dotenv - load_dotenv(Path(__file__).parent.parent / ".env.example") -from app.config import encrypt_value, decrypt_value # noqa: E402 -from app.db.database import Base, get_db -from app.db.db_models import DbProject, DbTask +from app.config import decrypt_value, encrypt_value # noqa: E402 +from app.db.database import Base, get_db # noqa: E402 +from app.db.db_models import DbProject, DbTask # noqa: E402 class DbQrCode(Base): @@ -37,17 +38,15 @@ class DbQrCode(Base): class TaskPlusQR(DbTask): """Task plus QR code foreign key.""" - qr_code_id = Column(Integer, ForeignKey('qr_code.id'), index=True) - qr_code = relationship( - DbQrCode, cascade="all", single_parent=True - ) + + qr_code_id = Column(Integer, ForeignKey("qr_code.id"), index=True) + qr_code = relationship(DbQrCode, cascade="all", single_parent=True) if not isinstance(DbTask.odk_token, InstrumentedAttribute): odk_token = Column(String, nullable=True) def odktoken_to_qr(): """Extract odk_token field from db and convert to QR codes.""" - db = next(get_db()) projects = db.query(DbProject).all() @@ -89,7 +88,6 @@ def odktoken_to_qr(): def qr_to_odktoken(): """Extract QR codes from db and convert to odk_token field.""" - db = next(get_db()) tasks = db.query(TaskPlusQR).all() @@ -100,7 +98,11 @@ def qr_to_odktoken(): # Base64/zlib decoded decoded_qr = zlib.decompress(base64.b64decode(qr_data)) - odk_token = json.loads(decoded_qr.decode("utf-8")).get("general", {}).get("server_url") + odk_token = ( + json.loads(decoded_qr.decode("utf-8")) + .get("general", {}) + .get("server_url") + ) task.odk_token = encrypt_value(odk_token) print(f"Added odk token for task {task.id}") @@ -109,7 +111,6 @@ def qr_to_odktoken(): def encrypt_odk_creds(): """Encrypt project odk password in the db.""" - db = next(get_db()) projects = db.query(DbProject).all() @@ -121,7 +122,6 @@ def encrypt_odk_creds(): def decrypt_odk_creds(): """Decrypt project odk password in the db.""" - db = next(get_db()) projects = db.query(DbProject).all() @@ -132,10 +132,20 @@ def decrypt_odk_creds(): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Apply or revert changes to QR codes and odk tokens.") - parser.add_argument("--apply", action="store_true", help="Apply changes (convert QR codes to odk tokens).") - parser.add_argument("--revert", action="store_true", help="Revert changes (convert odk tokens to QR codes).") - + parser = argparse.ArgumentParser( + description="Apply or revert changes to QR codes and odk tokens." + ) + parser.add_argument( + "--apply", + action="store_true", + help="Apply changes (convert QR codes to odk tokens).", + ) + parser.add_argument( + "--revert", + action="store_true", + help="Revert changes (convert odk tokens to QR codes).", + ) + args = parser.parse_args() if args.apply: From b45139320709cf2ec59dad9d77d7e49dc84bb874 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 30 Jan 2024 11:06:39 +0000 Subject: [PATCH 17/20] refactor: remove assigned vars when not used --- src/backend/app/central/central_crud.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index c0be1abd13..3eeb0f2ef0 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -193,7 +193,7 @@ def upload_xform_media( status_code=500, detail={"message": "Connection failed to odk central"} ) from e - result = xform.uploadMedia(project_id, title, filespec) + xform.uploadMedia(project_id, title, filespec) result = xform.publishForm(project_id, title) return result @@ -235,9 +235,7 @@ def create_odk_xform( # This modifies an existing published XForm to be in draft mode. # An XForm must be in draft mode to upload an attachment. if upload_media: - result = xform.uploadMedia( - project_id, title, data, convert_to_draft_when_publishing - ) + xform.uploadMedia(project_id, title, data, convert_to_draft_when_publishing) result = xform.publishForm(project_id, title) return result From 751b8d905ed837a0946b7676be35f5c1a4ab3c37 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 30 Jan 2024 11:48:53 +0000 Subject: [PATCH 18/20] refactor: move password decrypt to model_post_init --- .../app/organisations/organisation_schemas.py | 16 +++++++--------- src/backend/app/projects/project_schemas.py | 10 ++++------ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/backend/app/organisations/organisation_schemas.py b/src/backend/app/organisations/organisation_schemas.py index 832021e8e0..3e0c035fcb 100644 --- a/src/backend/app/organisations/organisation_schemas.py +++ b/src/backend/app/organisations/organisation_schemas.py @@ -22,7 +22,6 @@ from fastapi import Form from pydantic import BaseModel, Field, HttpUrl, computed_field -from pydantic.functional_serializers import field_serializer from pydantic.functional_validators import field_validator from app.config import decrypt_value, encrypt_value @@ -98,22 +97,21 @@ class OrganisationOut(BaseModel): slug: Optional[str] url: Optional[str] type: OrganisationType + odk_central_url: Optional[str] = None class OrganisationOutWithCreds(BaseModel): """Organisation plus decrypted ODK Central password. - WARNING Do not display this to the user. + WARNING this model is for illustration only. + WARNING do not display this to the user. WARNING contains decrypted credentials. """ - odk_central_url: Optional[str] = None odk_central_user: Optional[str] = None odk_central_password: Optional[str] = None - @field_serializer("odk_central_password") - def decrypt_password(self, value: str) -> str: - """Decrypt the database password value.""" - if not value: - return "" - return decrypt_value(value) + def model_post_init(self, ctx): + """Run logic after model object instantiated.""" + # Decrypt odk central password from database + self.odk_central_password = decrypt_value(self.odk_central_password) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 05d950b012..d1c4e281a6 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -41,12 +41,10 @@ class ODKCentral(BaseModel): odk_central_user: str odk_central_password: str - @field_serializer("odk_central_password") - def decrypt_password(self, value: str) -> str: - """Decrypt the database password value.""" - if not value: - return "" - return decrypt_value(value) + def model_post_init(self, ctx): + """Run logic after model object instantiated.""" + # Decrypt odk central password from database + self.odk_central_password = decrypt_value(self.odk_central_password) @field_validator("odk_central_password", mode="before") @classmethod From 7a1c7bdea237a65fb08ddccf7fd427719c256466 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 30 Jan 2024 11:51:08 +0000 Subject: [PATCH 19/20] build: set central-db restart policy unless-stopped --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2e8597d398..c1cdef9cd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -239,7 +239,7 @@ services: - "5434:5432" networks: - fmtm-net - restart: "on-failure:3" + restart: "unless-stopped" healthcheck: test: pg_isready -U ${CENTRAL_DB_USER:-odk} -d ${CENTRAL_DB_NAME:-odk} start_period: 5s From e06a6d1b083fce484a41304f978a54aae79c30e4 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Tue, 30 Jan 2024 13:38:42 +0000 Subject: [PATCH 20/20] refactor: update odk password type to obfuscated SecretStr --- src/backend/app/central/central_crud.py | 6 ++-- .../app/organisations/organisation_schemas.py | 36 ++++++++++--------- src/backend/app/projects/project_crud.py | 22 +++++++++--- src/backend/app/projects/project_routes.py | 7 ---- src/backend/app/projects/project_schemas.py | 22 ++++++++---- 5 files changed, 54 insertions(+), 39 deletions(-) diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index 3eeb0f2ef0..185f557760 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -40,7 +40,7 @@ def get_odk_project(odk_central: project_schemas.ODKCentral = None): if odk_central: url = odk_central.odk_central_url user = odk_central.odk_central_user - pw = odk_central.odk_central_password + pw = odk_central.odk_central_password.get_secret_value() else: log.debug("ODKCentral connection variables not set in function") log.debug("Attempting extraction from environment variables") @@ -65,7 +65,7 @@ def get_odk_form(odk_central: project_schemas.ODKCentral = None): if odk_central: url = odk_central.odk_central_url user = odk_central.odk_central_user - pw = odk_central.odk_central_password + pw = odk_central.odk_central_password.get_secret_value() else: log.debug("ODKCentral connection variables not set in function") @@ -91,7 +91,7 @@ def get_odk_app_user(odk_central: project_schemas.ODKCentral = None): if odk_central: url = odk_central.odk_central_url user = odk_central.odk_central_user - pw = odk_central.odk_central_password + pw = odk_central.odk_central_password.get_secret_value() else: log.debug("ODKCentral connection variables not set in function") log.debug("Attempting extraction from environment variables") diff --git a/src/backend/app/organisations/organisation_schemas.py b/src/backend/app/organisations/organisation_schemas.py index 3e0c035fcb..adf9dc6bca 100644 --- a/src/backend/app/organisations/organisation_schemas.py +++ b/src/backend/app/organisations/organisation_schemas.py @@ -21,7 +21,7 @@ from typing import Optional from fastapi import Form -from pydantic import BaseModel, Field, HttpUrl, computed_field +from pydantic import BaseModel, Field, HttpUrl, SecretStr, computed_field from pydantic.functional_validators import field_validator from app.config import decrypt_value, encrypt_value @@ -45,7 +45,7 @@ class OrganisationIn(BaseModel): odk_central_user: Optional[str] = Field( Form(None, description="Organisation default ODK User") ) - odk_central_password: Optional[str] = Field( + odk_central_password: Optional[SecretStr] = Field( Form(None, description="Organisation default ODK Password") ) @@ -64,20 +64,21 @@ def convert_url_to_str(cls, value: HttpUrl) -> str: @property def slug(self) -> str: """Sanitise the organisation name for use in a URL.""" - if self.name: - # Remove special characters and replace spaces with hyphens - slug = sub(r"[^\w\s-]", "", self.name).strip().lower().replace(" ", "-") - # Remove consecutive hyphens - slug = sub(r"[-\s]+", "-", slug) - return slug + if not self.name: + return "" + # Remove special characters and replace spaces with hyphens + slug = sub(r"[^\w\s-]", "", self.name).strip().lower().replace(" ", "-") + # Remove consecutive hyphens + slug = sub(r"[-\s]+", "-", slug) + return slug @field_validator("odk_central_password", mode="before") @classmethod - def encrypt_odk_password(cls, value: str) -> str: + def encrypt_odk_password(cls, value: str) -> Optional[SecretStr]: """Encrypt the ODK Central password before db insertion.""" if not value: - return "" - return encrypt_value(value) + return None + return SecretStr(encrypt_value(value)) class OrganisationEdit(OrganisationIn): @@ -101,17 +102,18 @@ class OrganisationOut(BaseModel): class OrganisationOutWithCreds(BaseModel): - """Organisation plus decrypted ODK Central password. + """Organisation plus ODK Central credentials. - WARNING this model is for illustration only. - WARNING do not display this to the user. - WARNING contains decrypted credentials. + Note: the password is obsfucated as SecretStr. """ odk_central_user: Optional[str] = None - odk_central_password: Optional[str] = None + odk_central_password: Optional[SecretStr] = None def model_post_init(self, ctx): """Run logic after model object instantiated.""" # Decrypt odk central password from database - self.odk_central_password = decrypt_value(self.odk_central_password) + if self.odk_central_password: + self.odk_central_password = SecretStr( + decrypt_value(self.odk_central_password.get_secret_value()) + ) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index b9e83b344a..c1c22824c9 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -62,6 +62,7 @@ from app.db import db_models from app.db.database import get_db from app.db.postgis_utils import geojson_to_flatgeobuf, geometry_to_geojson +from app.models.enums import HTTPStatus from app.projects import project_schemas from app.s3 import add_obj_to_bucket, get_obj_from_bucket from app.tasks import tasks_crud @@ -255,6 +256,8 @@ async def create_project_with_project_info( """Create a new project, including all associated info.""" # FIXME the ProjectUpload model should be converted to the db model directly # FIXME we don't need to extract each variable and pass manually + # project_data = project_metadata.model_dump() + project_user = project_metadata.author project_info = project_metadata.project_info xform_title = project_metadata.xform_title @@ -268,11 +271,20 @@ async def create_project_with_project_info( # verify data coming in if not project_user: - raise HTTPException("User details are missing") + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + detail="User details are missing", + ) if not project_info: - raise HTTPException("Project info is missing") + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + detail="Project info is missing", + ) if not odk_project_id: - raise HTTPException("ODK Central project id is missing") + raise HTTPException( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + detail="ODK Central project id is missing", + ) log.debug( "Creating project in FMTM database with vars: " @@ -287,7 +299,7 @@ async def create_project_with_project_info( if odk_credentials: url = odk_credentials.odk_central_url user = odk_credentials.odk_central_user - pw = odk_credentials.odk_central_password + pw = odk_credentials.odk_central_password.get_secret_value() else: log.debug("ODKCentral connection variables not set in function") @@ -1182,7 +1194,7 @@ def generate_task_files( appuser = OdkAppUser( odk_credentials.odk_central_url, odk_credentials.odk_central_user, - odk_credentials.odk_central_password, + odk_credentials.odk_central_password.get_secret_value(), ) appuser_json = appuser.create(odk_id, appuser_name) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index e93eba1509..0c0c3b4676 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -249,17 +249,10 @@ async def create_project( """ log.debug(f"Creating project {project_info.project_info.name}") - if project_info.odk_central.odk_central_url.endswith("/"): - project_info.odk_central.odk_central_url = ( - project_info.odk_central.odk_central_url[:-1] - ) - odkproject = central_crud.create_odk_project( project_info.project_info.name, project_info.odk_central ) - # TODO check token against user or use token instead of passing user - # project_info.project_name_prefix = project_info.project_info.name project = await project_crud.create_project_with_project_info( db, project_info, odkproject["id"] ) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index d1c4e281a6..f7a02854e0 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -23,7 +23,7 @@ from dateutil import parser from geojson_pydantic import Feature as GeojsonFeature -from pydantic import BaseModel +from pydantic import BaseModel, SecretStr from pydantic.functional_serializers import field_serializer from pydantic.functional_validators import field_validator @@ -39,20 +39,28 @@ class ODKCentral(BaseModel): odk_central_url: str odk_central_user: str - odk_central_password: str + odk_central_password: SecretStr def model_post_init(self, ctx): """Run logic after model object instantiated.""" # Decrypt odk central password from database - self.odk_central_password = decrypt_value(self.odk_central_password) + self.odk_central_password = SecretStr( + decrypt_value(self.odk_central_password.get_secret_value()) + ) @field_validator("odk_central_password", mode="before") @classmethod - def encrypt_odk_password(cls, value: str) -> str: + def encrypt_odk_password(cls, value: str) -> SecretStr: """Encrypt the ODK Central password before db insertion.""" - if not value: - return "" - return encrypt_value(value) + return SecretStr(encrypt_value(value)) + + @field_validator("odk_central_url", mode="before") + @classmethod + def remove_trailing_slash(cls, value: str) -> str: + """Remove trailing slash from ODK Central URL.""" + if value.endswith("/"): + return value[:-1] + return value class ProjectInfo(BaseModel):