Merge branch 'staging'

.all-contributorsrc

@@ -141,6 +141,13 @@
       "avatar_url": "https://avatars.githubusercontent.com/u/75356640?v=4",
       "profile": "https://www.el-cordovez.com",
       "contributions": ["doc"]
+    },
+    {
+      "login": "Prajwalism",
+      "name": "Prajwal Khadgi",
+      "avatar_url": "https://avatars.githubusercontent.com/u/123072058?v=4",
+      "profile": "https://github.com/Prajwalism",
+      "contributions": ["code"]
     }
   ],
   "contributorsPerLine": 7,

README.md

@@ -287,6 +287,7 @@ Thanks goes to these wonderful people:
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/Balofire"><img src="https://avatars.githubusercontent.com/u/102294666?v=4?s=100" width="100px;" alt="Ahmeed Etti-Balogun"/><br /><sub><b>Ahmeed Etti-Balogun</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=Balofire" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/Roseford"><img src="https://avatars.githubusercontent.com/u/75838716?v=4?s=100" width="100px;" alt="Uju"/><br /><sub><b>Uju</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=Roseford" title="Documentation">📖</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://www.el-cordovez.com"><img src="https://avatars.githubusercontent.com/u/75356640?v=4?s=100" width="100px;" alt="JC CorMan"/><br /><sub><b>JC CorMan</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=cordovez" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Prajwalism"><img src="https://avatars.githubusercontent.com/u/123072058?v=4?s=100" width="100px;" alt="Prajwal Khadgi"/><br /><sub><b>Prajwal Khadgi</b></sub></a><br /><a href="https://github.com/hotosm/fmtm/commits?author=Prajwalism" title="Code">💻</a></td>
     </tr>
   </tbody>
   <tfoot>

contrib/encrypter/Dockerfile

@@ -0,0 +1,60 @@
+# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# This file is part of FMTM.
+#
+#     FMTM is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     FMTM is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
+#
+ARG PYTHON_IMG_TAG=3.10
+
+
+# Includes all labels and timezone info to extend from
+FROM docker.io/python:${PYTHON_IMG_TAG}-slim-bookworm as base
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends "locales" "ca-certificates" \
+    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && update-ca-certificates
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+
+# Build stage will all dependencies required to build Python wheels
+FROM base as build
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends \
+        "build-essential" \
+        "gcc" \
+    && rm -rf /var/lib/apt/lists/*
+RUN pip install --user --no-warn-script-location \
+    --no-cache-dir cryptography==42.0.5
+
+
+# Run stage will minimal dependencies required to run Python libraries
+FROM base as runtime
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONFAULTHANDLER=1
+# Copy Python deps from build to runtime
+COPY --from=build \
+    /root/.local \
+    /root/.local
+WORKDIR /code
+COPY encrypter.py .
+ENTRYPOINT ["python", "encrypter.py"]

contrib/encrypter/README.md

@@ -0,0 +1,42 @@
+# Encrypter Util
+
+FMTM encrypts the ODK access token & ODK passwords in the database.
+
+Sometimes these must be manually encrypted / decrypted for the database.
+
+This util makes that process easier.
+
+```bash
+usage: encrypter.py [-h] [--encrypt] [--decrypt] key value
+
+Encrypt or decrypt string values.
+
+positional arguments:
+  key         Encryption key.
+  value       Value to encrypt or decrypt.
+
+options:
+  -h, --help  show this help message and exit
+  --encrypt   Encrypt the value.
+  --decrypt   Decrypt the value.
+```
+
+> Note the values may need to be quoted, as they often contain special chars.
+
+## Encrypt a value
+
+```bash
+docker run -i --rm ghcr.io/hotosm/fmtm/encrypter:latest \
+  --encrypt \
+  your_encryption_token \
+  some_value_to_encrypt
+```
+
+## Decrypt a value
+
+```bash
+docker run -i --rm ghcr.io/hotosm/fmtm/encrypter:latest \
+  --decrypt \
+  your_encryption_token \
+  some_value_to_encrypt
+```

contrib/encrypter/build.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+docker build . -t ghcr.io/hotosm/fmtm/encrypter:latest
+
+docker push ghcr.io/hotosm/fmtm/encrypter:latest

contrib/encrypter/encrypter.py

@@ -0,0 +1,76 @@
+"""Convert between JSON and QRCode formats."""
+
+import sys
+import argparse
+import base64
+from cryptography.fernet import Fernet
+
+
+
+def get_cipher_suite(key):
+    """Cache cypher suite."""
+    return Fernet(key)
+
+
+def encrypt_value(key: str, value: str) -> str:
+    """Encrypt value before going to the DB."""
+    cipher_suite = get_cipher_suite(key)
+    encrypted_password = cipher_suite.encrypt(value.encode("utf-8"))
+    return base64.b64encode(encrypted_password).decode("utf-8")
+
+
+def decrypt_value(key: str, value: str) -> str:
+    """Decrypt the database value."""
+    cipher_suite = get_cipher_suite(key)
+    encrypted_password = base64.b64decode(value.encode("utf-8"))
+    decrypted_password = cipher_suite.decrypt(encrypted_password)
+    return decrypted_password.decode("utf-8")
+
+
+def display_value(value: str) -> None:
+    """Pretty print the final value."""
+    print("")
+    print("Value:")
+    print("")
+    print(value)
+    print("")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Encrypt or decrypt string values."
+    )
+    parser.add_argument(
+        "key",
+        help="Encryption key.",
+    )
+    parser.add_argument(
+        "value",
+        help="Value to encrypt or decrypt.",
+    )
+    parser.add_argument(
+        "--encrypt",
+        action="store_true",
+        help="Encrypt the value.",
+    )
+    parser.add_argument(
+        "--decrypt",
+        action="store_true",
+        help="Decrypt the value.",
+    )
+
+    args = parser.parse_args()
+
+    if args.encrypt and args.decrypt:
+        print("Cannot both encrypt and decrypt at the same time.")
+        print("Pass one of either: --encrypt or --decrypt")
+        sys.exit(1)
+
+    if args.encrypt:
+        result = encrypt_value(args.key, args.value)
+        display_value(result)
+    elif args.decrypt:
+        result = decrypt_value(args.key, args.value)
+        display_value(result)
+    else:
+        print("Please provide either --encrypt or --decrypt flag.")  

contrib/qrcode_util/README.md

@@ -1,5 +1,18 @@
 # QR Code Utils
 
+Small util for encoding and decoding QR code content.
+
+```bash
+usage: qrcode_util.py [-h] [--write] [--read]
+
+Encode or decode QR code data.
+
+options:
+  -h, --help  show this help message and exit
+  --write     Write data from STDIN to a QRCode.
+  --read      Read QRCode data from STDIN and print to terminal.
+```
+
 ## Convert QR Code to JSON
 
 ```bash

docs/qrcode-util.md

@@ -0,0 +1,26 @@
+<!-- markdownlint-disable -->
+<style>
+    h1 {
+        /* Remove space for heading */
+        display: none;
+    }
+
+    /* .md-main {
+    } */
+
+    iframe {
+        width: 100vw;
+        height: 80vh;
+        border: none;
+        margin: 0;
+        padding: 0;
+        overflow: hidden;
+    }
+</style>
+
+<!-- Hide title -->
+<h1></h1>
+
+<iframe src="https://hotosm.github.io/qrcodes" scrolling="no">
+</iframe>
+<!-- markdownlint-enable -->

mkdocs.yml

@@ -15,12 +15,16 @@ extra:
       link: "https://twitter.com/hotosm"
     - icon: "fontawesome/solid/globe"
       link: "https://www.hotosm.org"
+  copyright: Copyright © 2010 HOTOSM
+  generator: false
 
 theme:
   name: material
   palette:
     primary: custom
   language: en
+  favicon: images/favicon.png
+  logo: images/hot_logo.png
 
 extra_css:
   - css/extra.css
@@ -83,3 +87,4 @@ nav:
   - API: https://hotosm.github.io/swagger/?url=https://hotosm.github.io/fmtm/openapi.json
   - Class Hierarchy: apidocs/html/inherits.html
   - Code Coverage: coverage.html
+  - QRCode Converter: qrcode-util.md

src/backend/app/central/central_crud.py

@@ -357,45 +357,42 @@ def download_submissions(
     return fixed.splitlines()
 
 
-async def test_form_validity(xform_content: str, form_type: str):
+async def test_form_validity(xform_content: bytes, form_type: str):
     """Validate an XForm.
 
     Args:
         xform_content (str): form to be tested
         form_type (str): type of form (xls or xlsx).
     """
     try:
-        xlsform_path = f"/tmp/validate_form.{form_type}"
-        outfile = "/tmp/outfile.xml"
-
-        with open(xlsform_path, "wb") as f:
-            f.write(xform_content)
-
-        xls2xform_convert(xlsform_path=xlsform_path, xform_path=outfile, validate=False)
-
-        namespaces = {
-            "h": "http://www.w3.org/1999/xhtml",
-            "odk": "http://www.opendatakit.org/xforms",
-            "xforms": "http://www.w3.org/2002/xforms",
-        }
-
-        with open(outfile, "r") as xml:
-            data = xml.read()
-
-        root = ElementTree.fromstring(data)
-        instances = root.findall(".//xforms:instance[@src]", namespaces)
-
-        geojson_list = []
-        for inst in instances:
-            try:
-                if "src" in inst.attrib:
-                    if (inst.attrib["src"].split("."))[1] == "geojson":
-                        parts = (inst.attrib["src"].split("."))[0].split("/")
-                        geojson_name = parts[-1]
-                        geojson_list.append(geojson_name)
-            except Exception:
-                continue
+        if form_type != "xml":
+            # Write xform_content to a temporary file
+            with open(f"/tmp/xform_temp.{form_type}", "wb") as f:
+                f.write(xform_content)
+        else:
+            with open(f"/tmp/xlsform.{form_type}", "wb") as f:
+                f.write(xform_content)
+            # Convert XLSForm to XForm
+            xls2xform_convert(
+                xlsform_path="/tmp/xlsform.xls",
+                xform_path="/tmp/xform_temp.xml",
+                validate=False,
+            )
+
+        # Parse XForm
+        namespaces = {"xforms": "http://www.w3.org/2002/xforms"}
+        tree = ElementTree.parse("/tmp/xform_temp.xml")
+        root = tree.getroot()
+
+        # Extract geojson filenames
+        geojson_list = [
+            os.path.splitext(inst.attrib["src"].split("/")[-1])[0]
+            for inst in root.findall(".//xforms:instance[@src]", namespaces)
+            if inst.attrib.get("src", "").endswith(".geojson")
+        ]
+
         return {"required media": geojson_list, "message": "Your form is valid"}
+
     except Exception as e:
         return JSONResponse(
             content={"message": "Your form is invalid", "possible_reason": str(e)},

src/backend/app/main.py

@@ -45,6 +45,7 @@
 
 # Add sentry tracing only in prod
 if not settings.DEBUG:
+    log.info("Adding Sentry tracing")
     sentry_sdk.init(
         dsn=settings.SENTRY_DSN,
         traces_sample_rate=0.1,