Merge pull request #186 from hotosm/build/update-pyproject

Update pyproject as template for others

.dockerignore

@@ -0,0 +1,10 @@
+# Ignore everything
+**
+
+# Allow files and directories
+!osm_fieldwork
+!pyproject.toml
+!pdm.lock
+!entrypoint.sh
+!README.md
+!LICENSE.md

.github/workflows/build.yml

@@ -0,0 +1,54 @@
+name: Build and Deploy
+
+on:
+  # Push includes PR merge
+  push:
+    branches:
+      - main
+    paths:
+      # Workflow is triggered only if src changes
+      - "src/**"
+  # Allow manual trigger
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  GIT_BRANCH: ${{ github.ref_name }}
+
+jobs:
+  build-and-push-images:
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ github.ref_name }}
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract version
+        id: extract_api_version
+        run: |
+          cd src/backend
+          echo "PKG_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)')" >> $GITHUB_ENV
+
+      - name: Build image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          target: prod
+          push: true
+          tags: |
+            "ghcr.io/hotosm/fmtm/osm-fieldwork:${{ env.PKG_VERSION }}"
+            "ghcr.io/hotosm/fmtm/osm-fieldwork:latest"
+          build-args: |
+            PKG_VERSION=${{ env.PKG_VERSION }}

.github/workflows/docs.yml

@@ -13,6 +13,21 @@ permissions:
   contents: write
 
 jobs:
+  build_doxygen:
+    name: Build Doxygen Docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup Doxygen
+        run: sudo apt-get update && sudo apt-get install -y doxygen
+
+      - name: Build Doxygen Docs
+        run: |
+          cd docs
+          doxygen
+
   publish_docs:
     name: Publish Docs
     runs-on: ubuntu-latest
@@ -24,7 +39,7 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.x
-      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV 
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
 
       - name: Read cache
         uses: actions/cache@v3
@@ -40,5 +55,5 @@ jobs:
 
       - name: Publish
         run: |
-          pdm install -G docs
+          pdm install -G docs --no-default
           pdm run mkdocs gh-deploy --force

.github/workflows/pytest.yml

@@ -0,0 +1,33 @@
+name: pytest
+
+on:
+  # Run tests on all pushed branches
+  push:
+    branches:
+      - "*"
+  # Run tests on PR, prior to merge to main & development.
+  pull_request:
+    branches:
+      - main
+  # Allow manual trigger (workflow_dispatch)
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ github.ref_name }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Start CI Container
+        run: >
+          docker run -d --name fmtm_pytest
+          -v "$GITHUB_WORKSPACE/osm_fieldwork:/home/appuser/.local/lib/python3.10/site-packages/osm_fieldwork"
+          -v "$GITHUB_WORKSPACE/tests:/data/tests"
+          ghcr.io/hotosm/osm-fieldwork:ci
+          sleep infinity
+
+      - name: Run PyTest as appuser
+        run: docker exec fmtm_pytest gosu appuser pytest

.gitignore

@@ -54,7 +54,6 @@ NEWS
 
 # PDM
 .pdm.toml
-pdm.lock
 .pdm-python
 
 # Python builds

.pre-commit-config.yaml

@@ -1,36 +1,35 @@
 repos:
   # Versioning: Commit messages & changelog
   - repo: https://github.com/commitizen-tools/commitizen
-    rev: v2.27.1
+    rev: v3.6.0
     hooks:
       - id: commitizen
         stages: [commit-msg]
 
   # Autoformat: Python code
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 23.7.0
     hooks:
       - id: black
-        args: [--target-version=py39]
 
   # Lint / autoformat: Python code
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.0.248"
+    rev: "v0.0.285"
     hooks:
       - id: ruff
-        args: [--fix, --exit-non-zero-on-fix, --target-version, py39]
+        args: [--exit-non-zero-on-fix]
 
   # Autoformat: YAML, JSON, Markdown, etc.
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.6.1
+    rev: v3.0.2
     hooks:
       - id: prettier
         args: [--ignore-unknown, --no-error-on-unmatched-pattern, "!chart/**"]
 
   # Lint: Markdown
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.31.1
+    rev: v0.35.0
     hooks:
       - id: markdownlint
         args: [--fix]

Dockerfile

@@ -0,0 +1,148 @@
+# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# This file is part of osm-fieldwork.
+#
+#     This program is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     This program is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with osm-fieldwork.  If not, see <https:#www.gnu.org/licenses/>.
+#
+ARG PYTHON_IMG_TAG=3.10
+
+
+FROM docker.io/python:${PYTHON_IMG_TAG}-slim-bookworm as base
+ARG PYTHON_IMG_TAG
+ARG PKG_VERSION
+ARG [email protected]
+LABEL org.hotosm.osm-fieldwork.python-img-tag="${PYTHON_IMG_TAG}" \
+      org.hotosm.osm-fieldwork.maintainer="${MAINTAINER}" \
+      org.hotosm.osm-fieldwork.version="${PKG_VERSION}"
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends "locales" "ca-certificates" \
+    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && update-ca-certificates
+# Set locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+
+
+FROM base as extract-deps
+WORKDIR /opt/python
+COPY pyproject.toml pdm.lock /opt/python/
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir pdm==2.6.1
+RUN pdm export --prod > requirements.txt \
+    && pdm export -G debug -G test \
+        --no-default > requirements-test.txt
+
+
+
+FROM base as build-wheel
+ARG PKG_VERSION
+WORKDIR /build
+COPY . .
+RUN pip install pdm==2.6.1 \
+    && pdm build
+
+
+
+FROM base as build
+ARG PKG_VERSION
+WORKDIR /opt/python
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends \
+        "build-essential" \
+        "gcc" \
+        "libpcre3-dev" \
+        "libspatialindex-dev" \
+        "libproj-dev" \
+        "libgeos-dev" \
+        "libgdal-dev" \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=extract-deps --chown=appuser \
+    /opt/python/requirements.txt /opt/python/
+RUN pip install --user --no-warn-script-location \
+    --no-cache-dir -r ./requirements.txt
+COPY --from=build-wheel --chown=appuser \
+    "/build/dist/osm_fieldwork-$PKG_VERSION-py3-none-any.whl" .
+RUN pip install --user --no-warn-script-location \
+    --no-cache-dir "/opt/python/osm_fieldwork-$PKG_VERSION-py3-none-any.whl"
+
+
+
+FROM base as runtime
+ARG PYTHON_IMG_TAG
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONFAULTHANDLER=1 \
+    PATH="/home/appuser/.local/bin:$PATH" \
+    PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_IMG_TAG/site-packages" \
+    SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
+    REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
+    CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+RUN set -ex \
+    && apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install \
+    -y --no-install-recommends \
+        "nano" \
+        "curl" \
+        "gosu" \
+        "libpcre3" \
+        "libglib2.0-0" \
+        "libspatialindex-c6" \
+        "libproj25" \
+        "libgeos-c1v5" \
+        "libgdal32" \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=build \
+    /root/.local \
+    /home/appuser/.local
+WORKDIR /data
+COPY entrypoint.sh /container-entrypoint.sh
+# Add non-root user (match Github runner), permissions
+RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser \
+    && chown -R appuser:appuser /home/appuser \
+    && chmod +x /container-entrypoint.sh \
+    && chmod -R 777 /data
+# Change to non-root user (for compile below)
+USER appuser
+
+
+
+FROM runtime as ci
+COPY --from=extract-deps --chown=appuser \
+    /opt/python/requirements-test.txt /opt/python/
+RUN pip install --user --no-warn-script-location \
+    --no-cache-dir -r /opt/python/requirements-test.txt
+# Pre-compile packages to .pyc (init speed gains)
+RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
+# Set entrypoint and cmd from default for CI
+ENTRYPOINT [""]
+CMD [""]
+# Change to root, use gosu at runtime
+USER root
+
+
+
+FROM runtime as prod
+# Pre-compile packages to .pyc (init speed gains)
+RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
+# Change to root, use gosu at runtime
+USER root
+ENTRYPOINT ["/container-entrypoint.sh"]
+CMD ["bash"]

README.md

@@ -1,5 +1,7 @@
 # OSM Fieldwork Project
 
+🕮 [Documentation](https://hotosm.github.io/osm-fieldwork/)
+
 Osm-Fieldwork is a project for processing data collection using
 OpenDataKit into OpenStreetMap format. It includes several utility
 programs that automate part of the data flow like creating satellite
@@ -29,7 +31,7 @@ To install osm-fieldwork, you can use pip. Here are two options:
   `pip install git+https://github.com/hotosm/osm-fieldwork.git`
 
 - Latest on PyPi:
-  `pip install Osm-Fieldwork`
+  `pip install osm-fieldwork`
 
 ### Configure
 
@@ -64,6 +66,25 @@ course for this to work. That file looks like this:
 **ODK_CENTRAL_SECURE**
 > If set to False, will allow insecure connections to the ODKCentral API. Else defaults to True.
 
+## Using the Container Image
+
+- osm-fieldwork scripts can be used via the pre-built container images.
+- These images come with all dependencies bundled, so are simple to run.
+
+Run a specific command:
+
+```bash
+docker run --rm -v $PWD:/data ghcr.io/hotosm/osm-fieldwork:0.3.5 json2osm
+```
+
+Run interactively (to use multiple commands):
+
+```bash
+docker run --rm -it -v $PWD:/data ghcr.io/hotosm/osm-fieldwork:0.3.5
+```
+
+> Note: the output directory should always be /data/... to persist data.
+
 # Utility Programs
 
 These programs are more fully documented [in this](wiki/programs.md)