setup artificial data generation

.cruft.json

@@ -0,0 +1,14 @@
+{
+  "template": "https://github.com/robert-koch-institut/mex-template",
+  "commit": "a4f25ab84f9e485ad77eb03663a9cf486f7a5826",
+  "checkout": null,
+  "context": {
+    "cookiecutter": {
+      "project_name": "artificial",
+      "short_summary": "Create artificial data for the MEx project.",
+      "long_summary": "Create artificial extracted items, transform them into merged items and write the results into a configured sink.",
+      "_template": "https://github.com/robert-koch-institut/mex-template"
+    }
+  },
+  "directory": null
+}

.dockerignore

@@ -0,0 +1,149 @@
+# Byte-compiled files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.eggs/
+.installed.cfg
+.pdm-build
+.pdm-python
+.Python
+.web/
+*.egg
+*.egg-info/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+lib/
+lib64/
+MANIFEST
+parts/
+sdist/
+share/python-wheels/
+var/
+wheels/
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+tests/
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+test*.jpeg
+test*.png
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+pytestdebug.log
+
+# Translations
+*.mo
+*.pot
+
+# Scrapy stuff
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# PEP 582; used by e.g. pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Environments
+*.env
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# ruff
+.ruff_cache
+
+# PyCharm
+.idea/
+.idea_modules/
+
+# VisualStudioCode
+.vscode/*
+.history/
+*.code-workspace
+
+# SQLite databases
+*.db
+
+# Default exports
+*.ndjson
+data/
+identity.csv
+schema.json
+tmp*/
+work/
+
+# Docker specifics
+**/.dockerignore
+**/.git
+**/.github
+**/.gitignore
+**/compose*
+**/docker-compose*
+**/Dockerfile*
+
+# Setup files
+.bandit
+.isort.cfg
+.mypy.ini
+.pdm-python
+.pre-commit-config.yaml
+.pydocstyle
+Makefile
+mex.bat
+*.lock
+poetry.toml
+pytest.ini
+requirements.txt

.github/pull_request_template.md

@@ -0,0 +1,20 @@
+# PR Context
+<!-- Additional info for the reviewer -->
+
+# Added
+<!-- New features and interfaces -->
+
+# Changes
+<!-- Changes in existing functionality -->
+
+# Deprecated
+<!-- Soon-to-be removed features -->
+
+# Removed
+<!-- Definitely removed features -->
+
+# Fixed
+<!-- Fixed bugs -->
+
+# Security
+<!-- Fixed vulnerabilities -->

.github/workflows/cookiecutter.yml

@@ -0,0 +1,88 @@
+name: Cookiecutter
+
+on:
+  push:
+    branches: ["main"]
+    tags: ["**"]
+  schedule:
+    - cron: "14 3 * * 1-5"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  PIP_NO_OPTION: on
+  PIP_NO_CLEAN: on
+  PIP_PREFER_BINARY: on
+
+jobs:
+  cruft:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.WORKFLOW_TOKEN }}
+
+      - name: Cache requirements
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-requirements
+        with:
+          path: ~/.cache/pip
+          key: ${{ env.cache-name }}-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ env.cache-name }}-
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install requirements
+        run: make setup
+
+      - name: Configure git
+        env:
+          MEX_BOT_EMAIL: ${{ vars.MEX_BOT_EMAIL }}
+          MEX_BOT_USER: ${{ vars.MEX_BOT_USER }}
+          SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
+          SIGNING_PUB: ${{ secrets.SIGNING_PUB }}
+        run: |
+          eval "$(ssh-agent -s)"
+          pdm setup-commit-signing
+
+      - name: Update template
+        env:
+          GH_TOKEN: ${{ secrets.WORKFLOW_TOKEN }}
+        run: |
+          if cruft check; then
+            echo template is up to date
+            exit 0
+          fi
+          if [[ $(gh pr list --label cruft | wc -c) -ne 0 ]]; then
+            echo already seeing pull request
+            exit 0
+          fi
+          template_url=$(python -c "print(__import__('json').load(open('.cruft.json'))['template'])")
+          template_ref=$(git ls-remote ${template_url} --heads main --exit-code | cut -c -6)
+          git checkout main
+          git checkout -b cruft/cookiecutter-template-${template_ref}
+          cruft update --skip-apply-ask
+          printf '# Changes\n\n- bumped cookiecutter template to %s/commit/%s\n' "$template_url" "$template_ref" > .cruft-pr-body
+          if [[ $(find . -type f -name "*.rej" | wc -l) -ne 0 ]]; then
+            printf '\n# Conflicts\n' >> .cruft-pr-body
+          fi
+          find . -type f -name "*.rej" | while read -r line ; do
+            printf '\n```' >> .cruft-pr-body
+            cat ${line} >> .cruft-pr-body
+            printf '```\n' >> .cruft-pr-body
+          done
+          git add --all --verbose
+          git commit --message "Bump cookiecutter template to $template_ref" --verbose
+          git push --set-upstream origin cruft/cookiecutter-template-${template_ref} --force --verbose
+          gh pr create --title "Bump cookiecutter template to $template_ref" --body-file .cruft-pr-body --label cruft --assignee ${{ vars.MEX_BOT_USER }}

.github/workflows/cve-scan.yml

@@ -0,0 +1,77 @@
+name: CVE Scan
+
+on:
+  push:
+    branches: ["main"]
+    tags: ["**"]
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+  schedule:
+    - cron: "14 3 * * 1-5"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Cache requirements
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-requirements
+        with:
+          path: ~/.cache/pip
+          key: ${{ env.cache-name }}-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            ${{ env.cache-name }}-
+
+      - name: Cache environment
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-environment
+        with:
+          path: ~/.cache/pip
+          key: ${{ env.cache-name }}-${{ hashFiles('*.lock') }}
+          restore-keys: |
+            ${{ env.cache-name }}-
+
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Install requirements
+        run: make setup
+
+      - name: Export dependencies
+        run: |
+          mkdir --parents pdm
+          pdm export-all > pdm/requirements.txt
+
+      - name: Run trivy
+        uses: aquasecurity/trivy-action@master
+        with:
+          format: 'sarif'
+          list-all-pkgs: 'true'
+          output: 'trivy-results.sarif'
+          scan-ref: '.'
+          scan-type: 'fs'
+          severity: 'MEDIUM,HIGH,CRITICAL'
+
+      - name: Publish results
+        uses: github/codeql-action/upload-sarif@v3
+        if: always()
+        with:
+          sarif_file: 'trivy-results.sarif'