MRG: Merge pull request #69 from aerosense-ai/feature/multiple-node-s…

…upport

Support multiple nodes

.devcontainer/.gitignore

@@ -0,0 +1 @@
+.zsh_history

.devcontainer/Dockerfile

@@ -0,0 +1,40 @@
+# A useful base repository based on ms devcontainer but with a bunch of fixes
+# and useful installs (geo tools are unnecessary for this project, but all the other tweaks are helpful)
+FROM windpioneers/gdal-python:familiar-catshark-gdal-2.4.1-python-3.9-dev
+
+# Tell zsh where you want to store history
+#     We leave you to decide, but if you put this into a folder that's been mapped
+#     into the container, then history will persist over container rebuilds :)
+#
+#     !!!IMPORTANT!!!
+#     Make sure your .zsh_history file is NOT committed into your repository, as it can contain
+#     sensitive information. So in this case, you should add
+#         .devcontainer/.zsh_history
+#     to your .gitignore file.
+#
+ENV HISTFILE="/workspaces/data-gateway/.devcontainer/.zsh_history"
+
+# Switch to vscode user
+USER vscode
+WORKDIR /workspaces/data-gateway
+
+# Install the rust toolchain and give permission for all users to use it
+ENV RUST_INSTALL_DIR=/home/vscode/.rust
+ENV RUSTUP_HOME="${RUST_INSTALL_DIR}/rustup"
+ENV CARGO_HOME="${RUST_INSTALL_DIR}/cargo"
+RUN mkdir -p ${RUSTUP_HOME} && \
+    mkdir -p ${CARGO_HOME}
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+ENV PATH="${CARGO_HOME}/bin:${PATH}"
+RUN chmod -R ugo+rwx ${RUST_INSTALL_DIR}
+
+# Install poetry
+RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
+ENV PATH "/home/vscode/.poetry/bin:$PATH"
+RUN poetry config virtualenvs.create false
+
+# Install python dependencies. Note that poetry installs any root packages by default,
+# But this is not available at this stage of caching dependencies. So we do a dependency-only install here
+# to cache the dependencies, then a full poetry install post-create to install any root packages.
+COPY pyproject.toml poetry.lock ./
+RUN poetry install --no-ansi --no-interaction --no-root

.devcontainer/devcontainer.json

@@ -0,0 +1,84 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.187.0/containers/python-3
+{
+  "name": "Data-Gateway Devcontainer",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": ".."
+  },
+  // Set *default* container specific settings.json values on container create.
+  "settings": {
+    "austin.mode": "Wall time",
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.formatOnSave": true,
+    "esbonio.server.enabled": true,
+    "esbonio.sphinx.confDir": "${workspaceFolder}/docs/source",
+    "jupyter.widgetScriptSources": ["jsdelivr.com", "unpkg.com"],
+    "prettier.prettierPath": "/usr/local/prettier",
+    "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+    "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+    "python.formatting.provider": "black",
+    "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+    "python.languageServer": "Pylance",
+    "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+    "python.linting.enabled": true,
+    "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+    "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+    "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+    "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+    // Line length to match black settings
+    // Disabling specific messages:
+    //  - To find the details do: /usr/local/py-utils/bin/pylint --list-msgs
+    //  - Disable missing-module-docstring (C0114) because we don't document modules routinely, just their members
+    //  - Disable invalid-name (C0103) because pylint thinks that eg 'x', 'df', 'np' are invalid due to their lengths
+    "python.linting.pylintArgs": [
+      "--max-line-length=120",
+      "--disable=missing-module-docstring,invalid-name"
+    ],
+    "python.linting.pylintEnabled": true,
+    "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
+    "python.pythonPath": "/usr/local/bin/python",
+    // Scrolling the editor is a nice idea but it doesn't work: always out of sync and impossible to manage
+    "restructuredtext.preview.scrollEditorWithPreview": false,
+    "restructuredtext.preview.scrollPreviewWithEditor": false,
+    "restructuredtext.linter.doc8.extraArgs": ["--max-line-length 180"],
+    "terminal.integrated.defaultProfile.linux": "zsh"
+  },
+
+  // Add the IDs of extensions you want installed when the container is created.
+  "extensions": [
+    "bungcip.better-toml",
+    "[email protected]",
+    "irongeek.vscode-env",
+    "lextudio.restructuredtext",
+    "me-dutour-mathieu.vscode-github-actions",
+    "mikestead.dotenv",
+    "ms-python.python",
+    "ms-python.vscode-pylance",
+    "ms-toolsai.jupyter",
+    "ms-toolsai.jupyter-renderers",
+    "ms-toolsai.jupyter-keymap",
+    "ms-vsliveshare.vsliveshare",
+    "p403n1x87.austin-vscode",
+    "ritwickdey.liveserver",
+    "trond-snekvik.simple-rst"
+  ],
+
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  "forwardPorts": [80, 443, 5000, 7045, 7046, 7047, 7048, 7049, 8000, 8080],
+
+  // Poetry install *with* the root, which can't be cached in the docker layers (see dockerfile)
+  "postCreateCommand": "poetry install && pre-commit install && pre-commit install -t commit-msg",
+
+  // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+  "remoteUser": "vscode",
+
+  // Allow ptrace based debuggers (like austin) to work in the container
+  "runArgs": [
+    "--env-file",
+    "${localWorkspaceFolder}/.env",
+    "--cap-add=SYS_PTRACE",
+    "--security-opt",
+    "seccomp=unconfined"
+  ]
+}

.dockerignore

@@ -0,0 +1,9 @@
+**/.DS_Store
+.coverage
+.devcontainer/
+.env
+.pytest_cache/
+.tox/
+.venv/
+.vscode/
+*.wg.conf

.github/PULL_REQUEST_TEMPLATE.md

@@ -3,12 +3,12 @@
 <!-- However, any part of the final PR description (i.e. the description at the point of the final commit to the PR) can be changed and will be retained when producing the release notes -->
 <!-- Please don't add features not discussed in an issue first -->
 
-## Summary
+# Summary
 
 <!--- START AUTOGENERATED NOTES --->
 <!--- END AUTOGENERATED NOTES --->
 
-## Quality Checklist
+# Quality Checklist
 
 - [ ] New features are fully tested (No matter how much Coverage Karma you have)
 - [ ] **[v0.2 onward]** New features are included in the documentation

.github/workflows/balena.yml

@@ -0,0 +1,17 @@
+name: balena
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  balena-push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/[email protected]
+      - uses: theaccordance/[email protected]
+        with:
+          api-token: ${{secrets.BALENA_API_TOKEN}}
+          application-name: ${{secrets.BALENA_APPLICATION_NAME}}

.github/workflows/gcloud-deploy.yml

@@ -0,0 +1,68 @@
+name: gcloud-deploy
+
+on:
+  push:
+    branches:
+      - main
+      - test
+
+jobs:
+  deploy:
+    permissions:
+      contents: "read"
+      id-token: "write"
+    runs-on: ubuntu-latest
+    steps:
+      - id: checkout
+        uses: actions/checkout@v2
+
+      - name: Get prefix (test- or nothing for production)
+        id: prefix
+        run: |
+          if [ "${{ github.ref }}" = "main" ]; then
+            echo "::set-output name=name_prefix::"
+          else
+            echo "::set-output name=name_prefix::test-"
+          fi
+
+      - id: auth
+        uses: google-github-actions/auth@v0
+        with:
+          workload_identity_provider: "projects/885434704038/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider"
+          service_account: "[email protected]"
+
+      - id: deploy-add-sensor-type
+        uses: google-github-actions/deploy-cloud-functions@v0
+        with:
+          name: ${{ steps.prefix.outputs.name_prefix }}add-sensor-type
+          description: Allows addition of a new sensor type whose values will be accepted into the database
+          entry_point: add_sensor_type
+          runtime: python39
+          region: europe-west6
+          env_vars: BIG_QUERY_DATASET_NAME=${{ steps.prefix.outputs.name_prefix }}greta,COMPUTE_PROVIDER=GOOGLE_CLOUD_FUNCTION,DESTINATION_PROJECT_NAME=aerosense-twined
+          source_dir: cloud_functions
+
+      - id: deploy-create-installation
+        uses: google-github-actions/deploy-cloud-functions@v0
+        with:
+          name: ${{ steps.prefix.outputs.name_prefix }}create-installation
+          description: Allows creation of a new installation
+          entry_point: create_installation
+          runtime: python39
+          region: europe-west6
+          env_vars: BIG_QUERY_DATASET_NAME=${{ steps.prefix.outputs.name_prefix }}greta,COMPUTE_PROVIDER=GOOGLE_CLOUD_FUNCTION,DESTINATION_PROJECT_NAME=aerosense-twined
+          source_dir: cloud_functions
+
+      - id: deploy-ingress-eu
+        uses: google-github-actions/deploy-cloud-functions@v0
+        with:
+          name: ${{ steps.prefix.outputs.name_prefix }}ingress-eu
+          entry_point: upload_window
+          runtime: python39
+          region: europe-west6
+          memory: 1GB
+          env_vars: BIG_QUERY_DATASET_NAME=${{ steps.prefix.outputs.name_prefix }}greta,COMPUTE_PROVIDER=GOOGLE_CLOUD_FUNCTION,DESTINATION_PROJECT_NAME=aerosense-twined,DESTINATION_BUCKET_NAME=data-gateway-processed-data,SOURCE_PROJECT_NAME=aerosense-twined
+          source_dir: cloud_functions
+          event_trigger_type: google.storage.object.finalize
+          event_trigger_resource: projects/_/buckets/${{ steps.prefix.outputs.name_prefix }}aerosense-ingress-eu
+          event_trigger_service: storage.googleapis.com

.gitignore

@@ -105,3 +105,9 @@ ENV/
 
 .DS_store
 .pytest_cache
+
+# Wireguard credentials
+*.wg.conf
+
+# VSCode local settings
+.vscode

.pre-commit-config.yaml

@@ -1,8 +1,8 @@
-exclude: 'build|docs|node_modules|.git|.tox|dist|docs|data_gateway.egg-info'
+exclude: "build|docs|node_modules|.git|.tox|dist|docs|data_gateway.egg-info"
 default_stages: [commit]
 fail_fast: true
 default_language_version:
-  python: python3  # force all unspecified python hooks to run python3
+  python: python3 # force all unspecified python hooks to run python3
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
@@ -18,58 +18,60 @@ repos:
       - id: isort
 
   - repo: https://github.com/psf/black
-    rev: 21.6b0
+    rev: 22.3.0
     hooks:
       - id: black
-        args: ['--line-length', '120']
+        args: ["--line-length", "120"]
 
   - repo: https://gitlab.com/pycqa/flake8
     rev: 3.9.2
     hooks:
       - id: flake8
         language_version: python3
         additional_dependencies:
-          - 'pep8-naming'
+          - "pep8-naming"
         args:
           - --ignore-names=setUp,tearDown,setUpClass,tearDownClass,asyncSetUp,asyncTearDown,setUpTestData,failureException,longMessage,maxDiff,startTestRun,stopTestRun
 
   - repo: https://github.com/pycqa/pydocstyle
     rev: 6.1.1
     hooks:
-    - id: pydocstyle
+      - id: pydocstyle
 
   - repo: https://github.com/thclark/pre-commit-sphinx
     rev: 0.0.3
     hooks:
       - id: build-docs
         language_version: python3
         additional_dependencies:
-          - 'poetry>=1,<2'
-          - 'Sphinx>=4,<5'
-          - 'sphinx-rtd-theme>=1,<2'
-          - 'sphinx-tabs>=3,<4'
-          - 'sphinx-autoapi==1.8.4'
+          - "poetry>=1,<2"
+          - "Sphinx>=4,<5"
+          - "sphinx-rtd-theme>=1,<2"
+          - "sphinx-tabs>=3,<4"
+          - "sphinx-autoapi==1.8.4"
 
   - repo: https://github.com/windpioneers/pre-commit-hooks
     rev: 0.0.5
     hooks:
       - id: check-branch-name
         language_version: python3
         args:
-          - '^main$'
-          - '^development$'
-          - '^devops/([a-z][a-z0-9]*)(-[a-z0-9]+)*$'
-          - '^doc/([a-z][a-z0-9]*)(-[a-z0-9]+)*$'
-          - '^feature/([a-z][a-z0-9]*)(-[a-z0-9]+)*$'
-          - '^fix/([a-z][a-z0-9]*)(-[a-z0-9]+)*$'
-          - '^hotfix/([a-z][a-z0-9]*)(-[a-z0-9]+)*$'
-          - '^review/([a-z][a-z0-9]*)(-[a-z0-9]+)*$'
-          - '^refactor/([a-z][a-z0-9]*)(-[a-z0-9]+)*$'
-          - '^enhancement/([a-z][a-z0-9]*)(-[a-z0-9]+)*$'
-          - '^test/([a-z][a-z0-9]*)(-[a-z0-9]+)*$'
+          - "^main$"
+          - "^test$"
+          - "^devops/([a-z][a-z0-9]*)(-[a-z0-9]+)*$"
+          - "^doc/([a-z][a-z0-9]*)(-[a-z0-9]+)*$"
+          - "^feature/([a-z][a-z0-9]*)(-[a-z0-9]+)*$"
+          - "^fix/([a-z][a-z0-9]*)(-[a-z0-9]+)*$"
+          - "^hotfix/([a-z][a-z0-9]*)(-[a-z0-9]+)*$"
+          - "^review/([a-z][a-z0-9]*)(-[a-z0-9]+)*$"
+          - "^refactor/([a-z][a-z0-9]*)(-[a-z0-9]+)*$"
+          - "^enhancement/([a-z][a-z0-9]*)(-[a-z0-9]+)*$"
+          - "^dependencies/([a-z][a-z0-9]*)(-[a-z0-9]+)*$"
 
-  - repo: https://github.com/octue/conventional-commits
-    rev: 0.3.1
+  - repo: https://github.com/octue/pre-commit-hooks
+    rev: 0.6.0
     hooks:
       - id: check-commit-message-is-conventional
         stages: [commit-msg]
+        args:
+          - --maximum-body-line-length=2000

Dockerfile.template

@@ -0,0 +1,26 @@
+# base-image for python on any machine using a template variable,
+# see more about dockerfile templates here: https://www.balena.io/docs/learn/develop/dockerfile/
+FROM balenalib/%%BALENA_MACHINE_NAME%%-debian-python:3.8-bullseye-run
+
+# Install git for development purposes
+RUN install_packages git
+
+# Set our working directory
+RUN mkdir -p /usr/aerosense/data-gateway
+WORKDIR /usr/aerosense/data-gateway
+
+# Copy requirements.txt first for better cache on later pushes
+COPY requirements-pi.txt requirements-pi.txt
+
+# Install python deps on the resin.io build server
+RUN pip install -r requirements-pi.txt
+
+# Copy and install the project, to source the gateway CLI
+COPY . ./
+RUN pip install .
+
+# Enable udevd so that plugged dynamic hardware devices show up in our container.
+ENV UDEV=1
+
+# Keep the container alive after start, so we can ssh in and use the gateway
+CMD ["sleep", "infinity"]