MRG: Merge pull request #1 from aerosense-ai/fix/narrow-exception-cat…

…ching-when-uploading-to-cloud

Uncover error raised during cloud upload; add GitHub workflows

.gitlab/issue_templates/bug_report.md → .github/ISSUE_TEMPLATE/bug_report.md

@@ -1,3 +1,8 @@
+---
+name: Bug report
+about: Report a bug
+---
+
 ## Bug report
 
 ### About me

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

.gitlab/issue_templates/feature_request.md → .github/ISSUE_TEMPLATE/feature_request.md

@@ -1,3 +1,8 @@
+---
+name: Feature request
+about: Request a feature
+---
+
 ## Feature request
 
 ### About me

.gitlab/issue_templates/support_request.md → .github/ISSUE_TEMPLATE/support_request.md

@@ -1,3 +1,8 @@
+---
+name: Support request
+about: Request support using the code
+---
+
 ## Support request
 
 ### About me

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+<!-- PRs into main are released as soon as they are merged. Their descriptions will be used directly to create release notes, so make sure they contain everything! -->
+<!-- Anything added between the auto-generation marker comments will be replaced on every pushed commit, so make sure to add anything you want to add outside of them. -->
+<!-- However, any part of the final PR description (i.e. the description at the point of the final commit to the PR) can be changed and will be retained when producing the release notes -->
+<!-- Please don't add features not discussed in an issue first -->
+
+## Summary
+
+<!--- START AUTOGENERATED NOTES --->
+<!--- END AUTOGENERATED NOTES --->
+
+## Quality Checklist
+
+- [ ] New features are fully tested (No matter how much Coverage Karma you have)
+- [ ] **[v0.2 onward]** New features are included in the documentation
+- [ ] **[v0.2 onward]** Breaking changes are documented with clear instructions of what 
+
+### Coverage Karma
+
+- [ ] If your PR decreases test coverage, do you feel you have built enough `Coverage Karma`* to justify it?
+
+*Coverage Karma can be earned by disproportionally increasing test coverage in the rest of your contributions.
+It's an honesty policy - you keep count of your own.

.github/workflows/python-ci.yml

@@ -0,0 +1,59 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions.
+#
+# On successful test, the package will be published. For candidate releases, the package will be
+#  published to test.pypi.org server (to ensure the process works). For merges to master, the
+#  package will be published live.
+
+name: python-ci
+
+on: [push]
+
+jobs:
+
+  check-semantic-version:
+    if: "!contains(github.event.head_commit.message, 'skipci')"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # Set fetch-depth to 0 to fetch all tags (necessary for git-mkver to determine the correct semantic version).
+          fetch-depth: 0
+      - uses: actions/setup-python@v2
+      - name: Install git-mkver
+        run: |
+          curl -L https://github.com/idc101/git-mkver/releases/download/v1.2.1/git-mkver-linux-amd64-1.2.1.tar.gz \
+          | tar xvz \
+          && sudo mv git-mkver /usr/local/bin
+      - name: Install semantic version checker
+        run: pip install git+https://github.com/octue/conventional-commits
+      - name: Check version
+        run: check-semantic-version setup.py
+
+  run-tests:
+    if: "!contains(github.event.head_commit.message, 'skipci')"
+    runs-on: ${{ matrix.os }}
+    env:
+      USING_COVERAGE: '3.8'
+    strategy:
+      matrix:
+        python: [3.8]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install tox
+        run: pip install tox
+      - name: Run tests
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
+        run: tox
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          file: coverage.xml
+          fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,28 @@
+name: Release the package on merge into main
+
+# Only trigger when a pull request into main branch is closed.
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+
+jobs:
+  release:
+    # This job will only run if the PR has been merged (and not closed without merging).
+    if: "github.event.pull_request.merged == true && !contains(github.event.pull_request.head.message, 'skipci')"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Get package version
+      run: echo "PACKAGE_VERSION=$(python setup.py --version)" >> $GITHUB_ENV
+    - name: Create Release
+      uses: actions/create-release@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, no need to create your own.
+      with:
+        tag_name: ${{ env.PACKAGE_VERSION }}
+        release_name: ${{ github.event.pull_request.title }}
+        body: ${{ github.event.pull_request.body }}
+        draft: false
+        prerelease: false

.github/workflows/update-pull-request.yml

@@ -0,0 +1,34 @@
+# This workflow updates the pull request description with an auto-generated section containing the categorised commit
+# message headers of the commits since the last pull request merged into main. The auto generated section is enveloped
+# between two comments: "<!--- START AUTOGENERATED NOTES --->" and "<!--- END AUTOGENERATED NOTES --->". Anything
+# outside these in the description is left untouched. Auto-generated updates can be skipped for a commit if
+# "skip_ci_update_description" is in the commit message body.
+
+name: update-pull-request
+
+# Only trigger for pull requests into main branch.
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  description:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        # Set fetch-depth to 0 to fetch all commit history (necessary for compiling pull request description).
+        fetch-depth: 0
+    - name: Install release note compiler
+      run: pip install git+https://github.com/octue/conventional-commits
+    - name: Compile new pull request description
+      run: |
+        echo 'PULL_REQUEST_NOTES<<EOF' >> $GITHUB_ENV
+        echo "$(compile-release-notes PULL_REQUEST_START --pull-request-url=${{ github.event.pull_request.url }} --api-token=${{ secrets.GITHUB_TOKEN }})" >> $GITHUB_ENV
+        echo EOF >> $GITHUB_ENV
+    - name: Update pull request body
+      uses: riskledger/update-pr-description@v2
+      with:
+        body: ${{ env.PULL_REQUEST_NOTES }}
+        token: ${{ secrets.GITHUB_TOKEN }}

data_gateway/persistence.py

@@ -264,9 +264,11 @@ def _persist_window(self):
                 timeout=self.upload_timeout,
             )
 
-        except Exception:  # noqa
+        except Exception as e:
+            logger.exception(e)
+
             logger.warning(
-                "Upload of window failed - writing to disk at %r instead.",
+                "Upload of window may have failed - writing to disk at %r.",
                 self._backup_writer._generate_window_path(),
             )
 

mkver.conf

@@ -0,0 +1,41 @@
+tagPrefix: ""
+defaults {
+  tag: false
+  tagMessageFormat: "Release/{Tag}"
+  preReleaseFormat: "RC{PreReleaseNumber}"
+  buildMetaDataFormat: "{Branch}.{ShortHash}"
+  includeBuildMetaData: false
+  whenNoValidCommitMessages: IncrementPatch
+  patches: [setup.py]
+}
+
+patches: [
+  {
+    name: setup.py
+    filePatterns: ["setup.py"]
+    replacements: [
+      {
+        find: "version=\"{VersionRegex}\""
+        replace: "version=\"{Version}\""
+      }
+    ]
+  }
+]
+
+commitMessageActions: [
+  # Disable major version increments while package is still in beta (i.e. keep the version below 1.0.0).
+  {
+    pattern: "BREAKING CHANGE"
+    action: IncrementMinor
+  }
+  {
+    pattern: "BREAKING-CHANGE"
+    action: IncrementMinor
+  }
+
+  # All new features require a minor version increase.
+  {
+    pattern: "FEA:"
+    action: IncrementMinor
+  }
+]

setup.py

@@ -16,7 +16,7 @@
 
 setup(
     name="data_gateway",
-    version="0.7.1",
+    version="0.7.2",
     install_requires=[
         "click>=7.1.2",
         "pyserial==3.5",

tests/test_cloud_functions/base.py

@@ -0,0 +1,40 @@
+import json
+import os
+
+from octue.cloud.credentials import GCPCredentialsManager
+
+
+class CredentialsEnvironmentVariableAsFile:
+    """Temporarily store JSON credentials from the `GOOGLE_APPLICATION_CREDENTIALS` environment variable in a file for
+    use during the test class's test run. This is useful on GitHub where a file cannot be created for a secret but
+    tests that require credentials to be present as a file are run.
+    """
+
+    credentials_path = "temporary_file.json"
+    current_google_application_credentials_variable_value = None
+
+    @classmethod
+    def setUpClass(cls):
+        """Temporarily write the credentials to a file so that the tests can run on GitHub where the credentials are
+        only provided as JSON in an environment variable. Set the credentials environment variable to point to this
+        file instead of the credentials JSON.
+
+        :return None:
+        """
+        cls.current_google_application_credentials_variable_value = os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
+
+        credentials = GCPCredentialsManager().get_credentials(as_dict=True)
+
+        with open(cls.credentials_path, "w") as f:
+            json.dump(credentials, f)
+
+        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cls.credentials_path
+
+    @classmethod
+    def tearDownClass(cls):
+        """Remove the temporary credentials file and restore the credentials environment variable to its original value.
+
+        :return None:
+        """
+        os.remove(cls.credentials_path)
+        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cls.current_google_application_credentials_variable_value

tests/test_cloud_functions/test_big_query.py

@@ -5,6 +5,7 @@
 
 from tests.base import BaseTestCase
 from tests.test_cloud_functions import REPOSITORY_ROOT
+from tests.test_cloud_functions.base import CredentialsEnvironmentVariableAsFile
 
 
 # Manually add the cloud_functions package to the path (its imports have to be done in a certain way for Google Cloud
@@ -17,7 +18,7 @@
 )
 
 
-class TestBigQueryDataset(BaseTestCase):
+class TestBigQueryDataset(CredentialsEnvironmentVariableAsFile, BaseTestCase):
     def test_insert_sensor_data(self):
         """Test that sensor data can be sent to BigQuery for insertion."""
         data = {