Merge branch 'main' into validate-allocation-input

Deltares · Feb 1, 2024 · 6965280 · 6965280
2 parents 764f23d + db6e94c
commit 6965280
Show file tree

Hide file tree

Showing 88 changed files with 10,566 additions and 7,497 deletions.
diff --git a/.docker/compose.yml b/.docker/compose.yml
@@ -9,3 +9,4 @@ services:
     environment:
       - CI=true
       - DISPLAY=:99
+    tty: true
diff --git a/.docker/start.sh b/.docker/start.sh
@@ -4,8 +4,6 @@ set -euxo pipefail
 source .env
 
 docker compose -f compose.yml up -d --force-recreate --remove-orphans
-echo "Wait 10 seconds"
-sleep 10
 echo "Installation of the plugin Ribasim"
 docker exec -t qgis sh -c "qgis_setup.sh ${PLUGIN_NAME}"
 echo "Containers are running"
diff --git a/.docker/test.sh b/.docker/test.sh
@@ -3,4 +3,4 @@ set -euxo pipefail
 
 source .env
 
-docker exec -t qgis sh -c "cd /tests_directory && qgis_testrunner.sh ${PLUGIN_NAME}.tests"
+docker exec -t qgis sh -c "cd /tests_directory && xvfb-run -a qgis_testrunner.sh ${PLUGIN_NAME}.tests"
diff --git a/.github/workflows/core_tests.yml b/.github/workflows/core_tests.yml
@@ -1,7 +1,7 @@
 name: Julia Tests
 on:
   push:
-    branches: [main]
+    branches: [main, update/pixi-lock, update/julia-manifest]
     paths-ignore: [".teamcity/**"]
     tags: ["*"]
   pull_request:
@@ -33,7 +33,7 @@ jobs:
         with:
           cache-compiled: "true"
           cache-registries: "true"
-      - uses: prefix-dev/[email protected].1
+      - uses: prefix-dev/[email protected].3
         with:
           pixi-version: "latest"
       - name: Prepare pixi

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -1,7 +1,7 @@
 name: Docs
 on:
   push:
-    branches: [main]
+    branches: [main, update/pixi-lock, update/julia-manifest]
     paths-ignore: [".teamcity/**"]
   pull_request:
   merge_group:
@@ -24,7 +24,7 @@ jobs:
         with:
           cache-compiled: "true"
           cache-registries: "true"
-      - uses: prefix-dev/[email protected].1
+      - uses: prefix-dev/[email protected].3
         with:
           pixi-version: "latest"
       - name: Prepare pixi

diff --git a/.github/workflows/julia_auto_update.yml b/.github/workflows/julia_auto_update.yml
@@ -0,0 +1,31 @@
+name: Julia auto update
+
+on:
+  schedule:
+  # At 03:00 on day 1 of the month
+    - cron: "0 3 1 * *"
+  # on demand
+  workflow_dispatch:
+
+jobs:
+  auto-update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: "latest"
+      - name: Update Julia manifest file
+        run: |
+          pixi run update-manifest-julia
+          pixi install
+      - uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          branch: update/julia-manifest
+          title: Update Julia manifest
+          commit-message: "Update Julia manifest"
+          body: Update Julia dependencies to the latest version.
+          author: "GitHub <[email protected]>"
diff --git a/.github/workflows/pixi_auto_update.yml b/.github/workflows/pixi_auto_update.yml
@@ -0,0 +1,32 @@
+name: Pixi auto update
+
+on:
+  schedule:
+  # At 03:00 on day 1 of the month
+    - cron: "0 3 1 * *"
+  # on demand
+  workflow_dispatch:
+
+jobs:
+  auto-update:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: "latest"
+          cache: false
+      - name: Update pixi lock file
+        run: |
+          rm pixi.lock
+          pixi install
+      - uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          branch: update/pixi-lock
+          title: Update pixi lock file
+          commit-message: "Update `pixi.lock`"
+          body: Update pixi dependencies to the latest version.
+          author: "GitHub <[email protected]>"
diff --git a/.github/workflows/pre-commit_check.yml b/.github/workflows/pre-commit_check.yml
@@ -4,7 +4,7 @@ on:
   pull_request:
   merge_group:
   push:
-    branches: [main, update/pre-commit-hooks]
+    branches: [main, update/pre-commit-hooks, update/pixi-lock]
 
 jobs:
   check:

diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml
@@ -1,7 +1,7 @@
 name: Python Lint
 on:
   push:
-    branches: [main]
+    branches: [main, update/pixi-lock]
     paths-ignore: [".teamcity/**"]
     tags: ["*"]
   pull_request:
@@ -16,7 +16,7 @@ jobs:
     continue-on-error: true
     steps:
       - uses: actions/checkout@v4
-      - uses: prefix-dev/[email protected].1
+      - uses: prefix-dev/[email protected].3
         with:
           pixi-version: "latest"
       - name: Prepare pixi

diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml
@@ -1,7 +1,7 @@
 name: Ribasim Python Tests
 on:
   push:
-    branches: [main]
+    branches: [main, update/pixi-lock]
     paths-ignore: [".teamcity/**"]
     tags: ["*"]
   pull_request:

diff --git a/.github/workflows/qgis.yml b/.github/workflows/qgis.yml
@@ -2,7 +2,7 @@ name: QGIS Tests
 
 on:
     push:
-      branches: [main]
+      branches: [main, update/pixi-lock]
       paths-ignore: [".teamcity/**"]
       tags: ["*"]
     pull_request:
@@ -11,6 +11,7 @@ jobs:
     test-qgis:
         name: "Test"
         runs-on: ubuntu-latest
+        if: false # Disable tests until we have time to fix them
         defaults:
           run:
             working-directory: .docker

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -14,7 +14,7 @@ repos:
         exclude: '.teamcity'
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.9
+    rev: v0.1.15
     hooks:
       - id: ruff
         types_or: [python, pyi, jupyter]
@@ -26,6 +26,6 @@ repos:
     hooks:
       - id: nbstripout
   - repo: https://github.com/crate-ci/typos
-    rev: v1.16.26
+    rev: v1.17.2
     hooks:
       - id: typos
diff --git a/.teamcity/Ribasim/buildTypes/Ribasim_GenerateTestmodels.xml b/.teamcity/Ribasim/buildTypes/Ribasim_GenerateTestmodels.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<build-type xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" uuid="e2ca72cd-0597-45e8-823b-3db665a1f46a" xsi:noNamespaceSchemaLocation="https://www.jetbrains.com/teamcity/schemas/2021.1/project-config.xsd">
+  <name>Generate Testmodels</name>
+  <description />
+  <settings>
+    <options>
+      <option name="artifactRules" value="ribasim\generated_testmodels =&gt; generated_testmodels.zip" />
+      <option name="cleanBuild" value="true" />
+      <option name="executionTimeoutMin" value="120" />
+      <option name="publishArtifactCondition" value="SUCCESSFUL" />
+    </options>
+    <parameters />
+    <build-runners>
+      <runner id="RUNNER_2415" name="Set up pixi" type="simpleRunner">
+        <parameters>
+          <param name="script.content"><![CDATA[#!/bin/bash
+# black magic
+source /usr/share/Modules/init/bash
+
+module load pixi
+pixi --version
+pixi run install-without-pre-commit]]></param>
+          <param name="teamcity.build.workingDir" value="ribasim" />
+          <param name="teamcity.step.mode" value="default" />
+          <param name="use.custom.script" value="true" />
+        </parameters>
+      </runner>
+      <runner id="RUNNER_2416" name="Generate testmodels" type="simpleRunner">
+        <parameters>
+          <param name="script.content"><![CDATA[#!/bin/bash
+# black magic
+source /usr/share/Modules/init/bash
+
+module load pixi
+pixi run generate-testmodels]]></param>
+          <param name="teamcity.build.workingDir" value="ribasim" />
+          <param name="teamcity.step.mode" value="default" />
+          <param name="use.custom.script" value="true" />
+        </parameters>
+      </runner>
+    </build-runners>
+    <vcs-settings>
+      <vcs-entry-ref root-id="Ribasim_Ribasim">
+        <checkout-rule rule=". =&gt; ribasim" />
+      </vcs-entry-ref>
+    </vcs-settings>
+    <requirements>
+      <does-not-equal id="RQ_275" name="env.OS" value="Windows_NT" />
+    </requirements>
+    <build-triggers>
+      <build-trigger id="TRIGGER_646" type="vcsTrigger">
+        <parameters>
+          <param name="branchFilter" value="+:*" />
+          <param name="enableQueueOptimization" value="true" />
+          <param name="quietPeriodMode" value="DO_NOT_USE" />
+        </parameters>
+      </build-trigger>
+    </build-triggers>
+    <build-extensions>
+      <extension id="BUILD_EXT_295" type="commit-status-publisher">
+        <parameters>
+          <param name="github_authentication_type" value="token" />
+          <param name="github_host" value="https://api.github.com" />
+          <param name="publisherId" value="githubStatusPublisher" />
+          <param name="secure:github_access_token" value="credentialsJSON:71420214-373c-4ccd-ba32-2ea886843f62" />
+        </parameters>
+      </extension>
+    </build-extensions>
+    <cleanup />
+  </settings>
+</build-type>
+
diff --git a/.teamcity/Ribasim/buildTypes/Ribasim_Ribasim_MakeGitHubRelease.xml b/.teamcity/Ribasim/buildTypes/Ribasim_Ribasim_MakeGitHubRelease.xml
@@ -26,7 +26,8 @@ if [ -n "$tag_name" ]; then
         --generate-notes \
         ribasim_cli_linux.zip \
         ribasim_cli_windows.zip \
-        ribasim_qgis.zip
+        ribasim_qgis.zip \
+        generated_testmodels.zip
 
     echo "Release created successfully."
 
@@ -54,12 +55,6 @@ fi]]></param>
       </build-trigger>
     </build-triggers>
     <build-extensions>
-      <extension id="PROVIDE_AWS_CREDS" type="PROVIDE_AWS_CREDS">
-        <parameters>
-          <param name="awsConnectionId" value="AmazonWebServicesAws" />
-          <param name="awsSessionDuration" value="60" />
-        </parameters>
-      </extension>
       <extension id="KEEP_RULE_10" type="keepRules">
         <parameters>
           <param name="filters.1.status" value="successful" />
@@ -73,17 +68,21 @@ fi]]></param>
       </extension>
     </build-extensions>
     <artifact-dependencies>
+      <dependency id="ARTIFACT_DEPENDENCY_685" sourceBuildTypeId="Ribasim_GenerateTestmodels" cleanDestination="false">
+        <revisionRule name="sameChainOrLastFinished" revision="latest.sameChainOrLastFinished" />
+        <artifact sourcePath="generated_testmodels.zip" />
+      </dependency>
       <dependency id="ARTIFACT_DEPENDENCY_603" sourceBuildTypeId="Ribasim_Ribasim_MakeQgisPlugin" cleanDestination="false">
         <revisionRule name="sameChainOrLastFinished" revision="latest.sameChainOrLastFinished" />
         <artifact sourcePath="ribasim_qgis.zip" />
       </dependency>
       <dependency id="ARTIFACT_DEPENDENCY_684" sourceBuildTypeId="Ribasim_Linux_BuildRibasimCli" cleanDestination="false">
         <revisionRule name="sameChainOrLastFinished" revision="latest.sameChainOrLastFinished" />
-        <artifact sourcePath="ribasim_cli.zip =&gt; ribasim_cli_linux.zip" />
+        <artifact sourcePath="ribasim_cli_linux.zip" />
       </dependency>
       <dependency id="ARTIFACT_DEPENDENCY_157" sourceBuildTypeId="Ribasim_Windows_BuildRibasimCli" cleanDestination="false">
         <revisionRule name="sameChainOrLastFinished" revision="latest.sameChainOrLastFinished" />
-        <artifact sourcePath="ribasim_cli.zip =&gt; ribasim_cli_windows.zip" />
+        <artifact sourcePath="ribasim_cli_windows.zip" />
       </dependency>
     </artifact-dependencies>
     <dependencies>
@@ -99,9 +98,26 @@ fi]]></param>
           <option name="take-successful-builds-only" value="true" />
         </options>
       </depend-on>
+      <depend-on sourceBuildTypeId="Ribasim_Linux_TestRibasimCli">
+        <options>
+          <option name="take-started-build-with-same-revisions" value="true" />
+          <option name="take-successful-builds-only" value="true" />
+        </options>
+      </depend-on>
+      <depend-on sourceBuildTypeId="Ribasim_GenerateTestmodels">
+        <options>
+          <option name="take-started-build-with-same-revisions" value="true" />
+          <option name="take-successful-builds-only" value="true" />
+        </options>
+      </depend-on>
       <depend-on sourceBuildTypeId="Ribasim_Linux_BuildRibasimCli">
         <options>
-          <option name="run-build-if-dependency-failed" value="RUN_ADD_PROBLEM" />
+          <option name="take-started-build-with-same-revisions" value="true" />
+          <option name="take-successful-builds-only" value="true" />
+        </options>
+      </depend-on>
+      <depend-on sourceBuildTypeId="Ribasim_Windows_TestRibasimCli">
+        <options>
           <option name="take-started-build-with-same-revisions" value="true" />
           <option name="take-successful-builds-only" value="true" />
         </options>

diff --git a/.teamcity/Ribasim/buildTypes/Ribasim_Ribasim_PushSuccessfulNightlyBuildToS3.xml b/.teamcity/Ribasim/buildTypes/Ribasim_Ribasim_PushSuccessfulNightlyBuildToS3.xml
@@ -11,7 +11,8 @@
 set -euxo pipefail
 . /usr/share/Modules/init/bash
 module load aws
-aws s3 cp ribasim_cli.zip s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWindows/latest/ribasim_cli.zip
+aws s3 cp ribasim_cli_windows.zip s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWindows/latest/ribasim_cli_windows.zip
+aws s3 cp ribasim_cli_linux.zip s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWindows/latest/ribasim_cli_linux.zip
 aws s3 cp ribasim_qgis.zip s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWindows/latest/ribasim_qgis.zip
 aws s3 cp ribasim*.whl s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWindows/latest/]]></param>
           <param name="teamcity.step.mode" value="default" />
@@ -45,14 +46,18 @@ aws s3 cp ribasim*.whl s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWind
         <revisionRule name="lastSuccessful" revision="latest.lastSuccessful" />
         <artifact sourcePath="ribasim*.whl" />
       </dependency>
-      <dependency id="ARTIFACT_DEPENDENCY_157" sourceBuildTypeId="Ribasim_Windows_BuildRibasimCli" cleanDestination="true">
-        <revisionRule name="lastSuccessful" revision="latest.lastSuccessful" />
-        <artifact sourcePath="ribasim_cli.zip" />
-      </dependency>
       <dependency id="ARTIFACT_DEPENDENCY_603" sourceBuildTypeId="Ribasim_Ribasim_MakeQgisPlugin" cleanDestination="false">
         <revisionRule name="lastSuccessful" revision="latest.lastSuccessful" />
         <artifact sourcePath="ribasim_qgis.zip" />
       </dependency>
+      <dependency id="ARTIFACT_DEPENDENCY_1176" sourceBuildTypeId="Ribasim_Linux_BuildRibasimCli" cleanDestination="false">
+        <revisionRule name="lastSuccessful" revision="latest.lastSuccessful" />
+        <artifact sourcePath="ribasim_cli_linux.zip" />
+      </dependency>
+      <dependency id="ARTIFACT_DEPENDENCY_157" sourceBuildTypeId="Ribasim_Windows_BuildRibasimCli" cleanDestination="true">
+        <revisionRule name="lastSuccessful" revision="latest.lastSuccessful" />
+        <artifact sourcePath="ribasim_cli_windows.zip" />
+      </dependency>
     </artifact-dependencies>
     <cleanup />
   </settings>

diff --git a/.teamcity/Ribasim/project-config.xml b/.teamcity/Ribasim/project-config.xml
@@ -46,6 +46,20 @@
         <param name="storage.type" value="S3_storage" />
       </parameters>
     </extension>
+    <extension id="AmazonWebServicesAws" type="OAuthProvider">
+      <parameters>
+        <param name="awsAccessKeyId" value="AKIAQBIN2MPWXSD2IZ5F" />
+        <param name="awsCredentialsType" value="awsAccessKeys" />
+        <param name="awsIamRoleSessionName" value="TeamCity-session" />
+        <param name="awsRegionName" value="eu-west-3" />
+        <param name="awsSessionCredentials" value="false" />
+        <param name="awsSessionDuration" value="60" />
+        <param name="awsStsEndpoint" value="https://sts.eu-west-3.amazonaws.com" />
+        <param name="displayName" value="Amazon Web Services (AWS)" />
+        <param name="providerType" value="AWS" />
+        <param name="secure:awsSecretAccessKey" value="credentialsJSON:dba90026-9856-4f87-94d9-bab91f3f2d5c" />
+      </parameters>
+    </extension>
   </project-extensions>
   <cleanup />
 </project>
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3,4 +3,4 @@ set -euxo pipefail

		source .env

		docker exec -t qgis sh -c "cd /tests_directory && qgis_testrunner.sh ${PLUGIN_NAME}.tests"
		docker exec -t qgis sh -c "cd /tests_directory && xvfb-run -a qgis_testrunner.sh ${PLUGIN_NAME}.tests"