Merge branch 'main' into main_allocation_network

Deltares · Dec 18, 2023 · 86e470b · 86e470b
2 parents b79e10c + 7ce31cd
commit 86e470b
Show file tree

Hide file tree

Showing 74 changed files with 1,457 additions and 416 deletions.
diff --git a/.github/workflows/core_tests.yml b/.github/workflows/core_tests.yml
@@ -9,6 +9,10 @@ on:
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
+# needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
 jobs:
   test:
     name: Julia ${{ matrix.julia_version }} - ${{ matrix.os }} - ${{ matrix.arch }}
@@ -20,34 +24,25 @@ jobs:
           - "~1.10.0-0"
         os:
           - ubuntu-latest
-          - macOS-latest
+          # https://github.com/Deltares/Ribasim/issues/825
+          # - macOS-latest
           - windows-latest
         arch:
           - x64
     steps:
       - uses: actions/checkout@v4
+      - uses: julia-actions/cache@v1
+        with:
+          cache-compiled: "true"
+          cache-registries: "true"
       - uses: prefix-dev/[email protected]
         with:
           pixi-version: "latest"
       - name: Prepare pixi
         run: pixi run install-without-pre-commit
-      - name: Prepare model input
+      - name: Test Ribasim Core
         run: |
-          pixi run generate-testmodels
-      - uses: julia-actions/setup-julia@v1
-        with:
-          version: ${{ matrix.julia_version }}
-          arch: ${{ matrix.arch }}
-      - uses: julia-actions/cache@v1
-        with:
-          cache-compiled: "true"
-          cache-registries: "true"
-      - uses: julia-actions/julia-buildpkg@v1
-        with:
-          project: core
-      - uses: julia-actions/julia-runtest@v1
-        with:
-          project: core
+          pixi run test-ribasim-core-cov
       - uses: julia-actions/julia-processcoverage@v1
         with:
           directories: core/src

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -8,6 +8,10 @@ on:
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
+# needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
 jobs:
   publish:
     name: Docs Julia ${{ matrix.julia_version }} - ${{ matrix.arch }}
@@ -22,21 +26,10 @@ jobs:
           - x64
     steps:
       - uses: actions/checkout@v4
-
-      - uses: julia-actions/setup-julia@v1
-        with:
-          version: ${{ matrix.julia_version }}
-          arch: ${{ matrix.arch }}
-
-      - uses: julia-actions/julia-buildpkg@v1
-        with:
-          project: core
-
       - uses: julia-actions/cache@v1
         with:
           cache-compiled: "true"
           cache-registries: "true"
-
       - uses: prefix-dev/[email protected]
         with:
           pixi-version: "latest"

diff --git a/.github/workflows/pre-commit_auto_update.yml b/.github/workflows/pre-commit_auto_update.yml
@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
       - uses: browniebroke/pre-commit-autoupdate-action@main

diff --git a/.github/workflows/pre-commit_check.yml b/.github/workflows/pre-commit_check.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.12"
       - uses: pre-commit/[email protected]
diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml
@@ -23,13 +23,14 @@ jobs:
         python-version:
           - "3.10"
           - "3.11"
+          - "3.12"
         arch:
           - x86
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "${{ matrix.python-version }}"
 

diff --git a/.gitignore b/.gitignore
@@ -150,3 +150,4 @@ dmypy.json
 build/ribasim_cli/tests/temp/
 python/ribasim_api/tests/temp/
 report.xml
+/juliaup
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -14,7 +14,7 @@ repos:
         exclude: '.teamcity'
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.3
+    rev: v0.1.6
     hooks:
       - id: ruff
         types_or: [python, pyi, jupyter]
@@ -26,6 +26,6 @@ repos:
     hooks:
       - id: nbstripout
   - repo: https://github.com/crate-ci/typos
-    rev: v1.16.21
+    rev: v1.16.23
     hooks:
       - id: typos
diff --git a/.teamcity/Ribasim_Ribasim/buildTypes/BuildRibasimCliWindows.xml b/.teamcity/Ribasim_Ribasim/buildTypes/BuildRibasimCliWindows.xml
@@ -8,9 +8,6 @@
       <option name="cleanBuild" value="true" />
       <option name="executionTimeoutMin" value="120" />
     </options>
-    <parameters>
-      <param name="julia_dir" value="%env.JULIA_DIR_1_10%" />
-    </parameters>
     <build-runners>
       <runner id="RUNNER_2418" name="Set up pixi" type="simpleRunner">
         <parameters>
@@ -22,8 +19,8 @@
       </runner>
       <runner id="RUNNER_2420" name="Build binary" type="simpleRunner">
         <parameters>
-          <param name="script.content"><![CDATA[set Path=%julia_dir%;%env.Path%
-            pixi run build-ribasim-cli]]></param>
+          <param name="script.content"><![CDATA[pixi run remove-artifacts
+pixi run build-ribasim-cli]]></param>
           <param name="teamcity.build.workingDir" value="ribasim" />
           <param name="teamcity.step.mode" value="default" />
           <param name="use.custom.script" value="true" />

diff --git a/.teamcity/Ribasim_Ribasim/buildTypes/Ribasim_RibasimJl_BuildLibribasimWindows.xml b/.teamcity/Ribasim_Ribasim/buildTypes/Ribasim_RibasimJl_BuildLibribasimWindows.xml
@@ -12,9 +12,6 @@
     <disabled-settings>
       <setting-ref ref="RQ_271" />
     </disabled-settings>
-    <parameters>
-      <param name="julia_dir" value="%env.JULIA_DIR_1_10%" />
-    </parameters>
     <build-runners>
       <runner id="RUNNER_2415" name="Set up pixi" type="simpleRunner">
         <parameters>
@@ -26,7 +23,7 @@
       </runner>
       <runner id="RUNNER_2416" name="Build binary" type="simpleRunner">
         <parameters>
-          <param name="script.content"><![CDATA[set Path=%julia_dir%;%env.Path%
+          <param name="script.content"><![CDATA[pixi run remove-artifacts
 pixi run build-libribasim]]></param>
           <param name="teamcity.build.workingDir" value="ribasim" />
           <param name="teamcity.step.mode" value="default" />

diff --git a/.teamcity/Ribasim_Ribasim/buildTypes/Ribasim_Ribasim_BuildLibribasimLinux.xml b/.teamcity/Ribasim_Ribasim/buildTypes/Ribasim_Ribasim_BuildLibribasimLinux.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<build-type xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" uuid="a7c17a38-6995-4b58-80f4-d91581b9b802" paused="true" xsi:noNamespaceSchemaLocation="https://www.jetbrains.com/teamcity/schemas/2021.1/project-config.xsd">
+  <name>Build libribasim - Linux</name>
+  <description />
+  <settings>
+    <options>
+      <option name="artifactRules" value="ribasim\build\create_binaries\libribasim =&gt; libribasim.zip" />
+      <option name="cleanBuild" value="true" />
+      <option name="executionTimeoutMin" value="120" />
+      <option name="publishArtifactCondition" value="SUCCESSFUL" />
+    </options>
+    <parameters />
+    <build-runners>
+      <runner id="RUNNER_2415" name="Set up pixi" type="simpleRunner">
+        <parameters>
+          <param name="script.content"><![CDATA[#!/bin/bash
+# black magic
+source /usr/share/Modules/init/bash
+
+module load pixi
+pixi --version
+pixi run install-without-pre-commit]]></param>
+          <param name="teamcity.build.workingDir" value="ribasim" />
+          <param name="teamcity.step.mode" value="default" />
+          <param name="use.custom.script" value="true" />
+        </parameters>
+      </runner>
+      <runner id="RUNNER_2416" name="Build binary" type="simpleRunner">
+        <parameters>
+          <param name="script.content"><![CDATA[#!/bin/bash
+# black magic
+source /usr/share/Modules/init/bash
+
+module load pixi
+pixi run remove-artifacts
+pixi run build-libribasim]]></param>
+          <param name="teamcity.build.workingDir" value="ribasim" />
+          <param name="teamcity.step.mode" value="default" />
+          <param name="use.custom.script" value="true" />
+        </parameters>
+      </runner>
+    </build-runners>
+    <vcs-settings>
+      <vcs-entry-ref root-id="Ribasim_Ribasim_Ribasim">
+        <checkout-rule rule=". =&gt; ribasim" />
+      </vcs-entry-ref>
+    </vcs-settings>
+    <requirements>
+      <does-not-equal id="RQ_275" name="env.OS" value="Windows_NT" />
+    </requirements>
+    <build-triggers>
+      <build-trigger id="TRIGGER_497" type="schedulingTrigger">
+        <parameters>
+          <param name="branchFilter"><![CDATA[+:<default>
++:v*]]></param>
+          <param name="cronExpression_dm" value="*" />
+          <param name="cronExpression_dw" value="?" />
+          <param name="cronExpression_hour" value="*" />
+          <param name="cronExpression_min" value="0" />
+          <param name="cronExpression_month" value="*" />
+          <param name="cronExpression_sec" value="0" />
+          <param name="cronExpression_year" value="*" />
+          <param name="dayOfWeek" value="Sunday" />
+          <param name="enableQueueOptimization" value="true" />
+          <param name="hour" value="3" />
+          <param name="minute" value="0" />
+          <param name="promoteWatchedBuild" value="true" />
+          <param name="revisionRule" value="lastFinished" />
+          <param name="revisionRuleBuildBranch" value="+:&lt;default&gt;" />
+          <param name="schedulingPolicy" value="daily" />
+          <param name="timezone" value="SERVER" />
+          <param name="triggerBuildWithPendingChangesOnly" value="true" />
+        </parameters>
+      </build-trigger>
+    </build-triggers>
+    <build-extensions>
+      <extension id="BUILD_EXT_295" type="commit-status-publisher">
+        <parameters>
+          <param name="github_authentication_type" value="token" />
+          <param name="github_host" value="https://api.github.com" />
+          <param name="publisherId" value="githubStatusPublisher" />
+          <param name="secure:github_access_token" value="credentialsJSON:71420214-373c-4ccd-ba32-2ea886843f62" />
+        </parameters>
+      </extension>
+    </build-extensions>
+    <cleanup />
+  </settings>
+</build-type>
+
diff --git a/.../Ribasim_Ribasim_PushReleaseBuildToS3.xml → ...pes/Ribasim_Ribasim_MakeGitHubRelease.xml b/.../Ribasim_Ribasim_PushReleaseBuildToS3.xml → ...pes/Ribasim_Ribasim_MakeGitHubRelease.xml
@@ -1,35 +1,55 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <build-type xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" uuid="5fa3a42d-6a7b-4820-95c1-c550d6648ae7" xsi:noNamespaceSchemaLocation="https://www.jetbrains.com/teamcity/schemas/2021.1/project-config.xsd">
-  <name>Push release build to S3</name>
+  <name>Make GitHub Release</name>
   <description />
   <settings>
-    <parameters />
+    <parameters>
+      <param name="env.GITHUB_TOKEN" value="%github_teamcity-deltares_public_access_token%" />
+    </parameters>
     <build-runners>
-      <runner id="RUNNER_537" name="Push release to S3" type="simpleRunner">
+      <runner id="RUNNER_2523" name="Push release to GitHub" type="simpleRunner">
         <parameters>
           <param name="script.content"><![CDATA[#!/usr/bin/env bash
 set -euxo pipefail
 . /usr/share/Modules/init/bash
 
-module load aws
-aws s3 cp ribasim_cli.zip s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWindows/%teamcity.build.branch%/ribasim_cli.zip
-aws s3 cp ribasim_qgis.zip s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWindows/%teamcity.build.branch%/ribasim_qgis.zip
-aws s3 cp ribasim*.whl s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWindows/%teamcity.build.branch%/]]></param>
+module load github
+# Get the name of the currently checked out tag
+tag_name=$(git describe --tags --exact-match 2>/dev/null)
+
+# Check if a tag is checked out
+if [ -n "$tag_name" ]; then
+    echo "Currently checked out tag: $tag_name"
+
+    # Create a release using gh
+    gh release create "$tag_name" \
+        --generate-notes \
+        ribasim*.whl \
+        ribasim_cli.zip \
+        ribasim_qgis.zip
+
+    echo "Release created successfully."
+
+else
+    echo "No tag is currently checked out."
+fi]]></param>
           <param name="teamcity.step.mode" value="default" />
           <param name="use.custom.script" value="true" />
         </parameters>
       </runner>
     </build-runners>
-    <vcs-settings />
+    <vcs-settings>
+      <vcs-entry-ref root-id="Ribasim_Ribasim_Ribasim" />
+    </vcs-settings>
     <requirements>
       <equals id="RQ_294" name="teamcity.agent.jvm.os.name" value="Linux" />
     </requirements>
     <build-triggers>
-      <build-trigger id="TRIGGER_492" type="buildDependencyTrigger">
+      <build-trigger id="TRIGGER_637" type="vcsTrigger">
         <parameters>
-          <param name="afterSuccessfulBuildOnly" value="true" />
-          <param name="branchFilter" value="+:v*" />
-          <param name="dependsOn" value="BuildRibasimCliWindows" />
+          <param name="branchFilter" value="+:v20*" />
+          <param name="enableQueueOptimization" value="true" />
+          <param name="quietPeriodMode" value="DO_NOT_USE" />
         </parameters>
       </build-trigger>
     </build-triggers>

diff --git a/.teamcity/Ribasim_Ribasim/buildTypes/Ribasim_Ribasim_PushSuccessfulNightlyBuildToS3.xml b/.teamcity/Ribasim_Ribasim/buildTypes/Ribasim_Ribasim_PushSuccessfulNightlyBuildToS3.xml
@@ -10,7 +10,6 @@
           <param name="script.content"><![CDATA[#!/usr/bin/env bash
 set -euxo pipefail
 . /usr/share/Modules/init/bash
-
 module load aws
 aws s3 cp ribasim_cli.zip s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWindows/latest/ribasim_cli.zip
 aws s3 cp ribasim_qgis.zip s3://ribasim/teamcity/Ribasim_Ribasim/BuildRibasimCliWindows/latest/ribasim_qgis.zip

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -0,0 +1,13 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Ribasim QGIS: Attach to QGIS",
+            "type": "python",
+            "request": "attach",
+            "host": "localhost",
+            "port": 5678,
+            "justMyCode": false
+        }
+    ]
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -11,11 +11,16 @@
         "editor.defaultFormatter": "charliermarsh.ruff",
         "editor.formatOnSave": true,
         "editor.codeActionsOnSave": {
-            "source.fixAll": true
+            "source.fixAll": "explicit"
         }
     },
     "python.analysis.extraPaths": [
         ".pixi/env/Library/python"
     ],
-    "mypy-type-checker.importStrategy": "fromEnvironment"
+    "mypy-type-checker.importStrategy": "fromEnvironment",
+    "python.testing.pytestArgs": [
+        "."
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
 }