Merge branch 'master' into xcm-dry-run-call

.github/scripts/deny-git-deps.py

@@ -0,0 +1,40 @@
+"""
+Script to deny Git dependencies in the Cargo workspace. Can be passed one optional argument for the
+root folder. If not provided, it will use the cwd.
+
+## Usage
+	python3 .github/scripts/deny-git-deps.py polkadot-sdk
+"""
+
+import os
+import sys
+
+from cargo_workspace import Workspace, DependencyLocation
+
+KNOWN_BAD_GIT_DEPS = {
+	'simple-mermaid': ['xcm-docs'],
+	# Fix in <https://github.com/paritytech/polkadot-sdk/issues/2922>
+	'bandersnatch_vrfs': ['sp-core'],
+}
+
+root = sys.argv[1] if len(sys.argv) > 1 else os.getcwd()
+workspace = Workspace.from_path(root)
+
+def check_dep(dep, used_by):
+	if dep.location != DependencyLocation.GIT:
+		return
+
+	if used_by in KNOWN_BAD_GIT_DEPS.get(dep.name, []):
+		print(f'🤨 Ignoring git dependency {dep.name} in {used_by}')
+	else:
+		print(f'🚫 Found git dependency {dep.name} in {used_by}')
+		sys.exit(1)	
+
+# Check the workspace dependencies that can be inherited:
+for dep in workspace.dependencies:
+	check_dep(dep, "workspace")
+
+# And the dependencies of each crate:
+for crate in workspace.crates:
+	for dep in crate.dependencies:
+		check_dep(dep, crate.name)

.github/workflows/check-changed-files.yml

@@ -0,0 +1,57 @@
+# Reusable workflow to perform checks and generate conditions for other workflows. 
+# Currently it checks if any Rust (build-related) file is changed
+# and if the current (caller) workflow file is changed.
+# Example:
+#
+# jobs:
+#   changes:
+#     permissions:
+#       pull-requests: read
+#     uses: ./.github/workflows/check-changed-files.yml
+#   some-job:
+#     needs: changes
+#     if: ${{ needs.changes.outputs.rust }}
+#  ....... 
+
+name: Check changes files
+
+on:
+  workflow_call:
+    # Map the workflow outputs to job outputs
+    outputs:
+      rust: 
+        value: ${{ jobs.changes.outputs.rust }}
+        description: 'true if any of the build-related OR current (caller) workflow files have changed'
+      current-workflow: 
+        value: ${{ jobs.changes.outputs.current-workflow }}
+        description: 'true if current (caller) workflow file has changed'      
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      # true if current workflow (caller) file is changed
+      rust: ${{ steps.filter.outputs.rust == 'true' || steps.filter.outputs.current-workflow == 'true' }}
+      current-workflow: ${{ steps.filter.outputs.current-workflow }}
+    steps:
+    - id: current-file
+      run: echo "current-workflow-file=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT
+    - run: echo "${{ steps.current-file.outputs.current-workflow-file }}"
+      # For pull requests it's not necessary to checkout the code
+    - id: filter
+      uses: dorny/paths-filter@v3
+      with:
+        predicate-quantifier: 'every'
+        # current-workflow - check if the current (caller) workflow file is changed
+        # rust - check if any Rust (build-related) file is changed
+        filters: |
+          current-workflow:
+            - '${{ steps.current-file.outputs.current-workflow-file }}'
+          rust:
+            - '**/*'
+            - '!.github/**/*'
+            - '!prdoc/**/*'
+            - '!docs/**/*'
+    #

.github/workflows/checks-quick.yml

@@ -87,13 +87,15 @@ jobs:
       - name: install python deps
         run: |
           sudo apt-get update && sudo apt-get install -y python3-pip python3
-          pip3 install toml
+          pip3 install toml "cargo-workspace>=1.2.6"
       - name: check integrity
         run: >
           python3 .github/scripts/check-workspace.py .
           --exclude
           "substrate/frame/contracts/fixtures/build"
           "substrate/frame/contracts/fixtures/contracts/common"
+      - name: deny git deps
+        run: python3 .github/scripts/deny-git-deps.py .
   check-markdown:
     runs-on: ubuntu-latest
     timeout-minutes: 10

.github/workflows/misc-sync-templates.yml

@@ -104,10 +104,6 @@ jobs:
 
           toml set templates/${{ matrix.template }}/Cargo.toml 'workspace.package.edition' "$(toml get --raw Cargo.toml 'workspace.package.edition')" > Cargo.temp
           mv Cargo.temp ./templates/${{ matrix.template }}/Cargo.toml
-
-          toml get Cargo.toml 'workspace.lints' --output-toml >> ./templates/${{ matrix.template }}/Cargo.toml
-
-          toml get Cargo.toml 'workspace.dependencies' --output-toml >> ./templates/${{ matrix.template }}/Cargo.toml
         working-directory: polkadot-sdk
       - name: Print the result Cargo.tomls for debugging
         if: runner.debug == '1'
@@ -120,6 +116,18 @@ jobs:
       - name: Copy over the new changes
         run: |
           cp -r polkadot-sdk/templates/${{ matrix.template }}/* "${{ env.template-path }}/"
+      - name: Copy over required workspace dependencies
+        run: |
+          echo -e "\n[workspace.dependencies]" >> Cargo.toml
+          set +e
+          # If a workspace dependency is required..
+          while cargo tree --depth 1 --prefix none --no-dedupe 2>&1 | grep 'was not found in `workspace.dependencies`'; do
+            # Get its name..
+            missing_dep=$(cargo tree --depth 1 --prefix none --no-dedupe 2>&1 | grep 'was not found in `workspace.dependencies`' | sed -E 's/(.*)`dependency.(.*)` was not found in `workspace.dependencies`/\2/')
+            # And copy the dependency from the monorepo.
+            toml get ../polkadot-sdk/Cargo.toml 'workspace.dependencies' --output-toml | grep "^${missing_dep} = " >> Cargo.toml
+          done;
+        working-directory: "${{ env.template-path }}"
 
       # 3. Verify the build. Push the changes or create a PR.
 
@@ -148,12 +156,12 @@ jobs:
           token: ${{ steps.app_token.outputs.token }}
           add-paths: |
             ./*
-          title: "[Don't merge] Update the ${{ matrix.template }} template"
+          title: "[Don't merge] Update the ${{ matrix.template }} template to ${{ github.event.inputs.crate_release_version }}"
           body: "The template has NOT been successfully built and needs to be inspected."
-          branch: "update-template/${{ github.event_name }}"
+          branch: "update-template/${{ github.event.inputs.crate_release_version }}"
       - name: Push changes
         run: |
           git add -A .
-          git commit --allow-empty -m "Update template triggered by ${{ github.event_name }}"
+          git commit --allow-empty -m "Update to ${{ github.event.inputs.crate_release_version }} triggered by ${{ github.event_name }}"
           git push
         working-directory: "${{ env.template-path }}"

.github/workflows/tests-linux-stable.yml

@@ -20,10 +20,18 @@ env:
   FORKLIFT_metrics_pushEndpoint: ${{ secrets.FORKLIFT_metrics_pushEndpoint }}
 
 jobs:
+
+  changes:
+    permissions:
+      pull-requests: read
+    uses: ./.github/workflows/check-changed-files.yml
+
   set-image:
     # GitHub Actions allows using 'env' in a container context.
     # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
     # This workaround sets the container image for each job using 'set-image' job output.
+    needs: changes
+    if: ${{ needs.changes.outputs.rust }}      
     runs-on: ubuntu-latest
     outputs:
       IMAGE: ${{ steps.set_image.outputs.IMAGE }}
@@ -32,10 +40,12 @@ jobs:
         uses: actions/checkout@v4
       - id: set_image
         run: cat .github/env >> $GITHUB_OUTPUT
+
   test-linux-stable-int:
+    needs: [set-image, changes]
+    if: ${{ needs.changes.outputs.rust }}  
     runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
-    needs: [set-image]
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     env:
@@ -50,11 +60,13 @@ jobs:
         uses: actions/checkout@v4
       - name: script
         run: WASM_BUILD_NO_COLOR=1 time forklift cargo test -p staging-node-cli --release --locked -- --ignored
+
   # https://github.com/paritytech/ci_cd/issues/864
   test-linux-stable-runtime-benchmarks:
+    needs: [set-image, changes]
+    if: ${{ needs.changes.outputs.rust }}    
     runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
-    needs: [set-image]
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     env:

.github/workflows/tests.yml

@@ -19,6 +19,12 @@ env:
   FORKLIFT_metrics_pushEndpoint: ${{ secrets.FORKLIFT_metrics_pushEndpoint }}
 
 jobs:
+
+  changes:
+    permissions:
+      pull-requests: read
+    uses: ./.github/workflows/check-changed-files.yml
+
   set-image:
     # GitHub Actions allows using 'env' in a container context.
     # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
@@ -31,10 +37,12 @@ jobs:
         uses: actions/checkout@v4
       - id: set_image
         run: cat .github/env >> $GITHUB_OUTPUT
+
   quick-benchmarks:
+    needs: [set-image, changes]
+    if: ${{ needs.changes.outputs.rust }}    
     runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
-    needs: [set-image]
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     env:
@@ -47,11 +55,13 @@ jobs:
         uses: actions/checkout@v4
       - name: script
         run: time forklift cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet
+
   # cf https://github.com/paritytech/polkadot-sdk/issues/1652
   test-syscalls:
+    needs: [set-image, changes]
+    if: ${{ needs.changes.outputs.rust }}    
     runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
-    needs: [set-image]
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     continue-on-error: true # this rarely triggers in practice
@@ -71,10 +81,12 @@ jobs:
     # - if [[ "$CI_JOB_STATUS" == "failed" ]]; then
     #   printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n";
     #   fi
+
   cargo-check-all-benches:
+    needs: [set-image, changes]
+    if: ${{ needs.changes.outputs.rust }}    
     runs-on: arc-runners-polkadot-sdk-beefy
     timeout-minutes: 30
-    needs: [set-image]
     container:
       image: ${{ needs.set-image.outputs.IMAGE }}
     env:

.gitlab/pipeline/publish.yml

@@ -76,6 +76,8 @@ publish-subsystem-benchmarks:
       artifacts: true
     - job: subsystem-benchmark-approval-voting
       artifacts: true
+    - job: subsystem-benchmark-statement-distribution
+      artifacts: true
     - job: publish-rustdoc
       artifacts: false
   script:
@@ -119,6 +121,8 @@ trigger_workflow:
       artifacts: true
     - job: subsystem-benchmark-approval-voting
       artifacts: true
+    - job: subsystem-benchmark-statement-distribution
+      artifacts: true
   script:
     - echo "Triggering workflow"
     - >

.gitlab/pipeline/test.yml

@@ -630,3 +630,10 @@ subsystem-benchmark-approval-voting:
   script:
     - cargo bench -p polkadot-node-core-approval-voting --bench approval-voting-regression-bench --features subsystem-benchmarks
   allow_failure: true
+
+subsystem-benchmark-statement-distribution:
+  extends:
+    - .subsystem-benchmark-template
+  script:
+    - cargo bench -p polkadot-statement-distribution --bench statement-distribution-regression-bench --features subsystem-benchmarks
+  allow_failure: true

.gitlab/pipeline/zombienet.yml

@@ -1,7 +1,7 @@
 .zombienet-refs:
   extends: .build-refs
   variables:
-    ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.104"
+    ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.105"
     PUSHGATEWAY_URL: "http://zombienet-prometheus-pushgateway.managed-monitoring:9091/metrics/job/zombie-metrics"
     DEBUG: "zombie,zombie::network-node,zombie::kube::client::logs"
 

.gitlab/pipeline/zombienet/polkadot.yml

@@ -183,6 +183,22 @@ zombienet-polkadot-functional-0012-spam-statement-distribution-requests:
       --local-dir="${LOCAL_DIR}/functional"
       --test="0012-spam-statement-distribution-requests.zndsl"
 
+zombienet-polkadot-functional-0013-systematic-chunk-recovery:
+  extends:
+    - .zombienet-polkadot-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
+      --local-dir="${LOCAL_DIR}/functional"
+      --test="0013-systematic-chunk-recovery.zndsl"
+
+zombienet-polkadot-functional-0014-chunk-fetching-network-compatibility:
+  extends:
+    - .zombienet-polkadot-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
+      --local-dir="${LOCAL_DIR}/functional"
+      --test="0014-chunk-fetching-network-compatibility.zndsl"
+
 zombienet-polkadot-smoke-0001-parachains-smoke-test:
   extends:
     - .zombienet-polkadot-common