Merge branch 'master' into para-scraping

paritytech · Nov 22, 2023 · 01f0f77 · 01f0f77
2 parents 5d2392d + 4987488
commit 01f0f77
Show file tree

Hide file tree

Showing 136 changed files with 3,041 additions and 2,166 deletions.
diff --git a/.github/workflows/merge-queue.yml b/.github/workflows/merge-queue.yml
@@ -0,0 +1,24 @@
+name: Merge-Queue
+
+on:
+  merge_group:
+
+jobs:
+  trigger-merge-queue-action:
+    runs-on: ubuntu-latest
+    environment: master
+    steps:
+      - name: Generate token
+        id: app_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        with:
+          app_id: ${{ secrets.REVIEW_APP_ID }}
+          private_key: ${{ secrets.REVIEW_APP_KEY }}
+      - name: Add Merge Queue status check
+        uses: billyjbryant/create-status-check@3e6fa0ac599d10d9588cf9516ca4330ef669b858 # v2
+        with:
+          authToken: ${{ steps.app_token.outputs.token }}
+          context: 'review-bot'
+          description: 'PRs for merge queue gets approved'
+          state: 'success'
+          sha: ${{ github.event.merge_group.head_commit.id }}
diff --git a/.github/workflows/release-50_publish-docker.yml b/.github/workflows/release-50_publish-docker.yml
@@ -268,7 +268,7 @@ jobs:
 
       - name: Build and push
         id:   docker_build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 #  v5.0.0
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 #  v5.1.0
         with:
           push: true
           file: docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -30,8 +30,7 @@ variables:
   RUSTY_CACHIER_COMPRESSION_METHOD: zstd
   NEXTEST_FAILURE_OUTPUT: immediate-final
   NEXTEST_SUCCESS_OUTPUT: final
-  ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.80"
-  DOCKER_IMAGES_VERSION: "${CI_COMMIT_SHA}"
+  DOCKER_IMAGES_VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
 
 default:
   retry:
@@ -199,9 +198,6 @@ default:
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
     - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues
 
-.zombienet-refs:
-  extends: .build-refs
-
 include:
   # check jobs
   - .gitlab/pipeline/check.yml

diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml
@@ -151,6 +151,31 @@ check-runtime-migration-asset-hub-westend:
     PACKAGE: "asset-hub-westend-runtime"
     WASM: "asset_hub_westend_runtime.compact.compressed.wasm"
     URI: "wss://westend-asset-hub-rpc.polkadot.io:443"
+
+check-runtime-migration-asset-hub-rococo:
+  stage: check
+  extends:
+    - .docker-env
+    - .test-pr-refs
+    - .check-runtime-migration
+  variables:
+    NETWORK: "asset-hub-rococo"
+    PACKAGE: "asset-hub-rococo-runtime"
+    WASM: "asset_hub_rococo_runtime.compact.compressed.wasm"
+    URI: "wss://rococo-asset-hub-rpc.polkadot.io:443"
+
+# Check runtime migrations for Parity managed bridge hub chains
+check-runtime-migration-bridge-hub-westend:
+  stage: check
+  extends:
+    - .docker-env
+    - .test-pr-refs
+    - .check-runtime-migration
+  variables:
+    NETWORK: "bridge-hub-westend"
+    PACKAGE: "bridge-hub-westend-runtime"
+    WASM: "bridge_hub_westend_runtime.compact.compressed.wasm"
+    URI: "wss://westend-bridge-hub-rpc.polkadot.io:443"
 
 check-runtime-migration-bridge-hub-rococo:
   stage: check

diff --git a/.gitlab/pipeline/publish.yml b/.gitlab/pipeline/publish.yml
@@ -63,16 +63,16 @@ publish-rustdoc:
   after_script:
     - rm -rf .git/ ./*
 
-# cumulus
-
+# note: images are used not only in zombienet but also in rococo, wococo and versi
 .build-push-image:
   image: $BUILDAH_IMAGE
   variables:
     DOCKERFILE: "" # docker/path-to.Dockerfile
     IMAGE_NAME: "" # docker.io/paritypr/image_name
   script:
-    # Exit if the job is not running in a merge queue
-    # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi
+    # Dockertag should differ in a merge queue
+    # TODO: test this
+    # - if [[ $CI_COMMIT_REF_NAME == *"gh-readonly-queue"* ]]; export DOCKER_IMAGES_VERSION="${CI_COMMIT_SHORT_SHA}"; fi
     - $BUILDAH_COMMAND build
       --format=docker
       --build-arg VCS_REF="${CI_COMMIT_SHA}"
@@ -112,59 +112,6 @@ build-push-image-test-parachain:
   variables:
     DOCKERFILE: "docker/dockerfiles/test-parachain_injected.Dockerfile"
     IMAGE_NAME: "docker.io/paritypr/test-parachain"
-# publish-s3:
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .publish-refs
-#   image: paritytech/awscli:latest
-#   needs:
-#     - job: build-linux-stable-cumulus
-#       artifacts: true
-#   variables:
-#     GIT_STRATEGY: none
-#     BUCKET: "releases.parity.io"
-#     PREFIX: "cumulus/${ARCH}-${DOCKER_OS}"
-#   script:
-#     - echo "___Publishing a binary with debug assertions!___"
-#     - echo "___VERSION = $(cat ./artifacts/VERSION) ___"
-#     - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/
-#     - echo "___Updating objects in latest path___"
-#     - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
-#   after_script:
-#     - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
-#       --recursive --human-readable --summarize
-
-# publish-benchmarks-assets-s3: &publish-benchmarks
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .benchmarks-refs
-#   image: paritytech/awscli:latest
-#   needs:
-#     - job: benchmarks-assets
-#       artifacts: true
-#   variables:
-#     GIT_STRATEGY: none
-#     BUCKET: "releases.parity.io"
-#     PREFIX: "cumulus/$CI_COMMIT_REF_NAME/benchmarks-assets"
-#   script:
-#     - echo "___Publishing benchmark results___"
-#     - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/
-#   after_script:
-#     - aws s3 ls s3://${BUCKET}/${PREFIX}/ --recursive --human-readable --summarize
-
-# publish-benchmarks-collectives-s3:
-#   <<: *publish-benchmarks
-#   variables:
-#     GIT_STRATEGY: none
-#     BUCKET: "releases.parity.io"
-#     PREFIX: "cumulus/$CI_COMMIT_REF_NAME/benchmarks-collectives"
-#   needs:
-#     - job: benchmarks-collectives
-#       artifacts: true
-
-### Polkadot
 
 build-push-image-polkadot-debug:
   stage: publish
@@ -217,143 +164,3 @@ build-push-image-substrate-pr:
   variables:
     DOCKERFILE: "docker/dockerfiles/substrate_injected.Dockerfile"
     IMAGE_NAME: "docker.io/paritypr/substrate"
-# old way
-
-# .build-push-image-polkadot:
-#   before_script:
-#     # - test -s ./artifacts/VERSION || exit 1
-#     # - test -s ./artifacts/EXTRATAG || exit 1
-#     - VERSION="$(cat ./artifacts/VERSION)"
-#     - EXTRATAG="$(cat ./artifacts/EXTRATAG)"
-#     - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
-#   script:
-#     # - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
-#     #   ( echo "no docker credentials provided"; exit 1 )
-#     - cd ./artifacts
-#     - $BUILDAH_COMMAND build
-#       --format=docker
-#       --build-arg VCS_REF="${CI_COMMIT_SHA}"
-#       --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
-#       --build-arg IMAGE_NAME="${IMAGE_NAME}"
-#       --tag "$IMAGE_NAME:$VERSION"
-#       --tag "$IMAGE_NAME:$EXTRATAG"
-#       --file ${DOCKERFILE} .
-#     # The job will success only on the protected branch
-#     # - echo "$DOCKER_PASS" |
-#     #   buildah login --username "$DOCKER_USER" --password-stdin docker.io
-#     # - $BUILDAH_COMMAND info
-#     # - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$VERSION"
-#     # - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$EXTRATAG"
-#   after_script:
-#     - buildah logout --all
-
-# publish-polkadot-debug-image:
-#   stage: publish
-#   image: ${BUILDAH_IMAGE}
-#   extends:
-#     - .kubernetes-env
-#     - .build-push-image-polkadot
-#   rules:
-#     - if: $CI_PIPELINE_SOURCE == "web"
-#     - if: $CI_PIPELINE_SOURCE == "schedule"
-#     - if: $CI_COMMIT_REF_NAME == "master"
-#     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
-#     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
-#   variables:
-#     GIT_STRATEGY: none
-#     DOCKER_USER: ${PARITYPR_USER}
-#     DOCKER_PASS: ${PARITYPR_PASS}
-#     # docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile
-#     DOCKERFILE: polkadot_injected_debug.Dockerfile
-#     IMAGE_NAME: docker.io/paritypr/polkadot-debug
-#   needs:
-#     - job: build-linux-stable
-#       artifacts: true
-#   after_script:
-#     # pass artifacts to the zombienet-tests job
-#     # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
-#     - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env
-#     - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env
-#   artifacts:
-#     reports:
-#       # this artifact is used in zombienet-tests job
-#       dotenv: ./artifacts/parachains.env
-#     expire_in: 1 days
-
-# publish-test-collators-image:
-#   # service image for zombienet
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .build-push-image-polkadot
-#     - .zombienet-refs
-#   variables:
-#     CI_IMAGE: ${BUILDAH_IMAGE}
-#     GIT_STRATEGY: none
-#     DOCKER_USER: ${PARITYPR_USER}
-#     DOCKER_PASS: ${PARITYPR_PASS}
-#     # docker/dockerfiles/collator_injected.Dockerfile
-#     DOCKERFILE: collator_injected.Dockerfile
-#     IMAGE_NAME: docker.io/paritypr/colander
-#   needs:
-#     - job: build-test-collators
-#       artifacts: true
-#   after_script:
-#     - buildah logout --all
-#     # pass artifacts to the zombienet-tests job
-#     - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env
-#     - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env
-#   artifacts:
-#     reports:
-#       # this artifact is used in zombienet-tests job
-#       dotenv: ./artifacts/collator.env
-
-# publish-malus-image:
-#   # service image for Simnet
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .build-push-image-polkadot
-#     - .zombienet-refs
-#   variables:
-#     CI_IMAGE: ${BUILDAH_IMAGE}
-#     GIT_STRATEGY: none
-#     DOCKER_USER: ${PARITYPR_USER}
-#     DOCKER_PASS: ${PARITYPR_PASS}
-#     # docker/dockerfiles/malus_injected.Dockerfile
-#     DOCKERFILE: malus_injected.Dockerfile
-#     IMAGE_NAME: docker.io/paritypr/malus
-#   needs:
-#     - job: build-malus
-#       artifacts: true
-#   after_script:
-#     - buildah logout "$IMAGE_NAME"
-#     # pass artifacts to the zombienet-tests job
-#     - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env
-#     - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env
-#   artifacts:
-#     reports:
-#       # this artifact is used in zombienet-tests job
-#       dotenv: ./artifacts/malus.env
-
-# substrate
-
-# publish-substrate-image-pr:
-#   # service image for zombienet
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .build-push-image-polkadot
-#     - .zombienet-refs
-#   variables:
-#     CI_IMAGE: ${BUILDAH_IMAGE}
-#     GIT_STRATEGY: none
-#     DOCKER_USER: ${PARITYPR_USER}
-#     DOCKER_PASS: ${PARITYPR_PASS}
-#     DOCKERFILE: substrate_injected.Dockerfile
-#     IMAGE_NAME: docker.io/paritypr/substrate
-#   needs:
-#     - job: build-linux-substrate
-#       artifacts: true
-#   after_script:
-#     - buildah logout "$IMAGE_NAME"
diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml
@@ -29,7 +29,7 @@ test-linux-stable:
         --locked \
         --release \
         --no-fail-fast \
-        --features try-runtime,experimental \
+        --features try-runtime,experimental,ci-only-tests \
         --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
     # Upload tests results to Elasticsearch
     - echo "Upload test results to Elasticsearch"

diff --git a/.gitlab/pipeline/zombienet.yml b/.gitlab/pipeline/zombienet.yml
@@ -1,3 +1,8 @@
+.zombienet-refs:
+  extends: .build-refs
+  variables:
+    ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.82"
+
 include:
   # substrate tests
   - .gitlab/pipeline/zombienet/substrate.yml

diff --git a/.gitlab/pipeline/zombienet/cumulus.yml b/.gitlab/pipeline/zombienet/cumulus.yml
@@ -28,7 +28,7 @@
     - job: build-push-image-test-parachain
       artifacts: true
   variables:
-    POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master"
+    POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:${DOCKER_IMAGES_VERSION}"
     GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet/tests"
     LOCAL_DIR: "/builds/parity/mirrors/polkadot-sdk/cumulus/zombienet/tests"
     COL_IMAGE: "docker.io/paritypr/test-parachain:${DOCKER_IMAGES_VERSION}"

diff --git a/.gitlab/pipeline/zombienet/substrate.yml b/.gitlab/pipeline/zombienet/substrate.yml
@@ -40,14 +40,13 @@
   tags:
     - zombienet-polkadot-integration-test
 
-# Skip this one until PolkadotJS includes `SkipCheckIfFeeless` extension
-# zombienet-substrate-0000-block-building:
-#   extends:
-#     - .zombienet-substrate-common
-#   script:
-#     - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
-#       --local-dir="${LOCAL_DIR}/0000-block-building"
-#       --test="block-building.zndsl"
+zombienet-substrate-0000-block-building:
+  extends:
+    - .zombienet-substrate-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
+      --local-dir="${LOCAL_DIR}/0000-block-building"
+      --test="block-building.zndsl"
 
 zombienet-substrate-0001-basic-warp-sync:
   extends: