Sync feature/atree-inlining-cadence-v0.42 with latest master

.github/workflows/builds.yml

@@ -1,6 +1,6 @@
+name: Build Node Docker Images
 # This workflow is used to build and push one-off images for specific node types. This is useful
 # when deploying hotfixes or any time a change is not needed for all node roles.
-name: Build Node Docker Images
 
 on:
   workflow_dispatch:
@@ -38,9 +38,10 @@ on:
         type: boolean
         description: 'Observer'
         required: false
-      include_without_netgo:
+      # GHA allows only up to 10 inputs - regroup two entries in one
+      include_alternative_builds:
         type: boolean
-        description: 'Build `without_netgo` images'
+        description: 'Build amd64 `without_adx` and `without_netgo_without_adx` images, and arm64 images'
         required: false
 
 jobs:
@@ -111,17 +112,22 @@ jobs:
       run: |
         gcloud auth configure-docker
 
-    - name: Build/Push ${{ matrix.role }} images
+    - name: Build/Push ${{ matrix.role }} amd64 images with adx (default)
       env:
         IMAGE_TAG: ${{ inputs.docker_tag }}
         CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
       run: |
-        make docker-build-${{ matrix.role }} docker-push-${{ matrix.role }}
+        make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx
 
-    - name: Build/Push ${{ matrix.role }} without_netgo images
-      if: ${{ inputs.include_without_netgo }}
+    - name: Build/Push ${{ matrix.role }} amd64 images without netgo and without adx, arm64 images
+      if: ${{ inputs.include_alternative_builds }}
       env:
         IMAGE_TAG: ${{ inputs.docker_tag }}
         CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
       run: |
-        make docker-build-${{ matrix.role }}-without-netgo docker-push-${{ matrix.role }}-without-netgo
+        make docker-build-${{ matrix.role }}-without-adx docker-push-${{ matrix.role }}-without-adx \
+          docker-build-${{ matrix.role }}-without-netgo-without-adx docker-push-${{ matrix.role }}-without-netgo-without-adx \
+          docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm
+
+
+

.github/workflows/cd.yml

@@ -33,9 +33,13 @@ jobs:
       env:
         CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
       run: |
-        make docker-build-flow
-        make docker-build-flow-without-netgo
+        make docker-build-flow-with-adx
+        make docker-build-flow-without-adx
+        make docker-build-flow-without-netgo-without-adx
+        make docker-cross-build-flow-arm
     - name: Docker push
       run: |
-        make docker-push-flow
-        make docker-push-flow-without-netgo
+        make docker-push-flow-with-adx
+        make docker-push-flow-without-adx
+        make docker-push-flow-without-netgo-without-adx
+        make docker-push-flow-arm

.github/workflows/ci.yml

@@ -89,7 +89,43 @@ jobs:
           cache: true
       - name: Set Test Matrix
         id: set-test-matrix
-        run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine/access engine/collection engine/common engine/consensus engine/execution/ingestion:buildjet-8vcpu-ubuntu-2204 engine/execution/computation engine/execution engine/verification engine:buildjet-4vcpu-ubuntu-2204 fvm ledger module/dkg module:buildjet-4vcpu-ubuntu-2204 network/alsp network/test/cohort1:buildjet-16vcpu-ubuntu-2204 network/test/cohort2:buildjet-4vcpu-ubuntu-2204 network/p2p/connection network/p2p/node:buildjet-4vcpu-ubuntu-2204 network/p2p/scoring network/p2p network state storage utils
+        run: go run tools/test_matrix_generator/matrix.go
+
+  create-insecure-dynamic-test-matrix:
+    name: Create Dynamic Unit Test Insecure Package Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      - name: Set Test Matrix
+        id: set-test-matrix
+        run: go run tools/test_matrix_generator/matrix.go -c insecure
+
+  create-integration-dynamic-test-matrix:
+    name: Create Dynamic Integration Test Package Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      - name: Set Test Matrix
+        id: set-test-matrix
+        run: go run tools/test_matrix_generator/matrix.go -c integration
 
   unit-test:
     name: Unit Tests (${{ matrix.targets.name }})
@@ -127,23 +163,15 @@ jobs:
         flags: unittests
         name: codecov-umbrella
 
-  unit-test-modules:
-    name: Unit Tests (Modules)
+  unit-test-insecure:
+    name: Unit Tests Insecure (${{ matrix.targets.name }})
+    needs: create-insecure-dynamic-test-matrix
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - name: insecure
-            setup: install-tools
-            retries: 5
-            race: 0
-            runner: buildjet-4vcpu-ubuntu-2204
-          - name: integration
-            setup: install-tools
-            retries: 5
-            race: 0
-            runner: buildjet-4vcpu-ubuntu-2204
-    runs-on: ${{ matrix.runner }}
+        targets: ${{ fromJSON(needs.create-insecure-dynamic-test-matrix.outputs.dynamic-matrix)}}
+    ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
+    runs-on: ${{ matrix.targets.runner }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
@@ -153,17 +181,17 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
           cache: true
-      - name: Setup tests (${{ matrix.name }})
-        run: make ${{ matrix.setup }}
-      - name: Run tests (${{ matrix.name }})
-        env:
-          RACE_DETECTOR: ${{ matrix.race }}
+      - name: Setup tests (${{ matrix.targets.name }})
+        run: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools
+      - name: Run tests (${{ matrix.targets.name }})
         uses: nick-fields/retry@v2
         with:
           timeout_minutes: 35
-          max_attempts: ${{ matrix.retries }}
-          # run test target inside each module's root
-          command: VERBOSE=1 make -C ${{ matrix.name }} test
+          max_attempts: 5
+          command: VERBOSE=1 make -C ./insecure -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test
+        # TODO(rbtz): re-enable when we fix exisiting races.
+        #env:
+        #  RACE_DETECTOR: 1
       - name: Upload coverage report
         uses: codecov/codecov-action@v3
         with:
@@ -191,7 +219,7 @@ jobs:
       - name: Docker build
         env:
           CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }}
-        run: make docker-build-flow docker-build-flow-corrupt
+        run: make docker-native-build-flow docker-native-build-flow-corrupt
       - name: Save Docker images
         run: |
           docker save \
@@ -212,6 +240,42 @@ jobs:
           # use the workflow run id as part of the cache key to ensure these docker images will only be used for a single workflow run
           key: flow-docker-images-${{ hashFiles('**/Dockerfile') }}-${{ github.run_id }}
 
+  integration-test-others:
+    name: Integration Tests Others (${{ matrix.targets.name }})
+    needs: create-integration-dynamic-test-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        targets: ${{ fromJSON(needs.create-integration-dynamic-test-matrix.outputs.dynamic-matrix)}}
+    ## need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
+    runs-on: ${{ matrix.targets.runner }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: true
+      - name: Setup tests (${{ matrix.targets.name }})
+        run: VERBOSE=1 make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools
+      - name: Run tests (${{ matrix.targets.name }})
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 35
+          max_attempts: 5
+          command: VERBOSE=1 make -C ./integration -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" test
+        # TODO(rbtz): re-enable when we fix exisiting races.
+        #env:
+        #  RACE_DETECTOR: 1
+      - name: Upload coverage report
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.txt
+          flags: unittests
+          name: codecov-umbrella
+
   integration-test:
     name: Integration Tests
     needs: docker-build

.github/workflows/flaky-test-monitor.yml

@@ -165,7 +165,7 @@ jobs:
           go-version: ${{ env.GO_VERSION }}
           cache: true
       - name: Docker build
-        run: make docker-build-flow docker-build-flow-corrupt
+        run: make docker-native-build-flow docker-native-build-flow-corrupt
       - name: Run tests
         run: make -es -C integration ${{ matrix.target }} > test-output
         timeout-minutes: 100

.gitignore

@@ -7,6 +7,8 @@
 /cmd/util/util
 /cmd/bootstrap/bootstrap
 
+# Test ouput of  bootstrapping CLI
+cmd/bootstrap/bootstrap-example
 
 # Test binary, build with `go test -c`
 *.test