From 6d31624b65dd2a3273a3cd9e40fee86809acee61 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 30 Sep 2023 00:07:49 -0400 Subject: [PATCH 1/4] ci cleanup --- .github/workflows/ci.yml | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e11fef128..4cbda2289 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,8 +99,8 @@ jobs: name: build-${{ matrix.target }} path: target_releases/* - build-cross: - name: Cross-platform builds + cross-build: + name: Cross-build for other platforms runs-on: ubuntu-latest env: TARGETS: "aarch64-unknown-linux-musl x86_64-unknown-linux-musl" @@ -132,7 +132,7 @@ jobs: - name: Save build artifacts to build-${{ matrix.target }} uses: actions/upload-artifact@v3 with: - name: build-cross + name: cross-build path: target_releases/* test: @@ -338,7 +338,7 @@ jobs: docker: name: Build docker images runs-on: ubuntu-latest - needs: [ build, build-cross ] + needs: [ cross-build ] env: # PG_* variables are used by psql PGDATABASE: test @@ -389,22 +389,21 @@ jobs: platforms: linux/amd64,linux/arm64 - run: rm -rf target_releases2 - - name: Download build-cross artifacts + - name: Download cross-build artifacts uses: actions/download-artifact@v3 with: - name: build-cross - path: target_releases2 + name: cross-build + path: target_releases_tmp - name: Reorganize build artifacts run: | - chmod -R +x target_releases2/ + chmod -R +x target_releases_tmp/ mkdir -p target_releases/linux/arm64 - mv target_releases2/aarch64-unknown-linux-musl/* target_releases/linux/arm64/ + mv target_releases_tmp/aarch64-unknown-linux-musl/* target_releases/linux/arm64/ mkdir -p target_releases/linux/amd64 - mv target_releases2/x86_64-unknown-linux-musl/* target_releases/linux/amd64/ - rm -rf target_releases2 + mv target_releases_tmp/x86_64-unknown-linux-musl/* target_releases/linux/amd64/ + rm -rf target_releases_tmp - name: Build linux/arm64 Docker image - id: docker_aarch64-unknown-linux-gnu uses: docker/build-push-action@v5 # https://github.com/docker/build-push-action with: @@ -426,7 +425,6 @@ jobs: DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require - name: Build linux/amd64 Docker image - id: docker_x86_64-unknown-linux-gnu uses: docker/build-push-action@v5 # https://github.com/docker/build-push-action with: @@ -476,7 +474,7 @@ jobs: package: name: Package ${{ matrix.target }} runs-on: ${{ matrix.os }} - needs: [ test, test-legacy ] + needs: [ test, test-legacy, docker, cross-build ] strategy: fail-fast: true matrix: @@ -488,9 +486,6 @@ jobs: - target: debian-x86_64 os: ubuntu-latest name: martin-Debian-x86_64.deb - # - target: aarch64-unknown-linux-gnu - # os: ubuntu-latest - # name: martin-Linux-aarch64.tar.gz - target: x86_64-apple-darwin os: macOS-latest name: martin-Darwin-x86_64.tar.gz @@ -502,7 +497,9 @@ jobs: - target: x86_64-unknown-linux-gnu os: ubuntu-latest name: martin-Linux-x86_64.tar.gz - # From the cross build + # + # From the cross-build + # - target: aarch64-unknown-linux-musl os: ubuntu-latest cross: 'true' @@ -524,7 +521,7 @@ jobs: if: matrix.cross == 'true' uses: actions/download-artifact@v3 with: - name: build-cross + name: cross-build path: target/ - name: Package run: | From 5fd2ace5e718a02243f8a10f12d863027241e69e Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 30 Sep 2023 00:15:47 -0400 Subject: [PATCH 2/4] CI: combine cross with docker builds --- .github/workflows/ci.yml | 79 ++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4cbda2289..3a5628b67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,42 +99,6 @@ jobs: name: build-${{ matrix.target }} path: target_releases/* - cross-build: - name: Cross-build for other platforms - runs-on: ubuntu-latest - env: - TARGETS: "aarch64-unknown-linux-musl x86_64-unknown-linux-musl" - # TODO: aarch64-unknown-linux-gnu - steps: - - name: Checkout sources - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 - - name: Install cross - run: | - cargo install cross - # Install latest cross version from git (disabled as it is probably less stable) - # cargo install cross --git https://github.com/cross-rs/cross - cross --version - - name: Build targets - run: | - for target in $TARGETS; do - echo -e "\n----------------------------------------------" - echo "Building $target" - - export "CARGO_TARGET_$(echo $target | tr 'a-z-' 'A-Z_')_RUSTFLAGS"='-C strip=debuginfo' - cross build --release --target $target --package martin-mbtiles - cross build --release --target $target --package martin --features=vendored-openssl - - mkdir -p target_releases/$target - mv target/$target/release/mbtiles target_releases/$target - mv target/$target/release/martin target_releases/$target - done - - name: Save build artifacts to build-${{ matrix.target }} - uses: actions/upload-artifact@v3 - with: - name: cross-build - path: target_releases/* - test: name: Test ${{ matrix.target }} runs-on: ${{ matrix.os }} @@ -336,15 +300,16 @@ jobs: retention-days: 5 docker: - name: Build docker images + name: Build and test docker images runs-on: ubuntu-latest - needs: [ cross-build ] env: # PG_* variables are used by psql PGDATABASE: test PGHOST: localhost PGUSER: postgres PGPASSWORD: postgres + TARGETS: "aarch64-unknown-linux-musl x86_64-unknown-linux-musl" + # TODO: aarch64-unknown-linux-gnu services: postgres: image: postgis/postgis:15-3.3 @@ -371,6 +336,12 @@ jobs: - name: Checkout sources uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 + - name: Install cross + run: | + cargo install cross + # Install latest cross version from git (disabled as it is probably less stable) + # cargo install cross --git https://github.com/cross-rs/cross + cross --version - name: Setup database run: tests/fixtures/initdb.sh env: @@ -388,20 +359,32 @@ jobs: install: true platforms: linux/amd64,linux/arm64 - - run: rm -rf target_releases2 - - name: Download cross-build artifacts - uses: actions/download-artifact@v3 + - name: Build targets + run: | + for target in $TARGETS; do + echo -e "\n----------------------------------------------" + echo "Building $target" + + export "CARGO_TARGET_$(echo $target | tr 'a-z-' 'A-Z_')_RUSTFLAGS"='-C strip=debuginfo' + cross build --release --target $target --package martin-mbtiles + cross build --release --target $target --package martin --features=vendored-openssl + + mkdir -p target_releases/$target + mv target/$target/release/mbtiles target_releases/$target + mv target/$target/release/martin target_releases/$target + done + + - name: Save build artifacts to build-${{ matrix.target }} + uses: actions/upload-artifact@v3 with: name: cross-build - path: target_releases_tmp - - name: Reorganize build artifacts + path: target_releases/* + - name: Reorganize artifacts for docker build run: | - chmod -R +x target_releases_tmp/ mkdir -p target_releases/linux/arm64 - mv target_releases_tmp/aarch64-unknown-linux-musl/* target_releases/linux/arm64/ + mv target_releases/aarch64-unknown-linux-musl/* target_releases/linux/arm64/ mkdir -p target_releases/linux/amd64 - mv target_releases_tmp/x86_64-unknown-linux-musl/* target_releases/linux/amd64/ - rm -rf target_releases_tmp + mv target_releases/x86_64-unknown-linux-musl/* target_releases/linux/amd64/ - name: Build linux/arm64 Docker image uses: docker/build-push-action@v5 @@ -474,7 +457,7 @@ jobs: package: name: Package ${{ matrix.target }} runs-on: ${{ matrix.os }} - needs: [ test, test-legacy, docker, cross-build ] + needs: [ test, test-legacy, docker ] strategy: fail-fast: true matrix: From f27941ed59fc16f9c1cf394b7e9348b92e5806f7 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 30 Sep 2023 00:33:29 -0400 Subject: [PATCH 3/4] cleanup --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a5628b67..3e580bcde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -538,7 +538,7 @@ jobs: done: name: CI Finished runs-on: ubuntu-latest - needs: [ docker, package ] + needs: [ package ] steps: - name: Finished run: echo "CI finished successfully" From 8bc7ca514ad61bd7659c8bb1c2c706e69b423655 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 30 Sep 2023 00:43:26 -0400 Subject: [PATCH 4/4] reorg --- .github/workflows/ci.yml | 312 +++++++++++++++++++-------------------- 1 file changed, 156 insertions(+), 156 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e580bcde..ed5715f4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,161 @@ defaults: shell: bash jobs: + docker: + name: Build and test docker images + runs-on: ubuntu-latest + env: + # PG_* variables are used by psql + PGDATABASE: test + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + TARGETS: "aarch64-unknown-linux-musl x86_64-unknown-linux-musl" + # TODO: aarch64-unknown-linux-gnu + services: + postgres: + image: postgis/postgis:15-3.3 + ports: + # will assign a random free host port + - 5432/tcp + # Sadly there is currently no way to pass arguments to the service image other than this hack + # See also https://stackoverflow.com/a/62720566/177275 + options: >- + -e POSTGRES_DB=test + -e POSTGRES_USER=postgres + -e POSTGRES_PASSWORD=postgres + -e PGDATABASE=test + -e PGUSER=postgres + -e PGPASSWORD=postgres + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + --entrypoint sh + postgis/postgis:15-3.3 + -c "exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key" + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + - name: Install cross + run: | + cargo install cross + # Install latest cross version from git (disabled as it is probably less stable) + # cargo install cross --git https://github.com/cross-rs/cross + cross --version + - name: Setup database + run: tests/fixtures/initdb.sh + env: + PGPORT: ${{ job.services.postgres.ports[5432] }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + # https://github.com/docker/setup-qemu-action + with: + platforms: linux/amd64,linux/arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + # https://github.com/docker/setup-buildx-action + with: + install: true + platforms: linux/amd64,linux/arm64 + + - name: Build targets + run: | + for target in $TARGETS; do + echo -e "\n----------------------------------------------" + echo "Building $target" + + export "CARGO_TARGET_$(echo $target | tr 'a-z-' 'A-Z_')_RUSTFLAGS"='-C strip=debuginfo' + cross build --release --target $target --package martin-mbtiles + cross build --release --target $target --package martin --features=vendored-openssl + + mkdir -p target_releases/$target + mv target/$target/release/mbtiles target_releases/$target + mv target/$target/release/martin target_releases/$target + done + + - name: Save build artifacts to build-${{ matrix.target }} + uses: actions/upload-artifact@v3 + with: + name: cross-build + path: target_releases/* + - name: Reorganize artifacts for docker build + run: | + mkdir -p target_releases/linux/arm64 + mv target_releases/aarch64-unknown-linux-musl/* target_releases/linux/arm64/ + mkdir -p target_releases/linux/amd64 + mv target_releases/x86_64-unknown-linux-musl/* target_releases/linux/amd64/ + + - name: Build linux/arm64 Docker image + uses: docker/build-push-action@v5 + # https://github.com/docker/build-push-action + with: + context: . + file: multi-platform.Dockerfile + load: true + tags: ${{ github.repository }}:linux-arm64 + platforms: linux/arm64 + - name: Test linux/arm64 Docker image + run: | + PLATFORM=linux/arm64 + TAG=${{ github.repository }}:linux-arm64 + export MBTILES_BUILD=- + export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG" + export MARTIN_BUILD=- + export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG" + tests/test.sh + env: + DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require + + - name: Build linux/amd64 Docker image + uses: docker/build-push-action@v5 + # https://github.com/docker/build-push-action + with: + context: . + file: multi-platform.Dockerfile + load: true + tags: ${{ github.repository }}:linux-amd64 + platforms: linux/amd64 + - name: Test linux/amd64 Docker image + run: | + PLATFORM=linux/amd64 + TAG=${{ github.repository }}:linux-amd64 + export MBTILES_BUILD=- + export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG" + export MARTIN_BUILD=- + export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG" + tests/test.sh + env: + DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require + + - name: Login to GitHub Docker registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + # https://github.com/docker/login-action + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v5 + # https://github.com/docker/metadata-action + with: + images: ghcr.io/${{ github.repository }} + - name: Push the Docker image + if: github.event_name != 'pull_request' + uses: docker/build-push-action@v5 + with: + context: . + file: multi-platform.Dockerfile + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 + build: name: Build ${{ matrix.target }} runs-on: ${{ matrix.os }} @@ -299,165 +454,10 @@ jobs: path: tests/output/* retention-days: 5 - docker: - name: Build and test docker images - runs-on: ubuntu-latest - env: - # PG_* variables are used by psql - PGDATABASE: test - PGHOST: localhost - PGUSER: postgres - PGPASSWORD: postgres - TARGETS: "aarch64-unknown-linux-musl x86_64-unknown-linux-musl" - # TODO: aarch64-unknown-linux-gnu - services: - postgres: - image: postgis/postgis:15-3.3 - ports: - # will assign a random free host port - - 5432/tcp - # Sadly there is currently no way to pass arguments to the service image other than this hack - # See also https://stackoverflow.com/a/62720566/177275 - options: >- - -e POSTGRES_DB=test - -e POSTGRES_USER=postgres - -e POSTGRES_PASSWORD=postgres - -e PGDATABASE=test - -e PGUSER=postgres - -e PGPASSWORD=postgres - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - --entrypoint sh - postgis/postgis:15-3.3 - -c "exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key" - steps: - - name: Checkout sources - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 - - name: Install cross - run: | - cargo install cross - # Install latest cross version from git (disabled as it is probably less stable) - # cargo install cross --git https://github.com/cross-rs/cross - cross --version - - name: Setup database - run: tests/fixtures/initdb.sh - env: - PGPORT: ${{ job.services.postgres.ports[5432] }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - # https://github.com/docker/setup-qemu-action - with: - platforms: linux/amd64,linux/arm64 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - # https://github.com/docker/setup-buildx-action - with: - install: true - platforms: linux/amd64,linux/arm64 - - - name: Build targets - run: | - for target in $TARGETS; do - echo -e "\n----------------------------------------------" - echo "Building $target" - - export "CARGO_TARGET_$(echo $target | tr 'a-z-' 'A-Z_')_RUSTFLAGS"='-C strip=debuginfo' - cross build --release --target $target --package martin-mbtiles - cross build --release --target $target --package martin --features=vendored-openssl - - mkdir -p target_releases/$target - mv target/$target/release/mbtiles target_releases/$target - mv target/$target/release/martin target_releases/$target - done - - - name: Save build artifacts to build-${{ matrix.target }} - uses: actions/upload-artifact@v3 - with: - name: cross-build - path: target_releases/* - - name: Reorganize artifacts for docker build - run: | - mkdir -p target_releases/linux/arm64 - mv target_releases/aarch64-unknown-linux-musl/* target_releases/linux/arm64/ - mkdir -p target_releases/linux/amd64 - mv target_releases/x86_64-unknown-linux-musl/* target_releases/linux/amd64/ - - - name: Build linux/arm64 Docker image - uses: docker/build-push-action@v5 - # https://github.com/docker/build-push-action - with: - context: . - file: multi-platform.Dockerfile - load: true - tags: ${{ github.repository }}:linux-arm64 - platforms: linux/arm64 - - name: Test linux/arm64 Docker image - run: | - PLATFORM=linux/arm64 - TAG=${{ github.repository }}:linux-arm64 - export MBTILES_BUILD=- - export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG" - export MARTIN_BUILD=- - export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG" - tests/test.sh - env: - DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require - - - name: Build linux/amd64 Docker image - uses: docker/build-push-action@v5 - # https://github.com/docker/build-push-action - with: - context: . - file: multi-platform.Dockerfile - load: true - tags: ${{ github.repository }}:linux-amd64 - platforms: linux/amd64 - - name: Test linux/amd64 Docker image - run: | - PLATFORM=linux/amd64 - TAG=${{ github.repository }}:linux-amd64 - export MBTILES_BUILD=- - export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG" - export MARTIN_BUILD=- - export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG" - tests/test.sh - env: - DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require - - - name: Login to GitHub Docker registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - # https://github.com/docker/login-action - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker meta - id: docker_meta - uses: docker/metadata-action@v5 - # https://github.com/docker/metadata-action - with: - images: ghcr.io/${{ github.repository }} - - name: Push the Docker image - if: github.event_name != 'pull_request' - uses: docker/build-push-action@v5 - with: - context: . - file: multi-platform.Dockerfile - push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 - package: name: Package ${{ matrix.target }} runs-on: ${{ matrix.os }} - needs: [ test, test-legacy, docker ] + needs: [ docker, test, test-legacy ] strategy: fail-fast: true matrix: