Skip to content

Commit

Permalink
CI: Combine cross docker builds (#910)
Browse files Browse the repository at this point in the history 
Minor CI cleanup
  • Loading branch information
@nyurik
nyurik authored Sep 30, 2023
1 parent 976a850 commit 48164d7
Showing 1 changed file with 161 additions and 181 deletions.
342 changes: 161 additions & 181 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,161 @@ defaults:
shell: bash

jobs:
docker:
name: Build and test docker images
runs-on: ubuntu-latest
env:
# PG_* variables are used by psql
PGDATABASE: test
PGHOST: localhost
PGUSER: postgres
PGPASSWORD: postgres
TARGETS: "aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
# TODO: aarch64-unknown-linux-gnu
services:
postgres:
image: postgis/postgis:15-3.3
ports:
# will assign a random free host port
- 5432/tcp
# Sadly there is currently no way to pass arguments to the service image other than this hack
# See also https://stackoverflow.com/a/62720566/177275
options: >-
-e POSTGRES_DB=test
-e POSTGRES_USER=postgres
-e POSTGRES_PASSWORD=postgres
-e PGDATABASE=test
-e PGUSER=postgres
-e PGPASSWORD=postgres
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
--entrypoint sh
postgis/postgis:15-3.3
-c "exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key"
steps:
- name: Checkout sources
uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
- name: Install cross
run: |
cargo install cross
# Install latest cross version from git (disabled as it is probably less stable)
# cargo install cross --git https://github.com/cross-rs/cross
cross --version
- name: Setup database
run: tests/fixtures/initdb.sh
env:
PGPORT: ${{ job.services.postgres.ports[5432] }}

- name: Set up QEMU
uses: docker/setup-qemu-action@v3
# https://github.com/docker/setup-qemu-action
with:
platforms: linux/amd64,linux/arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# https://github.com/docker/setup-buildx-action
with:
install: true
platforms: linux/amd64,linux/arm64

- name: Build targets
run: |
for target in $TARGETS; do
echo -e "\n----------------------------------------------"
echo "Building $target"
export "CARGO_TARGET_$(echo $target | tr 'a-z-' 'A-Z_')_RUSTFLAGS"='-C strip=debuginfo'
cross build --release --target $target --package martin-mbtiles
cross build --release --target $target --package martin --features=vendored-openssl
mkdir -p target_releases/$target
mv target/$target/release/mbtiles target_releases/$target
mv target/$target/release/martin target_releases/$target
done
- name: Save build artifacts to build-${{ matrix.target }}
uses: actions/upload-artifact@v3
with:
name: cross-build
path: target_releases/*
- name: Reorganize artifacts for docker build
run: |
mkdir -p target_releases/linux/arm64
mv target_releases/aarch64-unknown-linux-musl/* target_releases/linux/arm64/
mkdir -p target_releases/linux/amd64
mv target_releases/x86_64-unknown-linux-musl/* target_releases/linux/amd64/
- name: Build linux/arm64 Docker image
uses: docker/build-push-action@v5
# https://github.com/docker/build-push-action
with:
context: .
file: multi-platform.Dockerfile
load: true
tags: ${{ github.repository }}:linux-arm64
platforms: linux/arm64
- name: Test linux/arm64 Docker image
run: |
PLATFORM=linux/arm64
TAG=${{ github.repository }}:linux-arm64
export MBTILES_BUILD=-
export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
export MARTIN_BUILD=-
export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG"
tests/test.sh
env:
DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require

- name: Build linux/amd64 Docker image
uses: docker/build-push-action@v5
# https://github.com/docker/build-push-action
with:
context: .
file: multi-platform.Dockerfile
load: true
tags: ${{ github.repository }}:linux-amd64
platforms: linux/amd64
- name: Test linux/amd64 Docker image
run: |
PLATFORM=linux/amd64
TAG=${{ github.repository }}:linux-amd64
export MBTILES_BUILD=-
export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
export MARTIN_BUILD=-
export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG"
tests/test.sh
env:
DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require

- name: Login to GitHub Docker registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
# https://github.com/docker/login-action
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
# https://github.com/docker/metadata-action
with:
images: ghcr.io/${{ github.repository }}
- name: Push the Docker image
if: github.event_name != 'pull_request'
uses: docker/build-push-action@v5
with:
context: .
file: multi-platform.Dockerfile
push: true
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
platforms: linux/amd64,linux/arm64

build:
name: Build ${{ matrix.target }}
runs-on: ${{ matrix.os }}
Expand Down Expand Up @@ -99,42 +254,6 @@ jobs:
name: build-${{ matrix.target }}
path: target_releases/*

build-cross:
name: Cross-platform builds
runs-on: ubuntu-latest
env:
TARGETS: "aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
# TODO: aarch64-unknown-linux-gnu
steps:
- name: Checkout sources
uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
- name: Install cross
run: |
cargo install cross
# Install latest cross version from git (disabled as it is probably less stable)
# cargo install cross --git https://github.com/cross-rs/cross
cross --version
- name: Build targets
run: |
for target in $TARGETS; do
echo -e "\n----------------------------------------------"
echo "Building $target"
export "CARGO_TARGET_$(echo $target | tr 'a-z-' 'A-Z_')_RUSTFLAGS"='-C strip=debuginfo'
cross build --release --target $target --package martin-mbtiles
cross build --release --target $target --package martin --features=vendored-openssl
mkdir -p target_releases/$target
mv target/$target/release/mbtiles target_releases/$target
mv target/$target/release/martin target_releases/$target
done
- name: Save build artifacts to build-${{ matrix.target }}
uses: actions/upload-artifact@v3
with:
name: build-cross
path: target_releases/*

test:
name: Test ${{ matrix.target }}
runs-on: ${{ matrix.os }}
Expand Down Expand Up @@ -335,148 +454,10 @@ jobs:
path: tests/output/*
retention-days: 5

docker:
name: Build docker images
runs-on: ubuntu-latest
needs: [ build, build-cross ]
env:
# PG_* variables are used by psql
PGDATABASE: test
PGHOST: localhost
PGUSER: postgres
PGPASSWORD: postgres
services:
postgres:
image: postgis/postgis:15-3.3
ports:
# will assign a random free host port
- 5432/tcp
# Sadly there is currently no way to pass arguments to the service image other than this hack
# See also https://stackoverflow.com/a/62720566/177275
options: >-
-e POSTGRES_DB=test
-e POSTGRES_USER=postgres
-e POSTGRES_PASSWORD=postgres
-e PGDATABASE=test
-e PGUSER=postgres
-e PGPASSWORD=postgres
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
--entrypoint sh
postgis/postgis:15-3.3
-c "exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key"
steps:
- name: Checkout sources
uses: actions/checkout@v4
- uses: Swatinem/rust-cache@v2
- name: Setup database
run: tests/fixtures/initdb.sh
env:
PGPORT: ${{ job.services.postgres.ports[5432] }}

- name: Set up QEMU
uses: docker/setup-qemu-action@v3
# https://github.com/docker/setup-qemu-action
with:
platforms: linux/amd64,linux/arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# https://github.com/docker/setup-buildx-action
with:
install: true
platforms: linux/amd64,linux/arm64

- run: rm -rf target_releases2
- name: Download build-cross artifacts
uses: actions/download-artifact@v3
with:
name: build-cross
path: target_releases2
- name: Reorganize build artifacts
run: |
chmod -R +x target_releases2/
mkdir -p target_releases/linux/arm64
mv target_releases2/aarch64-unknown-linux-musl/* target_releases/linux/arm64/
mkdir -p target_releases/linux/amd64
mv target_releases2/x86_64-unknown-linux-musl/* target_releases/linux/amd64/
rm -rf target_releases2
- name: Build linux/arm64 Docker image
id: docker_aarch64-unknown-linux-gnu
uses: docker/build-push-action@v5
# https://github.com/docker/build-push-action
with:
context: .
file: multi-platform.Dockerfile
load: true
tags: ${{ github.repository }}:linux-arm64
platforms: linux/arm64
- name: Test linux/arm64 Docker image
run: |
PLATFORM=linux/arm64
TAG=${{ github.repository }}:linux-arm64
export MBTILES_BUILD=-
export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
export MARTIN_BUILD=-
export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG"
tests/test.sh
env:
DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require

- name: Build linux/amd64 Docker image
id: docker_x86_64-unknown-linux-gnu
uses: docker/build-push-action@v5
# https://github.com/docker/build-push-action
with:
context: .
file: multi-platform.Dockerfile
load: true
tags: ${{ github.repository }}:linux-amd64
platforms: linux/amd64
- name: Test linux/amd64 Docker image
run: |
PLATFORM=linux/amd64
TAG=${{ github.repository }}:linux-amd64
export MBTILES_BUILD=-
export MBTILES_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests --entrypoint /usr/local/bin/mbtiles $TAG"
export MARTIN_BUILD=-
export MARTIN_BIN="docker run --rm --net host --platform $PLATFORM -e DATABASE_URL -v $PWD/tests:/tests $TAG"
tests/test.sh
env:
DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=require

- name: Login to GitHub Docker registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
# https://github.com/docker/login-action
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
# https://github.com/docker/metadata-action
with:
images: ghcr.io/${{ github.repository }}
- name: Push the Docker image
if: github.event_name != 'pull_request'
uses: docker/build-push-action@v5
with:
context: .
file: multi-platform.Dockerfile
push: true
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}
platforms: linux/amd64,linux/arm64

package:
name: Package ${{ matrix.target }}
runs-on: ${{ matrix.os }}
needs: [ test, test-legacy ]
needs: [ docker, test, test-legacy ]
strategy:
fail-fast: true
matrix:
Expand All @@ -488,9 +469,6 @@ jobs:
- target: debian-x86_64
os: ubuntu-latest
name: martin-Debian-x86_64.deb
# - target: aarch64-unknown-linux-gnu
# os: ubuntu-latest
# name: martin-Linux-aarch64.tar.gz
- target: x86_64-apple-darwin
os: macOS-latest
name: martin-Darwin-x86_64.tar.gz
Expand All @@ -502,7 +480,9 @@ jobs:
- target: x86_64-unknown-linux-gnu
os: ubuntu-latest
name: martin-Linux-x86_64.tar.gz
# From the cross build
#
# From the cross-build
#
- target: aarch64-unknown-linux-musl
os: ubuntu-latest
cross: 'true'
Expand All @@ -524,7 +504,7 @@ jobs:
if: matrix.cross == 'true'
uses: actions/download-artifact@v3
with:
name: build-cross
name: cross-build
path: target/
- name: Package
run: |
Expand Down Expand Up @@ -558,7 +538,7 @@ jobs:
done:
name: CI Finished
runs-on: ubuntu-latest
needs: [ docker, package ]
needs: [ package ]
steps:
- name: Finished
run: echo "CI finished successfully"

0 comments on commit 48164d7

Please sign in to comment.