feat: add new ansible build matrix

.github/workflows/docker.yml

@@ -0,0 +1,187 @@
+name: Build docker images
+on:
+  push:
+env:
+  REGISTRY_IMAGE: ghcr.io/${{ github.repository }}
+concurrency:
+  group: docker-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  matrix:
+    name: Compute build matrix from pypi API
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: './build-matrix/go.mod'
+
+      - name: Run matrix generator tests
+        working-directory: build-matrix
+        run: go test ./
+
+      - id: matrix
+        working-directory: build-matrix
+        run: |
+          MATRIX=$(go run ./)
+          echo ${MATRIX} | jq
+          echo 'matrix=[{"ansible":"10.2","additional_tags":["10"]},{"ansible":"10.1","additional_tags":[]}]' >> $GITHUB_OUTPUT
+          #echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT
+  build:
+    needs: [ matrix ]
+    runs-on: ubuntu-latest
+    name: Build ansible ${{ matrix.versions.ansible }}/${{ matrix.platform }}
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+        versions: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      # Allows to build arm64 images
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      # This is not required but recommended using it to be able to build multi-platform images, export cache, etc.
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Github Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build Docker images
+        uses: docker/build-push-action@v6
+        id: build
+        with:
+          build-args: |
+            ANSIBLE_VERSION=${{ matrix.versions.ansible }}
+          platforms: ${{ matrix.platform }}
+          outputs: type=docker,dest=/tmp/${{ matrix.versions.ansible }}-${{ env.PLATFORM_PAIR }}.tar
+          tags: ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-${{ github.sha }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-${{ env.PLATFORM_PAIR }}
+          path: /tmp/${{ matrix.versions.ansible }}-${{ env.PLATFORM_PAIR }}.tar
+          retention-days: 1
+          if-no-files-found: error
+
+  test:
+    needs: [ matrix, build ]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform:
+          - linux/amd64
+        versions: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-${{ env.PLATFORM_PAIR }}
+          path: /tmp
+      - name: Load image
+        run: |
+          docker load --input /tmp/${{ matrix.versions.ansible }}-${{ env.PLATFORM_PAIR }}.tar
+          docker image ls -a
+      - name: Test ansible version
+        run:
+          docker run ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-${{ github.sha }} ansible-community --version | grep 'Ansible community version ${{ matrix.versions.ansible }}'
+
+  deploy:
+    needs: [ matrix, test ]
+    runs-on: ubuntu-latest
+    env:
+      AWS_REGION: "us-east-1"
+    strategy:
+      matrix:
+        versions: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/ansible_build_matrix' }} # TODO(eliecharra): Remove condition
+    permissions:
+      id-token: write
+      packages: write
+      contents: read
+    steps:
+      - name: Download amd64 artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-linux-amd64
+          path: /tmp
+
+      - name: Download arm64 artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ansible-runner-${{ github.sha }}-${{ matrix.versions.ansible }}-linux-arm64
+          path: /tmp
+
+      - name: Load image
+        run: |
+          docker load --input /tmp/${{ matrix.versions.ansible }}-linux-amd64.tar
+          docker load --input /tmp/${{ matrix.versions.ansible }}-linux-arm64.tar
+          docker image ls -a
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ env.AWS_REGION }}
+          role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+          role-duration-seconds: 900
+
+      - name: Login to Amazon ECR
+        run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${REPOSITORY_PATH}
+        env:
+          REPOSITORY_PATH: ${{ secrets.PUBLIC_RUNNER_ANSIBLE_ECR_REPOSITORY_URL }}
+
+      - name: Log in to Github Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Tag images
+        run: |
+          echo "Tagging image to ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}"
+          docker tag ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}-${{ github.sha }} ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}
+          for tag in ${{ join(fromJSON(matrix.versions.additional_tags), ' ') }}
+          do
+            echo "Tagging image to $tag"
+            docker tag ghcr.io/${{ github.repository }}:${tag}
+          done
+          # TODO Tag ECR
+          docker image ls -a
+
+      - name: Push images
+        run: |
+          echo "Pushing image ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}"
+          docker push ghcr.io/${{ github.repository }}:${{ matrix.versions.ansible }}
+          for tag in ${{ join(fromJSON(matrix.versions.additional_tags), ' ') }}
+          do
+            echo "Pushing image $tag"
+            docker push ghcr.io/${{ github.repository }}:${tag}
+          done
+          # TODO Push to ECR

Dockerfile

@@ -0,0 +1,3 @@
+FROM python:3.12
+ARG ANSIBLE_VERSION=10.0
+RUN pip install ansible==${ANSIBLE_VERSION}.* ansible-runner~=2.4

build-matrix/go.mod

@@ -0,0 +1,11 @@
+module github.com/spacelift-io/build-matrix
+
+go 1.22.1
+
+require (
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

build-matrix/go.sum

@@ -0,0 +1,11 @@
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

build-matrix/main.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"sort"
+
+	"github.com/Masterminds/semver/v3"
+)
+
+const (
+	// Define the oldest major version we care about, we do not want to build image starting ansible 1.0
+	minSupportedMajor = 7
+)
+
+type ReleaseResponse struct {
+	Releases map[string]any `json:"releases"`
+}
+
+type matrixVersion struct {
+	Ansible        string   `json:"ansible"`
+	AdditionalTags []string `json:"additional_tags"`
+}
+type Matrix []matrixVersion
+
+func main() {
+	resp, err := http.Get("https://pypi.org/pypi/ansible/json")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	matrixOutput := GenerateBuildMatrix(resp.Body, minSupportedMajor)
+
+	output, err := json.Marshal(matrixOutput)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Print(string(output))
+}
+
+func GenerateBuildMatrix(reader io.Reader, minSupportedMajor uint64) Matrix {
+	releases := ReleaseResponse{}
+	if err := json.NewDecoder(reader).Decode(&releases); err != nil {
+		log.Fatal(err)
+	}
+
+	var versions []*semver.Version
+
+	for v := range releases.Releases {
+		version, err := semver.NewVersion(v)
+		if err != nil {
+			log.Printf("Unable to parse version %s\n", v)
+			continue
+		}
+		versions = append(versions, version)
+	}
+
+	sort.Slice(versions, func(i, j int) bool {
+		return versions[j].LessThan(versions[i])
+	})
+
+	versionGroupedByMajor := make(map[int][]*semver.Version)
+	// Just used for stable ordering
+	var majorVersions []int
+
+	for _, version := range versions {
+		if version.Major() < minSupportedMajor {
+			break
+		}
+		major := int(version.Major())
+		if _, exists := versionGroupedByMajor[major]; !exists {
+			majorVersions = append(majorVersions, major)
+		}
+		versionGroupedByMajor[major] = append(versionGroupedByMajor[major], version)
+	}
+
+	sort.Sort(sort.Reverse(sort.IntSlice(majorVersions)))
+
+	minorVersionDeduplication := map[string]any{}
+	matrix := Matrix{}
+	for _, majorVersion := range majorVersions {
+		for i, version := range versionGroupedByMajor[majorVersion] {
+			key := fmt.Sprintf("%d.%d", version.Major(), version.Minor())
+			if _, exists := minorVersionDeduplication[key]; exists {
+				continue
+			}
+			additionalTags := make([]string, 0)
+			if i == 0 {
+				additionalTags = append(additionalTags, fmt.Sprintf("%d", version.Major()))
+			}
+			minorVersionDeduplication[key] = struct{}{}
+			matrix = append(matrix, matrixVersion{
+				Ansible:        key,
+				AdditionalTags: additionalTags,
+			})
+		}
+	}
+
+	return matrix
+}

build-matrix/main_test.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGenerateBuildMatrix(t *testing.T) {
+	fakePythonVersions := ReleaseResponse{
+		Releases: map[string]any{
+			"1.1.0":  struct{}{},
+			"1.1.1":  struct{}{},
+			"2.10.0": struct{}{},
+			"2.11.0": struct{}{},
+			"2.11.2": struct{}{},
+			"3.1.0":  struct{}{},
+			"3.1.1":  struct{}{},
+			"3.2.0":  struct{}{},
+		},
+	}
+
+	fakeJsonResponse, err := json.Marshal(fakePythonVersions)
+	require.NoError(t, err)
+
+	matrix := GenerateBuildMatrix(bytes.NewReader(fakeJsonResponse), 2)
+	expectedMatrix := Matrix{
+		{
+			Ansible:        "3.2",
+			AdditionalTags: []string{"3"},
+		},
+		{
+			Ansible:        "3.1",
+			AdditionalTags: []string{},
+		},
+		{
+			Ansible:        "2.11",
+			AdditionalTags: []string{"2"},
+		},
+		{
+			Ansible:        "2.10",
+			AdditionalTags: []string{},
+		},
+	}
+	assert.Equal(t, expectedMatrix, matrix)
+}