Feature: parallelize dockerized builds GitHub actions

.github/workflows/develop.yaml

@@ -23,58 +23,102 @@ jobs:
       - name: Golang test
         run: make test
 
-  build:
-    name: Build CLI Binaries
+  dependencies:
+    name: Build Dependencies
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
         with:
           # https://github.com/actions/checkout/pull/258 needs to fetch all tags so that Makefile can make the correct version
           fetch-depth: 0
 
-      - name: Dockerized Cross-compile Build
-        run: make build
-
-      # Produces the binaries at the directory ./dist
-      - name: Dockerized Binary Distribution
-        run: make dist
+      - name: Build OS and Golang Dependencies
+        run: make build-dependencies
+        # outputs: the docker image with dependencies that can be used as cache for the binaries
+        # marcellodesales/cloner/dependencies:20.09.12
 
       # https://github.community/t/cache-a-docker-image-built-in-workflow/16260/9
       # Produces the docker image at the directory ./dist/docker-image.raw
-      - name: Save Raw Docker Image for Reuse
-        run: make save-docker-image
+      - name: Save Raw Docker Image for Dependencies
+        run: make save-dependencies-docker-image
+        # outputs: cloner-dependencies.dockerimage
 
-      # https://docs.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts#passing-data-between-jobs-in-a-workflow
-      - name: Upload MacOS Binary
+      # Local cache of docker images
+      - name: Upload Docker Image for Dependencies
         uses: actions/upload-artifact@v2
         with:
-          name: cloner-darwin-amd64
-          path: dist/cloner-darwin-amd64
+          name: cloner.dockerimage
+          path: dist/cloner-dependencies.dockerimage
 
-      - name: Upload Linux Binary
-        uses: actions/upload-artifact@v2
+  # https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34
+  compile:
+    name: Compile CLI as Binaries
+    runs-on: ${{ matrix.os }}
+    needs: dependencies
+    strategy:
+      matrix:
+        os: [darwin, linux, windows]
+        include:
+          - os: darwin
+            extension:
+          - os: linux
+            extension:
+          - os: windows
+            extension: .exe
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # https://github.com/actions/checkout/pull/258 needs to fetch all tags so that Makefile can make the correct version
+          fetch-depth: 0
+
+      - name: Download Docker Image for Dependencies as cache
+        uses: actions/download-artifact@v2
         with:
-          name: cloner-linux-amd64
-          path: dist/cloner-linux-amd64
+          name: cloner-dependencies.dockerimage
+
+      - name: Load Docker Image Dependencies for cache to compile
+        run: |
+          ls -la ./cloner.dockerimage
+          docker load -i ./cloner-dependencies.dockerimage
+
+      # Compiles for OS specific dependencies through Dockerkit
+      - name: Compile for ${{matrix.os}}
+        env:
+          OS_NAME: ${{ matrix.os }}
+        run: make compile-${OS_NAME}
+
+      # Compiles for OS specific dependencies through Dockerkit
+      - name: Get distribution binaries for ${{matrix.os}}
+        env:
+          OS_NAME: ${{ matrix.os }}
+        run: make dist-${OS_NAME}
 
-      - name: Upload Windows Binary
+      - name: Upload ${{ matrix.os }} Binary
         uses: actions/upload-artifact@v2
         with:
-          name: cloner-windows-amd64.exe
-          path: dist/cloner-windows-amd64.exe
+          name: cloner-${{ matrix.os }}-amd64.${{ matrix.extension }}
+          path: dist/cloner-${{ matrix.os }}-amd64.${{ matrix.extension }}
 
-      # Local cache of docker images
-      - name: Upload Docker Image
+      # Compiles for OS specific dependencies through Dockerkit
+      - name: Save docker runtime image for ${{matrix.os}}
+        if: matrix.os == 'linux'
+        env:
+          OS_NAME: ${{ matrix.os }}
+        run: make save-docker-image
+        # output: docker image saved at ./dist/cloner.dockerimage
+
+      - name: Upload runtime docker image
         uses: actions/upload-artifact@v2
+        if: matrix.os == 'linux'
         with:
           name: cloner.dockerimage
-          path: dist/cloner.dockerimage
+          path: ./dist/cloner.dockerimage
 
   # https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34
   verify:
     name: Verify CLI Binaries
     runs-on: ${{ matrix.os }}
-    needs: build
+    needs: compile
     strategy:
       matrix:
         os: [ubuntu-latest, macOS-latest, windows-latest]
@@ -106,7 +150,7 @@ jobs:
         if: matrix.os == 'windows-latest'
         env:
           BIN_NAME: ${{ matrix.bin_name }}
-        # https://stackoverflow.com/questions/53961802/how-to-use-an-environment-variable-in-powershell-command/53963070#53963070
+        # start-process -nonewwindow https://stackoverflow.com/questions/53961802/how-to-use-an-environment-variable-in-powershell-command/53963070#53963070
         run: |
           dir
           echo $pwd\$env:BIN_NAME
@@ -115,7 +159,7 @@ jobs:
   e2e:
     name: Verify Dockerized E2E Test
     runs-on: ubuntu-latest
-    needs: build
+    needs: compile
     steps:
       - uses: actions/checkout@v2
         with:
@@ -146,7 +190,7 @@ jobs:
   push:
     name: Push CLI Docker Images
     runs-on: ubuntu-latest
-    needs: build
+    needs: compile
     steps:
       - uses: actions/checkout@v2
         with:

Dockerfile

@@ -1,5 +1,5 @@
 #first stage - builder
-FROM golang:1.13.0-stretch as builder
+FROM golang:1.13.0-stretch AS dependencies
 
 WORKDIR /build
 
@@ -18,6 +18,8 @@ COPY go.sum /build/go.sum
 ENV GO111MODULE=on
 RUN go mod download
 
+FROM dependencies AS compiler
+
 # Add the main
 COPY main.go /build/main.go
 
@@ -33,11 +35,13 @@ COPY util /build/util
 
 ARG BIN_NAME
 ARG BIN_VERSION
+ARG PLATFORMS
 
 # Cross-compile all versions
 ENV BIN_NAME ${BIN_NAME:-unknown}
 ENV BUILD_VERSION ${BIN_VERSION:-0.1.0}
-ENV PLATFORMS "darwin linux windows"
+ENV PLATFORMS ${PLATFORMS:-darwin linux windows}
+RUN echo "Building for ${PLATFORMS}"
 ENV ARCHS "amd64"
 #ENV ARCHS "386 amd64"
 
@@ -58,14 +62,14 @@ RUN export export FULL_NAME_GIT=$(git -C /build/.git remote -v | grep fetch | aw
 #RUN upx --lzma /build/${BIN_NAME}*
 
 # Build the main container (Linux Runtime)
-FROM alpine:latest
+FROM alpine:latest AS runtime
 WORKDIR /root/
 
 ARG BIN_NAME
 ENV BIN_NAME ${BIN_NAME:-unknown}
 
 # Copy the linux amd64 binary, based on the arg (or else all files are copied) inspect with https://github.com/wagoodman/dive
-COPY --from=builder /build/${BIN_NAME}* /usr/local/bin/
+COPY --from=compiler /build/${BIN_NAME}* /usr/local/bin/
 
 # Move the bin to /usr/local/bin and make the entrypoint to point to it passing the params
 # https://stackoverflow.com/questions/33439230/how-to-write-commands-with-multiple-lines-in-dockerfile-while-preserving-the-new/33439625#33439625

Makefile

@@ -40,6 +40,35 @@ build: clean ## Builds the docker image with binaries
 	@echo "Building next version $(BIN_VERSION)"
 	BIN_VERSION=$(BIN_VERSION) docker-compose build --build-arg BIN_VERSION=$(BIN_VERSION) cli
 
+build-dependencies: clean ## Builds the docker image only with dependencies using buildkit
+	@echo "Building dependencies for version $(BIN_VERSION) - Dependencies ONLY"
+	DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) docker-compose build dependencies
+
+save-dependencies-docker-image: ## Saves the raw docker image of dependencies for cache
+ifndef GITHUB_ACTION
+	$(error GITHUB_ACTION is undefined. This must run only by Github Actions)
+endif
+	$(eval BUILD_IMAGE_TAG=$(shell BIN_VERSION=$(BIN_VERSION) docker-compose config | grep image: | grep dependencies | awk '{print $$2}'))
+	docker save -o ./dist/$(APP_NAME)-dependencies.dockerimage $(BUILD_IMAGE_TAG)
+	ls -la ./dist/$(APP_NAME)-dependencies.dockerimage
+
+compile-%: build-dependencies ## Compiles for (darwin, linux, windows)
+	$(eval PLATFORM=$(shell echo $@ | awk -F"-" '{print $$2}'))
+	@echo "Compiling version $(BIN_VERSION) for $(PLATFORM)"
+	DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=$(PLATFORM) docker-compose build binaries
+
+build-docker-runtime:
+	@echo "Building linux runtime for version $(BIN_VERSION)"
+	DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=linux docker-compose build runtime
+
+dist-%: ## Makes the dir ./dist with binaries from docker image
+	$(eval PLATFORM=$(shell echo $@ | awk -F"-" '{print $$2}'))
+	@echo "$(PLATFORM) Distribution binary for version $(BIN_VERSION)"
+	$(eval DIST_IMAGE=$(shell DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=$(PLATFORM) docker-compose config | grep image: | grep binaries | awk '{print $$2}'))
+	docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(DIST_IMAGE) -c "cp /build/$(APP_NAME)-$(PLATFORM)-amd64s /bins" || true
+	docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(DIST_IMAGE) -c "cp /build/$(APP_NAME)-$(PLATFORM)-amd64.exe /bins" || true
+	ls -la $(PWD)/$(DIST_DIR)
+
 dist: build ## Makes the dir ./dist with binaries from docker image
 	@echo "Distribution libraries for version $(BIN_VERSION)"
 	docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(ORG)/$(APP_NAME):$(BIN_VERSION) -c "cp /usr/local/bin/$(APP_NAME)-darwin-amd64 /bins"

docker-compose.yaml

@@ -1,11 +1,36 @@
-version: "3"
+version: "3.8"
 
 services:
 
-  cli:
+  # Usually takes time to download, and can be reused for all platforms
+  dependencies:
+    image: marcellodesales/cloner/dependencies:${BIN_VERSION:-0.1.0}
+    build:
+      context: .
+      target: dependencies
+
+  # Reuses the dependencies image as cache so we can parallelize binary builds
+  binaries:
+    image: marcellodesales/cloner/binaries-${PLATFORMS:-all}:${BIN_VERSION:-0.1.0}
+    build:
+      context: .
+      args:
+        - BIN_NAME=cloner
+        - BIN_VERSION=${BIN_VERSION:-0.1.0}
+        - PLATFORMS=${PLATFORMS:-darwin linux windows}
+      cache_from:
+        - marcellodesales/cloner/dependencies:${BIN_VERSION:-0.1.0}
+      target: compiler
+
+  # Reuses the dependencies image as cache so we can parallelize binary builds
+  runtime:
     image: marcellodesales/cloner:${BIN_VERSION:-0.1.0}
     build:
       context: .
       args:
         - BIN_NAME=cloner
         - BIN_VERSION=${BIN_VERSION:-0.1.0}
+        - PLATFORMS=linux
+      cache_from:
+        - marcellodesales/cloner/binaries-linux:${BIN_VERSION:-0.1.0}
+      target: runtime