🔨 Workflows: build paralle docker images BUILDKIT

marcellodesales · Sep 23, 2020 · 6c481b1 · 6c481b1
1 parent adf787b
commit 6c481b1
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 48 deletions.
diff --git a/.github/workflows/develop.yaml b/.github/workflows/develop.yaml
@@ -50,58 +50,75 @@ jobs:
           name: cloner.dockerimage
           path: dist/cloner-dependencies.dockerimage
 
-  build:
-    name: Build CLI Binaries
-    runs-on: ubuntu-latest
+  # https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34
+  compile:
+    name: Compile CLI as Binaries
+    runs-on: ${{ matrix.os }}
+    needs: dependencies
+    strategy:
+      matrix:
+        os: [darwin, linux, windows]
+        include:
+          - os: darwin
+            extension:
+          - os: linux
+            extension:
+          - os: windows
+            extension: .exe
     steps:
       - uses: actions/checkout@v2
         with:
           # https://github.com/actions/checkout/pull/258 needs to fetch all tags so that Makefile can make the correct version
           fetch-depth: 0
 
-      - name: Dockerized Cross-compile Build
-        run: make build
+      - name: Download Docker Image for Dependencies as cache
+        uses: actions/download-artifact@v2
+        with:
+          name: cloner-dependencies.dockerimage
 
-      # Produces the binaries at the directory ./dist
-      - name: Dockerized Binary Distribution
-        run: make dist
+      - name: Load Docker Image Dependencies for cache to compile
+        run: |
+          ls -la ./cloner.dockerimage
+          docker load -i ./cloner-dependencies.dockerimage
 
-      # https://github.community/t/cache-a-docker-image-built-in-workflow/16260/9
-      # Produces the docker image at the directory ./dist/docker-image.raw
-      - name: Save Raw Docker Image for Reuse
-        run: make save-docker-image
+      # Compiles for OS specific dependencies through Dockerkit
+      - name: Compile for ${{matrix.os}}
+        env:
+          OS_NAME: ${{ matrix.os }}
+        run: make compile-${OS_NAME}
 
-      # https://docs.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts#passing-data-between-jobs-in-a-workflow
-      - name: Upload MacOS Binary
-        uses: actions/upload-artifact@v2
-        with:
-          name: cloner-darwin-amd64
-          path: dist/cloner-darwin-amd64
+      # Compiles for OS specific dependencies through Dockerkit
+      - name: Get distribution binaries for ${{matrix.os}}
+        env:
+          OS_NAME: ${{ matrix.os }}
+        run: make dist-${OS_NAME}
 
-      - name: Upload Linux Binary
+      - name: Upload ${{ matrix.os }} Binary
         uses: actions/upload-artifact@v2
         with:
-          name: cloner-linux-amd64
-          path: dist/cloner-linux-amd64
+          name: cloner-${{ matrix.os }}-amd64.${{ matrix.extension }}
+          path: dist/cloner-${{ matrix.os }}-amd64.${{ matrix.extension }}
 
-      - name: Upload Windows Binary
-        uses: actions/upload-artifact@v2
-        with:
-          name: cloner-windows-amd64.exe
-          path: dist/cloner-windows-amd64.exe
+      # Compiles for OS specific dependencies through Dockerkit
+      - name: Save docker runtime image for ${{matrix.os}}
+        if: matrix.os == 'linux'
+        env:
+          OS_NAME: ${{ matrix.os }}
+        run: make save-docker-image
+        # output: docker image saved at ./dist/cloner.dockerimage
 
-      # Local cache of docker images
-      - name: Upload Docker Image
+      - name: Upload runtime docker image
         uses: actions/upload-artifact@v2
+        if: matrix.os == 'linux'
         with:
           name: cloner.dockerimage
-          path: dist/cloner.dockerimage
+          path: ./dist/cloner.dockerimage
 
   # https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34
   verify:
     name: Verify CLI Binaries
     runs-on: ${{ matrix.os }}
-    needs: build
+    needs: compile
     strategy:
       matrix:
         os: [ubuntu-latest, macOS-latest, windows-latest]
@@ -133,7 +150,7 @@ jobs:
         if: matrix.os == 'windows-latest'
         env:
           BIN_NAME: ${{ matrix.bin_name }}
-        # https://stackoverflow.com/questions/53961802/how-to-use-an-environment-variable-in-powershell-command/53963070#53963070
+        # start-process -nonewwindow https://stackoverflow.com/questions/53961802/how-to-use-an-environment-variable-in-powershell-command/53963070#53963070
         run: |
           dir
           echo $pwd\$env:BIN_NAME
@@ -142,7 +159,7 @@ jobs:
   e2e:
     name: Verify Dockerized E2E Test
     runs-on: ubuntu-latest
-    needs: build
+    needs: compile
     steps:
       - uses: actions/checkout@v2
         with:
@@ -173,7 +190,7 @@ jobs:
   push:
     name: Push CLI Docker Images
     runs-on: ubuntu-latest
-    needs: build
+    needs: compile
     steps:
       - uses: actions/checkout@v2
         with:

diff --git a/Makefile b/Makefile
@@ -44,22 +44,31 @@ build-dependencies: clean ## Builds the docker image only with dependencies usin
 	@echo "Building dependencies for version $(BIN_VERSION) - Dependencies ONLY"
 	DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) docker-compose build dependencies
 
-compile-linux: build-dependencies ## Compiles for Linux
-	@echo "Compiling version $(BIN_VERSION) for linux"
-	DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=linux docker-compose build binaries
-
-compile-darwin: build-dependencies ## Compiles for MacOS
-	@echo "Compiling version $(BIN_VERSION) for darwin"
-	DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=darwin docker-compose build binaries
+save-dependencies-docker-image: ## Saves the raw docker image of dependencies for cache
+ifndef GITHUB_ACTION
+	$(error GITHUB_ACTION is undefined. This must run only by Github Actions)
+endif
+	$(eval BUILD_IMAGE_TAG=$(shell BIN_VERSION=$(BIN_VERSION) docker-compose config | grep image: | grep dependencies | awk '{print $$2}'))
+	docker save -o ./dist/$(APP_NAME)-dependencies.dockerimage $(BUILD_IMAGE_TAG)
+	ls -la ./dist/$(APP_NAME)-dependencies.dockerimage
 
-compile-windows: build-dependencies ## Compiles for Windows
-	@echo "Compiling version $(BIN_VERSION) for windows"
-	DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=windows docker-compose build binaries
+compile-%: build-dependencies ## Compiles for (darwin, linux, windows)
+	$(eval PLATFORM=$(shell echo $@ | awk -F"-" '{print $$2}'))
+	@echo "Compiling version $(BIN_VERSION) for $(PLATFORM)"
+	DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=$(PLATFORM) docker-compose build binaries
 
 build-docker-runtime:
 	@echo "Building linux runtime for version $(BIN_VERSION)"
 	DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=linux docker-compose build runtime
 
+dist-%: ## Makes the dir ./dist with binaries from docker image
+	$(eval PLATFORM=$(shell echo $@ | awk -F"-" '{print $$2}'))
+	@echo "$(PLATFORM) Distribution binary for version $(BIN_VERSION)"
+	$(eval DIST_IMAGE=$(shell DOCKER_BUILDKIT=1 BIN_VERSION=$(BIN_VERSION) PLATFORMS=$(PLATFORM) docker-compose config | grep image: | grep binaries | awk '{print $$2}'))
+	docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(DIST_IMAGE) -c "cp /build/$(APP_NAME)-$(PLATFORM)-amd64s /bins" || true
+	docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(DIST_IMAGE) -c "cp /build/$(APP_NAME)-$(PLATFORM)-amd64.exe /bins" || true
+	ls -la $(PWD)/$(DIST_DIR)
+
 dist: build ## Makes the dir ./dist with binaries from docker image
 	@echo "Distribution libraries for version $(BIN_VERSION)"
 	docker run --rm --entrypoint sh -v $(PWD)/$(DIST_DIR):/bins $(ORG)/$(APP_NAME):$(BIN_VERSION) -c "cp /usr/local/bin/$(APP_NAME)-darwin-amd64 /bins"

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -4,22 +4,22 @@ services:
 
   # Usually takes time to download, and can be reused for all platforms
   dependencies:
-    image: marcellodesales/cloner-dependencies:${BIN_VERSION:-0.1.0}
+    image: marcellodesales/cloner/dependencies:${BIN_VERSION:-0.1.0}
     build:
       context: .
       target: dependencies
 
   # Reuses the dependencies image as cache so we can parallelize binary builds
   binaries:
-    image: marcellodesales/cloner-binaries-${PLATFORMS}:${BIN_VERSION:-0.1.0}
+    image: marcellodesales/cloner/binaries-${PLATFORMS:-all}:${BIN_VERSION:-0.1.0}
     build:
       context: .
       args:
         - BIN_NAME=cloner
         - BIN_VERSION=${BIN_VERSION:-0.1.0}
-        - PLATFORMS=${PLATFORMS}
+        - PLATFORMS=${PLATFORMS:-darwin linux windows}
       cache_from:
-        - marcellodesales/cloner-dependencies:${BIN_VERSION:-0.1.0}
+        - marcellodesales/cloner/dependencies:${BIN_VERSION:-0.1.0}
       target: compiler
 
   # Reuses the dependencies image as cache so we can parallelize binary builds
@@ -32,5 +32,5 @@ services:
         - BIN_VERSION=${BIN_VERSION:-0.1.0}
         - PLATFORMS=linux
       cache_from:
-        - marcellodesales/cloner-binaries-linux:${BIN_VERSION:-0.1.0}
+        - marcellodesales/cloner/binaries-linux:${BIN_VERSION:-0.1.0}
       target: runtime