Merge pull request #21 from marcellodesales/develop

Next release: Dockerized CLI + Test Cases
marcellodesales · Sep 22, 2020 · 89890c9 · 89890c9
2 parents 997e7f8 + 3e0aaa3
commit 89890c9
Show file tree

Hide file tree

Showing 19 changed files with 603 additions and 65 deletions.
diff --git a/.github/scripts/id_cloner_test.pgp b/.github/scripts/id_cloner_test.pgp
diff --git a/.github/workflows/develop.yaml b/.github/workflows/develop.yaml
@@ -6,13 +6,17 @@ on:
     branches:
       - develop
       - feature/**
+      - bugfix/**
+      - hotfix/**
+    paths-ignore:
+      - '**/README.md' # https://stackoverflow.com/questions/62968897/is-it-possible-to-not-run-github-action-for-readme-updates/62972393#62972393
 
 jobs:
   test:
     name: Run Test Cases
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
         with:
           fetch-depth: 1
 
@@ -23,12 +27,10 @@ jobs:
     name: Build CLI Binaries
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
         with:
-          fetch-depth: 1
-
-      - name: Golang build
-        run: make local
+          # https://github.com/actions/checkout/pull/258 needs to fetch all tags so that Makefile can make the correct version
+          fetch-depth: 0
 
       - name: Dockerized Cross-compile Build
         run: make build
@@ -37,6 +39,11 @@ jobs:
       - name: Dockerized Binary Distribution
         run: make dist
 
+      # https://github.community/t/cache-a-docker-image-built-in-workflow/16260/9
+      # Produces the docker image at the directory ./dist/docker-image.raw
+      - name: Save Raw Docker Image for Reuse
+        run: make save-docker-image
+
       # https://docs.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts#passing-data-between-jobs-in-a-workflow
       - name: Upload MacOS Binary
         uses: actions/upload-artifact@v2
@@ -56,6 +63,13 @@ jobs:
           name: cloner-windows-amd64.exe
           path: dist/cloner-windows-amd64.exe
 
+      # Local cache of docker images
+      - name: Upload Docker Image
+        uses: actions/upload-artifact@v2
+        with:
+          name: cloner.dockerimage
+          path: dist/cloner.dockerimage
+
   # https://github.com/nightlark/ninja/blob/f1a33131154ae7d9648aa82afac462859535fb62/.github/workflows/release-ninja-binaries.yml#L8-L34
   verify:
     name: Verify CLI Binaries
@@ -77,7 +91,7 @@ jobs:
         with:
           name: ${{ matrix.bin_name }}
 
-      # Install OS specific dependencies
+      # Verify OS specific dependencies
       - name: Unix Binary Validation
         if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
         env:
@@ -96,4 +110,67 @@ jobs:
         run: |
           dir
           echo $pwd\$env:BIN_NAME
-          start-process -nonewwindow $pwd\$env:BIN_NAME
+          start-process -nonewwindow $pwd\$env:BIN_NAME
+
+  e2e:
+    name: Verify Dockerized E2E Test
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # https://github.com/actions/checkout/pull/258 needs to fetch all tags so that Makefile can make the correct version
+          fetch-depth: 0
+
+      - name: Download artifact cloner.dockerimage from build
+        uses: actions/download-artifact@v2
+        with:
+          name: cloner.dockerimage
+
+      - name: Load Docker Image Binary for cache
+        run: |
+          ls -la ./cloner.dockerimage
+          docker load -i ./cloner.dockerimage
+
+      # https://docs.github.com/en/actions/reference/encrypted-secrets as secrets can't be written to files directly
+      # Moved the decrypt function to Makefile (test-e2e)
+      # Not using https://github.com/webfactory/ssh-agent creates a .known_hosts with the keys
+      - name: Run cloner from Docker with Private Key
+        env:
+          ID_CLONER_TEST_PASSPHRASE: ${{ secrets.ID_CLONER_TEST_PASSPHRASE }}
+        run: make test-e2e
+
+      - name: Show cloned files scructure
+        run: docker container run --rm -v $(pwd):$(pwd) iankoulski/tree $(pwd)/.github/test-cloned-repos
+
+  push:
+    name: Push CLI Docker Images
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # The command will compute the current version to get the docker image name, so it needs all tags
+          fetch-depth: 0
+
+      - name: Download artifact cloner.dockerimage from build
+        uses: actions/download-artifact@v2
+        with:
+          name: cloner.dockerimage
+
+      - name: Load Docker Image Binary for cache
+        run: |
+          ls -la ./cloner.dockerimage
+          docker load -i ./cloner.dockerimage
+
+      # https://github.com/marcellodesales/cloner/packages?package_type=Docker
+      - name: Login to GitHub Packages Docker Registry
+        uses: docker/login-action@v1
+        with:
+          registry: docker.pkg.github.com
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.REGISTRY_GITHUB_TOKEN }}
+
+      # Publishes the Docker Images to Github Container Registry
+      - name: Push Docker Development Images
+        run: make docker-push-develop
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -15,6 +15,7 @@ jobs:
           # https://github.com/actions/checkout/pull/258 needs to fetch all tags
           fetch-depth: 0
 
+        # Publishes the binaries to the Github Release section
       - name: Makefile + Dockerfile dockerized release
         env:
           PUBLISH_GITHUB_TOKEN: ${{ secrets.PUBLISH_GITHUB_TOKEN }}

diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,5 @@
 .idea/
 dist/
+.github/scripts/.ssh/
+.github/test-cloned-repos/
+
diff --git a/DEVELOP.md b/DEVELOP.md
@@ -0,0 +1,171 @@
+# Make local binary
+
+```
+make local
+```
+
+# Private Repos - git@host:org/repo
+
+1. Create a test SSH key
+  - Convert the OPENSSH to RSA
+2. Add the public key as a `Deploy Key` 
+  - `Settings -> Deploy Keys`
+  - Make sure to create a deploy key *with clone permission only*!
+3. Run the container providing the private key as param
+4. References
+
+## Create a test SSH key
+
+* Just create a test key with empty passphrase
+
+```
+$ ssh-keygen -t rsa -b 4096 -C "[email protected]" -N ""
+Generating public/private rsa key pair.
+Enter file in which to save the key (/Users/marcellodesales/.ssh/id_rsa): id_cloner_test
+Your identification has been saved in id_cloner_test.
+Your public key has been saved in id_cloner_test.pub.
+The key fingerprint is:
+SHA256:XP89ufanVLXe63vYJSrRYGzWX0cAV+1TJAhgffs3lUw [email protected]
+The key's randomart image is:
++---[RSA 4096]----+
+|        oo...o+++|
+|       .  . o. E+|
+|         ..o .oo+|
+|       . .*.o  +B|
+|        S+ o.o o=|
+|          . ..=+=|
+|           . .oO=|
+|          . ....B|
+|           .  +**|
++----[SHA256]-----+
+```
+
+* 2 files are created: `id_cloner_test` and `id_cloner_test.pub`.
+
+```
+$ cat id_cloner_test
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAACFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAgEAt/DB1OZFiWqwHXjRItqFnxOIZ2+st6OTkF+b7OGD8vnMdm058s1K
+...
+...
+PL1esI1hYGI9Z6II+g3iQwxkszTaBRs6vLPBsKqkcSH4CvlvOPbSDJwPgVvvcgRTZPF7rZ
+u31hhB/MXHnN4yDFjiTqFHZWQgTlUWTvrqdZKkO8XGeisu0Z9vzn7XHjDdgN69jrjPAANh
+UTASRHJrLmqzAAAAFnRlc3RAY2xvbmVyLmdpdGh1Yi5jb20BAgME
+-----END OPENSSH PRIVATE KEY-----
+
+$ cat id_cloner_test.pub
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC38MHU5kWJarAdeNEi2oWfE4hnb6y3o5OQX5vs4YPy+cx2bTnyzUpJZZ1mQmRupTIz+tssp12Co/HFxAOd9KNqIpdN42FuFVAUO1waGcIqRELfF+/kYMJPk4ZxKMWe+gJ9KHqPGlFeUlsmVoGERYuHFUOIXptKLqxrkTh+fEVzT7eUyY8DXnjOlggjoBCm9VTcHvlETndlJu6LH9CtXMuP18noIXAB9mqWIe4dqf1jdXbOANiCLYId8H84mnjjML6aFgE4JO3D/WBQFwI5Vr1WNL0g45NbJ+j3MKVm2MXQAzd4RE2mH6Rv5wr4cLzvHTfy2x+Cm6fq5kml3N4uku+4BBVWXVCxd+AXf42z5/kmOPQwo58Kk++CQhnaP49UroXW8NmQ6zMUsB5mZqypQ4NPLrLCXQcK4jbUSjG3fLquEv3pmzXSiqQ6618SwRKDsC0qIGanDsZmKthm3PabG2/qEzkDES6axFrJAQQGSGmScJP+ZSxNyvsiSzbQqRfPMcHYOW9OmGpGvGK4SO+46alcBhEUhqGyZBvKJdWfCG8qeEZ0M9uyLmZrMJ6i+aMpNReFmhAx43sGd6SpGHhb4uGUkGiSABLD51ZVW/I1rOIEytPImBVZa+5wkwK3JmFs1savVUqnX7Q0SH9N9JL7KSih1W/jCYHvOw8h0bro4JmUOQ== [email protected]
+```
+
+* Convert the key from `OPENSSH` to RSA
+
+```
+$ ssh-keygen -p -m PEM -f id_cloner_test
+Key has comment '[email protected]'
+Enter new passphrase (empty for no passphrase):
+Enter same passphrase again:
+Your identification has been saved with the new passphrase.
+```
+
+* The file is updated in place
+
+```
+$ cat id_cloner_test
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAt/DB1OZFiWqwHXjRItqFnxOIZ2+st6OTkF+b7OGD8vnMdm05
+8s1KSWWdZkJkbqUyM/rbLKddgqPxxcQDnfSjaiKXTeNhbhVQFDtcGhnCKkRC3xfv
+5GDCT5OGcSjFnvoCfSh6jxpRXlJbJlaBhEWLhxVDiF6bSi6sa5E4fnxFc0+3lMmP
+...
+...
+GmiYxMLb1gQYVH+R9wxIb0LaBl+XLfjKjU8Z+aW0lVVXcAIgCIwG0qdL5CiY0rG4
+coclH2OmlSN2G+fS8jK/dcAHu7oQZptFUZ9Q98PuVsi9csTfxmN/y1mcq5o=
+-----END RSA PRIVATE KEY-----
+```
+
+## Add a Deploy Key
+
+* https://developer.github.com/v3/guides/managing-deploy-keys/#deploy-keys
+  * https://github.com/marcellodesales/cloner/settings/keys/new
+
+Use the public key so that CI systems can clone it
+
+> **ATTENTION**: Make sure the key is listed as `Read-only`.
+
+## Run the container to clone
+
+* Execute the command
+
+```
+$ docker run -ti -v $(pwd):/data marcellodesales/cloner:20.09.7 git \
+         --repo [email protected]:marcellodesales/unmazedboot.git -v debug -k /data/id_cloner_test
+DEBU[2020-09-18T02:17:38Z] config.git.cloneBaseDir=/root/cloner
+INFO[2020-09-18T02:17:38Z] Cloning into '/root/cloner/github.com/marcellodesales/unmazedboot'
+DEBU[2020-09-18T02:17:38Z] Attempting to clone repo '[email protected]:marcellodesales/unmazedboot.git' => '/root/cloner/github.com/marcellodesales/unmazedboot'
+DEBU[2020-09-18T02:17:38Z] Authenticating using the key
+Enumerating objects: 16, done.
+Counting objects: 100% (16/16), done.
+Compressing objects: 100% (16/16), done.
+Total 144 (delta 9), reused 2 (delta 0), pack-reused 128
+INFO[2020-09-18T02:17:41Z] Done...
+INFO[2020-09-18T02:17:41Z]
+/root/cloner/github.com/marcellodesales/unmazedboot
+└── .env
+└── CHANGELOG
+└── LICENSE
+└── README.md
+└── builder
+│   ├── gradle.Dockerfile
+│   ├── maven.Dockerfile
+```
+
+## References
+
+* https://stackoverflow.com/questions/44269142/golang-ssh-getting-must-specify-hoskeycallback-error-despite-setting-it-to-n/63308243#63308243
+* https://skarlso.github.io/2019/02/17/go-ssh-with-host-key-verification/
+* https://github.com/src-d/go-git/issues/637#issuecomment-543015125
+
+> NOTE: FOR VERIFICATION with agent-based execution
+
+Needs to execute the agent https://github.com/src-d/go-git/issues/550#issuecomment-323075887
+
+In order to test, run a docker container as follows:
+
+1. Start an ssh-agent
+
+In order to avoid errors like the following:
+
+```
+ERRO[2020-09-17T21:39:33Z] can't clone the repo at 'github.com/marcellodesales/unmazedboot': 
+       error creating SSH agent: "SSH agent requested but SSH_AUTH_SOCK not-specified"
+```
+
+* Test in Container
+
+```
+$ docker run -ti -v $(pwd):/data --entrypoint sh alpine/git
+/git # cd /data/
+/data # eval `ssh-agent`
+Agent pid 19
+/data # ssh-add ~/.ssh/id_cloner_test
+Identity added: /root/.ssh/id_cloner_test (/root/.ssh/id_cloner_test)
+```
+
+2. If the known_hosts is empty, then errors will show
+
+```
+ERRO[2020-09-17T22:26:53Z] can't clone the repo at 'github.com/marcellodesales/unmazedboot': 
+       ssh: handshake failed: knownhosts: key is unknown
+```
+
+3. Copy the known hosts file to verify
+
+```
+/data # cp known_hosts ~/.ssh/known_hosts
+```
+
+For this reason, validation on the ssh key is disabled when the file does not exist
+
+4. If the file exists, there's no verification.
+
+Build local and test.
diff --git a/Dockerfile b/Dockerfile
@@ -47,9 +47,11 @@ COPY .git/ /build/.git/
 # Injecting version info into the golang build https://github.com/Ropes/go-linker-vars-example
 # https://github.com/Ropes/go-linker-vars-example, https://stackoverflow.com/questions/11354518/application-auto-build-versioning/11355611#11355611
 # https://medium.com/@joshroppo/setting-go-1-5-variables-at-compile-time-for-versioning-5b30a965d33e
-RUN export GO_MODULE_FULL_NAME=$(git -C /build/.git remote -v | grep fetch | awk '{print $2}' | sed -En "s/git@//p" | sed -En "s/.git//p" | sed -En "s/:/\//p") && \
-        export BUILD_GIT_SHA=$(git rev-parse --short HEAD) && \
-        for GOOS in ${PLATFORMS}; do for GOARCH in ${ARCHS}; do BINARY="${BIN_NAME}-$GOOS-$GOARCH"; if [ $GOOS = "windows" ]; then BINARY="${BINARY}.exe"; fi; export BUILD_TIME="$(date -u +"%Y-%m-%d_%H:%M:%S_GMT")"; echo "Cross-compiling $GO_MODULE_FULL_NAME@$BUILD_GIT_SHA as ${BINARY}@$BUILD_VERSION at $BUILD_TIME"; GO_MODULE_FULL_NAME=$GO_MODULE_FULL_NAME BUILD_GIT_SHA=$BUILD_GIT_SHA BUILD_VERSION=$BUILD_VERSION BUILD_TIME=$BUILD_TIME GO111MODULE=$GO111MODULE CGO_ENABLED=0 GOARCH=$GOARCH GOOS=$GOOS GOPRIVATE=$GOPRIVATE go build -o ${BINARY} -a -ldflags "-X ${GO_MODULE_FULL_NAME}/config.BuildModule=${GO_MODULE_FULL_NAME} -X ${GO_MODULE_FULL_NAME}/config.BuildVersion=${BUILD_VERSION} -X ${GO_MODULE_FULL_NAME}/config.BuildGitSha=${BUILD_GIT_SHA} -X ${GO_MODULE_FULL_NAME}/config.BuildTime=${BUILD_TIME}"; ls -la "/build/${BINARY}"; file "/build/${BINARY}"; chmod +x "/build/${BINARY}"; if [ $GOOS = "linux" ]; then sh -c "/build/${BINARY}"; fi; done; done
+RUN export export FULL_NAME_GIT=$(git -C /build/.git remote -v | grep fetch | awk '{print $2}' | sed -En "s/git@//p" | sed -En "s/.git//p" | sed -En "s/:/\//p") && \
+    export export FULL_NAME_HTTP=$(git -C /build/.git remote -v | grep fetch | awk '{print $2}' | sed -En "s/https:\/\///p") && \
+    export GO_MODULE_FULL_NAME=$(if [ ! -z "$FULL_NAME_GIT" ]; then echo "$FULL_NAME_GIT"; else echo "$FULL_NAME_HTTP"; fi) && \
+    export BUILD_GIT_SHA=$(git rev-parse --short HEAD) && \
+    for GOOS in ${PLATFORMS}; do for GOARCH in ${ARCHS}; do BINARY="${BIN_NAME}-$GOOS-$GOARCH"; if [ $GOOS = "windows" ]; then BINARY="${BINARY}.exe"; fi; export BUILD_TIME="$(date -u +"%Y-%m-%d_%H:%M:%S_GMT")"; echo "Cross-compiling $GO_MODULE_FULL_NAME@$BUILD_GIT_SHA as ${BINARY}@$BUILD_VERSION at $BUILD_TIME"; GO_MODULE_FULL_NAME=$GO_MODULE_FULL_NAME BUILD_GIT_SHA=$BUILD_GIT_SHA BUILD_VERSION=$BUILD_VERSION BUILD_TIME=$BUILD_TIME GO111MODULE=$GO111MODULE CGO_ENABLED=0 GOARCH=$GOARCH GOOS=$GOOS GOPRIVATE=$GOPRIVATE go build -o ${BINARY} -a -ldflags "-X ${GO_MODULE_FULL_NAME}/config.VersionBuildGoModule=${GO_MODULE_FULL_NAME} -X ${GO_MODULE_FULL_NAME}/config.VersionBuildNumber=${BUILD_VERSION} -X ${GO_MODULE_FULL_NAME}/config.VersionBuildGitSha=${BUILD_GIT_SHA} -X ${GO_MODULE_FULL_NAME}/config.VersionBuildTime=${BUILD_TIME}"; ls -la "/build/${BINARY}"; file "/build/${BINARY}"; chmod +x "/build/${BINARY}"; if [ $GOOS = "linux" ]; then sh -c "/build/${BINARY}"; sh -c "/build/${BINARY} version"; fi; done; done
 
 # Compress the binaries
 # It is not working with errors like https://github.com/golang/go/issues/19625
@@ -59,7 +61,20 @@ RUN export GO_MODULE_FULL_NAME=$(git -C /build/.git remote -v | grep fetch | awk
 FROM alpine:latest
 WORKDIR /root/
 
-# Copy the linux amd64 binary
-COPY --from=builder /build/${BIN_NAME}* /bin/
+ARG BIN_NAME
+ENV BIN_NAME ${BIN_NAME:-unknown}
+
+# Copy the linux amd64 binary, based on the arg (or else all files are copied) inspect with https://github.com/wagoodman/dive
+COPY --from=builder /build/${BIN_NAME}* /usr/local/bin/
+
+# Move the bin to /usr/local/bin and make the entrypoint to point to it passing the params
+# https://stackoverflow.com/questions/33439230/how-to-write-commands-with-multiple-lines-in-dockerfile-while-preserving-the-new/33439625#33439625
+RUN echo "#!/bin/sh" > /usr/local/bin/entrypoint.sh && \
+    # https://stackoverflow.com/questions/32727594/how-to-pass-arguments-to-shell-script-through-docker-run/40312311#40312311
+    echo "${BIN_NAME} \$@" >> /usr/local/bin/entrypoint.sh && \
+    chmod +x /usr/local/bin/entrypoint.sh && \
+    ln -s /usr/local/bin/${BIN_NAME}-linux-amd64 /usr/local/bin/${BIN_NAME}
+    # Delete all binarines that are not the linux one https://www.cyberciti.biz/faq/find-command-exclude-ignore-files/
+    # find /bin -type f -name "cloner-*" ! -path "*linux*" | xargs rm -f && \
 
-ENTRYPOINT /bin/${ARG BIN_NAME}-linux-amd64
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]