build_packages.yaml

name: Build packages

on:
  push:
    tags:
  pull_request:
  release:
    types:
      - published

jobs:
    build:
        runs-on: ubuntu-latest

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - uses: docker/setup-buildx-action@v1
        - uses: docker/setup-qemu-action@v1
          with:
            image: tonistiigi/binfmt:latest
            platforms: all
        - name: build
          run: |
            make all_pkgs
            cd _packages && for var in $(ls); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
        - uses: actions/upload-artifact@v1
          with:
            name: packages
            path: _packages/.

    build-on-mac:
        runs-on: macos-latest

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - uses: actions/setup-go@v2
          with:
            go-version: '1.15.1'
        - name: prepare
          run: |
              brew install curl zip unzip gnu-sed upx pkg-config zmq
              echo "/usr/local/bin:$PATH" >> ~/.bashrc
        - name: build
          run: |
            make pkg
            cd _packages && for var in $(ls); do openssl dgst -sha256 $var | awk '{print $2}' > $var.sha256; done && cd -
        - uses: actions/upload-artifact@v1
          with:
            name: packages-mac
            path: _packages/.

    build-docker-images:
        runs-on: ubuntu-latest

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - uses: docker/setup-buildx-action@v1
        - uses: docker/setup-qemu-action@v1
          with:
            image: tonistiigi/binfmt:latest
            platforms: all
        - name: build docker image
          run: |
            make docker -j4
            docker save lfedge/ekuiper:$(git describe --tags --always)-slim > kuiper-image.tar.gz
        - name: test docker image
          run: |
            for image_id in $(docker images lfedge/ekuiper -q); do
              container_id=$(docker run -d $image_id)
              ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_id)
              sleep 5
              if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
            done
        - uses: actions/upload-artifact@v2
          with:
            name: kuiper-image
            path: "kuiper-image.tar.gz"

    build-plugins:
        runs-on: ubuntu-latest

        needs: build-docker-images

        strategy:
            fail-fast: false
            matrix:
                plugin:
                - sinks/file
                - sinks/image
                - sinks/influx
                - sinks/tdengine
                - sinks/zmq
                - sinks/redis
                - sources/random
                - sources/zmq
                - functions/accumulateWordCount
                - functions/countPlusOne
                - functions/echo
                - functions/image
                - functions/geohash
                - functions/labelImage

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - uses: docker/setup-buildx-action@v1
        - uses: docker/setup-qemu-action@v1
          with:
            image: tonistiigi/binfmt:latest
            platforms: all
        - name: build debian plugins
          env:
            PLUGIN: ${{ matrix.plugin }}
          run: |
              make ${PLUGIN}
        - uses: actions/download-artifact@v1
          with:
            name: kuiper-image
            path: .
        - name: test docker and plugins
          env:
            PLUGIN: ${{ matrix.plugin }}
          run: |
            set -e -x -u
            docker load < kuiper-image.tar.gz
            plugin_type=$(echo ${PLUGIN%%/*})
            plugin_name=$(echo ${PLUGIN##*/})

            for image_id in $(docker images lfedge/ekuiper -q); do
                container_id=$(docker run -d -v $(pwd)/_plugins:/var/plugins $image_id)
                ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_id)
                os=$(docker exec -i ${container_id} sh -c "sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g'" )
                sleep 5
                if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
                if [ ${os} = alpine ]; then continue; fi
                if [ "${plugin_name}" = "tdengine" ]; then
                    curl \
                    ${ip_address}:9081/plugins/${plugin_type} \
                    -X POST \
                    -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"shellParas\": [\"2.0.3.1\"]}"
                elif [ "${plugin_name}" = "image" ]; then
                     curl \
                     ${ip_address}:9081/plugins/${plugin_type} \
                     -X POST \
                     -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"functions\": [\"resize\",\"thumbnail\"]}"
                elif [ "${plugin_name}" = "geohash" ]; then
                     curl \
                     ${ip_address}:9081/plugins/${plugin_type} \
                     -X POST \
                     -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"functions\": [\"geohashEncode\", \"geohashEncodeInt\", \"geohashDecode\", \"geohashDecodeInt\", \"geohashBoundingBox\", \"geohashBoundingBoxInt\", \"geohashNeighbor\", \"geohashNeighborInt\", \"geohashNeighbors\", \"geohashNeighborsInt\"]}"
                else
                    curl \
                    ${ip_address}:9081/plugins/${plugin_type} \
                    -X POST \
                    -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\"}"
                fi
                docker logs ${container_id}
                [ $plugin_name = $(curl ${ip_address}:9081/plugins/${plugin_type}/${plugin_name} | jq '.name'| sed 's/\"//g' ) ] || exit 1
            done
        - uses: actions/upload-artifact@v2
          with:
            name: plugins
            path: "_plugins/"

    build-docker-manifest-images:
        runs-on: ubuntu-latest

        needs: build-docker-images

        if: github.event_name == 'release'

        strategy:
            matrix:
                suffix:
                - fat
                - slim
                - alpine

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - uses: docker/setup-buildx-action@v1
        - uses: docker/setup-qemu-action@v1
          with:
            image: tonistiigi/binfmt:latest
            platforms: all
        - uses: docker/login-action@v1
          with:
            username: ${{ secrets.DOCKER_HUB_USER }}
            password: ${{ secrets.DOCKER_HUB_TOKEN }}
        - name: cross build docker images
          if: matrix.suffix == 'fat'
          run: |
            set -e -u -x
            docker buildx build --no-cache \
            --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386 \
            -t lfedge/ekuiper:$(git describe --tags --always) \
            -f deploy/docker/Dockerfile . \
            --push
        - name: cross build docker images
          if: matrix.suffix != 'fat'
          env:
              SUFFIX: ${{ matrix.suffix }}
          run: |
            set -e -u -x
            docker buildx build --no-cache \
            --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386 \
            -t lfedge/ekuiper:$(git describe --tags --always)-$SUFFIX \
            -f deploy/docker/Dockerfile-$SUFFIX . \
            --push

    build-kubernetes-tool:
        runs-on: ubuntu-latest

        steps:
        - uses: actions/checkout@v2
          with:
            fetch-depth: 0
        - uses: docker/setup-buildx-action@v1
        - uses: docker/setup-qemu-action@v1
          with:
            image: tonistiigi/binfmt:latest
            platforms: all
        - name: build docker image
          run: docker build --no-cache -t lfedge/ekuiper-kubernetes-tool:$(git describe --tags --always) -f deploy/docker/Dockerfile-kubernetes-tool .
        - name: test docker image
          run: |
            set -e -u -x
            docker run -d --name kuiper-kubernetes-tool lfedge/ekuiper-kubernetes-tool:$(git describe --tags --always)
            sleep 5
            if [[ "$(docker logs kuiper-kubernetes-tool)" != *"Kuiper kubernetes tool is started successfully!"*  ]]; then exit 1; fi
        - uses: docker/login-action@v1
          if: github.event_name == 'release'
          with:
            username: ${{ secrets.DOCKER_HUB_USER }}
            password: ${{ secrets.DOCKER_HUB_TOKEN }}
        - name: buildx docker image
          if: github.event_name == 'release'
          run: |
            docker buildx build --no-cache \
                --platform=linux/amd64,linux/arm64,linux/arm/v7,linux/386 \
                -t lfedge/ekuiper-kubernetes-tool:$(git describe --tags --always) \
                -f deploy/docker/Dockerfile-kubernetes-tool . \
                --push

    release:
        runs-on: ubuntu-latest

        needs:
        - build
        - build-on-mac
        - build-plugins

        steps:
        - uses: actions/checkout@v2
        - uses: actions/download-artifact@v1
          with:
            name: packages
            path: _packages
        - uses: actions/download-artifact@v1
          with:
            name: packages-mac
            path: _packages
        - uses: actions/download-artifact@v1
          with:
            name: plugins
            path: _plugins
        - name: check packages
          run: |
            cd _packages && for var in $( ls |grep -v sha256); do
              echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
            done
        - uses: Rory-Z/upload-release-asset@v1
          if: github.event_name == 'release'
          with:
            repo: ekuiper
            path: "_packages/kuiper-*"
            token: ${{ secrets.GITHUB_TOKEN }}
        - name: upload plugins to s3
          if: github.event_name == 'release'
          run: |
            version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
            aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
            aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
            aws configure set default.region us-west-2
            aws s3 rm --quiet --recursive s3://packages.emqx/kuiper/$version
            aws s3 cp --quiet --recursive ./_packages s3://packages.emqx/kuiper/$version
            aws s3 cp --quiet --recursive ./_plugins s3://packages.emqx/kuiper-plugins/$version
            aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/kuiper/$version/*,/kuiper-plugins/$version/*"