build_packages.yaml

name: Build packages

concurrency:
  group: build-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

on:
  workflow_dispatch:
  pull_request:
  release:
    types:
      - published

jobs:
  prepare:
    runs-on: ubuntu-latest
    outputs:
      arch: ${{ steps.arch.outputs.arch }}

    steps:
      - id: arch
        run: |
          if ${{ contains(fromJSON('["release", "workflow_dispatch"]'), github.event_name) }}; then
            echo "arch=[\"linux/amd64\", \"linux/arm64\", \"linux/arm/v7\"]" >> $GITHUB_OUTPUT
          else
            echo "arch=[\"linux/amd64\"]" >> $GITHUB_OUTPUT
          fi

  build:
    runs-on: ubuntu-latest
    needs: prepare

    strategy:
      matrix:
        golang:
          - 1.21.3
        arch: ${{fromJson(needs.prepare.outputs.arch)}}
        os:
          - debian
          - centos
        exclude:
          - os: centos
            arch: linux/arm/v7

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - uses: docker/setup-qemu-action@v2
    - uses: docker/setup-buildx-action@v2
    - name: build
      if: matrix.os == 'debian'
      run: |
        docker run -i --rm \
        -v $(pwd):/ekuiper \
        --workdir /ekuiper \
        --env KUIPER_SOURCE='/ekuiper' \
        --platform ${{ matrix.arch }} \
        ghcr.io/lf-edge/ekuiper/base:${{ matrix.golang }}-${{ matrix.os }} \
        bash -euc "git config --global --add safe.directory /ekuiper && make pkg && make pkg_core && make pkg_full && .github/scripts/test.sh"
    - name: build
      if: matrix.os == 'centos'
      run: |
        docker run -i --rm \
        -v $(pwd):/ekuiper \
        --workdir /ekuiper \
        --env KUIPER_SOURCE='/ekuiper' \
        --platform ${{ matrix.arch }} \
        ghcr.io/lf-edge/ekuiper/base:${{ matrix.golang }}-${{ matrix.os }} \
        bash -euc "make -C deploy/packages && .github/scripts/test.sh"
    - name: create sha file
      run: |
        cd _packages && for var in $(ls); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
    - uses: actions/upload-artifact@v3
      with:
        name: packages
        path: _packages/

  build-on-mac:
    runs-on: macos-latest

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - uses: actions/setup-go@v3
      with:
        go-version: '1.21.3'
    - name: prepare
      run: |
          brew install curl zip unzip gnu-sed pkg-config zmq
          echo "/usr/local/bin:$PATH" >> ~/.bashrc
    - name: build
      run: |
        make pkg
        cd _packages && for var in $(ls); do openssl dgst -sha256 $var | awk '{print $2}' > $var.sha256; done && cd -
    - uses: actions/upload-artifact@v3
      with:
        name: packages
        path: _packages/

  build-docker-images:
    runs-on: ubuntu-latest

    strategy:
        fail-fast: false
        matrix:
            suffix:
            - ""
            - "-alpine"
            - "-dev"
            - "-slim"
            - "-slim-python"
            - "-full"
            golang:
            - 1.21.3

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - uses: docker/setup-buildx-action@v2
    - uses: docker/setup-qemu-action@v2
      with:
        image: tonistiigi/binfmt:latest
        platforms: all
    - name: Build single platform image
      if: endsWith( matrix.suffix, 'python') == false
      uses: docker/build-push-action@v3
      with:
        context: .
        platforms: linux/amd64
        push: false
        load: true
        tags: docker.io/lfedge/ekuiper
        build-args: GO_VERSION=${{ matrix.golang }}
        file: deploy/docker/Dockerfile${{ matrix.suffix }}
    - name: Test docker image
      run: |
        docker run -d --name ekuiper docker.io/lfedge/ekuiper
        ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ekuiper)
        sleep 5
        if ! curl ${ip_address}:9081  >/dev/null 2>&1; then
            echo "docker image failed"
            docker logs ekuiper
            exit 1
        fi
    - uses: docker/metadata-action@v4
      id: meta
      with:
        images: docker.io/lfedge/ekuiper
        flavor: |
          latest=${{ github.event_name == 'release' && matrix.suffix == '-alpine'  && !github.event.release.prerelease}}
          suffix=${{ matrix.suffix }}
        tags: |
          type=ref,event=branch
          type=ref,event=pr
          type=ref,event=tag
          type=semver,pattern={{version}}
          type=semver,pattern={{major}}.{{minor}}
    - uses: docker/login-action@v2
      if: github.event_name == 'release'
      with:
        username: ${{ secrets.DOCKER_HUB_USER }}
        password: ${{ secrets.DOCKER_HUB_TOKEN }}
    - name: Update version for setup.py
      if: github.event_name == 'release'
      run: sed -i -r "s|([ \t]*version=).*|\1'${GITHUB_REF#refs/tags/}',|1" sdk/python/setup.py
    - name: Build multi platform image
      if: contains(fromJSON('["release", "workflow_dispatch"]'), github.event_name)
      uses: docker/build-push-action@v3
      with:
        context: .
        platforms: linux/amd64,linux/arm64,linux/arm/v7
        push: ${{ github.event_name == 'release' }}
        tags: ${{ steps.meta.outputs.tags }}
        build-args: GO_VERSION=${{ matrix.golang }}
        labels: ${{ steps.meta.outputs.labels }}
        file: deploy/docker/Dockerfile${{ matrix.suffix }}

  build-plugins:
    runs-on: ubuntu-latest
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        plugin:
          - sinks/image
          - sinks/influx
          - sinks/influx2
          - sinks/tdengine
          - sinks/zmq
          - sinks/kafka
          - sinks/sql
          - sources/random
          - sources/zmq
          - sources/sql
          - sources/video
          - sources/kafka
          - functions/accumulateWordCount
          - functions/countPlusOne
          - functions/echo
          - functions/image
          - functions/geohash
          - functions/labelImage
          - functions/tfLite
        arch: ${{fromJson(needs.prepare.outputs.arch)}}
        os:
          - [debian, "slim"]
          - [alpine, "alpine"]
        golang:
          - 1.21.3
        exclude:
          - arch: linux/arm/v7
          - os: [ alpine,"alpine" ]
            plugin: functions/labelImage
          - os: [ alpine,"alpine" ]
            plugin: functions/tfLite
          - os: [alpine,"alpine"]
            plugin: sinks/tdengine

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - uses: docker/setup-buildx-action@v2
    - uses: docker/setup-qemu-action@v2
      with:
        image: tonistiigi/binfmt:latest
        platforms: all
    - name: build plugins
      run: |
        docker run -i --rm \
            -v $(pwd):/ekuiper \
            --workdir /ekuiper \
            --platform ${{ matrix.arch }} \
            ghcr.io/lf-edge/ekuiper/base:${{ matrix.golang }}-${{ matrix.os[0] }} \
            bash -euc "git config --global --add safe.directory /ekuiper && make ${{ matrix.plugin }}"
    - name: Build ekuiper image
      uses: docker/build-push-action@v3
      if: matrix.arch == 'linux/amd64'
      with:
        context: .
        platforms: linux/amd64
        push: false
        load: true
        tags: docker.io/lfedge/ekuiper-${{ matrix.os[0] }}
        file: deploy/docker/Dockerfile-${{ matrix.os[1] }}
    - name: test docker and plugins
      if: matrix.arch == 'linux/amd64'
      env:
        PLUGIN: ${{ matrix.plugin }}
      run: |
        set -e -x -u
        plugin_type=$(echo ${PLUGIN%%/*})
        plugin_name=$(echo ${PLUGIN##*/})

        container_id=$(docker run -u 0 -d -v $(pwd)/_plugins:/var/plugins docker.io/lfedge/ekuiper-${{ matrix.os[0] }})
        ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_id)
        os=$(docker exec -i ${container_id} sh -c "sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g'" )
        sleep 5
        if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
        if [ "${plugin_name}" = "tdengine" ]; then
            curl \
            ${ip_address}:9081/plugins/${plugin_type} \
            -X POST \
            -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"shellParas\": [\"2.4.0.18\"]}"
        elif [ "${plugin_name}" = "image" ]; then
            curl \
            ${ip_address}:9081/plugins/${plugin_type} \
            -X POST \
            -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"functions\": [\"resize\",\"thumbnail\"]}"
        elif [ "${plugin_name}" = "geohash" ]; then
            curl \
            ${ip_address}:9081/plugins/${plugin_type} \
            -X POST \
            -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"functions\": [\"geohashEncode\", \"geohashEncodeInt\", \"geohashDecode\", \"geohashDecodeInt\", \"geohashBoundingBox\", \"geohashBoundingBoxInt\", \"geohashNeighbor\", \"geohashNeighborInt\", \"geohashNeighbors\", \"geohashNeighborsInt\"]}"
        else
            curl \
            ${ip_address}:9081/plugins/${plugin_type} \
            -X POST \
            -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\"}"
        fi
        docker logs ${container_id}
        [ $plugin_name = $(curl ${ip_address}:9081/plugins/${plugin_type}/${plugin_name} | jq '.name'| sed 's/\"//g' ) ] || exit 1
    - uses: actions/upload-artifact@v3
      with:
        name: plugins
        path: _plugins/

  build-kubernetes-tool:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - uses: docker/setup-buildx-action@v2
    - uses: docker/setup-qemu-action@v2
      with:
        image: tonistiigi/binfmt:latest
        platforms: all
    - name: Build single platform image
      uses: docker/build-push-action@v3
      with:
        context: .
        platforms: linux/amd64
        push: false
        load: true
        tags: docker.io/lfedge/ekuiper-kubernetes-tool
        file: deploy/docker/Dockerfile-kubernetes-tool
    - name: Test docker image
      run: |
        docker run -d --name kuiper-kubernetes-tool docker.io/lfedge/ekuiper-kubernetes-tool
        sleep 5
        if [[ "$(docker logs kuiper-kubernetes-tool)" != *"Kuiper kubernetes tool is started successfully!"*  ]]; then exit 1; fi
    - uses: docker/metadata-action@v4
      id: meta
      with:
        images: docker.io/lfedge/ekuiper-kubernetes-tool
        flavor: |
          latest=${{ github.event_name == 'release' && !github.event.release.prerelease}}
        tags: |
          type=ref,event=branch
          type=ref,event=pr
          type=ref,event=tag
          type=semver,pattern={{version}}
          type=semver,pattern={{major}}.{{minor}}
    - uses: docker/login-action@v2
      if: github.event_name == 'release'
      with:
        username: ${{ secrets.DOCKER_HUB_USER }}
        password: ${{ secrets.DOCKER_HUB_TOKEN }}
    - name: Build multi platform image
      uses: docker/build-push-action@v3
      with:
        context: .
        platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
        push: ${{ github.event_name == 'release' }}
        tags: ${{ steps.meta.outputs.tags }}
        labels: ${{ steps.meta.outputs.labels }}
        file: deploy/docker/Dockerfile-kubernetes-tool

  release:
    runs-on: ubuntu-latest

    needs:
    - build
    - build-on-mac
    - build-plugins

    steps:
    - uses: actions/checkout@v3
    - uses: actions/download-artifact@v1
      with:
        name: packages
        path: _packages
    - uses: actions/download-artifact@v1
      with:
        name: plugins
        path: _plugins
    - name: check packages
      run: |
        cd _packages && for var in $( ls |grep -v sha256); do
          echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
        done
    - uses: Rory-Z/upload-release-asset@v1
      if: github.event_name == 'release'
      with:
        repo: ekuiper
        path: "_packages/kuiper*"
        token: ${{ secrets.GITHUB_TOKEN }}
    - name: upload packages to s3
      if: github.event_name == 'release'
      run: |
        version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
        aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws configure set default.region us-west-2
        aws s3 rm --quiet --recursive s3://packages.emqx/kuiper/$version
        aws s3 cp --quiet --recursive ./_packages s3://packages.emqx/kuiper/$version
        aws s3 cp --quiet --recursive ./_plugins s3://packages.emqx/kuiper-plugins/$version
        aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/kuiper/$version/*,/kuiper-plugins/$version/*"
    - name: update ekuiper.org
      if: github.event_name == 'release'
      run: |
        set -e -x -u
        curl -fvs -w %{http_code} \
          --insecure \
          -H "Content-Type: application/json" \
          -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
          -X POST \
          -d "{\"repo\":\"lf-edge/ekuiper\", \"tag\": \"${{ github.ref_name }}\" }" \
          ${{ secrets.EMQX_IO_RELEASE_API }}
    - uses: geekyeggo/delete-artifact@v2
      with:
        name: packages
    - uses: geekyeggo/delete-artifact@v2
      with:
        name: plugins