add GHA test (#1235)

.github/workflows/copyright_check.yml

@@ -0,0 +1,61 @@
+name: Copyright Check
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - intel_extension_for_transformers/**
+      - setup.py
+      - .github/workflows/format_scan.yml
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  CODE_SCAN_LOG_PATH: "${{ github.workspace }}/log"
+
+jobs:
+  format-scan:
+    runs-on: itrex-node-spell
+    strategy:
+      matrix:
+        job_name: ["copyright"]
+      fail-fast: false
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+
+      - name: CopyRight check
+        run: |
+          source ${{ github.workspace }}/.github/workflows/script/change_color.sh
+          set -e
+          mkdir -p ${{ env.CODE_SCAN_LOG_PATH }}
+          supported_extensions=(py, sh, yaml)
+          git fetch
+          git --no-pager diff --name-only remotes/origin/${{ github.base_ref }} ${{ github.workspace }}/intel_extension_for_transformers> ${{ env.CODE_SCAN_LOG_PATH }}/diff.log
+          files=$(cat ${{ env.CODE_SCAN_LOG_PATH }}/diff.log | awk '!a[$0]++')
+          $LIGHT_PURPLE && echo " ----------------- checking ... --------------------------" && $RESET
+          for file in ${files}
+          do
+              if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then
+                  if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then
+                      echo ${file} >> ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
+                      $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------"
+                      cat ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
+                      $BOLD_YELLOW && echo " -----------------  Current log file output end --------------------------" && $RESET
+                      $BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET; exit 1
+                  fi
+              else
+                  $LIGHT_PURPLE && echo "Skipping ${file}"  && $RESET
+              fi
+          done
+
+      - name: Publish pipeline artifact
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.job_name }}
+          path: ${{ env.CODE_SCAN_LOG_PATH }}.*

.github/workflows/cpp-graph-test.yml

@@ -0,0 +1,124 @@
+name: CPP Graph Test
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+       - '.github/workflows/cpp-graph-test.yml'
+       - '.github/workflows/script/models/cpp_graph_inference.sh'
+       - 'intel_extension_for_transformers/backends/neural_engine/graph/**'
+       - '!intel_extension_for_transformers/backends/neural_engine/graph/README.md'
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  OUT_SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/script/models
+  SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/script
+  WORKING_DIR: ${{ github.workspace }}
+
+
+jobs:
+  CPP-Graph-Workflow:
+    runs-on: spr
+    strategy:
+      matrix:
+        include:
+          - modelName: "llama-7b-hf"
+            framework: "engine"
+            mode: "latency"
+            precision: "bf16,int8"
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Env build
+        run: |
+          bash ${{ github.workspace }}/.github/workflows/script/prepare_env_with_conda.sh "cpp-graph-test" "3.8"
+
+      - name: Binary build
+        run: |
+          cd ${{ github.workspace }}
+          conda activate cpp-graph-test || source activate cpp-graph-test 
+          pip install build --upgrade
+          python -m build -s -w
+          pip install dist/intel_extension_for_transformers*.whl
+          pip list
+
+      - name: BF16 Benchmark
+        run: |
+          cd ${{ github.workspace }}/.github/workflows/script/models
+          bash cpp_graph_inference.sh cpp-graph-test ${{ matrix.modelName }} 12.0.0
+  
+      - name: Publish pipeline artifact
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: cpp_graph
+          path: ${{ github.workspace }}/cpp_graph_summary.log
+          if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn`
+          retention-days: 60 # 1 <= retention-days <= 90
+
+  Genreate-Report:
+    runs-on: itrex-node-spell
+    needs: [CPP-Graph-Workflow]
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Download Summary Log
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.OUT_SCRIPT_PATH }}/generated/log
+
+      - name: Download Reference Artifact
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: cpp-graph-test.yml
+          name: FinalReport
+          run_id: ${{ vars.GRAPH_REF_ID }}
+          path: ${{ env.OUT_SCRIPT_PATH }}
+          name_is_regexp: true
+          repo: ${{ github.repository }}
+          check_artifacts: false
+          search_artifacts: false
+          skip_unpack: false
+          if_no_artifact_found: warn
+
+      - name: Display structure of downloaded files
+        run: cd ${{ env.OUT_SCRIPT_PATH }} && ls -R
+
+      - name: Generate report
+        run: |
+          echo "------ Generating final report.html ------"
+          cd ${{ env.OUT_SCRIPT_PATH }}
+          /usr/bin/bash generate_report.sh --workflow=deploy
+        env:
+          RUN_DISPLAY_URL: https://github.com/VincyZhang/intel-extension-for-transformers/actions/runs/${{ github.run_id }}
+          BUILD_NUMBER: ${{ github.run_id }}
+          JOB_STATUS: succeed
+          MR_source_branch: ${{ github.head_ref }}
+          ghprbActualCommit: ${{ github.event.pull_request.head.sha }}
+
+      - name: Publish Report
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: FinalReport
+          path: ${{ env.OUT_SCRIPT_PATH }}/generated
+
+      - name: Specify performance regression
+        run: |
+          if [ $(is_perf_reg) == 'true' ]; then
+            echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports."
+            exit 1
+          fi
+  

.github/workflows/deploy-test.yml

@@ -0,0 +1,207 @@
+name: Deploy Model Test
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - ".github/workflows/deploy-test.yml"
+      - ".github/workflows/script/models/run_deploy.sh"
+      - "intel_extension_for_transformers/backends/**"
+      - "!intel_extension_for_transformers/backends/neural_engine/kernels/**"
+      - "!intel_extension_for_transformers/backends/neural_engine/test/**"
+      - "!intel_extension_for_transformers/backends/neural_engine/graph/**"
+      - "!intel_extension_for_transformers/backends/neural_engine/third_party/**"
+      - "!intel_extension_for_transformers/backends/neural_engine/docs/**"
+
+  workflow_dispatch:
+
+# If there is a new commit, the previous jobs will be canceled
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  OUT_SCRIPT_PATH: ${{ github.workspace }}/.github/workflows/script/models
+  SCRIPT_PATH: /intel-extension-for-transformers/.github/workflows/script
+  DOCKER_CONFIG_NAME: "commonDockerConfig"
+  REPO_NAME: "intel-extension-for-transformers"
+  REPO_TAG: "py38"
+  DOCKER_FILE_NAME: "devel"
+  CONTAINER_NAME: "modelTest"
+  EXTRA_CONTAINER_NAME: "utTest"
+
+jobs:
+  Deploy-Workflow:
+    runs-on: itrex-node
+    strategy:
+      matrix:
+        include:
+          - modelName: "bert_mini_sparse"
+            framework: "engine"
+            mode: "accuracy,performance"
+            precision: "int8"
+          - modelName: "distilbert_base_squad_ipex"
+            framework: "ipex"
+            mode: "accuracy,performance"
+            precision: "int8"
+    steps:
+      - name: Docker Clean Up
+        run: |
+          docker ps -a
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
+              docker start $(docker ps -aq)
+              echo "remove left files through container ..."
+              docker exec ${{ env.CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+          fi
+          docker ps -a
+          if [[ $(docker ps -a | grep -i '${{ env.EXTRA_CONTAINER_NAME }}'$) ]]; then
+              docker start $(docker ps -aq)
+              echo "remove left files through container ..."
+              docker exec ${{ env.EXTRA_CONTAINER_NAME }} bash -c "ls -a /intel-extension-for-transformers && rm -fr /intel-extension-for-transformers/* && rm -fr /intel-extension-for-transformers/.* || true"
+          fi
+
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+      # We need this because GitHub needs to clone the branch to pipeline
+      - name: Docker Build
+        run: |
+          docker build -f ${{ github.workspace }}/.github/workflows/docker/${{ env.DOCKER_FILE_NAME }}.dockerfile -t ${{ env.REPO_NAME }}:${{ env.REPO_TAG }} .
+
+      - name: Docker Run
+        run: |
+          if [[ $(docker ps -a | grep -i '${{ env.CONTAINER_NAME }}'$) ]]; then
+            docker stop $(docker ps -aq)
+            docker rm -vf ${{ env.CONTAINER_NAME }} || true
+          fi
+          docker run -dit --disable-content-trust --privileged --name=${{ env.CONTAINER_NAME }} -v /dev/shm:/dev/shm \
+          -v ${{ github.workspace }}:/intel-extension-for-transformers \
+          ${{ env.REPO_NAME }}:${{ env.REPO_TAG }}
+
+      - name: Env build
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} \
+          bash /intel-extension-for-transformers/.github/workflows/script/prepare_env.sh
+
+      - name: Binary build
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} \
+          bash -c "cd /intel-extension-for-transformers/.github/workflows/script \
+          && bash install_binary.sh"
+
+      - name: Download Reference Artifact
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: deploy-test.yml
+          name: ${{ matrix.framework }}-${{ matrix.modelName }}
+          run_id: ${{ vars.DEPLOY_REF_ID }}
+          path: ${{ github.workspace }}/${{ matrix.framework }}_${{ matrix.modelName }}_refer_log
+          name_is_regexp: true
+          repo: ${{ github.repository }}
+          check_artifacts: false
+          search_artifacts: false
+          skip_unpack: false
+          if_no_artifact_found: warn
+
+      - name: Display structure of downloaded files
+        run: ls -R
+
+      - name: FP32 Benchmark
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} \
+          bash -c "cd /intel-extension-for-transformers/.github/workflows/script/models \
+          && bash run_deploy.sh --model=${{ matrix.modelName }} --framework=${{ matrix.framework }} --mode=${{ matrix.mode }} --precision=fp32 --PERF_STABLE_CHECK=${{ vars.PERF_STABLE_CHECK }}"
+
+      - name: INT8 Benchmark
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} \
+          bash -c "cd /intel-extension-for-transformers/.github/workflows/script/models \
+          && bash run_deploy.sh --model=${{ matrix.modelName }} --framework=${{ matrix.framework }} --mode=${{ matrix.mode }} --precision=int8 --PERF_STABLE_CHECK=${{ vars.PERF_STABLE_CHECK }}"
+
+      - name: Collect Log
+        run: |
+          docker exec ${{ env.CONTAINER_NAME }} \
+          bash -c "cd /intel-extension-for-transformers/.github/workflows/script/models \
+          && python collect_model_log.py --model=${{ matrix.modelName }} \
+           --framework=${{ matrix.framework }} \
+           --logs_dir=/intel-extension-for-transformers/${{matrix.framework}}_${{matrix.modelName}} \
+           --output_dir=/intel-extension-for-transformers/${{matrix.framework}}_${{matrix.modelName}} \
+           --build_id=${{ github.run_id }} \
+           --model_test_type=deploy"
+
+      - name: Publish pipeline artifact
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: ${{ matrix.framework }}-${{ matrix.modelName }}
+          path: ${{ github.workspace }}/${{ matrix.framework }}_${{ matrix.modelName }}
+          if-no-files-found: ignore # 'warn' or 'ignore' are also available, defaults to `warn`
+          retention-days: 60 # 1 <= retention-days <= 90
+
+  Genreate-Report:
+    runs-on: itrex-node-spell
+    needs: [Deploy-Workflow]
+    steps:
+      - name: Checkout out Repo
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+
+      - name: Download Summary Log
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.OUT_SCRIPT_PATH }}/log
+
+      - name: Analysis Summary
+        run: |
+          cd ${{ env.OUT_SCRIPT_PATH }}
+          mkdir generated
+          pip install requests
+          python summary.py --logs_dir ${{ env.OUT_SCRIPT_PATH }}/log --output_dir=generated
+
+      - name: Download Reference Artifact
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: deploy-test.yml
+          name: FinalReport
+          run_id: ${{ vars.DEPLOY_REF_ID }}
+          path: ${{ env.OUT_SCRIPT_PATH }}
+          name_is_regexp: true
+          repo: ${{ github.repository }}
+          check_artifacts: false
+          search_artifacts: false
+          skip_unpack: false
+          if_no_artifact_found: warn
+
+      - name: Display structure of downloaded files
+        run: cd ${{ env.OUT_SCRIPT_PATH }}/log && ls -R
+
+      - name: Generate report
+        run: |
+          echo "------ Generating final report.html ------"
+          cd ${{ env.OUT_SCRIPT_PATH }}
+          /usr/bin/bash generate_report.sh --workflow=deploy
+        env:
+          RUN_DISPLAY_URL: https://github.com/VincyZhang/intel-extension-for-transformers/actions/runs/${{ github.run_id }}
+          BUILD_NUMBER: ${{ github.run_id }}
+          JOB_STATUS: succeed
+          MR_source_branch: ${{ github.head_ref }}
+          ghprbActualCommit: ${{ github.event.pull_request.head.sha }}
+
+      - name: Publish Report
+        uses: actions/upload-artifact@v3
+        if: ${{ !cancelled() }}
+        with:
+          name: FinalReport
+          path: ${{ env.OUT_SCRIPT_PATH }}/generated
+
+      - name: Specify performance regression
+        if: ${{ !cancelled() }}
+        run: |
+          if [ ${{ env.is_perf_reg }} == 'true' ]; then
+            echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports."
+            exit 1
+          fi