Skip to content
This repository has been archived by the owner on Dec 12, 2024. It is now read-only.

Integrate LLVM at llvm/llvm-project@8598bcb9 (#425) #1353

Integrate LLVM at llvm/llvm-project@8598bcb9 (#425)

Integrate LLVM at llvm/llvm-project@8598bcb9 (#425) #1353

name: Build and test
on:
push:
paths-ignore:
- 'docs/**'
pull_request:
branches:
- main
permissions:
contents: read
env:
EMITC: emitc
LLVM: llvm
E2E: emitc-e2e
E2E_VERSION: 3
jobs:
build-llvm:
name: Build LLVM
runs-on: ubuntu-22.04
steps:
- name: Checkout EmitC
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
path: ${{ env.EMITC }}
submodules: 'true'
- name: Get LLVM hash
id: get-llvm-hash
run: echo "llvm_hash=$(cat ${{ env.EMITC }}/build_tools/llvm_version.txt)" >> $GITHUB_ENV
shell: bash
- name: Cache LLVM
id: cache-llvm
uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
with:
path: ${{ env.LLVM }}
key: ${{ runner.os }}-llvm-22.04-install-${{ env.llvm_hash }}
lookup-only: 'true'
- name: Checkout LLVM
if: steps.cache-llvm.outputs.cache-hit != 'true'
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
repository: llvm/llvm-project
path: ${{ env.LLVM }}
ref: ${{ env.llvm_hash }}
- name: Rebuild and install LLVM
if: steps.cache-llvm.outputs.cache-hit != 'true'
run: |
mkdir -p ${LLVM}/build
mkdir -p ${LLVM}/install
cd ${LLVM}/build
cmake ../llvm \
-DLLVM_INSTALL_UTILS=ON \
-DLLVM_ENABLE_LLD=ON \
-DLLVM_ENABLE_PROJECTS=mlir \
-DLLVM_TARGETS_TO_BUILD="host" \
-DLLVM_INCLUDE_TOOLS=ON \
-DLLVM_BUILD_TOOLS=OFF \
-DLLVM_INCLUDE_TESTS=OFF \
-DCMAKE_INSTALL_PREFIX=../install \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_ASSERTIONS=On \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++
cmake --build . --target install -- -j$(nproc)
prepare-e2e-test:
name: Prepare e2e test
runs-on: ubuntu-22.04
steps:
- name: Checkout EmitC
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
path: ${{ env.EMITC }}
submodules: 'false'
- name: Cache e2e
id: cache-e2e
uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
with:
path: ${{ env.E2E }}
key: ${{ runner.os }}-e2e-${{ hashFiles('emitc/scripts/*.py', 'emitc/scripts/requirements.txt', 'emitc/scripts/e2e*.sh') }}-${{ env.E2E_VERSION }}
lookup-only: 'true'
- name: Install TensorFlow
if: steps.cache-e2e.outputs.cache-hit != 'true'
run: python3 -m pip install -r ${EMITC}/scripts/requirements.txt
- name: Prepare model
if: steps.cache-e2e.outputs.cache-hit != 'true'
run: |
mkdir -p ${E2E}
mkdir -p tmp
cd tmp
python3 ../${EMITC}/scripts/get_mobilenet_v2.py
python3 ../${EMITC}/scripts/model_to_savedmodel_with_predict_function.py --batch-size 2 mobilenet_v2.h5 model
python3 ../${EMITC}/scripts/savedmodel_to_tf_dialect.py --exported-names predict model model_tf.mlir
python3 ../${EMITC}/scripts/optimize_tf_dialect.py model_tf.mlir model_tf_opt.mlir
python3 ../${EMITC}/scripts/tf_to_hlo_dialect.py --hlo-dialect stablehlo model_tf_opt.mlir ../${E2E}/model_stablehlo.mlir
python3 ../${EMITC}/scripts/tf_to_tosa_dialect.py model_tf_opt.mlir model_tosa.mlir
sed "s/tf._input_shapes =.*]//" model_tosa.mlir > ../${E2E}/model_tosa_noattr.mlir
sed -i "s/, }/}/" ../${E2E}/model_tosa_noattr.mlir
python3 ../${EMITC}/scripts/generate_testscases.py --file-format cpp --count 1 --batch-size 2 --seed 1234 mobilenet_v2.h5 ../${E2E}/
build-debug:
name: Build and test EmitC (Debug)
needs: build-llvm
runs-on: ubuntu-22.04
steps:
- name: Configure environment
run: echo "$GITHUB_WORKSPACE/${LLVM}/install/bin" >> $GITHUB_PATH
- name: Install dependencies
run: sudo apt-get install -y libeigen3-dev
- name: Checkout EmitC
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
path: ${{ env.EMITC }}
submodules: 'true'
- name: Get LLVM hash
id: get-llvm-hash
run: echo "llvm_hash=$(cat ${{ env.EMITC }}/build_tools/llvm_version.txt)" >> $GITHUB_ENV
shell: bash
- name: Cache LLVM
id: cache-llvm
uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
with:
path: ${{ env.LLVM }}
key: ${{ runner.os }}-llvm-22.04-install-${{ env.llvm_hash }}
- name: Build and test EmitC
id: build-emitc-debug
run: |
mkdir -p ${EMITC}/build_debug
cd ${EMITC}/build_debug
cmake .. \
-DCMAKE_BUILD_TYPE=Debug \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DMLIR_DIR=$GITHUB_WORKSPACE/${LLVM}/install/lib/cmake/mlir/ \
-DLLVM_DIR=$GITHUB_WORKSPACE/${LLVM}/install/lib/cmake/llvm/ \
-DCMAKE_LINKER=lld \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++ \
-DLLVM_EXTERNAL_LIT=`pwd`/../../${LLVM}/build/bin/llvm-lit \
-DEMITC_TOSA_USE_EIGEN=ON
cmake --build . --target check-emitc -- -j$(nproc)
cmake --build . --target MLIREmitCTests -- -j$(nproc)
cmake --build . --target MLIREmitCEigenTests -- -j$(nproc)
./reference-implementation/unittests/MLIREmitCTests
./reference-implementation/unittests/MLIREmitCEigenTests
build-release:
name: Build and test EmitC (Release)
needs: [build-llvm, prepare-e2e-test]
runs-on: ubuntu-22.04
steps:
- name: Configure environment
run: echo "$GITHUB_WORKSPACE/${LLVM}/install/bin" >> $GITHUB_PATH
- name: Install dependencies
run: sudo apt-get install -y libeigen3-dev
- name: Checkout EmitC
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
path: ${{ env.EMITC }}
submodules: 'true'
- name: Get LLVM hash
id: get-llvm-hash
run: echo "llvm_hash=$(cat ${{ env.EMITC }}/build_tools/llvm_version.txt)" >> $GITHUB_ENV
shell: bash
- name: Cache LLVM
id: cache-llvm
uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
with:
path: ${{ env.LLVM }}
key: ${{ runner.os }}-llvm-22.04-install-${{ env.llvm_hash }}
- name: Build and test EmitC
id: build-emitc-release
run: |
mkdir -p ${EMITC}/build_release
cd ${EMITC}/build_release
cmake .. \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_ASSERTIONS=OFF \
-DMLIR_DIR=$GITHUB_WORKSPACE/${LLVM}/install/lib/cmake/mlir/ \
-DLLVM_DIR=$GITHUB_WORKSPACE/${LLVM}/install/lib/cmake/llvm/ \
-DCMAKE_LINKER=lld \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++ \
-DLLVM_EXTERNAL_LIT=`pwd`/../../${LLVM}/build/bin/llvm-lit \
-DEMITC_TOSA_USE_EIGEN=ON
cmake --build . --target check-emitc -- -j$(nproc)
cmake --build . --target MLIREmitCTests -- -j$(nproc)
cmake --build . --target MLIREmitCEigenTests -- -j$(nproc)
./reference-implementation/unittests/MLIREmitCTests
./reference-implementation/unittests/MLIREmitCEigenTests
- name: Cache e2e
uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
with:
path: ${{ env.E2E }}
key: ${{ runner.os }}-e2e-${{ hashFiles('emitc/scripts/*.py', 'emitc/scripts/requirements.txt', 'emitc/scripts/e2e*.sh') }}-${{ env.E2E_VERSION }}
- name: Run StableHLO e2e test
run: |
mkdir tmp-stablehlo
cd tmp-stablehlo
../${EMITC}/build_release/bin/emitc-opt --canonicalize --inline --symbol-dce ../${E2E}/model_stablehlo.mlir > model_canon.mlir
FUNCTION_NAME=$(grep -oe "@[^(]*" model_canon.mlir)
FUNCTION_NAME="${FUNCTION_NAME:1}"
sed "s/$FUNCTION_NAME/predict/g" model_canon.mlir > model_fix_name.mlir
../${EMITC}/build_release/bin/emitc-opt \
--insert-emitc-stablehlo-include \
--convert-stablehlo-region-ops-to-emitc \
--convert-stablehlo-to-emitc \
model_fix_name.mlir > model_emitc.mlir
../${EMITC}/build_release/bin/emitc-translate --mlir-to-cpp model_emitc.mlir > model_generated.h
clang++ ../${E2E}/test.cpp -O3 -I `pwd`/../emitc/reference-implementation/include -I `pwd` -o test
./test
- name: Run TOSA e2e test
run: |
mkdir -p tmp-tosa
cd tmp-tosa
../${EMITC}/build_release/bin/emitc-opt --canonicalize --inline --symbol-dce ../${E2E}/model_tosa_noattr.mlir > model_canon.mlir
FUNCTION_NAME=$(grep -m 1 -oe "@[^(]*" model_canon.mlir)
FUNCTION_NAME="${FUNCTION_NAME:1}"
sed "s/$FUNCTION_NAME/predict/g" model_canon.mlir > model_fix_name.mlir
../${EMITC}/build_release/bin/emitc-opt \
--insert-emitc-tosa-include \
--convert-tosa-to-emitc \
model_fix_name.mlir > model_emitc.mlir
../${EMITC}/build_release/bin/emitc-translate --mlir-to-cpp model_emitc.mlir > model_generated.h
clang++ ../${E2E}/test.cpp -O3 -I `pwd`/../emitc/reference-implementation/include -I `pwd` -o test
./test
#- name: Run MHLO test
# run: |
# mkdir -p ${EMITC}/test_mhlo
# ./${EMITC}/build_release/bin/emitc-opt \
# --insert-emitc-mhlo-include \
# --convert-mhlo-region-ops-to-emitc \
# --convert-mhlo-to-emitc \
# "${EMITC}"/test/MobileNetV2_FakeWeights_mhlo.mlir > "${EMITC}"/test_mhlo/model_emitc.mlir
# ./${EMITC}/build_release/bin/emitc-translate --mlir-to-cpp ${EMITC}/test_mhlo/model_emitc.mlir > ${EMITC}/test_mhlo/model_generated.h
# cp ${EMITC}/test/MobileNetV2_FakeWeights_test.cpp ${EMITC}/test_mhlo/test.cpp
# cd ${EMITC}/test_mhlo/
# clang++ test.cpp -O3 -I `pwd`/../reference-implementation/include -I `pwd` -o test
# ./test
#- name: Run TOSA test
# run: |
# mkdir -p ${EMITC}/test_tosa
# ./${EMITC}/build_release/bin/emitc-opt \
# --insert-emitc-tosa-include \
# --convert-tosa-to-emitc \
# "${EMITC}"/test/MobileNetV2_FakeWeights_tosa.mlir > "${EMITC}"/test_tosa/model_emitc.mlir
# ./${EMITC}/build_release/bin/emitc-translate --mlir-to-cpp ${EMITC}/test_tosa/model_emitc.mlir > ${EMITC}/test_tosa/model_generated.h
# cp ${EMITC}/test/MobileNetV2_FakeWeights_test.cpp ${EMITC}/test_tosa/test.cpp
# cd ${EMITC}/test_tosa/
# clang++ test.cpp -O3 -I `pwd`/../reference-implementation/include -I `pwd` -o test
# ./test
build-release-no-hlo:
name: Build and test EmitC (Release, HLO disabled)
needs: build-llvm
runs-on: ubuntu-22.04
steps:
- name: Configure environment
run: echo "$GITHUB_WORKSPACE/${LLVM}/install/bin" >> $GITHUB_PATH
- name: Checkout EmitC
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
path: ${{ env.EMITC }}
submodules: 'true'
- name: Get LLVM hash
id: get-llvm-hash
run: echo "llvm_hash=$(cat ${{ env.EMITC }}/build_tools/llvm_version.txt)" >> $GITHUB_ENV
shell: bash
- name: Cache LLVM
id: cache-llvm
uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0
with:
path: ${{ env.LLVM }}
key: ${{ runner.os }}-llvm-22.04-install-${{ env.llvm_hash }}
- name: Build and test EmitC, HLO disabled
id: build-emitc-release
run: |
mkdir -p ${EMITC}/build_release_hlo_disabled
cd ${EMITC}/build_release_hlo_disabled
cmake .. \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_ASSERTIONS=OFF \
-DMLIR_DIR=$GITHUB_WORKSPACE/${LLVM}/install/lib/cmake/mlir/ \
-DLLVM_DIR=$GITHUB_WORKSPACE/${LLVM}/install/lib/cmake/llvm/ \
-DCMAKE_LINKER=lld \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++ \
-DLLVM_EXTERNAL_LIT=`pwd`/../../${LLVM}/build/bin/llvm-lit \
-DEMITC_ENABLE_HLO=OFF
cmake --build . --target check-emitc -- -j$(nproc)