refact folder stracture (#170)

.github/workflows/copyright_check.yml

@@ -38,6 +38,9 @@ jobs:
           git --no-pager diff --name-only remotes/origin/${{ github.base_ref }} ${{ github.workspace }}/intel_extension_for_transformers> ${{ env.CODE_SCAN_LOG_PATH }}/diff.log
           files=$(cat ${{ env.CODE_SCAN_LOG_PATH }}/diff.log | awk '!a[$0]++')
           $LIGHT_PURPLE && echo " ----------------- checking ... --------------------------" && $RESET
+          if [[ -f ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log ]]; then 
+            rm -f ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
+          fi
           for file in ${files}
           do
               if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then
@@ -46,12 +49,18 @@ jobs:
                       $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------"
                       cat ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
                       $BOLD_YELLOW && echo " -----------------  Current log file output end --------------------------" && $RESET
-                      $BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET; exit 1
+                      $BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET
                   fi
               else
                   $LIGHT_PURPLE && echo "Skipping ${file}"  && $RESET
               fi
           done
+          if [[ -f ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log ]]; then 
+            $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------"
+            cat ${{ env.CODE_SCAN_LOG_PATH }}/copyright_issue_summary.log
+            $BOLD_YELLOW && echo " -----------------  Current log file output end --------------------------" && $RESET
+            $BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET && exit 1
+          fi
 
       - name: Publish pipeline artifact
         if: ${{ failure() }}

.github/workflows/cpp-graph-test.yml

@@ -6,8 +6,8 @@ on:
     paths:
        - '.github/workflows/cpp-graph-test.yml'
        - '.github/workflows/script/models/cpp_graph_inference.sh'
-       - 'intel_extension_for_transformers/backends/neural_engine/graph/**'
-       - '!intel_extension_for_transformers/backends/neural_engine/graph/README.md'
+       - 'intel_extension_for_transformers/llm/runtime/graph/**'
+       - '!intel_extension_for_transformers/llm/runtime/graph/README.md'
   workflow_dispatch:
 
 # If there is a new commit, the previous jobs will be canceled

.github/workflows/deploy-test.yml

@@ -7,11 +7,11 @@ on:
       - ".github/workflows/deploy-test.yml"
       - ".github/workflows/script/models/run_deploy.sh"
       - "intel_extension_for_transformers/backends/**"
-      - "!intel_extension_for_transformers/backends/neural_engine/kernels/**"
-      - "!intel_extension_for_transformers/backends/neural_engine/test/**"
-      - "!intel_extension_for_transformers/backends/neural_engine/graph/**"
-      - "!intel_extension_for_transformers/backends/neural_engine/third_party/**"
-      - "!intel_extension_for_transformers/backends/neural_engine/docs/**"
+      - "!intel_extension_for_transformers/llm/runtime/kernels/**"
+      - "!intel_extension_for_transformers/llm/runtime/test/**"
+      - "!intel_extension_for_transformers/llm/runtime/graph/**"
+      - "!intel_extension_for_transformers/llm/runtime/third_party/**"
+      - "!intel_extension_for_transformers/llm/runtime/docs/**"
 
   workflow_dispatch:
 

.github/workflows/llm-test.yml

@@ -7,11 +7,11 @@ on:
        - '.github/workflows/llm-test.yml'
        - '.github/workflows/script/models/run_llm.sh'
        - "intel_extension_for_transformers/backends/**"
-       - "!intel_extension_for_transformers/backends/neural_engine/kernels/**"
-       - "!intel_extension_for_transformers/backends/neural_engine/graph/**"
-       - "!intel_extension_for_transformers/backends/neural_engine/test/**"
-       - "!intel_extension_for_transformers/backends/neural_engine/third_party/**"
-       - "!intel_extension_for_transformers/backends/neural_engine/docs/**"
+       - "!intel_extension_for_transformers/llm/runtime/kernels/**"
+       - "!intel_extension_for_transformers/llm/runtime/graph/**"
+       - "!intel_extension_for_transformers/llm/runtime/test/**"
+       - "!intel_extension_for_transformers/llm/runtime/third_party/**"
+       - "!intel_extension_for_transformers/llm/runtime/docs/**"
   workflow_dispatch:
 
 # If there is a new commit, the previous jobs will be canceled

.github/workflows/script/SparseLibCI/run_sparse_lib.sh

@@ -10,27 +10,27 @@ cp /intel-extension-for-transformers/.github/workflows/script/SparseLibCI/genera
 cp /intel-extension-for-transformers/.github/workflows/script/SparseLibCI/generate_sparse_lib.py /generate_sparse_lib.py
 pip install cmake
 
-rm -rf /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/build
+rm -rf /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/build
 cd /intel-extension-for-transformers
 
 git config --global --add safe.directory "*"
 git fetch
 git submodule update --init --recursive
-cd /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine
+cd /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime
 mkdir build && cd build
 
 CC=gcc CXX=g++ cmake .. -DNE_WITH_SPARSELIB=ON -DNE_WITH_TESTS=ON -DNE_WITH_SPARSELIB_BENCHMARK=ON -DPYTHON_EXECUTABLE=$(which python)
 make -j
 cd bin
-bash /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/build/bin/ci/run_ci.sh $cur_dir
+bash /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/build/bin/ci/run_ci.sh $cur_dir
 
 for caselog in $(find $cur_dir/*); do
     case_name=$(echo $caselog | sed -e 's/\.log$//')
     $BOLD_YELLOW && echo "[VAL INFO] write summary, case_name=$case_name" && $RESET
-    bash /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/test/kernels/benchmark/ci/to_summary.sh $caselog | tee -a "${case_name}_summary.log"
+    bash /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/test/kernels/benchmark/ci/to_summary.sh $caselog | tee -a "${case_name}_summary.log"
 done
 
-cd /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine
+cd /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime
 mkdir refer && cd refer
 git checkout -b refer origin/main
 git pull
@@ -39,10 +39,10 @@ make -j
 cd bin
 
 mkdir -p ${output_log_dir}/ref
-bash /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/refer/bin/ci/run_ci.sh $ref_dir
+bash /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/refer/bin/ci/run_ci.sh $ref_dir
 
 for caselog in $(find $ref_dir/*); do
     case_name=$(echo $caselog | sed -e 's/\.log$//')
     $BOLD_YELLOW && echo "[VAL INFO] write summary, case_name=$case_name" && $RESET
-    bash /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/test/kernels/benchmark/ci/to_summary.sh $caselog | tee -a "${case_name}_summary.log"
+    bash /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/test/kernels/benchmark/ci/to_summary.sh $caselog | tee -a "${case_name}_summary.log"
 done

.github/workflows/script/formatScan/clangformat.sh

@@ -8,7 +8,7 @@ log_path=${log_dir}/clangformat.log
 cd /intel-extension-for-transformers
 git config --global --add safe.directory "*"
 
-cd /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/kernels
+cd /intel-extension-for-transformers/intel_extension_for_transformers/llm/library/
 clang-format --style=file -i include/**/*.hpp
 clang-format --style=file -i src/**/*.hpp
 clang-format --style=file -i src/**/*.cpp

.github/workflows/script/formatScan/cpplint.sh

@@ -6,9 +6,9 @@ pip install cpplint
 REPO_DIR=/intel-extension-for-transformers
 log_dir=/intel-extension-for-transformers/.github/workflows/script/formatScan
 log_path=${log_dir}/cpplint.log
-cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/intel_extension_for_transformers/backends/neural_engine/compile 2>&1 | tee ${log_path}
-cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/intel_extension_for_transformers/backends/neural_engine/executor 2>&1 | tee -a ${log_path}
-cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/intel_extension_for_transformers/backends/neural_engine/test 2>&1 | tee -a ${log_path}
+cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/intel_extension_for_transformers/llm/runtime/compile 2>&1 | tee ${log_path}
+cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/intel_extension_for_transformers/llm/runtime/executor 2>&1 | tee -a ${log_path}
+cpplint --filter=-build/include_subdir,-build/header_guard --recursive --quiet --linelength=120 ${REPO_DIR}/intel_extension_for_transformers/llm/runtime/test 2>&1 | tee -a ${log_path}
 if [[ ! -f ${log_path} ]] || [[ $(grep -c "Total errors found:" ${log_path}) != 0 ]]; then
     exit 1
 fi

.github/workflows/script/formatScan/nlp_dict.txt

@@ -2283,4 +2283,5 @@ lkk
 luoyu
 lvliang
 xin
-zhenwei
+zhenwei
+llm

.github/workflows/script/formatScan/pylint.sh

@@ -1,6 +1,25 @@
 #!/bin/bash
 
 source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
+cd /intel-extension-for-transformers
+$BOLD_YELLOW && echo "---------------- git submodule update --init --recursive -------------" && $RESET
+git config --global --add safe.directory "*"
+git submodule update --init --recursive
+
+$BOLD_YELLOW && echo "---------------- run python setup.py sdist bdist_wheel -------------" && $RESET
+pip install build --upgrade
+python3 -m build -s -w
+$BOLD_YELLOW && echo "---------------- pip install binary -------------" && $RESET
+pip install dist/intel_extension_for_transformers*.whl
+pip list
+
+cd /intel-extension-for-transformers/intel_extension_for_transformers/neural_chat/
+if [ -f "requirements.txt" ]; then
+    python -m pip install --default-timeout=100 -r requirements.txt
+    pip list
+else
+    echo "Not found requirements.txt file."
+fi
 
 cd /intel-extension-for-transformers
 log_dir=/intel-extension-for-transformers/.github/workflows/script/formatScan
@@ -14,12 +33,16 @@ fi
 pip install accelerate intel_extension_for_pytorch nlpaug nltk
 pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@83dbfbf6070324f3e5872f63e49d49ff7ef4c9b3
 
+echo "[DEBUG] list pipdeptree..."
+pip install pipdeptree
+pipdeptree
+
 python -m pylint -f json --disable=R,C,W,E1129 \
     --enable=line-too-long \
     --max-line-length=120 \
     --extension-pkg-whitelist=numpy,nltk \
     --ignored-classes=TensorProto,NodeProto \
-    --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,neural_compressor,engine_py,neural_engine_py,intel_extension_for_transformers.neural_engine_py,neural_compressor.benchmark,intel_extension_for_transformers.lm_eval*,ntel_extension_for_transformers.evaluation.hf_eval* \
+    --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,neural_compressor,engine_py,neural_engine_py,intel_extension_for_transformers.neural_engine_py,neural_compressor.benchmark,intel_extension_for_transformers.transformers.modeling.modeling_causal \
     /intel-extension-for-transformers/intel_extension_for_transformers >${log_dir}/pylint.json
 exit_code=$?
 

.github/workflows/script/launch_benchmark.sh

@@ -34,11 +34,11 @@ if [[ "${ir_path}" == "sparse_.*_ir" ]]; then
 else
     cd ${working_dir_fullpath}
     if [[ "${precision}" == "fp32" ]]; then
-        python -c 'from intel_extension_for_transformers.backends.neural_engine.compile import compile; graph = compile("./model_and_tokenizer/fp32-model.onnx"); graph.save("./ir")'
+        python -c 'from intel_extension_for_transformers.llm.runtime.compile import compile; graph = compile("./model_and_tokenizer/fp32-model.onnx"); graph.save("./ir")'
     elif [[ "${precision}" == "bf16" ]]; then
-        python -c 'from intel_extension_for_transformers.backends.neural_engine.compile import compile; graph = compile("./model_and_tokenizer/bf16-model.onnx"); graph.save("./ir")'
+        python -c 'from intel_extension_for_transformers.llm.runtime.compile import compile; graph = compile("./model_and_tokenizer/bf16-model.onnx"); graph.save("./ir")'
     else
-        python -c 'from intel_extension_for_transformers.backends.neural_engine.compile import compile; graph = compile("./model_and_tokenizer/int8-model.onnx"); graph.save("./ir")'
+        python -c 'from intel_extension_for_transformers.llm.runtime.compile import compile; graph = compile("./model_and_tokenizer/int8-model.onnx"); graph.save("./ir")'
     fi
     echo "ir_path for model ${model} is ${ir_path}"
     cd -

.github/workflows/script/models/cpp_graph_inference.sh

@@ -12,7 +12,7 @@ function main() {
     conda_env="$1"
     model="$2"
     compiler_version="$3"
-    working_dir="${WORKING_DIR}/intel_extension_for_transformers/backends/neural_engine/graph"
+    working_dir="${WORKING_DIR}/intel_extension_for_transformers/llm/runtime/graph"
     # init params
     if [[ "${model}" == "llama-7b-hf" ]]; then
         convert_script="${working_dir}/scripts/convert_llama.py"

.github/workflows/script/unitTest/coverage/.optimize-coveragerc

@@ -5,8 +5,15 @@ branch = True
 omit =
  */**/fake*yaml
  */**/fake.py
- */intel_extension_for_transformers/backends/*
- */intel_extension_for_transformers/optimization/utils/get_throughput.py
+ */intel_extension_for_transformers/llm/amp*
+ */intel_extension_for_transformers/llm/finetuning*
+ */intel_extension_for_transformers/llm/inference*
+ */intel_extension_for_transformers/llm/library*
+ */intel_extension_for_transformers/llm/operator*
+ */intel_extension_for_transformers/llm/runtime*
+ */intel_extension_for_transformers/neural_chat*
+ */intel_extension_for_transformers/transformers/modeling*
+ */intel_extension_for_transformers/transformers/utils/get_throughput.py
 exclude_lines =
  pragma: no cover
  raise NotImplementedError

.github/workflows/script/unitTest/run_unit_test_engine.sh

@@ -9,10 +9,12 @@ function pytest() {
     local coverage_log_dir=$1
     JOB_NAME=unit_test
 
-    cd /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/test/pytest || exit 1
-
+    cd /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/test/pytest
+    if [[ -d /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/test/pytest ]] ; then
+      cd /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/test/pytest
+    fi
     engine_path=$(python -c 'import intel_extension_for_transformers; import os; print(os.path.dirname(intel_extension_for_transformers.__file__))')
-    engine_path="${engine_path}/backends/neural_engine"
+    engine_path="${engine_path}/llm/runtime"
     echo "engine path is ${engine_path}"
     find . -name "test*.py" | sed 's,\.\/,coverage run --source='"${engine_path}"' --append ,g' | sed 's/$/ --verbose/' >run.sh
     coverage erase
@@ -45,7 +47,7 @@ function gtest() {
     pip install cmake
     cmake_path=$(which cmake)
     ln -s ${cmake_path} ${cmake_path}3 || true
-    cd /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine
+    cd /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/
 
     mkdir build && cd build && cmake .. -DNE_WITH_SPARSELIB=ON -DNE_WITH_TESTS=ON -DPYTHON_EXECUTABLE=$(which python) && make -j 2>&1 |
         tee -a ${LOG_DIR}/gtest_cmake_build.log
@@ -57,7 +59,7 @@ function gtest() {
         ut_log_name=${LOG_DIR}/unit_test_gtest.log
     fi
 
-    cd /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/build
+    cd /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/build
     ctest -V -L "engine_test" 2>&1 | tee ${ut_log_name}
     if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] ||
         [ $(grep -c "PASSED" ${ut_log_name}) == 0 ] ||
@@ -83,6 +85,7 @@ function install_itrex_base() {
     git config --global --add safe.directory "*"
     git submodule update --init --recursive
     $BOLD_YELLOW && echo "---------------- pip install binary -------------" && $RESET
+    git clean -xdf
     pip install .
 }
 

.github/workflows/script/unitTest/run_unit_test_kernel.sh

@@ -9,7 +9,7 @@ function gtest() {
     pip install cmake
     cmake_path=$(which cmake)
     ln -s ${cmake_path} ${cmake_path}3 || true
-    cd /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/
+    cd /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/
 
     mkdir build && cd build && cmake .. -DNE_WITH_SPARSELIB=ON -DNE_WITH_TESTS=ON -DPYTHON_EXECUTABLE=$(which python) && make -j 2>&1 |
         tee -a ${LOG_DIR}/gtest_cmake_build.log
@@ -18,7 +18,7 @@ function gtest() {
 
 # -------------------engine test-------------------
 function engine_test() {
-    cd /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/build
+    cd /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/build
 
     if [[ ${test_install_backend} == "true" ]]; then
         local ut_log_name=${LOG_DIR}/unit_test_engine_gtest_backend_only.log
@@ -41,7 +41,7 @@ function engine_test() {
 
 # ------------------kernel test--------------------
 function kernel_test() {
-    cd /intel-extension-for-transformers/intel_extension_for_transformers/backends/neural_engine/build
+    cd /intel-extension-for-transformers/intel_extension_for_transformers/llm/runtime/build
 
     if [[ ${test_install_backend} == "true" ]]; then
         local ut_log_name=${LOG_DIR}/unit_test_kernel_gtest_backend_only.log

.github/workflows/script/unitTest/run_unit_test_optimize.sh

@@ -58,6 +58,7 @@ function install_itrex_base() {
     git config --global --add safe.directory "*"
     git submodule update --init --recursive
     $BOLD_YELLOW && echo "---------------- pip install binary -------------" && $RESET
+    git clean -xdf
     pip install .
 }
 

.github/workflows/sparse_lib_CI.yml

@@ -4,12 +4,12 @@ on:
   pull_request:
     branches: [main]
     paths:
-      - 'intel_extension_for_transformers/backends/neural_engine/kernels/**'
-      - 'intel_extension_for_transformers/backends/neural_engine/test/kernels/benchmark/**'
+      - 'intel_extension_for_transformers/llm/library/kernels/**'
+      - 'intel_extension_for_transformers/llm/runtime/test/kernels/benchmark/**'
       - '.github/workflows/sparse_lib_CI.yml'
       - '.github/workflows/script/SparseLibCI'
-      - '!intel_extension_for_transformers/backends/neural_engine/kernels/docs/**'
-      - '!intel_extension_for_transformers/backends/neural_engine/kernels/README.md'
+      - '!intel_extension_for_transformers/llm/library/kernels/docs/**'
+      - '!intel_extension_for_transformers/llm/library/kernels/README.md'
   workflow_dispatch:
 
 # If there is a new commit, the previous jobs will be canceled

.github/workflows/unit-test-engine.yml

@@ -6,11 +6,11 @@ on:
     paths:
       - ".github/workflows/unit-test-engine.yml"
       - ".github/workflows/script/unitTest/**"
-      - "intel_extension_for_transformers/backends/neural_engine/**"
-      - "!intel_extension_for_transformers/backends/neural_engine/kernels/**"
-      - "!intel_extension_for_transformers/backends/neural_engine/graph/**"
-      - "!intel_extension_for_transformers/backends/neural_engine/third_party/**"
-      - "!intel_extension_for_transformers/backends/neural_engine/docs/**"
+      - "intel_extension_for_transformers/llm/runtime/**"
+      - "!intel_extension_for_transformers/llm/runtime/kernels/**"
+      - "!intel_extension_for_transformers/llm/runtime/graph/**"
+      - "!intel_extension_for_transformers/llm/runtime/third_party/**"
+      - "!intel_extension_for_transformers/llm/runtime/docs/**"
   workflow_dispatch:
 
 # If there is a new commit, the previous jobs will be canceled