One step install (#358)

* This change is used to help users build and install DIPU from source code and pip in one step. * simplify the job of ci_nv_script.sh * fix a bug with the ENV DIPU_COMPILED_WITH_CXX11 * fix a bug with direct CMake and remove some unused import in setup.py * rebase to new main and add kineto in CMake * add DIOPI as an external project * add "import torch" in setup.py for setting related CMake args. Also pipeclean for "pip install ." * make some code improvement and precise dependency for ninja. * update build script for other device type * use two whitespace for indentation and add correct diopi build targets for other devices. * Split the CMake files to let the third_party/CMakeLists.txt build the third_party lib. * update kineto * code improvement based on git review conmments --------- Co-authored-by: caikun <[email protected]>
DeepLink-org · Nov 6, 2023 · 012f5fe · 012f5fe
1 parent 22457dd
commit 012f5fe
Show file tree

Hide file tree

Showing 11 changed files with 341 additions and 115 deletions.
diff --git a/dipu/CMakeLists.txt b/dipu/CMakeLists.txt
@@ -1,6 +1,7 @@
 cmake_minimum_required(VERSION 3.10)
 project(TorchDIPU)
 
+set(CMAKE_BUILD_TYPE "Release" CACHE STRING "default build type is Release")
 option(TESTS "Whether to build unit tests" OFF)
 option(LIBS "Whether to build dipu lib, default on" ON)
 
@@ -21,6 +22,40 @@ list(APPEND DEVICE_DROPLET "DROPLET" "droplet")
 
 execute_process(COMMAND git rev-parse --short HEAD
                 OUTPUT_VARIABLE DIPU_GIT_HASH)
+
+set(UsedVendor "")
+set(DIOPI_IMPL_OPT "")
+if (${DEVICE} IN_LIST DEVICE_CUDA)
+  set(USE_CUDA ON)
+  set(UsedVendor cuda)
+  set(DIOPI_IMPL_OPT "torch")
+elseif (${DEVICE} IN_LIST DEVICE_CAMB)
+  set(USE_CAMB ON)
+  set(UsedVendor camb)
+  set(DIOPI_IMPL_OPT "camb")
+elseif (${DEVICE} IN_LIST DEVICE_ASCEND)
+  set(USE_ASCEND ON)
+  set(UsedVendor ascend)
+  set(DIOPI_IMPL_OPT "ascend")
+elseif (${DEVICE} IN_LIST DEVICE_TOPSRIDER)
+  set(USE_TOPSRIDER ON)
+  set(UsedVendor topsrider)
+  set(DIOPI_IMPL_OPT "tops")
+elseif (${DEVICE} IN_LIST DEVICE_SUPA)
+  set(USE_SUPA ON)
+  set(UsedVendor supa)
+  set(DIOPI_IMPL_OPT "")
+  #SUPA DEVICE DOES NOT NEED TO BUILD DIOPI, so set the target to "" to control the workflow.
+elseif (${DEVICE} IN_LIST DEVICE_DROPLET)
+  set(USE_DROPLET ON)
+  set(UsedVendor droplet)
+  set(DIOPI_IMPL_OPT "droplet")
+else()
+  message(FATAL_ERROR "No implementation module is compiled, cmake requires option -DDEVICE=CAMB or CUDA or ASCEND or SUPA")
+endif()
+
+
+
 string(REGEX MATCH "[a-z0-9_]+" DIPU_GIT_HASH ${DIPU_GIT_HASH})
 execute_process(COMMAND sh -c "git status --porcelain | egrep '^(M| M)' | wc -l"
                 OUTPUT_VARIABLE DIPU_MODIFY_LEN)
@@ -37,25 +72,32 @@ execute_process(
     "grep -Po 'diopi[a-zA-Z0-9]+(?=\\()' ${CMAKE_CURRENT_SOURCE_DIR}/scripts/autogen_diopi_wrapper/diopi_functions.yaml | sort -uf > ${CMAKE_CURRENT_SOURCE_DIR}/SupportedDiopiFunctions.txt"
 )
 
-execute_process(
-  COMMAND
-    sh -x -c
-    "python -c 'import torch, builtins; print(next(item[-4:-2] for item in dir(builtins)      \
-        if \"__pybind11_internals_v4_gcc_libstdcpp_cxxabi10\" in item))'"
-  OUTPUT_VARIABLE DIPU_ABI_V)
-add_definitions(-fabi-version=${DIPU_ABI_V})
-message(STATUS "DIPU_ABI_V: ${DIPU_ABI_V}")
+if(NOT DEFINED DIPU_ABI_V)
+  execute_process(
+    COMMAND
+      sh -x -c
+      "python -c 'import torch, builtins; print(next(item[-4:-2] for item in dir(builtins)      \
+          if \"__pybind11_internals_v4_gcc_libstdcpp_cxxabi10\" in item))'"
+    OUTPUT_VARIABLE DIPU_ABI_V)
+endif()
 
-execute_process(
-  COMMAND
-    sh -x -c
-    "python -c 'import torch;print(1 if torch.compiled_with_cxx11_abi() else 0)'"
-  OUTPUT_VARIABLE DIPU_COMPILED_WITH_CXX11_ABI)
+if(NOT DEFINED DIPU_COMPILED_WITH_CXX11_ABI)    
+  execute_process(
+    COMMAND
+      sh -x -c
+      "python -c 'import torch;print(1 if torch.compiled_with_cxx11_abi() else 0)'"
+    OUTPUT_VARIABLE DIPU_COMPILED_WITH_CXX11_ABI)
+endif()
+
 if(DIPU_COMPILED_WITH_CXX11_ABI GREATER 0)
   set(DIPU_COMPILED_WITH_CXX11_ABI 1)
 else()
   set(DIPU_COMPILED_WITH_CXX11_ABI 0)
 endif()
+
+add_definitions(-fabi-version=${DIPU_ABI_V})
+message(STATUS "DIPU_ABI_V: ${DIPU_ABI_V}")
+
 set(CMAKE_CXX_FLAGS
     "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=${DIPU_COMPILED_WITH_CXX11_ABI}"
 )
@@ -67,43 +109,19 @@ execute_process(
   OUTPUT_VARIABLE PYTHON_INCLUDE_DIR)
 message(STATUS "PYTHON_INCLUDE_DIR: " ${PYTHON_INCLUDE_DIR})
 
-execute_process(
-  COMMAND sh -c "dirname $(python -c 'import torch;print(torch.__path__[0])')"
-  OUTPUT_VARIABLE PYTORCH_DIR)
-string(REPLACE "\n" "" PYTORCH_DIR "${PYTORCH_DIR}")
-string(REPLACE "\r" "" PYTORCH_DIR "${PYTORCH_DIR}")
+if(NOT DEFINED PYTORCH_DIR)
+  execute_process(
+    COMMAND sh -c "dirname $(python -c 'import torch;print(torch.__path__[0])')"
+    OUTPUT_VARIABLE PYTORCH_DIR
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+endif()
+
 message(STATUS "PYTORCH_DIR: " ${PYTORCH_DIR})
 
 # config
 include(cmake/BaseFuncions.cmake)
 _set_cpp_flags()
 
-set(UsedVendor "")
-if(${DEVICE} IN_LIST DEVICE_CUDA)
-  set(USE_CUDA ON)
-  set(UsedVendor cuda)
-elseif(${DEVICE} IN_LIST DEVICE_CAMB)
-  set(USE_CAMB ON)
-  set(UsedVendor camb)
-elseif(${DEVICE} IN_LIST DEVICE_ASCEND)
-  set(USE_ASCEND ON)
-  set(UsedVendor ascend)
-elseif(${DEVICE} IN_LIST DEVICE_TOPSRIDER)
-  set(USE_TOPSRIDER ON)
-  set(UsedVendor topsrider)
-elseif(${DEVICE} IN_LIST DEVICE_SUPA)
-  set(USE_SUPA ON)
-  set(UsedVendor supa)
-elseif(${DEVICE} IN_LIST DEVICE_DROPLET)
-  set(USE_DROPLET ON)
-  set(UsedVendor droplet)
-else()
-  message(
-    FATAL_ERROR
-      "No implementation module is compiled, cmake requires option -DDEVICE=CAMB or CUDA or ASCEND or SUPA"
-  )
-endif()
-
 if(ENABLE_COVERAGE)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage")
@@ -130,34 +148,7 @@ list(APPEND TORCH_INCLUDE_DIRS ${PYTORCH_DIR}/torch/include/
 include_directories(SYSTEM ${TORCH_INCLUDE_DIRS})
 message(STATUS "Torch TORCH_INCLUDE_DIRS: ${TORCH_INCLUDE_DIRS}")
 
-# diopi
-set(WITH_DIOPI
-    "${PROJECT_SOURCE_DIR}/third_party/DIOPI"
-    CACHE STRING "DIOPI project root path")
-# TODO(lljbash): use cache variable instead of env
-if(EXISTS $ENV{DIOPI_PATH})
-  set(DIOPI_PROTO_PATH $ENV{DIOPI_PATH})
-else()
-  set(DIOPI_PROTO_PATH "${WITH_DIOPI}/proto")
-endif()
-if(EXISTS $ENV{DIOPI_ROOT})
-  set(DIOPI_IMPL_LIB_PATH $ENV{DIOPI_ROOT})
-else()
-  set(DIOPI_IMPL_LIB_PATH "${WITH_DIOPI}/impl/lib")
-endif()
-message(STATUS "DIOPI_PROTO_PATH: ${DIOPI_PROTO_PATH}")
-message(STATUS "DIOPI_IMPL_LIB_PATH: ${DIOPI_IMPL_LIB_PATH}")
-add_library(diopi_impl SHARED IMPORTED)
-set_target_properties(
-  diopi_impl PROPERTIES IMPORTED_LOCATION
-                        "${DIOPI_IMPL_LIB_PATH}/libdiopi_impl.so")
-target_include_directories(diopi_impl INTERFACE "${DIOPI_PROTO_PATH}/include")
-target_compile_definitions(diopi_impl INTERFACE DIOPI_ATTR_WEAK)
-
-include_directories(${PROJECT_SOURCE_DIR}/third_party/kineto/libkineto/third_party/fmt/include)
-include_directories(${PROJECT_SOURCE_DIR}/third_party/kineto/libkineto/include)
-include_directories(${PROJECT_SOURCE_DIR}/third_party/kineto/libkineto/src)
-add_definitions(-DUSE_KINETO)
+add_subdirectory(third_party)
 
 if(LIBS)
   add_subdirectory(torch_dipu/csrc_dipu)

diff --git a/dipu/pyproject.toml b/dipu/pyproject.toml
@@ -0,0 +1,9 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "scikit-build>=0.13",
+    "cmake>=3.1",
+    "ninja",
+    "pyyaml"
+]
+build-backend = "setuptools.build_meta"
diff --git a/dipu/scripts/ci/ascend/ci_ascend_script.sh b/dipu/scripts/ci/ascend/ci_ascend_script.sh
@@ -16,19 +16,32 @@ function config_dipu_ascend_cmake() {
     cd ../
 }
 
+function config_all_ascend_cmake() {
+    mkdir -p build && cd ./build && rm -rf ./*
+    cmake ../  -DCMAKE_BUILD_TYPE=Debug \
+        -DDEVICE=ascend \
+        -DWITH_DIOPI=INTERNAL
+    cd ../
+}
+
 function build_dipu_lib() {
     echo "building dipu_lib:$(pwd)"
     echo  "DIOPI_ROOT:${DIOPI_ROOT}"
     config_dipu_ascend_cmake 2>&1 | tee ./build1.log
     cd build && make -j8  2>&1 | tee ./build1.log &&  cd ..
 }
 
+function build_all() {
+    echo "building dipu_lib:$(pwd)"
+    echo  "DIOPI_ROOT:${DIOPI_ROOT}"
+    config_all_ascend_cmake 2>&1 | tee ./build1.log
+    cd build && make -j8  2>&1 | tee ./build1.log &&  cd ..
+}
+
 case $1 in
     build_dipu)
         (
-            bash scripts/ci/ci_build_third_party.sh
-            build_diopi_lib
-            build_dipu_lib
+            build_all
         ) \
         || exit -1;;
     *)

diff --git a/dipu/scripts/ci/camb/ci_camb_script.sh b/dipu/scripts/ci/camb/ci_camb_script.sh
@@ -11,6 +11,14 @@ function config_dipu_camb_cmake() {
     cd ../
 }
 
+function config_all_camb_cmake() {
+    mkdir -p build && cd ./build && rm -rf ./*
+    cmake ../  -DCMAKE_BUILD_TYPE=Release \
+        -DDEVICE=camb  \
+        -DENABLE_COVERAGE=${USE_COVERAGE} \
+        -DWITH_DIOPI=INTERNAL
+    cd ../
+}
 
 function build_diopi_lib() {
     cd third_party/DIOPI/impl
@@ -26,18 +34,22 @@ function build_dipu_lib() {
     cd build && make -j8  2>&1 | tee ./build.log &&  cd ..
 }
 
+function build_all() {
+    echo "building dipu_lib:$(pwd)"
+    echo  "DIOPI_ROOT:${DIOPI_ROOT}"
+    config_all_camb_cmake 2>&1 | tee ./cmake_camb.log
+    cd build && make -j8  2>&1 | tee ./build.log &&  cd ..
+}
+
 case $1 in
     build_dipu)
         (
-            bash scripts/ci/ci_build_third_party.sh
-            build_diopi_lib
-            build_dipu_lib
+            build_all
         ) || exit -1;;
     build_diopi)
         build_diopi_lib || exit -1;;
     build_dipu_only)
         (
-            bash scripts/ci/ci_build_third_party.sh
             build_dipu_lib
         ) || exit -1;;
     *)

diff --git a/dipu/scripts/ci/droplet/ci_droplet_script.sh b/dipu/scripts/ci/droplet/ci_droplet_script.sh
@@ -16,6 +16,19 @@ function config_dipu_droplet_cmake() {
     cd ../
 }
 
+function config_all_droplet_cmake() {
+    mkdir -p build && cd ./build && rm -rf ./*
+    echo "config_dipu_nv_cmake PYTORCH_DIR: ${PYTORCH_DIR}"
+    echo "config_dipu_nv_cmake PYTHON_INCLUDE_DIR: ${PYTHON_INCLUDE_DIR}"
+    cmake ../  -DCMAKE_BUILD_TYPE=Debug \
+     -DDEVICE=${DIPU_DEVICE} -DPYTORCH_DIR=${PYTORCH_DIR} \
+     -DWITH_DIOPI=INTERNAL \
+     -DPYTHON_INCLUDE_DIR=${PYTHON_INCLUDE_DIR}
+      # -DCMAKE_C_FLAGS_DEBUG="-g -O0" \
+      # -DCMAKE_CXX_FLAGS_DEBUG="-g -O0"
+    cd ../
+}
+
 function build_diopi_lib() {
     echo "开始编译DIOPI..."
     cd third_party/DIOPI/
@@ -39,6 +52,16 @@ function build_dipu_lib() {
     cd build && make -j8  2>&1 | tee ./build.log &&  cd ..
 }
 
+function build_all() {
+    echo "building dipu_lib:$(pwd)"
+    echo  "DIOPI_ROOT:${DIOPI_ROOT}"
+    echo  "PYTORCH_DIR:${PYTORCH_DIR}"
+    echo  "PYTHON_INCLUDE_DIR:${PYTHON_INCLUDE_DIR}"
+    export LIBRARY_PATH=$DIOPI_ROOT:$LIBRARY_PATH;
+    config_all_droplet_cmake 2>&1 | tee ./cmake_droplet.log
+    cd build && make -j8  2>&1 | tee ./build.log &&  cd ..
+}
+
 case $1 in
     build_diopi)
         (
@@ -47,14 +70,11 @@ case $1 in
         || exit -1;;
     build_dipu)
         (
-            bash scripts/ci/ci_build_third_party.sh
-            build_diopi_lib
-            build_dipu_lib
+            build_all
         ) \
         || exit -1;;
     build_dipu_only)
         (
-            bash scripts/ci/ci_build_third_party.sh
             build_dipu_lib
         ) \
         || exit -1;;

diff --git a/dipu/scripts/ci/nv/ci_nv_script.sh b/dipu/scripts/ci/nv/ci_nv_script.sh
@@ -2,7 +2,6 @@
 set -e
 echo "pwd: $(pwd)"
 
-
 function config_dipu_nv_cmake() {
     # export NCCL_ROOT="you nccl path should exist"
 
@@ -13,15 +12,15 @@ function config_dipu_nv_cmake() {
     cd ../
 }
 
-function build_diopi_lib() {
-    cd third_party/DIOPI/
-    cd impl
-    which cmake
-    sh scripts/build_impl.sh clean
-    sh scripts/build_impl.sh torch || exit -1
-
-    cd ../../..
-    unset Torch_DIR
+function config_all_nv_cmake() {
+    # export NCCL_ROOT="you nccl path should exist"
+
+    mkdir -p build && cd ./build && rm -rf ./*
+    cmake ../  -DCMAKE_BUILD_TYPE=Release \
+        -DDEVICE=cuda \
+        -DENABLE_COVERAGE=${USE_COVERAGE} \
+        -DWITH_DIOPI=INTERNAL
+    cd ../
 }
 
 function build_dipu_lib() {
@@ -32,17 +31,21 @@ function build_dipu_lib() {
     cd build && make -j8  2>&1 | tee ./build.log &&  cd ..
 }
 
+function build_all(){
+    echo "building dipu_lib:$(pwd)"
+    echo  "DIOPI_ROOT:${DIOPI_ROOT}"
+    export DIOPI_BUILD_TESTRT=1
+    config_all_nv_cmake 2>&1 | tee ./cmake_nv.log
+    cd build && make -j8  2>&1 | tee ./build.log &&  cd ..
+}
 case $1 in
     build_dipu)
         (
-            bash scripts/ci/ci_build_third_party.sh
-            build_diopi_lib
-            build_dipu_lib
+            build_all
         ) \
         || exit -1;;
     build_dipu_only)
         (
-            bash scripts/ci/ci_build_third_party.sh
             build_dipu_lib
         ) \
         || exit -1;;