From 906c1a5a085f8d1cfd4524b275616c1beb0eb58d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 16 Jan 2024 21:37:39 -0800 Subject: [PATCH 1/6] [ci] [R-package] more closely mimic CRAN compiler configs in clang-devel jobs --- .appveyor.yml | 43 --- .ci/install-clang-devel.sh | 8 + .github/workflows/cuda.yml | 115 -------- .github/workflows/python_package.yml | 113 -------- .vsts-ci.yml | 400 --------------------------- 5 files changed, 8 insertions(+), 671 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 .github/workflows/cuda.yml delete mode 100644 .github/workflows/python_package.yml delete mode 100644 .vsts-ci.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 58a536d17a98..000000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: 4.2.0.99.{build} - -image: Visual Studio 2015 -platform: x64 -configuration: # a trick to construct a build matrix with multiple Python versions - - '3.8' - -# only build pull requests and -# commits to 'master' or any branch starting with 'release' -branches: - only: - - master - - /^release/ - -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python - -clone_depth: 5 - -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - set CONDA_ENV="test-env" - - ps: | - $env:MINICONDA = "C:\Miniconda3-x64" - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() - -build: false - -test_script: - - conda config --remove channels defaults - - conda config --add channels nodefaults - - conda config --add channels conda-forge - - conda config --set channel_priority strict - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.ci/install-clang-devel.sh b/.ci/install-clang-devel.sh index a175617852c7..d3fbe400a351 100755 --- a/.ci/install-clang-devel.sh +++ b/.ci/install-clang-devel.sh @@ -63,6 +63,14 @@ CXX11 += -stdlib=libc++ CXX14 += -stdlib=libc++ CXX17 += -stdlib=libc++ CXX20 += -stdlib=libc++ + +CFLAGS = -O3 -Wall -pedantic -Wp,-D_FORTIFY_SOURCE=3 +CPPFLAGS = -isystem /usr/local/clang/include +CXXFLAGS = -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX11FLAGS = \$(CXXFLAGS) +CXX14FLAGS = \$(CXXFLAGS) +CXX17FLAGS = \$(CXXFLAGS) +CXX120FLAGS = \$(CXXFLAGS) EOF echo "" diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml deleted file mode 100644 index 3120fb1fae9f..000000000000 --- a/.github/workflows/cuda.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: CUDA Version - -on: - push: - branches: - - master - pull_request: - branches: - - master - - release/* - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - github_actions: 'true' - os_name: linux - conda_env: test-env - -jobs: - test: - name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - method: wheel - compiler: gcc - python_version: "3.11" - cuda_version: "11.8.0" - task: cuda - - method: source - compiler: gcc - python_version: "3.9" - cuda_version: "12.2.0" - task: cuda - - method: pip - compiler: clang - python_version: "3.10" - cuda_version: "11.8.0" - task: cuda - steps: - - name: Setup or update software on host machine - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - git \ - gnupg-agent \ - lsb-release \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: Remove old folder with repository - run: sudo rm -rf $GITHUB_WORKSPACE - - name: Checkout repository - uses: actions/checkout@v1 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < '$(Build.ArtifactStagingDirectory)/commit.txt' - displayName: 'Add commit hash to artifacts archive' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: Linux_latest -########################################### - variables: - COMPILER: clang-17 - DEBIAN_FRONTEND: 'noninteractive' - IN_UBUNTU_BASE_CONTAINER: 'true' - OS_NAME: 'linux' - SETUP_CONDA: 'true' - pool: sh-mariner - container: ubuntu-latest - strategy: - matrix: - regular: - TASK: regular - sdist: - TASK: sdist - bdist: - TASK: bdist - PYTHON_VERSION: '3.9' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - mpi_pip: - TASK: mpi - METHOD: pip - PYTHON_VERSION: '3.10' - mpi_wheel: - TASK: mpi - METHOD: wheel - PYTHON_VERSION: '3.8' - gpu_source: - TASK: gpu - METHOD: source - PYTHON_VERSION: '3.10' - gpu_pip: - TASK: gpu - METHOD: pip - PYTHON_VERSION: '3.9' - gpu_wheel: - TASK: gpu - METHOD: wheel - PYTHON_VERSION: '3.8' - cpp_tests: - TASK: cpp-tests - METHOD: with-sanitizers - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - CONDA=$HOME/miniforge - echo "##vso[task.setvariable variable=CONDA]$CONDA" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 - - script: | - /tmp/docker exec -t -u 0 ci-container \ - sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: 'Install sudo' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test -########################################### -- job: QEMU_multiarch -########################################### - variables: - COMPILER: gcc - OS_NAME: 'linux' - PRODUCES_ARTIFACTS: 'true' - pool: - vmImage: ubuntu-22.04 - timeoutInMinutes: 180 - strategy: - matrix: - bdist: - TASK: bdist - ARCH: aarch64 - steps: - - script: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - binfmt-support \ - qemu \ - qemu-user \ - qemu-user-static - displayName: 'Install QEMU' - - script: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - displayName: 'Enable Docker multi-architecture support' - - script: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < Date: Tue, 16 Jan 2024 22:00:42 -0800 Subject: [PATCH 2/6] fix assignment --- .ci/install-clang-devel.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/install-clang-devel.sh b/.ci/install-clang-devel.sh index d3fbe400a351..84a3177e8b29 100755 --- a/.ci/install-clang-devel.sh +++ b/.ci/install-clang-devel.sh @@ -64,13 +64,13 @@ CXX14 += -stdlib=libc++ CXX17 += -stdlib=libc++ CXX20 += -stdlib=libc++ -CFLAGS = -O3 -Wall -pedantic -Wp,-D_FORTIFY_SOURCE=3 +CFLAGS := -O3 -Wall -pedantic -Wp,-D_FORTIFY_SOURCE=3 CPPFLAGS = -isystem /usr/local/clang/include -CXXFLAGS = -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 -CXX11FLAGS = \$(CXXFLAGS) -CXX14FLAGS = \$(CXXFLAGS) -CXX17FLAGS = \$(CXXFLAGS) -CXX120FLAGS = \$(CXXFLAGS) +CXXFLAGS := -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX11FLAGS := \$(CXXFLAGS) +CXX14FLAGS := \$(CXXFLAGS) +CXX17FLAGS := \$(CXXFLAGS) +CXX20FLAGS := \$(CXXFLAGS) EOF echo "" From f4cc502590f729e3b397ebed9452a5b1e20cbd9b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 16 Jan 2024 22:21:28 -0800 Subject: [PATCH 3/6] fix Makevars --- .ci/install-clang-devel.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/install-clang-devel.sh b/.ci/install-clang-devel.sh index 84a3177e8b29..3ab31609ad6a 100755 --- a/.ci/install-clang-devel.sh +++ b/.ci/install-clang-devel.sh @@ -64,13 +64,13 @@ CXX14 += -stdlib=libc++ CXX17 += -stdlib=libc++ CXX20 += -stdlib=libc++ -CFLAGS := -O3 -Wall -pedantic -Wp,-D_FORTIFY_SOURCE=3 -CPPFLAGS = -isystem /usr/local/clang/include -CXXFLAGS := -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 -CXX11FLAGS := \$(CXXFLAGS) -CXX14FLAGS := \$(CXXFLAGS) -CXX17FLAGS := \$(CXXFLAGS) -CXX20FLAGS := \$(CXXFLAGS) +CFLAGS += -O3 -Wall -pedantic -Wp,-D_FORTIFY_SOURCE=3 +CPPFLAGS += -isystem /usr/local/clang/include +CXXFLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX11FLAGS += \$(CXXFLAGS) +CXX14FLAGS += \$(CXXFLAGS) +CXX17FLAGS += \$(CXXFLAGS) +CXX20FLAGS += \$(CXXFLAGS) EOF echo "" From b963798a7ad60b5c845cb94805ae916d4a471328 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 16 Jan 2024 22:37:49 -0800 Subject: [PATCH 4/6] try repetition --- .ci/install-clang-devel.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/install-clang-devel.sh b/.ci/install-clang-devel.sh index 3ab31609ad6a..1f39f5eace82 100755 --- a/.ci/install-clang-devel.sh +++ b/.ci/install-clang-devel.sh @@ -64,13 +64,13 @@ CXX14 += -stdlib=libc++ CXX17 += -stdlib=libc++ CXX20 += -stdlib=libc++ -CFLAGS += -O3 -Wall -pedantic -Wp,-D_FORTIFY_SOURCE=3 -CPPFLAGS += -isystem /usr/local/clang/include -CXXFLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 -CXX11FLAGS += \$(CXXFLAGS) -CXX14FLAGS += \$(CXXFLAGS) -CXX17FLAGS += \$(CXXFLAGS) -CXX20FLAGS += \$(CXXFLAGS) +CFLAGS += -O3 -Wall -pedantic -Wp,-D_FORTIFY_SOURCE=3 +CPPFLAGS += -isystem /usr/local/clang/include +CXXFLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX11FLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX14FLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX17FLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX20FLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 EOF echo "" From 2159ba280910fd05fe511abbbdb8f962ae2e1e87 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 16 Jan 2024 22:39:49 -0800 Subject: [PATCH 5/6] simpler assignment --- .ci/install-clang-devel.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/install-clang-devel.sh b/.ci/install-clang-devel.sh index 1f39f5eace82..ed2256504f74 100755 --- a/.ci/install-clang-devel.sh +++ b/.ci/install-clang-devel.sh @@ -64,13 +64,13 @@ CXX14 += -stdlib=libc++ CXX17 += -stdlib=libc++ CXX20 += -stdlib=libc++ -CFLAGS += -O3 -Wall -pedantic -Wp,-D_FORTIFY_SOURCE=3 -CPPFLAGS += -isystem /usr/local/clang/include -CXXFLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 -CXX11FLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 -CXX14FLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 -CXX17FLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 -CXX20FLAGS += -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CFLAGS = -O3 -Wall -pedantic -Wp,-D_FORTIFY_SOURCE=3 +CPPFLAGS = -isystem /usr/local/clang/include +CXXFLAGS = -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX11FLAGS = \$CXXFLAGS +CXX14FLAGS = \$CXXFLAGS +CXX17FLAGS = \$CXXFLAGS +CXX20FLAGS = \$CXXFLAGS EOF echo "" From 706690977118f67a5556fb7d508e9242a92f41dd Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 16 Jan 2024 22:56:36 -0800 Subject: [PATCH 6/6] just copy everything --- .ci/install-clang-devel.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/install-clang-devel.sh b/.ci/install-clang-devel.sh index ed2256504f74..1722cb3169f3 100755 --- a/.ci/install-clang-devel.sh +++ b/.ci/install-clang-devel.sh @@ -67,10 +67,10 @@ CXX20 += -stdlib=libc++ CFLAGS = -O3 -Wall -pedantic -Wp,-D_FORTIFY_SOURCE=3 CPPFLAGS = -isystem /usr/local/clang/include CXXFLAGS = -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 -CXX11FLAGS = \$CXXFLAGS -CXX14FLAGS = \$CXXFLAGS -CXX17FLAGS = \$CXXFLAGS -CXX20FLAGS = \$CXXFLAGS +CXX11FLAGS = -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX14FLAGS = -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX17FLAGS = -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 +CXX20FLAGS = -O3 -Wall -pedantic -frtti -Wp,-D_FORTIFY_SOURCE=3 EOF echo ""