From 6a4826409c3eedb71b2b3c752f33b4c5527f7d54 Mon Sep 17 00:00:00 2001 From: Fabrice de Gans Date: Mon, 25 Nov 2024 16:30:34 -0800 Subject: [PATCH] Refactor matrix definitions The various build matrix definitions used to be duplicated across multiple jobs. This changes the matrix definitions to have a single definition for every OS-Arch target, removing duplication. In addition, this change makes it easier to set up a separate arm64 Windows build. The end goal is to have a single matrix definition for every platform and remove many of the inputs to the `swift-toolchain` sub-build. Once the Mac build is complete, it should be possible to merge both Windows and Mac matrix definitions, reducing build complexity. --- .github/workflows/build-toolchain.yml | 444 +++++++++++--------------- .github/workflows/swift-toolchain.yml | 64 ++-- setup-matrix.ps1 | 150 +++++++++ 3 files changed, 368 insertions(+), 290 deletions(-) create mode 100644 setup-matrix.ps1 diff --git a/.github/workflows/build-toolchain.yml b/.github/workflows/build-toolchain.yml index 0655b59e..8b276e3f 100644 --- a/.github/workflows/build-toolchain.yml +++ b/.github/workflows/build-toolchain.yml @@ -81,6 +81,15 @@ on: type: string required: false + build_setup: + description: 'JSON definition of the build setup' + type: string + required: false + default: >- + { + "Windows-amd64": [ "Windows-amd64", "Windows-arm64", "Android" ] + } + secrets: SYMBOL_SERVER_PAT: required: true @@ -155,14 +164,13 @@ jobs: swift_tag: ${{ steps.context.outputs.swift_tag }} windows_build_runner: ${{ steps.context.outputs.windows_build_runner }} windows_compilers_runner: ${{ steps.context.outputs.windows_compilers_runner }} - mac_build_runner: ${{ steps.context.outputs.mac_build_runner }} - windows_host_matrix: ${{ steps.setup-matrix.outputs.windows_host_matrix }} - windows_build_matrix: ${{ steps.setup-matrix.outputs.windows_build_matrix }} - windows_target_matrix: ${{ steps.setup-matrix.outputs.windows_target_matrix }} - darwin_host_matrix: ${{ steps.setup-matrix.outputs.darwin_host_matrix }} - darwin_build_matrix: ${{ steps.setup-matrix.outputs.darwin_build_matrix }} - darwin_target_matrix: ${{ steps.setup-matrix.outputs.darwin_target_matrix }} + build_matrix: ${{ steps.setup-matrix.outputs.build_matrix }} + host_matrix: ${{ steps.setup-matrix.outputs.host_matrix }} + target_matrix: ${{ steps.setup-matrix.outputs.compilers_matrix }} steps: + - name: Checkout + uses: actions/checkout@v4 + - id: context name: Generate Build Context run: | @@ -301,197 +309,184 @@ jobs: - name: Setup matrix id: setup-matrix env: - WINDOWS_HOST_MATRIX: >- + DEFAULT_BUILD_SETUP: >- + { + "Windows-amd64": [ "Windows-amd64", "Windows-arm64", "Android" ] + } + WINDOWS_X64: >- + { + "arch": "amd64", + "cpu" : "x86_64", + "os": "Windows", + "platform": "Windows", + "triple": "x86_64-unknown-windows-msvc", + "triple_no_api_level": "x86_64-unknown-windows-msvc", + "cc": "cl", + "cflags": "${{ steps.context.outputs.WINDOWS_CMAKE_C_FLAGS }}", + "cxx": "cl", + "cxxflags": "${{ steps.context.outputs.WINDOWS_CMAKE_CXX_FLAGS }}", + "swiftflags": "${{ steps.context.outputs.WINDOWS_CMAKE_Swift_FLAGS }}", + "cmake_linker_flags": "-D CMAKE_EXE_LINKER_FLAGS=\"${{ steps.context.outputs.WINDOWS_CMAKE_EXE_LINKER_FLAGS }}\" -D CMAKE_SHARED_LINKER_FLAGS=\"${{ steps.context.outputs.WINDOWS_CMAKE_EXE_LINKER_FLAGS }}\"", + "llvm_flags": "", + "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=AMD64 -D CMAKE_MT=mt" + } + WINDOWS_X86: >- + { + "arch": "x86", + "cpu" : "i686", + "os": "Windows", + "platform": "Windows", + "triple": "i686-unknown-windows-msvc", + "triple_no_api_level": "i686-unknown-windows-msvc", + "cc": "cl", + "cflags": "${{ steps.context.outputs.WINDOWS_CMAKE_C_FLAGS }}", + "cxx": "cl", + "cxxflags": "${{ steps.context.outputs.WINDOWS_CMAKE_CXX_FLAGS }}", + "swiftflags": "${{ steps.context.outputs.WINDOWS_CMAKE_Swift_FLAGS }}", + "cmake_linker_flags": "-D CMAKE_EXE_LINKER_FLAGS=\"${{ steps.context.outputs.WINDOWS_CMAKE_EXE_LINKER_FLAGS }}\" -D CMAKE_SHARED_LINKER_FLAGS=\"${{ steps.context.outputs.WINDOWS_CMAKE_EXE_LINKER_FLAGS }}\"", + "llvm_flags": "", + "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=X86 -D CMAKE_MT=mt" + } + WINDOWS_ARM64: >- { - "include": [ - { - "arch": "amd64", - "compiler_target": "x86_64-unknown-windows-msvc", - "os": "Windows", - "cc": "cl", - "cflags": "${{ steps.context.outputs.WINDOWS_CMAKE_C_FLAGS }}", - "cxx": "cl", - "cxxflags": "${{ steps.context.outputs.WINDOWS_CMAKE_CXX_FLAGS }}", - "swiftflags": "${{ steps.context.outputs.WINDOWS_CMAKE_Swift_FLAGS }}", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=AMD64 -D CMAKE_MT=mt" - }, - { - "arch": "arm64", - "compiler_target": "aarch64-unknown-windows-msvc", - "os": "Windows", - "cc": "cl", - "cflags": "${{ steps.context.outputs.WINDOWS_CMAKE_C_FLAGS }}", - "cxx": "cl", - "cxxflags": "${{ steps.context.outputs.WINDOWS_CMAKE_CXX_FLAGS }}", - "swiftflags": "${{ steps.context.outputs.WINDOWS_CMAKE_Swift_FLAGS }}", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=ARM64 -D CMAKE_MT=mt" - } - ] + "arch": "arm64", + "cpu" : "aarch64", + "os": "Windows", + "platform": "Windows", + "triple": "aarch64-unknown-windows-msvc", + "triple_no_api_level": "aarch64-unknown-windows-msvc", + "cc": "cl", + "cflags": "${{ steps.context.outputs.WINDOWS_CMAKE_C_FLAGS }}", + "cxx": "cl", + "cxxflags": "${{ steps.context.outputs.WINDOWS_CMAKE_CXX_FLAGS }}", + "swiftflags": "${{ steps.context.outputs.WINDOWS_CMAKE_Swift_FLAGS }}", + "cmake_linker_flags": "-D CMAKE_EXE_LINKER_FLAGS=\"${{ steps.context.outputs.WINDOWS_CMAKE_EXE_LINKER_FLAGS }}\" -D CMAKE_SHARED_LINKER_FLAGS=\"${{ steps.context.outputs.WINDOWS_CMAKE_EXE_LINKER_FLAGS }}\"", + "llvm_flags": "", + "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=ARM64 -D CMAKE_MT=mt" } - WINDOWS_BUILD_MATRIX: >- + ANDROID_ARM64: >- { - "include": [ - { - "arch": "amd64", - "os": "Windows", - "cc": "cl", - "cflags": "${{ steps.context.outputs.WINDOWS_CMAKE_C_FLAGS }}", - "cxx": "cl", - "cxxflags": "${{ steps.context.outputs.WINDOWS_CMAKE_CXX_FLAGS }}", - "swiftflags": "${{ steps.context.outputs.WINDOWS_CMAKE_Swift_FLAGS }}", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=AMD64 -D CMAKE_MT=mt" - } - ] + "arch": "arm64", + "cpu": "aarch64", + "platform": "Android", + "triple": "aarch64-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }}", + "triple_no_api_level": "aarch64-unknown-linux-android", + "os": "Android", + "cc": "clang", + "cflags": "${{ steps.context.outputs.ANDROID_CMAKE_C_FLAGS }}", + "cxx": "clang++", + "cxxflags": "${{ steps.context.outputs.ANDROID_CMAKE_CXX_FLAGS }}", + "swiftflags": "-sdk $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/sysroot -Xclang-linker -target -Xclang-linker aarch64-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }} -Xclang-linker --sysroot -Xclang-linker $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/sysroot -Xclang-linker -resource-dir -Xclang-linker $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/lib/clang/17 -L ${GithubWorkspace}/BinaryCache/swift/lib/swift/android -g", + "cmake_linker_flags": "-D CMAKE_EXE_LINKER_FLAGS=\"${{ steps.context.outputs.ANDROID_CMAKE_EXE_LINKER_FLAGS }}\" -D CMAKE_SHARED_LINKER_FLAGS=\"${{ steps.context.outputs.ANDROID_CMAKE_SHARED_LINKER_FLAGS }}\"", + "llvm_flags": "-D CMAKE_SYSTEM_PROCESSOR=aarch64 -D LLVM_HOST_TRIPLE=aarch64-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }}", + "extra_flags": "-D CMAKE_ANDROID_API=${{ steps.context.outputs.ANDROID_API_LEVEL }} -D CMAKE_ANDROID_ARCH_ABI=arm64-v8a" } - WINDOWS_TARGET_MATRIX: >- + ANDROID_ARM32: >- { - "include": [ - { - "arch": "amd64", - "os": "Windows", - "cc": "cl", - "cflags": "${{ steps.context.outputs.WINDOWS_CMAKE_C_FLAGS }}", - "cxx": "cl", - "cxxflags": "${{ steps.context.outputs.WINDOWS_CMAKE_CXX_FLAGS }}", - "swiftflags": "${{ steps.context.outputs.WINDOWS_CMAKE_Swift_FLAGS }}", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=AMD64 -D CMAKE_MT=mt" - }, - { - "arch": "arm64", - "os": "Windows", - "cc": "cl", - "cflags": "${{ steps.context.outputs.WINDOWS_CMAKE_C_FLAGS }}", - "cxx": "cl", - "cxxflags": "${{ steps.context.outputs.WINDOWS_CMAKE_CXX_FLAGS }}", - "swiftflags": "${{ steps.context.outputs.WINDOWS_CMAKE_Swift_FLAGS }}", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=ARM64 -D CMAKE_MT=mt" - }, - { - "arch": "x86", - "os": "Windows", - "cc": "cl", - "cflags": "${{ steps.context.outputs.WINDOWS_CMAKE_C_FLAGS }}", - "cxx": "cl", - "cxxflags": "${{ steps.context.outputs.WINDOWS_CMAKE_CXX_FLAGS }}", - "swiftflags": "", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=X86 -D CMAKE_MT=mt" - }, - { - "arch": "arm64", - "os": "Android", - "cc": "clang", - "cflags": "${{ steps.context.outputs.ANDROID_CMAKE_C_FLAGS }}", - "cxx": "clang++", - "cxxflags": "${{ steps.context.outputs.ANDROID_CMAKE_CXX_FLAGS }}", - "swiftflags": "", - "extra_flags": "-DCMAKE_ANDROID_API=${{ inputs.ANDROID_API_LEVEL }} -DCMAKE_ANDROID_ARCH_ABI=arm64-v8a" - }, - { - "arch": "armv7", - "os": "Android", - "cc": "clang", - "cflags": "${{ steps.context.outputs.ANDROID_CMAKE_C_FLAGS }}", - "cxx": "clang++", - "cxxflags": "${{ steps.context.outputs.ANDROID_CMAKE_CXX_FLAGS }}", - "swiftflags": "", - "extra_flags": "-DCMAKE_ANDROID_API=${{ inputs.ANDROID_API_LEVEL }} -DCMAKE_ANDROID_ARCH_ABI=armeabi-v7a" - }, - { - "arch": "i686", - "os": "Android", - "cc": "clang", - "cflags": "${{ steps.context.outputs.ANDROID_CMAKE_C_FLAGS }}", - "cxx": "clang++", - "cxxflags": "${{ steps.context.outputs.ANDROID_CMAKE_CXX_FLAGS }}", - "swiftflags": "", - "extra_flags": "-DCMAKE_ANDROID_API=${{ inputs.ANDROID_API_LEVEL }} -DCMAKE_ANDROID_ARCH_ABI=x86" - }, - { - "arch": "x86_64", - "os": "Android", - "cc": "clang", - "cflags": "${{ steps.context.outputs.ANDROID_CMAKE_C_FLAGS }}", - "cxx": "clang++", - "cxxflags": "${{ steps.context.outputs.ANDROID_CMAKE_CXX_FLAGS }}", - "swiftflags": "", - "extra_flags": "-DCMAKE_ANDROID_API=${{ inputs.ANDROID_API_LEVEL }} -DCMAKE_ANDROID_ARCH_ABI=x86_64" - } - ] + "arch": "armv7", + "cpu": "armv7", + "platform": "Android", + "triple": "armv7-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }}", + "triple_no_api_level": "armv7-unknown-linux-android", + "os": "Android", + "cc": "clang", + "cflags": "${{ steps.context.outputs.ANDROID_CMAKE_C_FLAGS }}", + "cxx": "clang++", + "cxxflags": "${{ steps.context.outputs.ANDROID_CMAKE_CXX_FLAGS }}", + "swiftflags": "-sdk $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/sysroot -Xclang-linker -target -Xclang-linker armv7a-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }} -Xclang-linker --sysroot -Xclang-linker $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/sysroot -Xclang-linker -resource-dir -Xclang-linker $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/lib/clang/17 -L ${GithubWorkspace}/BinaryCache/swift/lib/swift/android -g", + "cmake_linker_flags": "-D CMAKE_EXE_LINKER_FLAGS=\"${{ steps.context.outputs.ANDROID_CMAKE_EXE_LINKER_FLAGS }}\" -D CMAKE_SHARED_LINKER_FLAGS=\"${{ steps.context.outputs.ANDROID_CMAKE_SHARED_LINKER_FLAGS }}\"", + "llvm_flags": "-D CMAKE_SYSTEM_PROCESSOR=armv7 -D LLVM_HOST_TRIPLE=armv7-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }}", + "extra_flags": "-D CMAKE_ANDROID_API=${{ steps.context.outputs.ANDROID_API_LEVEL }} -D CMAKE_ANDROID_ARCH_ABI=armeabi-v7a" } - DARWIN_HOST_MATRIX: >- + ANDROID_X64: >- { - "include": [ - { - "arch": "x86_64", - "compiler_target": "x86_64-apple-macosx10.15", - "os": "Darwin", - "cc": "clang", - "cflags": "${{ steps.context.outputs.DARWIN_CMAKE_C_FLAGS }}", - "cxx": "clang++", - "cxxflags": "${{ steps.context.outputs.DARWIN_CMAKE_CXX_FLAGS }}", - "swiftflags": "${{ steps.context.outputs.DARWIN_CMAKE_Swift_FLAGS }}", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=x86_64 -D CMAKE_OSX_DEPLOYMENT_TARGET=\"10.15\" -D CMAKE_OSX_ARCHITECTURES=x86_64" - }, - { - "arch": "arm64", - "compiler_target": "arm64-apple-macosx10.15", - "os": "Darwin", - "cc": "clang", - "cflags": "${{ steps.context.outputs.DARWIN_CMAKE_C_FLAGS }}", - "cxx": "clang++", - "cxxflags": "${{ steps.context.outputs.DARWIN_CMAKE_CXX_FLAGS }}", - "swiftflags": "${{ steps.context.outputs.DARWIN_CMAKE_Swift_FLAGS }}", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=arm64 -D CMAKE_OSX_DEPLOYMENT_TARGET=\"10.15\" -D CMAKE_OSX_ARCHITECTURES=arm64" - } - ] + "arch": "x86_64", + "cpu": "x86_64", + "platform": "Android", + "triple": "x86_64-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }}", + "triple_no_api_level": "x86_64-unknown-linux-android", + "os": "Android", + "cc": "clang", + "cflags": "${{ steps.context.outputs.ANDROID_CMAKE_C_FLAGS }}", + "cxx": "clang++", + "cxxflags": "${{ steps.context.outputs.ANDROID_CMAKE_CXX_FLAGS }}", + "swiftflags": "-sdk $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/sysroot -Xclang-linker -target -Xclang-linker i686-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }} -Xclang-linker --sysroot -Xclang-linker $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/sysroot -Xclang-linker -resource-dir -Xclang-linker $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/lib/clang/17 -L ${GithubWorkspace}/BinaryCache/swift/lib/swift/android -g", + "cmake_linker_flags": "-D CMAKE_EXE_LINKER_FLAGS=\"${{ steps.context.outputs.ANDROID_CMAKE_EXE_LINKER_FLAGS }}\" -D CMAKE_SHARED_LINKER_FLAGS=\"${{ steps.context.outputs.ANDROID_CMAKE_SHARED_LINKER_FLAGS }}\"", + "llvm_flags": "-D CMAKE_SYSTEM_PROCESSOR=i686 -D LLVM_HOST_TRIPLE=i686-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }}", + "extra_flags": "-D CMAKE_ANDROID_API=${{ steps.context.outputs.ANDROID_API_LEVEL }} -D CMAKE_ANDROID_ARCH_ABI=x86_64" } - DARWIN_BUILD_MATRIX: >- + ANDROID_X86: >- { - "include": [ - { - "arch": "arm64", - "os": "Darwin", - "cc": "clang", - "cflags": "${{ steps.context.outputs.DARWIN_CMAKE_C_FLAGS }}", - "cxx": "clang++", - "cxxflags": "${{ steps.context.outputs.DARWIN_CMAKE_CXX_FLAGS }}" - "swiftflags": "${{ steps.context.outputs.DARWIN_CMAKE_Swift_FLAGS }}", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=arm64 -D CMAKE_OSX_DEPLOYMENT_TARGET=\"10.15\" -D CMAKE_OSX_ARCHITECTURES=arm64" - } - ] + "arch": "i686", + "cpu": "i686", + "platform": "Android", + "triple": "i686-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }}", + "triple_no_api_level": "i686-unknown-linux-android", + "os": "Android", + "cc": "clang", + "cflags": "${{ steps.context.outputs.ANDROID_CMAKE_C_FLAGS }}", + "cxx": "clang++", + "cxxflags": "${{ steps.context.outputs.ANDROID_CMAKE_CXX_FLAGS }}", + "swiftflags": "-sdk $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/sysroot -Xclang-linker -target -Xclang-linker x86_64-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }} -Xclang-linker --sysroot -Xclang-linker $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/sysroot -Xclang-linker -resource-dir -Xclang-linker $NDKPATH/toolchains/llvm/prebuilt/windows-x86_64/lib/clang/17 -L ${GithubWorkspace}/BinaryCache/swift/lib/swift/android -g", + "cmake_linker_flags": "-D CMAKE_EXE_LINKER_FLAGS=\"${{ steps.context.outputs.ANDROID_CMAKE_EXE_LINKER_FLAGS }}\" -D CMAKE_SHARED_LINKER_FLAGS=\"${{ steps.context.outputs.ANDROID_CMAKE_SHARED_LINKER_FLAGS }}\"", + "llvm_flags": "-D CMAKE_SYSTEM_PROCESSOR=x86_64 -D LLVM_HOST_TRIPLE=x86_64-unknown-linux-android${{ steps.context.outputs.ANDROID_API_LEVEL }}", + "extra_flags": "-D CMAKE_ANDROID_API=${{ steps.context.outputs.ANDROID_API_LEVEL }} -D CMAKE_ANDROID_ARCH_ABI=x86" } - DARWIN_TARGET_MATRIX: >- + DARWIN_ARM64: >- { - "include": [ - { - "arch": "x86_64", - "os": "Darwin", - "cc": "clang", - "cflags": "${{ steps.context.outputs.DARWIN_CMAKE_C_FLAGS }}", - "cxx": "clang++", - "cxxflags": "${{ steps.context.outputs.DARWIN_CMAKE_CXX_FLAGS }}", - "swiftflags": "${{ steps.context.outputs.DARWIN_CMAKE_Swift_FLAGS }}", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=x86_64 -D CMAKE_OSX_DEPLOYMENT_TARGET=\"10.15\" -D CMAKE_OSX_ARCHITECTURES=x86_64" - }, - { - "arch": "arm64", - "os": "Darwin", - "cc": "clang", - "cflags": "${{ steps.context.outputs.DARWIN_CMAKE_C_FLAGS }}", - "cxx": "clang++", - "cxxflags": "${{ steps.context.outputs.DARWIN_CMAKE_CXX_FLAGS }}", - "swiftflags": "${{ steps.context.outputs.DARWIN_CMAKE_Swift_FLAGS }}", - "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=arm64 -D CMAKE_OSX_DEPLOYMENT_TARGET=\"10.15\" -D CMAKE_OSX_ARCHITECTURES=arm64" - } - ] + "arch": "arm64", + "cpu": "arm64", + "platform": "Darwin", + "triple": "arm64-apple-macos15.0", + "tipple_no_api_level": "arm64-apple-macos15.0", + "os": "Darwin", + "cc": "clang", + "cflags": "${{ steps.context.outputs.DARWIN_CMAKE_C_FLAGS }}", + "cxx": "clang++", + "cxxflags": "${{ steps.context.outputs.DARWIN_CMAKE_CXX_FLAGS }}", + "swiftflags": "${{ steps.context.outputs.DARWIN_CMAKE_Swift_FLAGS }}", + "cmake_linker_flags": "", + "llvm_flags": "", + "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=arm64 -D CMAKE_OSX_DEPLOYMENT_TARGET=\"15.0\" -D CMAKE_OSX_ARCHITECTURES=arm64" } + DARWIN_X64: >- + { + "arch": "x86_64", + "cpu": "x86_64", + "platform": "Darwin", + "triple": "x86_64-apple-macos15.0", + "tipple_no_api_level": "x86_64-apple-macos15.0", + "os": "Darwin", + "cc": "clang", + "cflags": "${{ steps.context.outputs.DARWIN_CMAKE_C_FLAGS }}", + "cxx": "clang++", + "cxxflags": "${{ steps.context.outputs.DARWIN_CMAKE_CXX_FLAGS }}", + "swiftflags": "${{ steps.context.outputs.DARWIN_CMAKE_Swift_FLAGS }}", + "cmake_linker_flags": "", + "llvm_flags": "", + "extra_flags": "-D CMAKE_SYSTEM_PROCESSOR=x86_64 -D CMAKE_OSX_DEPLOYMENT_TARGET=\"15.0\" -D CMAKE_OSX_ARCHITECTURES=x86_64" + } + BUILD_WINDOWS_X64: >- + { + "build_os": "Windows", + "build_arch": "amd64", + "runner": "${{ steps.context.outputs.windows_build_runner }}", + "compilers_runner": "${{ steps.context.outputs.windows_compilers_runner }}" + } + BUILD_DARWIN: >- + { + "build_os": "Darwin", + "build_arch": "arm64", + "runner": "${{ steps.context.outputs.mac_build_runner }}", + "compilers_runner": "${{ steps.context.outputs.mac_build_runner }}" + } + shell: pwsh run: | - echo "windows_host_matrix=$(jq -r -c '.' <<< ${WINDOWS_HOST_MATRIX})" >> ${GITHUB_OUTPUT} - echo "windows_build_matrix=$(jq -r -c '.' <<< ${WINDOWS_BUILD_MATRIX})" >> ${GITHUB_OUTPUT} - echo "windows_target_matrix=$(jq -r -c '.' <<< ${WINDOWS_TARGET_MATRIX})" >> ${GITHUB_OUTPUT} - echo "darwin_host_matrix=$(jq -r -c '.' <<< ${DARWIN_HOST_MATRIX})" >> ${GITHUB_OUTPUT} - echo "darwin_build_matrix=$(jq -r -c '.' <<< ${DARWIN_BUILD_MATRIX})" >> ${GITHUB_OUTPUT} - echo "darwin_target_matrix=$(jq -r -c '.' <<< ${DARWIN_TARGET_MATRIX})" >> ${GITHUB_OUTPUT} + $buildSetup = "${{ inputs.build_setup }}" + if ($buildSetup -eq "") { + $buildSetup = $env:DEFAULT_BUILD_SETUP + } + ./setup-matrix.ps1 -BuildSetup $buildSetup windows-build: needs: [context] @@ -500,9 +495,9 @@ jobs: with: build_os: Windows build_arch: amd64 - build_matrix: ${{ needs.context.outputs.windows_build_matrix }} - host_matrix: ${{ needs.context.outputs.windows_host_matrix }} - target_matrix: ${{ needs.context.outputs.windows_target_matrix }} + build_matrix: ${{ needs.context.outputs.build_matrix }} + host_matrix: ${{ needs.context.outputs.host_matrix }} + target_matrix: ${{ needs.context.outputs.target_matrix }} curl_revision: ${{ needs.context.outputs.curl_revision }} curl_version: ${{ needs.context.outputs.curl_version }} ds2_revision: ${{ needs.context.outputs.ds2_revision }} @@ -564,79 +559,6 @@ jobs: CERTIFICATE: ${{ secrets.CERTIFICATE }} PASSPHRASE: ${{ secrets.PASSPHRASE }} - mac-build: - # TODO: Enable the mac build. - if: false - needs: [context] - name: macOS Swift Toolchains Build - uses: ./.github/workflows/swift-toolchain.yml - with: - build_os: Darwin - build_arch: aarch64 - build_matrix: ${{ needs.context.outputs.darwin_build_matrix }} - host_matrix: ${{ needs.context.outputs.darwin_host_matrix }} - target_matrix: ${{ needs.context.outputs.darwin_target_matrix }} - curl_revision: ${{ needs.context.outputs.curl_revision }} - curl_version: ${{ needs.context.outputs.curl_version }} - ds2_revision: ${{ needs.context.outputs.ds2_revision }} - indexstore_db_revision: ${{ needs.context.outputs.indexstore_db_revision }} - libxml2_revision: ${{ needs.context.outputs.libxml2_revision }} - libxml2_version: ${{ needs.context.outputs.libxml2_version }} - llvm_project_revision: ${{ needs.context.outputs.llvm_project_revision }} - mimalloc_revision: ${{ needs.context.outputs.mimalloc_revision }} - sourcekit_lsp_revision: ${{ needs.context.outputs.sourcekit_lsp_revision }} - swift_argument_parser_revision: ${{ needs.context.outputs.swift_argument_parser_revision }} - swift_asn1_revision: ${{ needs.context.outputs.swift_asn1_revision }} - swift_atomics_revision: ${{ needs.context.outputs.swift_atomics_revision }} - swift_certificates_revision: ${{ needs.context.outputs.swift_certificates_revision }} - swift_cmark_revision: ${{ needs.context.outputs.swift_cmark_revision }} - swift_cmark_version: ${{ needs.context.outputs.swift_cmark_version }} - swift_collections_revision: ${{ needs.context.outputs.swift_collections_revision }} - swift_corelibs_foundation_revision: ${{ needs.context.outputs.swift_corelibs_foundation_revision }} - swift_corelibs_libdispatch_revision: ${{ needs.context.outputs.swift_corelibs_libdispatch_revision }} - swift_corelibs_xctest_revision: ${{ needs.context.outputs.swift_corelibs_xctest_revision }} - swift_crypto_revision: ${{ needs.context.outputs.swift_crypto_revision }} - swift_driver_revision: ${{ needs.context.outputs.swift_driver_revision }} - swift_experimental_string_processing_revision: ${{ needs.context.outputs.swift_experimental_string_processing_revision }} - swift_format_revision: ${{ needs.context.outputs.swift_format_revision }} - swift_foundation_revision: ${{ needs.context.outputs.swift_foundation_revision }} - swift_foundation_icu_revision: ${{ needs.context.outputs.swift_foundation_icu_revision }} - swift_installer_scripts_revision: ${{ needs.context.outputs.swift_installer_scripts_revision }} - swift_llbuild_revision: ${{ needs.context.outputs.swift_llbuild_revision }} - swift_markdown_revision: ${{ needs.context.outputs.swift_markdown_revision }} - swift_package_manager_revision: ${{ needs.context.outputs.swift_package_manager_revision }} - swift_revision: ${{ needs.context.outputs.swift_revision }} - swift_syntax_revision: ${{ needs.context.outputs.swift_syntax_revision }} - swift_system_revision: ${{ needs.context.outputs.swift_system_revision }} - swift_testing_revision: ${{ needs.context.outputs.swift_testing_revision }} - swift_toolchain_sqlite_revision: ${{ needs.context.outputs.swift_toolchain_sqlite_revision }} - swift_toolchain_sqlite_version: ${{ needs.context.outputs.swift_toolchain_sqlite_version }} - swift_tools_support_core_revision: ${{ needs.context.outputs.swift_tools_support_core_revision }} - yams_revision: ${{ needs.context.outputs.yams_revision }} - zlib_revision: ${{ needs.context.outputs.zlib_revision }} - zlib_version: ${{ needs.context.outputs.zlib_version }} - ANDROID_API_LEVEL: ${{ needs.context.outputs.ANDROID_API_LEVEL }} - WINDOWS_CMAKE_C_FLAGS: ${{ needs.context.outputs.WINDOWS_CMAKE_C_FLAGS }} - WINDOWS_CMAKE_CXX_FLAGS: ${{ needs.context.outputs.WINDOWS_CMAKE_CXX_FLAGS }} - WINDOWS_CMAKE_EXE_LINKER_FLAGS: ${{ needs.context.outputs.WINDOWS_CMAKE_EXE_LINKER_FLAGS }} - WINDOWS_CMAKE_SHARED_LINKER_FLAGS: ${{ needs.context.outputs.WINDOWS_CMAKE_SHARED_LINKER_FLAGS }} - ANDROID_CMAKE_C_FLAGS: ${{ needs.context.outputs.ANDROID_CMAKE_C_FLAGS }} - ANDROID_CMAKE_CXX_FLAGS: ${{ needs.context.outputs.ANDROID_CMAKE_CXX_FLAGS }} - ANDROID_CMAKE_EXE_LINKER_FLAGS: ${{ needs.context.outputs.ANDROID_CMAKE_EXE_LINKER_FLAGS }} - ANDROID_CMAKE_SHARED_LINKER_FLAGS: ${{ needs.context.outputs.ANDROID_CMAKE_SHARED_LINKER_FLAGS }} - ANDROID_NDK_VERSION: ${{ needs.context.outputs.ANDROID_NDK_VERSION }} - CMAKE_Swift_FLAGS: ${{ needs.context.outputs.DARWIN_CMAKE_Swift_FLAGS }} - debug_info: ${{ fromJSON(needs.context.outputs.debug_info) }} - signed: ${{ fromJSON(needs.context.outputs.signed) }} - swift_version: ${{ needs.context.outputs.swift_version }} - swift_tag: ${{ needs.context.outputs.swift_tag }} - default_build_runner: ${{ needs.context.outputs.mac_build_runner }} - compilers_build_runner: ${{ needs.context.outputs.mac_build_runner }} - secrets: - SYMBOL_SERVER_PAT: ${{ secrets.SYMBOL_SERVER_PAT }} - CERTIFICATE: ${{ secrets.CERTIFICATE }} - PASSPHRASE: ${{ secrets.PASSPHRASE }} - snapshot: runs-on: ubuntu-latest needs: [context, windows-build] diff --git a/.github/workflows/swift-toolchain.yml b/.github/workflows/swift-toolchain.yml index 397fbda5..df05433e 100644 --- a/.github/workflows/swift-toolchain.yml +++ b/.github/workflows/swift-toolchain.yml @@ -264,11 +264,12 @@ defaults: jobs: sqlite: - runs-on: ${{ inputs.default_build_runner }} + runs-on: ${{ matrix.runner }} strategy: fail-fast: false - matrix: ${{ fromJSON(inputs.host_matrix) }} + matrix: + include: ${{ fromJSON(inputs.host_matrix) }} name: ${{ matrix.os }} ${{ matrix.arch }} SQLite3 @@ -287,12 +288,12 @@ jobs: - uses: compnerd/gha-setup-vsdevenv@f1ba60d553a3216ce1b89abe0201213536bc7557 # main with: - host_arch: ${{ inputs.build_arch }} + host_arch: ${{ matrix.build_arch }} components: 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64;Microsoft.VisualStudio.Component.VC.Tools.ARM64' arch: ${{ matrix.arch }} - uses: seanmiddleditch/gha-setup-ninja@96bed6edff20d1dd61ecff9b75cc519d516e6401 # master - if: inputs.build_os == 'Darwin' + if: matrix.build_os == 'Darwin' - name: Compute workspace hash id: workspace_hash @@ -492,11 +493,12 @@ jobs: path: ${{ github.workspace }}/BinaryCache/Library/Developer cmark_gfm: - runs-on: ${{ inputs.default_build_runner }} + runs-on: ${{ matrix.runner }} strategy: fail-fast: false - matrix: ${{ fromJSON(inputs.host_matrix) }} + matrix: + include: ${{ fromJSON(inputs.host_matrix) }} name: ${{ matrix.os }} ${{ matrix.arch }} CMark GFM @@ -510,12 +512,12 @@ jobs: - uses: compnerd/gha-setup-vsdevenv@f1ba60d553a3216ce1b89abe0201213536bc7557 # main with: - host_arch: ${{ inputs.build_arch }} + host_arch: ${{ matrix.build_arch }} components: 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64;Microsoft.VisualStudio.Component.VC.Tools.ARM64' arch: ${{ matrix.arch }} - uses: seanmiddleditch/gha-setup-ninja@96bed6edff20d1dd61ecff9b75cc519d516e6401 # master - if: inputs.build_os == 'Darwin' + if: matrix.build_os == 'Darwin' - name: Compute workspace hash id: workspace_hash @@ -564,11 +566,12 @@ jobs: build_tools: needs: [cmark_gfm] - runs-on: ${{ inputs.default_build_runner }} + runs-on: ${{ matrix.runner }} strategy: fail-fast: false - matrix: ${{ fromJSON(inputs.build_matrix) }} + matrix: + include: ${{ fromJSON(inputs.build_matrix) }} name: ${{ matrix.os }} ${{ matrix.arch }} Compiler Build Tools @@ -594,7 +597,7 @@ jobs: - uses: compnerd/gha-setup-vsdevenv@f1ba60d553a3216ce1b89abe0201213536bc7557 # main - uses: seanmiddleditch/gha-setup-ninja@96bed6edff20d1dd61ecff9b75cc519d516e6401 # master - if: inputs.build_os == 'Darwin' + if: matrix.build_os == 'Darwin' - name: Compute workspace hash id: workspace_hash @@ -702,11 +705,12 @@ jobs: # TODO: Build this on Windows too. if: inputs.build_os == 'Darwin' needs: [sqlite] - runs-on: ${{ inputs.default_build_runner }} + runs-on: ${{ matrix.runner }} strategy: fail-fast: false - matrix: ${{ fromJSON(inputs.host_matrix) }} + matrix: + include: ${{ fromJSON(inputs.host_matrix) }} name: ${{ matrix.os }} ${{ matrix.arch }} Early Swift Driver @@ -749,15 +753,15 @@ jobs: - uses: compnerd/gha-setup-vsdevenv@f1ba60d553a3216ce1b89abe0201213536bc7557 # main with: - host_arch: ${{ inputs.build_arch }} + host_arch: ${{ matrix.build_arch }} components: 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64;Microsoft.VisualStudio.Component.VC.Tools.ARM64' arch: ${{ matrix.arch }} - uses: seanmiddleditch/gha-setup-ninja@96bed6edff20d1dd61ecff9b75cc519d516e6401 # master - if: inputs.build_os == 'Darwin' + if: matrix.build_os == 'Darwin' - name: Install Swift Toolchain (Mac) - if: inputs.build_os == 'Darwin' + if: matrix.build_os == 'Darwin' uses: compnerd/gha-setup-swift@b6c5fc1ed5b5439ada8e7661985acb09ad8c3ba2 # main with: branch: swift-6.0.1-release @@ -832,7 +836,6 @@ jobs: - arch: 'amd64' cpu: 'x86_64' triple: 'x86_64-unknown-windows-msvc' - - arch: 'arm64' cpu: 'aarch64' triple: 'aarch64-unknown-windows-msvc' @@ -1093,11 +1096,12 @@ jobs: searchPattern: '**/*.exe' zlib: - runs-on: ${{ inputs.default_build_runner }} + runs-on: ${{ matrix.runner }} strategy: fail-fast: false - matrix: ${{ fromJSON(inputs.target_matrix) }} + matrix: + include: ${{ fromJSON(inputs.target_matrix) }} name: ${{ matrix.os }} ${{ matrix.arch }} zlib @@ -1116,7 +1120,7 @@ jobs: arch: ${{ matrix.arch }} - uses: seanmiddleditch/gha-setup-ninja@96bed6edff20d1dd61ecff9b75cc519d516e6401 # master - if: inputs.build_os == 'Darwin' + if: matrix.build_os == 'Darwin' - name: Compute workspace hash id: workspace_hash @@ -1144,7 +1148,7 @@ jobs: - name: Configure zlib run: | $NDKPATH = "${{ steps.setup-ndk.outputs.ndk-path }}" - if ( "${{ inputs.build_os }}" -eq "Windows" ) { + if ( "${{ matrix.build_os }}" -eq "Windows" ) { $NDKPATH = cygpath -m $NDKPATH } cmake -B ${{ github.workspace }}/BinaryCache/zlib-${{ inputs.zlib_version }} ` @@ -1176,11 +1180,12 @@ jobs: curl: needs: [zlib] - runs-on: ${{ inputs.default_build_runner }} + runs-on: ${{ matrix.runner }} strategy: fail-fast: false - matrix: ${{ fromJSON(inputs.target_matrix) }} + matrix: + include: ${{ fromJSON(inputs.target_matrix) }} name: ${{ matrix.os }} ${{ matrix.arch }} curl @@ -1204,7 +1209,7 @@ jobs: arch: ${{ matrix.arch }} - uses: seanmiddleditch/gha-setup-ninja@96bed6edff20d1dd61ecff9b75cc519d516e6401 # master - if: inputs.build_os == 'Darwin' + if: matrix.build_os == 'Darwin' - name: Compute workspace hash id: workspace_hash @@ -1232,7 +1237,7 @@ jobs: - name: Configure curl run: | $NDKPATH = "${{ steps.setup-ndk.outputs.ndk-path }}" - if ( "${{ inputs.build_os }}" -eq "Windows" ) { + if ( "${{ matrix.build_os }}" -eq "Windows" ) { $NDKPATH = cygpath -m $NDKPATH } cmake -B ${{ github.workspace }}/BinaryCache/curl-${{ inputs.curl_version }} ` @@ -1340,11 +1345,12 @@ jobs: path: ${{ github.workspace }}/BuildRoot/Library/curl-${{ inputs.curl_version }}/usr libxml2: - runs-on: ${{ inputs.default_build_runner }} + runs-on: ${{ matrix.runner }} strategy: fail-fast: false - matrix: ${{ fromJSON(inputs.target_matrix) }} + matrix: + include: ${{ fromJSON(inputs.target_matrix) }} name: ${{ matrix.os }} ${{ matrix.arch }} libxml2 @@ -1363,7 +1369,7 @@ jobs: arch: ${{ matrix.arch }} - uses: seanmiddleditch/gha-setup-ninja@96bed6edff20d1dd61ecff9b75cc519d516e6401 # master - if: inputs.build_os == 'Darwin' + if: matrix.build_os == 'Darwin' - name: Compute workspace hash id: workspace_hash @@ -1392,7 +1398,7 @@ jobs: - name: Configure libxml2 run: | $NDKPATH = "${{ steps.setup-ndk.outputs.ndk-path }}" - if ( "${{ inputs.build_os }}" -eq "Windows" ) { + if ( "${{ matrix.build_os }}" -eq "Windows" ) { $NDKPATH = cygpath -m $NDKPATH } cmake -B ${{ github.workspace }}/BinaryCache/libxml2-${{ inputs.libxml2_version }} ` diff --git a/setup-matrix.ps1 b/setup-matrix.ps1 new file mode 100644 index 00000000..7331e13b --- /dev/null +++ b/setup-matrix.ps1 @@ -0,0 +1,150 @@ +param ( + [string] $BuildSetup +) + +# Get the matrix information from the environment. +$windowsX64 = (Get-Item -Path Env:WINDOWS_X64).Value | ConvertFrom-Json -AsHashtable +$windowsX86 = (Get-Item -Path Env:WINDOWS_X86).Value | ConvertFrom-Json -AsHashtable +$windowsArm64 = (Get-Item -Path Env:WINDOWS_ARM64).Value | ConvertFrom-Json -AsHashtable +$darwinArm64 = (Get-Item -Path Env:DARWIN_ARM64).Value | ConvertFrom-Json -AsHashtable +$darwinX64 = (Get-Item -Path Env:DARWIN_X64).Value | ConvertFrom-Json -AsHashtable +$androidArm64 = (Get-Item -Path Env:ANDROID_ARM64).Value | ConvertFrom-Json -AsHashtable +$androidArm32 = (Get-Item -Path Env:ANDROID_ARM32).Value | ConvertFrom-Json -AsHashtable +$androidX64 = (Get-Item -Path Env:ANDROID_X64).Value | ConvertFrom-Json -AsHashtable +$androidX86 = (Get-Item -Path Env:ANDROID_X86).Value | ConvertFrom-Json -AsHashtable +$buildWindowsX64 = (Get-Item -Path Env:BUILD_WINDOWS_X64).Value | ConvertFrom-Json -AsHashtable +$buildDarwin = (Get-Item -Path Env:BUILD_DARWIN).Value | ConvertFrom-Json -AsHashtable + +# Name mapping for targets and builders. +$targetNameToVarName = @{ + 'Darwin' = @("darwinX64", "darwinArm64") + 'Windows-amd64' = @("windowsX64", "windowsX86") + 'Windows-arm64' = @("windowsArm64") + 'Android' = @("androidArm64", "androidArm32", "androidX64", "androidX86") +} +$builderNameToVarName = @{ + 'Windows-amd64' = 'buildWindowsX64' + 'Darwin' = 'buildDarwin' +} + + +# Parse the input JSON. +try { + $parsedJson = $BuildSetup | ConvertFrom-Json -AsHashtable +} catch { + Write-Error "Invalid JSON input" + exit 1 +} + +# Extract all builder lists. +if ($parsedJson.ContainsKey('Windows-amd64')) { + $windowsAmd64List = $parsedJson['Windows-amd64'] + if ($windowsAmd64List -isnot [System.Collections.IList]) { + Write-Output "::error::'Windows-amd64' value is not a list" + exit 1 + } +} else { + $windowsAmd64List = @() +} +if ($parsedJson.ContainsKey('Darwin')) { + $darwinList = $parsedJson['Darwin'] + if ($darwinList -isnot [System.Collections.IList]) { + Write-Output "::error::'Darwin' value is not a list" + exit 1 + } +} else { + $darwinList = @() +} + +# Exit early if no builders are found. +if ($windowsAmd64List.Count -eq 0 -and $darwinList.Count -eq 0) { + Write-Output "::error::No builders found in the input" + exit 1 +} + +# Build the target to builder mapping. +$targetToBuilder = @{} +foreach ($target in $windowsAmd64List) { + if (-not $targetNameToVarName.ContainsKey($target)) { + Write-Output "::error::Invalid target found: $target" + exit 1 + } + if ($targetToBuilder.ContainsKey($target)) { + Write-Output "::error::Duplicate builders found for target $target" + exit 1 + } + $targetToBuilder[$target] = 'Windows-amd64' +} + +foreach ($target in $darwinList) { + if (-not $targetNameToVarName.ContainsKey($target)) { + Write-Output "::error::Invalid target found: $target" + exit 1 + } + if ($targetToBuilder.ContainsKey($target)) { + Write-Output "::error::Duplicate builders found for target $target" + exit 1 + } + $targetToBuilder[$target] = 'Darwin' +} + +# Populate the builder information. +foreach ($kvp in $targetToBuilder.GetEnumerator()) { + $target = $kvp.Key + $builder = $kvp.Value + + # Get the builder information for this target. + $builderVarName = $builderNameToVarName[$builder] + $builderVar = Get-Variable -Name $builderVarName -ValueOnly + + # Append the builder information for this target. + foreach($targetVarName in $targetNameToVarName[$target]) { + $targetVar = Get-Variable -Name $targetVarName -ValueOnly + $targetVar += $builderVar + } +} + +# Build the host matrix. +$hostMatrix = @() +if ($windowsAmd64List.Count -gt 0) { + foreach ($hostName in @('windowsX64', 'windowsArm64')) { + $hostDict = Get-Variable -Name $hostName -ValueOnly + $hostMatrix += $hostDict + } +} +if ($darwinList.Count -gt 0) { + foreach ($hostName in @('darwinX64', 'darwinArm64')) { + $hostDict = Get-Variable -Name $hostName -ValueOnly + $hostMatrix += $hostDict + } +} + +# Build the build matrix. +$buildMatrix = @() +if ($windowsAmd64List.Count -gt 0) { + $buildMatrix += $windowsX64 +} +if ($darwinList.Count -gt 0) { + $buildMatrix += $darwinArm64 +} + +# Build the target matrix. +$targetMatrix = @() +foreach ($target in $windowsAmd64List + $darwinList) { + foreach ($targetVarName in $targetNameToVarName[$target]) { + $targetDict = Get-Variable -Name $targetVarName -ValueOnly + $targetMatrix += $targetDict + } +} + +# Output the matrix. +Write-Host "Host matrix:" +Write-Host $($hostMatrix | ConvertTo-Json -AsArray) +Write-Host "Build matrix:" +Write-Host $($buildMatrix | ConvertTo-Json -AsArray) +Write-Host "Target matrix:" +Write-Host $($targetMatrix | ConvertTo-Json -AsArray) + +Write-Output host_matrix=$($hostMatrix | ConvertTo-Json -AsArray -Compress )| Out-File -FilePath ${env:GITHUB_OUTPUT} -Encoding utf8 -Append +Write-Output build_matrix=$($buildMatrix | ConvertTo-Json -AsArray -Compress ) | Out-File -FilePath ${env:GITHUB_OUTPUT} -Encoding utf8 -Append +Write-Output target_matrix=$($targetMatrix | ConvertTo-Json -AsArray -Compress ) | Out-File -FilePath ${env:GITHUB_OUTPUT} -Encoding utf8 -Append