From 294a7c503c5f304062d3b1b1c6d5d30f8efbd486 Mon Sep 17 00:00:00 2001 From: Adrian Stanea Date: Wed, 6 Dec 2023 16:01:34 +0200 Subject: [PATCH] cleanup code - remove 32b windows build - Not supported in manylixnux_2_28 - Python bindings for 3.12 were tested and work fine in Linux. - Enable all azure-pipelines.yml jobs - NOTE: should artifacts be published when pushing code to staging/ci? - easier to test CI output --- .github/scripts/install_deps.sh | 42 +--- .github/workflows/build_wheels.yml | 8 +- CI/windows/install_deps.ps1 | 2 - azure-pipelines.yml | 339 ++++++++++++++--------------- 4 files changed, 178 insertions(+), 213 deletions(-) diff --git a/.github/scripts/install_deps.sh b/.github/scripts/install_deps.sh index f710cffc..43d36407 100755 --- a/.github/scripts/install_deps.sh +++ b/.github/scripts/install_deps.sh @@ -4,47 +4,16 @@ LIBIIO_VERSION=libiio-v0 PACKAGE_DIR=${1-build} echo $PACKAGE_DIR -echo "Calling update - SKIPPED" -# apt-get -y update yum -y update +echo "## Installing dependencies" +yum install -y bzip2 gzip rpm rpm-build git wget tar \ + libxml2-devel libusbx-devel libusb-devel doxygen libaio-devel avahi-devel avahi-tools graphviz swig flex bison \ + python3 python3-devel python3-setuptools python3-pip -echo "Installing dependencies" -# apt-get install -y git wget tar graphviz libavahi-common-dev libavahi-client-dev libaio-dev libusb-1.0-0-dev libxml2-dev rpm tar bzip2 gzip flex bison git swig python3 python3-dev python3-setuptools python3-pip python3-all libserialport-dev -yum -y groupinstall 'Development Tools' -yum -y install yum-utils -yum config-manager --set-enabled powertools # not available on manylinux2014 -yum -y install libxml2-devel libusbx-devel libusb-devel doxygen libaio-devel avahi-devel bzip2 gzip rpm rpm-build swig -# libusbx-devel libusb-devel # REPLACE ** libusb1-devel ** with this on centOS7 - -yum install -y git wget tar graphviz avahi-devel avahi-tools libaio-devel libusb1-devel libxml2-devel rpm tar bzip2 gzip flex bison git swig python3 python3-devel python3-setuptools python3-pip rpm-build - -# On CentOS 8, avahi-devel & doxygen are in this repo; enable it -# yum -y install yum-utils -# yum config-manager --set-enabled powertools -# yum -y install libxml2-devel libusb1-devel doxygen libaio-devel avahi-devel bzip2 gzip rpm rpm-build swig - - -# THE below deps work on centOS7 without USB backend -# yum install -y git wget tar graphviz avahi-devel avahi-tools libaio-devel libusbx-devel libusb1-devel libxml2-devel bzip2 gzip flex bison git swig python3 python3-devel python3-setuptools python3-pip # libserialport-dev - -# yum config-manager --set-enabled powertools -# yum install -y git wget tar graphviz avahi-devel avahi-tools libaio-devel libusb1-devel libxml2-devel rpm tar bzip2 gzip flex bison git swig python3 python3-devel python3-setuptools python3-pip rpm-build - -# pip3 install --upgrade pip # Needed to reproduce in local image -echo "Installing cmake with pip" python3 -m pip install cmake cmake --version -# Install libserialport => not available from yum`` -# yum install -y libtool autoconf -# git clone https://github.com/martinling/libserialport.git -# cd libserialport -# ./autogen.sh -# ./configure -# make -# make install - #Install libiio echo "## Building libiio $LIBIIO_VERSION" git clone -b $LIBIIO_VERSION --single-branch --depth 1 https://github.com/analogdevicesinc/libiio.git libiio @@ -78,8 +47,7 @@ make cat setup.py make install -echo "## Content from build folder" -ls +# Repair wheel step fails if it does not find the .so file echo "## Content from src folder" ls ./src echo "creating links to .so" diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 0ac51a07..cf9a53e2 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -32,16 +32,16 @@ jobs: cibuildwheel --output-dir ${{ github.workspace }}/wheelhouse ${{ github.workspace }}/build env: CIBW_DEBUG_KEEP_CONTAINER=: TRUE - CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 - CIBW_MANYLINUX_I686_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 # Skip building PyPy wheels on all platforms # Skip musllinux builds - CIBW_SKIP: "pp* *-musllinux*" + # Skip i686 builds: 32bit arch not supported in manylinux_2_28 + CIBW_SKIP: "pp* *-musllinux* *i686" CIBW_BUILD_VERBOSITY: 3 CIBW_BEFORE_ALL: > cd {project} && ./.github/scripts/install_deps.sh {package} - CIBW_BUILD: "cp37-* cp38-* cp39-* cp310-* cp311-*" + CIBW_BUILD: "cp37-* cp38-* cp39-* cp310-* cp311-* cp312-*" CIBW_ENVIRONMENT: COMPILE_BINDINGS="True" - uses: actions/upload-artifact@v3 with: diff --git a/CI/windows/install_deps.ps1 b/CI/windows/install_deps.ps1 index 3618774e..31d96473 100644 --- a/CI/windows/install_deps.ps1 +++ b/CI/windows/install_deps.ps1 @@ -124,8 +124,6 @@ function Install-Swig { Set-Location $DIR_PATH [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - # Potential alternative but still breaks - # Invoke-WebRequest -UserAgent "Wget" -Uri "https://sourceforge.net/projects/swig/files/swigwin/swigwin-4.0.0/swigwin-4.0.0.zip/download" -OutFile "swigwin-4.0.0.zip" $crnt_attempt=0 $max_attemps=10 do { diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7b29dcc4..256cf540 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -81,179 +81,178 @@ jobs: targetPath: '$(Build.ArtifactStagingDirectory)' artifactName: '$(artifactName)' -# - job: macOSBuilds -# workspace: -# clean: all -# strategy: -# matrix: -# macOS_11: -# poolName: 'Azure Pipelines' -# vmImage: 'macOS-11' -# agentName: 'Azure Pipelines 4' -# artifactName: 'macOS-11' -# macOS_12: -# poolName: 'Azure Pipelines' -# vmImage: 'macOS-12' -# agentName: 'Azure Pipelines 3' -# artifactName: 'macOS-12' -# macOS_13_x64: -# poolName: 'Azure Pipelines' -# vmImage: 'macOS-13' -# agentName: 'Azure Pipelines 2' -# artifactName: 'macOS-13-x64' -# # TODO: discuss with DevOps team how to setup ARM agent -# # macOS_13_arm64: -# # poolName: 'Default' -# # vmImage: -# # agentName: 'macOS_arm64' -# # artifactName: 'macOS-13-arm64' -# pool: -# name: $(poolName) -# vmImage: $(vmImage) -# demands: -# - agent.name -equals $(agentName) -# steps: -# - checkout: self -# fetchDepth: 1 -# clean: true -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: '3.11' -# addToPath: true -# - script: ./CI/macOS/install_deps -# displayName: "Install Dependencies" -# condition: ne(variables['agentName'],'macOS_arm64') -# - script: ./CI/macOS/make_macOS -# displayName: "Build" -# - ${{ each pyVersion in parameters.pythonVersions }}: -# - task: UsePythonVersion@0 -# displayName: Use Python ${{ pyVersion }} -# inputs: -# versionSpec: '${{ pyVersion }}' -# - script: | -# python3 --version -# python3 -m pip install wheel twine build virtualenv -# cd '$(Agent.BuildDirectory)/s/build/' -# mkdir -p wheelhouse -# export COMPILE_BINDINGS=True -# python3 -m build -# export COMPILE_BINDINGS= -# cp dist/*.whl wheelhouse -# displayName: Wheel Python ${{ pyVersion }} -# - task: CopyFiles@2 -# inputs: -# sourceFolder: '$(Agent.BuildDirectory)/s/build/' -# contents: '$(Agent.BuildDirectory)/s/build/?(*.pkg)' -# targetFolder: '$(Build.ArtifactStagingDirectory)' -# - task: CopyFiles@2 -# inputs: -# sourceFolder: '$(Agent.BuildDirectory)/s/build/' -# contents: '$(Agent.BuildDirectory)/s/build/wheelhouse/?(*.whl)' -# targetFolder: '$(Build.ArtifactStagingDirectory)' -# - task: CopyFiles@2 -# inputs: -# sourceFolder: '$(Agent.BuildDirectory)/s/build_tar/' -# contents: '$(Agent.BuildDirectory)/s/build_tar/?(*.tar.gz)' -# targetFolder: '$(Build.ArtifactStagingDirectory)' -# - task: PublishPipelineArtifact@1 -# # condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) -# condition: succeeded() -# inputs: -# targetPath: '$(Build.ArtifactStagingDirectory)' -# artifactName: '$(artifactName)' -# - script: | -# python -m pip install --upgrade pip twine -# cd '$(Agent.BuildDirectory)/s/build/wheelhouse/' -# python -m twine upload --repository "testpypi" *.whl -# condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) -# displayName: "Upload to PyPI" -# env: -# TWINE_USERNAME: '__token__' -# TWINE_PASSWORD: $(TWINE_PASSWORD) +- job: macOSBuilds + workspace: + clean: all + strategy: + matrix: + macOS_11: + poolName: 'Azure Pipelines' + vmImage: 'macOS-11' + agentName: 'Azure Pipelines 4' + artifactName: 'macOS-11' + macOS_12: + poolName: 'Azure Pipelines' + vmImage: 'macOS-12' + agentName: 'Azure Pipelines 3' + artifactName: 'macOS-12' + macOS_13_x64: + poolName: 'Azure Pipelines' + vmImage: 'macOS-13' + agentName: 'Azure Pipelines 2' + artifactName: 'macOS-13-x64' + # TODO: discuss with DevOps team how to setup ARM agent + # macOS_13_arm64: + # poolName: 'Default' + # vmImage: + # agentName: 'macOS_arm64' + # artifactName: 'macOS-13-arm64' + pool: + name: $(poolName) + vmImage: $(vmImage) + demands: + - agent.name -equals $(agentName) + steps: + - checkout: self + fetchDepth: 1 + clean: true + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.11' + addToPath: true + - script: ./CI/macOS/install_deps + displayName: "Install Dependencies" + condition: ne(variables['agentName'],'macOS_arm64') + - script: ./CI/macOS/make_macOS + displayName: "Build" + - ${{ each pyVersion in parameters.pythonVersions }}: + - task: UsePythonVersion@0 + displayName: Use Python ${{ pyVersion }} + inputs: + versionSpec: '${{ pyVersion }}' + - script: | + python3 --version + python3 -m pip install wheel twine build virtualenv + cd '$(Agent.BuildDirectory)/s/build/' + mkdir -p wheelhouse + export COMPILE_BINDINGS=True + python3 -m build + export COMPILE_BINDINGS= + cp dist/*.whl wheelhouse + displayName: Wheel Python ${{ pyVersion }} + - task: CopyFiles@2 + inputs: + sourceFolder: '$(Agent.BuildDirectory)/s/build/' + contents: '$(Agent.BuildDirectory)/s/build/?(*.pkg)' + targetFolder: '$(Build.ArtifactStagingDirectory)' + - task: CopyFiles@2 + inputs: + sourceFolder: '$(Agent.BuildDirectory)/s/build/' + contents: '$(Agent.BuildDirectory)/s/build/wheelhouse/?(*.whl)' + targetFolder: '$(Build.ArtifactStagingDirectory)' + - task: CopyFiles@2 + inputs: + sourceFolder: '$(Agent.BuildDirectory)/s/build_tar/' + contents: '$(Agent.BuildDirectory)/s/build_tar/?(*.tar.gz)' + targetFolder: '$(Build.ArtifactStagingDirectory)' + - task: PublishPipelineArtifact@1 + # condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + condition: succeeded() + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)' + artifactName: '$(artifactName)' + - script: | + python -m pip install --upgrade pip twine + cd '$(Agent.BuildDirectory)/s/build/wheelhouse/' + python -m twine upload --repository "testpypi" *.whl + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) + displayName: "Upload to PyPI" + env: + TWINE_USERNAME: '__token__' + TWINE_PASSWORD: $(TWINE_PASSWORD) -# - job: WindowsBuilds -# workspace: -# clean: all -# strategy: -# matrix: -# VS_2022: -# poolName: 'Azure Pipelines' -# vmImage: 'windows-2022' -# COMPILER: 'Visual Studio 17 2022' -# ARCH: 'x64' -# PLATFORM: 'win64' -# artifactName: 'Windows-VS-2022-x64' -# VS_2019: -# poolName: 'Azure Pipelines' -# vmImage: 'windows-2019' -# COMPILER: 'Visual Studio 16 2019' -# ARCH: 'x64' -# PLATFORM: 'win64' -# artifactName: 'Windows-VS-2019-x64' -# pool: -# vmImage: $[ variables['vmImage'] ] -# steps: -# - checkout: self -# fetchDepth: 1 -# clean: true -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: '3.11' -# - task: PowerShell@2 -# displayName: 'Install Dependencies' -# inputs: -# targetType: filePath -# filePath: ./CI/windows/install_deps.ps1 -# - task: PowerShell@2 -# displayName: "Build libm2k" -# inputs: -# targetType: 'filePath' -# filePath: ./CI/windows/make_windows.ps1 -# - ${{ each pyVersion in parameters.pythonVersions }}: -# - task: UsePythonVersion@0 -# displayName: Use Python ${{ pyVersion }} -# inputs: -# versionSpec: '${{ pyVersion }}' -# - powershell: | -# $SRC_DIR = Get-Item -Path $env:BUILD_SOURCESDIRECTORY -# $OUTSIDE_BUILD = $SRC_DIR.Parent.FullName +- job: WindowsBuilds + workspace: + clean: all + strategy: + matrix: + VS_2022: + poolName: 'Azure Pipelines' + vmImage: 'windows-2022' + COMPILER: 'Visual Studio 17 2022' + ARCH: 'x64' + PLATFORM: 'win64' + artifactName: 'Windows-VS-2022-x64' + VS_2019: + poolName: 'Azure Pipelines' + vmImage: 'windows-2019' + COMPILER: 'Visual Studio 16 2019' + ARCH: 'x64' + PLATFORM: 'win64' + artifactName: 'Windows-VS-2019-x64' + pool: + vmImage: $[ variables['vmImage'] ] + steps: + - checkout: self + fetchDepth: 1 + clean: true + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.11' + - task: PowerShell@2 + displayName: 'Install Dependencies' + inputs: + targetType: filePath + filePath: ./CI/windows/install_deps.ps1 + - task: PowerShell@2 + displayName: "Build libm2k" + inputs: + targetType: 'filePath' + filePath: ./CI/windows/make_windows.ps1 + - ${{ each pyVersion in parameters.pythonVersions }}: + - task: UsePythonVersion@0 + displayName: Use Python ${{ pyVersion }} + inputs: + versionSpec: '${{ pyVersion }}' + - powershell: | + $SRC_DIR = Get-Item -Path $env:BUILD_SOURCESDIRECTORY + $OUTSIDE_BUILD = $SRC_DIR.Parent.FullName -# $BUILD_DIR = Join-Path $OUTSIDE_BUILD "libm2k-$env:PLATFORM" -# $TEMP_BUILD_DIR = Join-Path $SRC_DIR "tmp-build-$env:PLATFORM" + $BUILD_DIR = Join-Path $OUTSIDE_BUILD "libm2k-$env:PLATFORM" + $TEMP_BUILD_DIR = Join-Path $SRC_DIR "tmp-build-$env:PLATFORM" -# Set-Location $TEMP_BUILD_DIR -# python.exe -m pip install --user --upgrade pip setuptools wheel twine build virtualenv -# $env:COMPILE_BINDINGS = $true -# python.exe -m build -# Remove-Item Env:COMPILE_BINDINGS -# Copy-Item -Path "dist/libm2k*.whl" -Destination (Join-Path $BUILD_DIR "dist") -# Set-Location $SRC_DIR -# displayName: Wheel Python ${{ pyVersion }} -# condition: eq('${{ pyVersion }}', '3.11') # TODO: to be removed in final version -# - task: PowerShell@2 -# displayName: 'Create libm2k-system-setup installer' -# # condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.JobName'], 'VS_2022')) -# condition: and(succeeded(), eq(variables['System.JobName'], 'VS_2022')) -# inputs: -# targetType: 'filePath' -# filePath: ./CI/windows/create_installer.ps1 -# - task: PublishPipelineArtifact@1 -# # condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.JobName'], 'VS_2022')) -# condition: and(succeeded(), eq(variables['System.JobName'], 'VS_2022')) -# inputs: -# targetPath: '$(Build.ArtifactStagingDirectory)/installer' -# artifactName: 'Libm2k-System-Setup-Exe' -# - task: PowerShell@2 -# displayName: "Create ZIPs" -# inputs: -# targetType: 'filePath' -# filePath: ./CI/windows/create_zips.ps1 -# - task: PublishPipelineArtifact@1 -# # condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) -# condition: succeeded() -# inputs: -# targetPath: '$(Build.ArtifactStagingDirectory)/libm2k-$(PLATFORM)' -# artifactName: '$(artifactName)' + Set-Location $TEMP_BUILD_DIR + python.exe -m pip install --user --upgrade pip setuptools wheel twine build virtualenv + $env:COMPILE_BINDINGS = $true + python.exe -m build + Remove-Item Env:COMPILE_BINDINGS + Copy-Item -Path "dist/libm2k*.whl" -Destination (Join-Path $BUILD_DIR "dist") + Set-Location $SRC_DIR + displayName: Wheel Python ${{ pyVersion }} + - task: PowerShell@2 + displayName: 'Create libm2k-system-setup installer' + # condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.JobName'], 'VS_2022')) + condition: and(succeeded(), eq(variables['System.JobName'], 'VS_2022')) + inputs: + targetType: 'filePath' + filePath: ./CI/windows/create_installer.ps1 + - task: PublishPipelineArtifact@1 + # condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.JobName'], 'VS_2022')) + condition: and(succeeded(), eq(variables['System.JobName'], 'VS_2022')) + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/installer' + artifactName: 'Libm2k-System-Setup-Exe' + - task: PowerShell@2 + displayName: "Create ZIPs" + inputs: + targetType: 'filePath' + filePath: ./CI/windows/create_zips.ps1 + - task: PublishPipelineArtifact@1 + # condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + condition: succeeded() + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)/libm2k-$(PLATFORM)' + artifactName: '$(artifactName)'