diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index ef0aa98e48..08b83e8a41 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -4,23 +4,25 @@ name: Upload Python Package on: - release: - types: [published] + push: + branches: [ main ] + pull_request: + branches: [ main ] jobs: deploy_with_bdist_wheel: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, macos-14, macos-latest] + os: [windows-latest, macos-13, macos-14, macos-latest] # FIXME: macos-latest will raise error now. # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -30,32 +32,26 @@ jobs: run: | make build - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: | - twine upload dist/* + ls dist deploy_with_manylinux: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Build wheel on Linux uses: RalfG/python-wheels-manylinux-build@v0.3.1-manylinux2010_x86_64 with: # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 - python-versions: 'cp37-cp37m cp38-cp38' + python-versions: 'cp38-cp38 cp39-cp39m cp310-cp310m cp311-cp311m cp312-cp312m' build-requirements: 'numpy cython' - - name: Set up Python - uses: actions/setup-python@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | pip install twine - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: | - twine upload dist/pyqlib-*-manylinux*.whl + ls dist diff --git a/.github/workflows/test_qlib_from_pip.yml b/.github/workflows/test_qlib_from_pip.yml deleted file mode 100644 index 65e31131a8..0000000000 --- a/.github/workflows/test_qlib_from_pip.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Test qlib from pip - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - timeout-minutes: 120 - - runs-on: ${{ matrix.os }} - strategy: - matrix: - # Since macos-latest changed from 12.7.4 to 14.4.1, - # the minimum python version that matches a 14.4.1 version of macos is 3.10, - # If you want to use python 3.7 in github action, then the latest macos system version is macos-13, - # after macos-13 python 3.7 is no longer supported. - # so we limit the macos version to macos-13. - os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13] - # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 - python-version: [3.7, 3.8] - - steps: - - name: Test qlib from pip - uses: actions/checkout@v3 - - # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error". - # So we make the version number of python 3.7 for MacOS more specific. - # refs: https://github.com/actions/setup-python/issues/682 - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.os == 'macos-latest' && matrix.python-version == '3.7') || (matrix.os == 'macos-13' && matrix.python-version == '3.7') - uses: actions/setup-python@v4 - with: - python-version: "3.7.16" - - - name: Set up Python ${{ matrix.python-version }} - if: (matrix.os != 'macos-latest' || matrix.python-version != '3.7') && (matrix.os != 'macos-13' || matrix.python-version != '3.7') - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Update pip to the latest version - run: | - python -m pip install --upgrade pip - - - name: Qlib installation test - run: | - # 2024-05-30 scs has released a new version: 3.2.4.post2, - # This will cause the CI to fail, so we have limited the version of scs for now. - python -m pip install "scs<=3.2.4" - python -m pip install pyqlib - - - name: Install Lightgbm for MacOS - if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-latest' }} - run: | - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)" - HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm - # FIX MacOS error: Segmentation fault - # reference: https://github.com/microsoft/LightGBM/issues/4229 - wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb - brew unlink libomp - brew install libomp.rb - - - name: Downloads dependencies data - run: | - cd .. - python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn - cd qlib - - - name: Test workflow by config - run: | - qrun examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml diff --git a/.github/workflows/test_qlib_from_source.yml b/.github/workflows/test_qlib_from_source.yml deleted file mode 100644 index 4dfa1f1cb2..0000000000 --- a/.github/workflows/test_qlib_from_source.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: Test qlib from source - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - timeout-minutes: 180 - # we may retry for 3 times for `Unit tests with Pytest` - - runs-on: ${{ matrix.os }} - strategy: - matrix: - # Since macos-latest changed from 12.7.4 to 14.4.1, - # the minimum python version that matches a 14.4.1 version of macos is 3.10, - # If you want to use python 3.7 in github action, then the latest macos system version is macos-13, - # after macos-13 python 3.7 is no longer supported. - # so we limit the macos version to macos-13. - os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, macos-14, macos-latest] - # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - - steps: - - name: Test qlib from source - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Update pip to the latest version - run: | - python -m pip install --upgrade pip - - - name: Installing pytorch for macos - if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-14' || matrix.os == 'macos-latest' }} - run: | - python -m pip install torch torchvision torchaudio - - - name: Installing pytorch for ubuntu - if: ${{ matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04' }} - run: | - python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu - - - name: Installing pytorch for windows - if: ${{ matrix.os == 'windows-latest' }} - run: | - python -m pip install torch torchvision torchaudio - - - name: Set up Python tools - run: | - make dev - - - name: Lint with Black - run: | - make black - - - name: Make html with sphinx - # Since read the docs builds on ubuntu 22.04, we only need to test that the build passes on ubuntu 22.04. - if: ${{ matrix.os == 'ubuntu-22.04' }} - run: | - make docs-gen - - - name: Check Qlib with pylint - run: | - make pylint - - - name: Check Qlib with flake8 - run: | - make flake8 - - - name: Check Qlib with mypy - run: | - make mypy - - - name: Check Qlib ipynb with nbqa - run: | - make nbqa - - - name: Test data downloads - run: | - python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn - python scripts/get_data.py download_data --file_name rl_data.zip --target_dir tests/.data/rl - - - name: Install Lightgbm for MacOS - if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-14' || matrix.os == 'macos-latest' }} - run: | - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)" - HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm - # FIX MacOS error: Segmentation fault - # reference: https://github.com/microsoft/LightGBM/issues/4229 - wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb - brew unlink libomp - brew install libomp.rb - - - name: Check Qlib ipynb with nbconvert - run: | - make nbconvert - - - name: Test workflow by config (install from source) - run: | - python -m pip install numba - python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml - - - name: Unit tests with Pytest - uses: nick-fields/retry@v2 - with: - timeout_minutes: 60 - max_attempts: 3 - command: | - cd tests - python -m pytest . -m "not slow" --durations=0 diff --git a/.github/workflows/test_qlib_from_source_slow.yml b/.github/workflows/test_qlib_from_source_slow.yml deleted file mode 100644 index d302fe3072..0000000000 --- a/.github/workflows/test_qlib_from_source_slow.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Test qlib from source slow - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - timeout-minutes: 720 - # we may retry for 3 times for `Unit tests with Pytest` - - runs-on: ${{ matrix.os }} - strategy: - matrix: - # Since macos-latest changed from 12.7.4 to 14.4.1, - # the minimum python version that matches a 14.4.1 version of macos is 3.10, - # If you want to use python 3.7 in github action, then the latest macos system version is macos-13, - # after macos-13 python 3.7 is no longer supported. - # so we limit the macos version to macos-13. - os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-13, macos-14, macos-latest] - # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129 - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - - steps: - - name: Test qlib from source slow - uses: actions/checkout@v3 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Set up Python tools - run: | - make dev - - - name: Downloads dependencies data - run: | - python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn - - - name: Install Lightgbm for MacOS - if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-14' || matrix.os == 'macos-latest' }} - run: | - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)" - HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm - # FIX MacOS error: Segmentation fault - # reference: https://github.com/microsoft/LightGBM/issues/4229 - wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb - brew unlink libomp - brew install libomp.rb - - - name: Unit tests with Pytest - uses: nick-fields/retry@v2 - with: - timeout_minutes: 240 - max_attempts: 3 - command: | - cd tests - python -m pytest . -m "slow" --durations=0