From 1fa7e68e319b7654ab59da3372fd35e089b4fb05 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 28 Feb 2021 22:15:22 -0600 Subject: [PATCH 1/8] [dask] [ci] add support for scikit-learn 0.24+ in tests (fixes #4031) --- tests/python_package_test/test_dask.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/python_package_test/test_dask.py b/tests/python_package_test/test_dask.py index 6b112b705a3e..a4e62c910e4f 100644 --- a/tests/python_package_test/test_dask.py +++ b/tests/python_package_test/test_dask.py @@ -28,12 +28,16 @@ from dask.array.utils import assert_eq from dask.distributed import Client, LocalCluster, default_client, wait from distributed.utils_test import client, cluster_fixture, gen_cluster, loop +from pkg_resources import parse_version from scipy.sparse import csr_matrix from scipy.stats import spearmanr +from sklearn import __version__ as sk_version from sklearn.datasets import make_blobs, make_regression from .utils import make_ranking +sk_version = parse_version(sk_version) + # time, in seconds, to wait for the Dask client to close. Used to avoid teardown errors # see https://distributed.dask.org/en/latest/api.html#distributed.Client.close CLIENT_CLOSE_TIMEOUT = 120 @@ -1253,5 +1257,9 @@ def test_sklearn_integration(estimator, check, client): # this test is separate because it takes a not-yet-constructed estimator @pytest.mark.parametrize("estimator", list(_tested_estimators())) def test_parameters_default_constructible(estimator): - name, Estimator = estimator.__class__.__name__, estimator.__class__ + name = estimator.__class__.__name__ + if sk_version > parse_version("0.23"): + Estimator = estimator + else: + Estimator = estimator.__class__ sklearn_checks.check_parameters_default_constructible(name, Estimator) From 821bfd4d47e966392d150938e736ab29354cca61 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 1 Mar 2021 08:25:41 -0600 Subject: [PATCH 2/8] Update tests/python_package_test/test_dask.py Co-authored-by: Nikita Titov --- tests/python_package_test/test_dask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python_package_test/test_dask.py b/tests/python_package_test/test_dask.py index a4e62c910e4f..5f7784190e4b 100644 --- a/tests/python_package_test/test_dask.py +++ b/tests/python_package_test/test_dask.py @@ -1258,7 +1258,7 @@ def test_sklearn_integration(estimator, check, client): @pytest.mark.parametrize("estimator", list(_tested_estimators())) def test_parameters_default_constructible(estimator): name = estimator.__class__.__name__ - if sk_version > parse_version("0.23"): + if sk_version >= parse_version("0.24"): Estimator = estimator else: Estimator = estimator.__class__ From 491ad95c1cec6ff892e246a8bacd06486490feba Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 1 Mar 2021 09:57:00 -0600 Subject: [PATCH 3/8] try upgrading mixtexsetup --- .ci/test_r_package_windows.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index e49a638b10b7..a718b03b30fa 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -104,9 +104,9 @@ Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); insta # # MiKTeX always needs to be built to test a CRAN package. if (($env:COMPILER -eq "MINGW") -or ($env:R_BUILD_TYPE -eq "cran")) { - Download-File-With-Retries "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/miktexsetup-4.0-x64.zip" -destfile "miktexsetup-x64.zip" + Download-File-With-Retries "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/miktexsetup-4.1-x86.zip" -destfile "miktexsetup.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup-x64.zip", "miktex") + [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup.zip", "miktex") Write-Output "Setting up MiKTeX" .\miktex\miktexsetup.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? Write-Output "Installing MiKTeX" From 6339a458b484b757bc3ed1a5659e4119b2628546 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 1 Mar 2021 10:11:37 -0600 Subject: [PATCH 4/8] they changed the executable name UGH --- .ci/test_r_package_windows.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index a718b03b30fa..938bcaca40e0 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -106,7 +106,7 @@ Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); insta if (($env:COMPILER -eq "MINGW") -or ($env:R_BUILD_TYPE -eq "cran")) { Download-File-With-Retries "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/miktexsetup-4.1-x86.zip" -destfile "miktexsetup.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup.zip", "miktex") + [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup.zip", "miktexsetup_standalone") Write-Output "Setting up MiKTeX" .\miktex\miktexsetup.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? Write-Output "Installing MiKTeX" From cf28a1fb3c645adbb3b26e37534ccb2524244382 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 1 Mar 2021 10:40:47 -0600 Subject: [PATCH 5/8] more changes for executable name --- .ci/test_r_package_windows.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 938bcaca40e0..67500747903f 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -106,11 +106,11 @@ Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); insta if (($env:COMPILER -eq "MINGW") -or ($env:R_BUILD_TYPE -eq "cran")) { Download-File-With-Retries "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/miktexsetup-4.1-x86.zip" -destfile "miktexsetup.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup.zip", "miktexsetup_standalone") + [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup.zip", "miktexsetup") Write-Output "Setting up MiKTeX" - .\miktex\miktexsetup.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? + .\miktex\miktexsetup_standalone.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? Write-Output "Installing MiKTeX" - .\miktex\download\miktexsetup.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --portable="$env:R_LIB_PATH/miktex" --quiet install ; Check-Output $? + .\miktex\download\miktexsetup_standalone.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --portable="$env:R_LIB_PATH/miktex" --quiet install ; Check-Output $? Write-Output "Done installing MiKTeX" Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'initexmf', args = c('--set-config-value', '[MPM]AutoInstall=1'), echo = TRUE, windows_verbatim_args = TRUE, error_on_status = TRUE)" ; Check-Output $? From e4c3570f390188acbd158ba74ba63599736c7c17 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 1 Mar 2021 10:48:11 -0600 Subject: [PATCH 6/8] another path change --- .ci/test_r_package_windows.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 67500747903f..d070cc40ea87 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -106,7 +106,7 @@ Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); insta if (($env:COMPILER -eq "MINGW") -or ($env:R_BUILD_TYPE -eq "cran")) { Download-File-With-Retries "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/miktexsetup-4.1-x86.zip" -destfile "miktexsetup.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup.zip", "miktexsetup") + [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup.zip", "miktex") Write-Output "Setting up MiKTeX" .\miktex\miktexsetup_standalone.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? Write-Output "Installing MiKTeX" From 4845136afa44045268b338047daa97f5ad66d441 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 1 Mar 2021 11:26:14 -0600 Subject: [PATCH 7/8] changing package mirrors --- .ci/test_r_package_windows.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index d070cc40ea87..9860e2d15411 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -57,7 +57,7 @@ $env:R_LIBS = "$env:R_LIB_PATH" $env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH $env:CRAN_MIRROR = "https://cloud.r-project.org/" $env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex" -$env:CTAN_PACKAGE_ARCHIVE = "$env:CTAN_MIRROR/tm/packages/" +$env:CTAN_PACKAGE_ARCHIVE = "https://mirror.las.iastate.edu/tex-archive/systems/win32/miktex/tm/packages/" # hack to get around this: # https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html From cd28b809978e190e4bbde7e2a17f3f1e45a329ec Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 1 Mar 2021 13:13:17 -0600 Subject: [PATCH 8/8] undo experiments --- .ci/test_r_package_windows.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 9860e2d15411..e49a638b10b7 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -57,7 +57,7 @@ $env:R_LIBS = "$env:R_LIB_PATH" $env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH $env:CRAN_MIRROR = "https://cloud.r-project.org/" $env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex" -$env:CTAN_PACKAGE_ARCHIVE = "https://mirror.las.iastate.edu/tex-archive/systems/win32/miktex/tm/packages/" +$env:CTAN_PACKAGE_ARCHIVE = "$env:CTAN_MIRROR/tm/packages/" # hack to get around this: # https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html @@ -104,13 +104,13 @@ Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); insta # # MiKTeX always needs to be built to test a CRAN package. if (($env:COMPILER -eq "MINGW") -or ($env:R_BUILD_TYPE -eq "cran")) { - Download-File-With-Retries "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/miktexsetup-4.1-x86.zip" -destfile "miktexsetup.zip" + Download-File-With-Retries "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/miktexsetup-4.0-x64.zip" -destfile "miktexsetup-x64.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup.zip", "miktex") + [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup-x64.zip", "miktex") Write-Output "Setting up MiKTeX" - .\miktex\miktexsetup_standalone.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? + .\miktex\miktexsetup.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? Write-Output "Installing MiKTeX" - .\miktex\download\miktexsetup_standalone.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --portable="$env:R_LIB_PATH/miktex" --quiet install ; Check-Output $? + .\miktex\download\miktexsetup.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --portable="$env:R_LIB_PATH/miktex" --quiet install ; Check-Output $? Write-Output "Done installing MiKTeX" Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'initexmf', args = c('--set-config-value', '[MPM]AutoInstall=1'), echo = TRUE, windows_verbatim_args = TRUE, error_on_status = TRUE)" ; Check-Output $?