From 227b8c5df4027cb73d10c0c8575d8a4067d6318b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 27 May 2020 20:16:51 -0500 Subject: [PATCH 1/6] [ci] make MiKTeX downloads more reliable --- .ci/download-miktex.R | 25 +++++++++++++++++++++++++ .ci/test_r_package_windows.ps1 | 10 +++++----- 2 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 .ci/download-miktex.R diff --git a/.ci/download-miktex.R b/.ci/download-miktex.R new file mode 100644 index 000000000000..f0d4b98cac5a --- /dev/null +++ b/.ci/download-miktex.R @@ -0,0 +1,25 @@ +# mirrors that host miktexsetup.zip do so only with explicitly-named +# files like miktexsetup-2.4.5.zip, so hard-coding a link to an archive as a +# way to peg to one mirror does not work +# +# this script will find the specific version of miktexsetup.zip at a given +# mirror +library(httr) +args <- commandArgs(trailingOnly = TRUE) +DESTFILE <- args[[1L]] +MIRROR <- "https://ctan.math.illinois.edu/systems/win32/miktex/setup/windows-x64/" +mirror_contents <- httr::content( + httr::RETRY("GET", MIRROR) + , as = "text" +) +content_lines <- strsplit(mirror_contents, "\n")[[1L]] +content_lines <- content_lines[grepl("miktexsetup-.*", content_lines)] +zip_loc <- regexpr(">miktexsetup-[0-9]+.*x64\\.zip", content_lines) +zip_name <- gsub(">", "", regmatches(content_lines, zip_loc)) +full_zip_url <- file.path(MIRROR, zip_name) +print(sprintf("downloading %s", full_zip_url)) +download.file( + url = full_zip_url + , destfile = DESTFILE +) +print(sprintf("MiKTeX setup downloaded to %s", DESTFILE)) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 966cc3dc0e91..6ebcdd6e18df 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -48,11 +48,15 @@ Write-Output "Installing Rtools" Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/Rtools" ; Check-Output $? Write-Output "Done installing Rtools" +Write-Output "Installing dependencies" +$packages = "c('data.table', 'httr', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" +Rscript --vanilla -e "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $? + # MiKTeX and pandoc can be skipped on non-MINGW builds, since we don't # build the package documentation for those if ($env:COMPILER -eq "MINGW") { Write-Output "Downloading MiKTeX" - Download-File-With-Retries -url "https://miktex.org/download/win/miktexsetup-x64.zip" -destfile "miktexsetup-x64.zip" + Rscript $env:BUILD_SOURCESDIRECTORY\.ci\download-miktex.R "miktexsetup-x64.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup-x64.zip", "miktex") Write-Output "Setting up MiKTeX" @@ -65,10 +69,6 @@ if ($env:COMPILER -eq "MINGW") { conda install -q -y --no-deps pandoc } -Write-Output "Installing dependencies" -$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" -Rscript --vanilla -e "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $? - Write-Output "Building R package" # R CMD check is not used for MSVC builds From 5a939f9ce5328cfab0e757d7d30a299af8d4d2bb Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 May 2020 10:57:29 -0500 Subject: [PATCH 2/6] switch to Powershell --- .ci/download-miktex.R | 25 ------------------------- .ci/test_r_package_windows.ps1 | 30 ++++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 31 deletions(-) delete mode 100644 .ci/download-miktex.R diff --git a/.ci/download-miktex.R b/.ci/download-miktex.R deleted file mode 100644 index f0d4b98cac5a..000000000000 --- a/.ci/download-miktex.R +++ /dev/null @@ -1,25 +0,0 @@ -# mirrors that host miktexsetup.zip do so only with explicitly-named -# files like miktexsetup-2.4.5.zip, so hard-coding a link to an archive as a -# way to peg to one mirror does not work -# -# this script will find the specific version of miktexsetup.zip at a given -# mirror -library(httr) -args <- commandArgs(trailingOnly = TRUE) -DESTFILE <- args[[1L]] -MIRROR <- "https://ctan.math.illinois.edu/systems/win32/miktex/setup/windows-x64/" -mirror_contents <- httr::content( - httr::RETRY("GET", MIRROR) - , as = "text" -) -content_lines <- strsplit(mirror_contents, "\n")[[1L]] -content_lines <- content_lines[grepl("miktexsetup-.*", content_lines)] -zip_loc <- regexpr(">miktexsetup-[0-9]+.*x64\\.zip", content_lines) -zip_name <- gsub(">", "", regmatches(content_lines, zip_loc)) -full_zip_url <- file.path(MIRROR, zip_name) -print(sprintf("downloading %s", full_zip_url)) -download.file( - url = full_zip_url - , destfile = DESTFILE -) -print(sprintf("MiKTeX setup downloaded to %s", DESTFILE)) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 6ebcdd6e18df..b8002a5c7984 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -12,12 +12,31 @@ function Download-File-With-Retries { } while(!$?); } +# mirrors that host miktexsetup.zip do so only with explicitly-named +# files like miktexsetup-2.4.5.zip, so hard-coding a link to an archive as a +# way to peg to one mirror does not work +# +# this function will find the specific version of miktexsetup.zip at a given +# given mirror and download it +function Download-Miktex-Setup { + param( + [string]$destfile + ) + $ARCHIVE_URL = "https://ctan.math.illinois.edu/systems/win32/miktex/setup/windows-x64/" + $PageContent = Invoke-WebRequest -Uri $ARCHIVE_URL -Method Get + $SetupExeFile = $PageContent.Links.href | Select-String -Pattern 'miktexsetup.*' + $FileToDownload = "${ARCHIVE_URL}/${SetupExeFile}" + Download-File-With-Retries $FileToDownload $destfile +} + $env:R_WINDOWS_VERSION = "3.6.3" $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:R_LIBS = "$env:R_LIB_PATH" $env:PATH = "$env:R_LIB_PATH/Rtools/bin;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH $env:CRAN_MIRROR = "https://cloud.r-project.org/" -$env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex/tm/packages/" +$env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex" +$env:CTAN_MIKTEX_ARCHIVE = "$env:CTAN_MIRROR/setup/windows-x64" +$env:CTAN_PACKAGE_ARCHIVE = "$env:CTAN_MIRROR/tm/packages/" if ($env:COMPILER -eq "MINGW") { $env:CXX = "$env:R_LIB_PATH/Rtools/mingw_64/bin/g++.exe" @@ -49,20 +68,19 @@ Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT Write-Output "Done installing Rtools" Write-Output "Installing dependencies" -$packages = "c('data.table', 'httr', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" +$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" Rscript --vanilla -e "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $? # MiKTeX and pandoc can be skipped on non-MINGW builds, since we don't # build the package documentation for those if ($env:COMPILER -eq "MINGW") { - Write-Output "Downloading MiKTeX" - Rscript $env:BUILD_SOURCESDIRECTORY\.ci\download-miktex.R "miktexsetup-x64.zip" + Download-Miktex-Setup "$env:CTAN_MIKTEX_ARCHIVE" "miktexsetup-x64.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup-x64.zip", "miktex") Write-Output "Setting up MiKTeX" - .\miktex\miktexsetup.exe --remote-package-repository="$env:CTAN_MIRROR" --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? + .\miktex\miktexsetup.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --local-package-repository=./miktex/download --package-set=essential --quiet download ; Check-Output $? Write-Output "Installing MiKTeX" - .\miktex\download\miktexsetup.exe --remote-package-repository="$env:CTAN_MIRROR" --portable="$env:R_LIB_PATH/miktex" --quiet install ; Check-Output $? + .\miktex\download\miktexsetup.exe --remote-package-repository="$env:CTAN_PACKAGE_ARCHIVE" --portable="$env:R_LIB_PATH/miktex" --quiet install ; Check-Output $? Write-Output "Done installing MiKTeX" initexmf --set-config-value [MPM]AutoInstall=1 From eb8ad633a5a9200f1591d5ac64e5264a582caad2 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 May 2020 13:32:55 -0500 Subject: [PATCH 3/6] remove extra slash --- .ci/test_r_package_windows.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index b8002a5c7984..a5e69bdc436f 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -22,7 +22,7 @@ function Download-Miktex-Setup { param( [string]$destfile ) - $ARCHIVE_URL = "https://ctan.math.illinois.edu/systems/win32/miktex/setup/windows-x64/" + $ARCHIVE_URL = "https://ctan.math.illinois.edu/systems/win32/miktex/setup/windows-x64" $PageContent = Invoke-WebRequest -Uri $ARCHIVE_URL -Method Get $SetupExeFile = $PageContent.Links.href | Select-String -Pattern 'miktexsetup.*' $FileToDownload = "${ARCHIVE_URL}/${SetupExeFile}" From b64e3a1b3cdeed43a2be78b3126336127d379e41 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 30 May 2020 13:54:13 -0500 Subject: [PATCH 4/6] fixing stuff --- .ci/test_r_package_windows.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index a5e69bdc436f..3a68b3bb4490 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -20,12 +20,12 @@ function Download-File-With-Retries { # given mirror and download it function Download-Miktex-Setup { param( + [string]$archive, [string]$destfile ) - $ARCHIVE_URL = "https://ctan.math.illinois.edu/systems/win32/miktex/setup/windows-x64" - $PageContent = Invoke-WebRequest -Uri $ARCHIVE_URL -Method Get + $PageContent = Invoke-WebRequest -Uri $archive -Method Get $SetupExeFile = $PageContent.Links.href | Select-String -Pattern 'miktexsetup.*' - $FileToDownload = "${ARCHIVE_URL}/${SetupExeFile}" + $FileToDownload = "${archive}/${SetupExeFile}" Download-File-With-Retries $FileToDownload $destfile } @@ -35,7 +35,7 @@ $env:R_LIBS = "$env:R_LIB_PATH" $env:PATH = "$env:R_LIB_PATH/Rtools/bin;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH $env:CRAN_MIRROR = "https://cloud.r-project.org/" $env:CTAN_MIRROR = "https://ctan.math.illinois.edu/systems/win32/miktex" -$env:CTAN_MIKTEX_ARCHIVE = "$env:CTAN_MIRROR/setup/windows-x64" +$env:CTAN_MIKTEX_ARCHIVE = "$env:CTAN_MIRROR/setup/windows-x64/" $env:CTAN_PACKAGE_ARCHIVE = "$env:CTAN_MIRROR/tm/packages/" if ($env:COMPILER -eq "MINGW") { From cab58f0e975e71a5a96acbbd6977263e008671cf Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 31 May 2020 18:28:39 -0500 Subject: [PATCH 5/6] revert install change --- .ci/test_r_package_windows.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 3a68b3bb4490..16542388f9b6 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -67,10 +67,6 @@ Write-Output "Installing Rtools" Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/Rtools" ; Check-Output $? Write-Output "Done installing Rtools" -Write-Output "Installing dependencies" -$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" -Rscript --vanilla -e "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $? - # MiKTeX and pandoc can be skipped on non-MINGW builds, since we don't # build the package documentation for those if ($env:COMPILER -eq "MINGW") { @@ -87,6 +83,10 @@ if ($env:COMPILER -eq "MINGW") { conda install -q -y --no-deps pandoc } +Write-Output "Installing dependencies" +$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" +Rscript --vanilla -e "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $? + Write-Output "Building R package" # R CMD check is not used for MSVC builds From cf93d50847c4f6920873659e3023655de0b40371 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 1 Jun 2020 00:26:00 +0100 Subject: [PATCH 6/6] Update .ci/test_r_package_windows.ps1 Co-authored-by: Nikita Titov --- .ci/test_r_package_windows.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 16542388f9b6..f0181d4709ae 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -17,7 +17,7 @@ function Download-File-With-Retries { # way to peg to one mirror does not work # # this function will find the specific version of miktexsetup.zip at a given -# given mirror and download it +# mirror and download it function Download-Miktex-Setup { param( [string]$archive,