diff --git a/.ci/ci.R b/.ci/ci.R index a165de818..f3a428566 100644 --- a/.ci/ci.R +++ b/.ci/ci.R @@ -185,7 +185,3 @@ function(pkgs, dp } -## set repositories for CI tests -if (as.logical(Sys.getenv("GITLAB_CI","false")) && identical(Sys.getenv("CI_PROJECT_NAME"), "data.table")) { - options("repos" = if (.Platform$OS.type == "windows") file.path("file://",getwd(),"bus/mirror-packages/cran") else file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE))) -} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 469d5ea76..e866d82db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,11 @@ stages: paths: - bus -mirror-packages: ## mirror all recursive dependencies, source and win.binary of data.table suggests from DESCRIPTION +## mirror packages +# download all recursive dependencies once to be used across multiple test jobs +# sources and binaries for r-release, r-devel and r-oldrel +# cache between runs +mirror-packages: stage: dependencies tags: - linux @@ -39,7 +43,10 @@ mirror-packages: ## mirror all recursive dependencies, source and win.binary of - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_REL_VERSION","R_DEVEL_VERSION","R_OLDREL_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))' <<: *artifacts -build: ## build data.table sources as tar.gz archive +## build +# sources as tar.gz archive +# build vignettes +build: stage: build tags: - linux @@ -57,27 +64,35 @@ build: ## build data.table sources as tar.gz archive - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/build/cran"), fields="Revision", addFiles=TRUE)' <<: *artifacts +## install deps aliases .test-install-deps: &install-deps - - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="most"), quiet=TRUE)' + - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), repos=file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE)), quiet=TRUE)' +.test-install-deps-win: &install-deps-win + - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'), repos=file.path("file://",getwd(),"bus/mirror-packages/cran"), quiet=TRUE)" +## copy data.table tar.gz from bus R repo to current directory .test-cp-src: &cp-src - cp $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) . .test-cp-src-win: &cp-src-win - cp.exe $(ls.exe -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head.exe -n 1) . +## move data.table tar.gz to bus .test-mv-src: &mv-src - mkdir -p bus/$CI_JOB_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME .test-mv-src-win: &mv-src-win - mkdir.exe -p bus/$CI_JOB_NAME; mv.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) bus/$CI_JOB_NAME +## move data.table binaries to bus R repo +.test-mv-bin-win: &mv-bin-win + - mkdir.exe -p cran/bin/windows/contrib/$R_VERSION; mv.exe $(ls.exe -1t data.table_*.zip | head.exe -n 1) cran/bin/windows/contrib/$R_VERSION + +## remove data.table tar.gz .test-rm-src: &rm-src - rm $(ls -1t data.table_*.tar.gz | head -n 1) .test-rm-src-win: &rm-src-win - rm.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) -.test-mv-bin-win: &mv-bin-win - - mkdir.exe -p cran/bin/windows/contrib/$R_VERSION; mv.exe $(ls.exe -1t data.table_*.zip | head.exe -n 1) cran/bin/windows/contrib/$R_VERSION - +## install R and Rtools on windows .test-install-r-rel-win: &install-r-rel-win - curl.exe -s -o ../R-rel.exe https://cloud.r-project.org/bin/windows/base/old/4.3.2/R-4.3.2-win.exe; Start-Process -FilePath ..\R-rel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait .test-install-r-devel-win: &install-r-devel-win @@ -95,21 +110,6 @@ build: ## build data.table sources as tar.gz archive tags: - linux -.test-cran-lin-template: &test-cran-lin - <<: *test-lin - variables: - _R_CHECK_CRAN_INCOMING_: "TRUE" - _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE" - before_script: - - *install-deps - - *cp-src - - rm -r bus - script: - - *mv-src - - cd bus/$CI_JOB_NAME - - R CMD check --as-cran --no-manual $(ls -1t data.table_*.tar.gz | head -n 1) - - *rm-src - .test-win-template: &test-win <<: *test tags: @@ -121,10 +121,14 @@ build: ## build data.table sources as tar.gz archive # tags: # - macosx -test-rel-lin: ## most comprehensive tests, force all suggests, also integration tests, using gcc -O3 -flto -fno-common -Wunused-result +## most comprehensive tests +# force all suggests +# flags: gcc -O3 -flto -fno-common -Wunused-result +# tests for compilation warnings +test-rel-lin: <<: *test-lin image: registry.gitlab.com/jangorecki/dockerfiles/r-builder - needs: ["mirror-packages","build"] # "mirror-other-packages" + needs: ["mirror-packages","build"] variables: _R_CHECK_CRAN_INCOMING_: "FALSE" _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE" @@ -132,7 +136,7 @@ test-rel-lin: ## most comprehensive tests, force all suggests, also integration _R_CHECK_TESTS_NLINES_: "0" OPENBLAS_MAIN_FREE: "1" before_script: - - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), quiet=TRUE)' + - *install-deps - *cp-src - rm -r bus - mkdir -p ~/.R @@ -145,7 +149,14 @@ test-rel-lin: ## most comprehensive tests, force all suggests, also integration - *rm-src - (! grep "warning:" data.table.Rcheck/00install.out) -test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, measure memory, using gcc -O0 -fno-openmp +## vanilla minimal +# TODO: no OS deps: libcurl zlib +# no suggested deps +# no vignettes or manuals +# no openmp +# measure memory +# flags: gcc -O0 -fno-openmp +test-rel-vanilla-lin: <<: *test-lin image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev variables: @@ -162,14 +173,17 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1) - *rm-src -test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual thus not from cran-lin template +## R-release on Linux +# strict checks for 0 NOTEs +# extra NOTEs check and build pdf manual thus not from cran-lin template +test-rel-cran-lin: <<: *test-lin image: registry.gitlab.com/jangorecki/dockerfiles/r-builder variables: _R_CHECK_CRAN_INCOMING_: "TRUE" ## stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though) _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE" ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284 - _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## effective from R 4.1.0 - #_R_CHECK_PKG_SIZES_THRESHOLD_: "7"## 'checking installed package size' NOTE - check if really needed before adding + _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" + #_R_CHECK_PKG_SIZES_THRESHOLD_: "7" ## 'checking installed package size' NOTE - possibly not yet needed bc -g0 (see flags below) before_script: - *install-deps - *cp-src @@ -185,7 +199,12 @@ test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual - >- Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(l)) else q("no")' -test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier --disable-long-double, check for new notes and compilation warnings, thus allow_failure +## R-devel on Linux +# TODO: --enable-strict-barrier --disable-long-double +# tests for compilation warnings +# tests for new notes +# thus allow_failure +test-dev-cran-lin: <<: *test-lin image: registry.gitlab.com/jangorecki/dockerfiles/r-devel allow_failure: true @@ -207,11 +226,24 @@ test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier - - >- Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 3 NOTEs")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 3 NOTEs"), " (size of tarball, installed package size, top-level files) but ", shQuote(l)) else q("no")' -test-310-cran-lin: ## R-3.1.0 on Linux, stated dependency of R - <<: *test-cran-lin +## R 3.1.0 +# stated dependency on R +test-310-cran-lin: image: registry.gitlab.com/jangorecki/dockerfiles/r-3.1.0 + <<: *test-lin + before_script: + - *install-deps + - *cp-src + - rm -r bus + script: + - *mv-src + - cd bus/$CI_JOB_NAME + - R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1) + - *rm-src -test-rel-win: ## R-release on Windows, test and build binaries +## R-release on Windows +# test and build binaries +test-rel-win: <<: *test-win variables: R_VERSION: "$R_REL_VERSION" @@ -219,7 +251,7 @@ test-rel-win: ## R-release on Windows, test and build binaries - *install-r-rel-win - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH" - - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)" + - *install-deps-win - *cp-src-win - rm.exe -r bus script: @@ -230,7 +262,9 @@ test-rel-win: ## R-release on Windows, test and build binaries - *rm-src-win - *mv-bin-win -test-dev-win: ## R-devel on Windows +## R-devel on Windows +# test and build binaries +test-dev-win: <<: *test-win variables: R_VERSION: "$R_DEVEL_VERSION" @@ -238,7 +272,7 @@ test-dev-win: ## R-devel on Windows - *install-r-devel-win - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH" - - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)" + - *install-deps-win - *cp-src-win - rm.exe -r bus script: @@ -249,7 +283,9 @@ test-dev-win: ## R-devel on Windows - *rm-src-win - *mv-bin-win -test-old-win: ## R-oldrel on Windows +## R-oldrel on Windows +# test and build binaries +test-old-win: <<: *test-win variables: R_VERSION: "$R_OLDREL_VERSION" @@ -257,7 +293,7 @@ test-old-win: ## R-oldrel on Windows - *install-r-oldrel-win - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools42/files/rtools42-5355-5357.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools42" -NoNewWindow -Wait - $ENV:PATH = "C:\R\bin;C:\rtools42\usr\bin;$ENV:PATH" - - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)" + - *install-deps-win - *cp-src-win - rm.exe -r bus script: @@ -268,7 +304,9 @@ test-old-win: ## R-oldrel on Windows - *rm-src-win - *mv-bin-win -#test-rel-mac: ## R-release on MacOS, no macosx runner yet +## R-release on MacOS +# no macosx runner set yet +#test-rel-mac: # <<: *test-mac # variables: # R_VERSION: "$R_REL_VERSION" @@ -286,7 +324,12 @@ test-old-win: ## R-oldrel on Windows # - *rm-src # - *mv-bin-mac -integration: ## merging all artifacts to produce single R repository, documentation and website +## integrate artifacts +# merging package tarballs and binaries into single R repository +# rendering documentation +# setting up CRAN-like structure +# generating pkgdown website +integration: stage: integration image: registry.gitlab.com/jangorecki/dockerfiles/r-pkgdown tags: