diff --git a/.ci/Dockerfile.in b/.ci/Dockerfile.in deleted file mode 100644 index 559bb9a40..000000000 --- a/.ci/Dockerfile.in +++ /dev/null @@ -1,9 +0,0 @@ -FROM registry.gitlab.com/jangorecki/dockerfiles/SRC_IMAGE_NAME - -MAINTAINER Jan Gorecki j.gorecki@wit.edu.pl - -COPY bus/build/cran/ /cran/ - -RUN Rscript -e 'install.packages("data.table", repos=file.path("file:","cran"))' - -CMD ["R"] diff --git a/.ci/ci.R b/.ci/ci.R index a165de818..f3a428566 100644 --- a/.ci/ci.R +++ b/.ci/ci.R @@ -185,7 +185,3 @@ function(pkgs, dp } -## set repositories for CI tests -if (as.logical(Sys.getenv("GITLAB_CI","false")) && identical(Sys.getenv("CI_PROJECT_NAME"), "data.table")) { - options("repos" = if (.Platform$OS.type == "windows") file.path("file://",getwd(),"bus/mirror-packages/cran") else file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE))) -} diff --git a/.ci/publish.R b/.ci/publish.R index 526d9bd80..16521fc0e 100644 --- a/.ci/publish.R +++ b/.ci/publish.R @@ -148,7 +148,7 @@ lib.copy <- function(lib.from, repodir="bus/integration/cran"){ pkg.copy <- function(pkg.from, lib.to) { pkg<-basename(pkg.from); dir.create(file.path(lib.to, pkg), recursive=TRUE) - lib.dirs<-intersect(c("html","doc"), all.lib.dirs<-list.dirs(pkg.from, full.names=FALSE)) + lib.dirs<-intersect(c("help","html","doc"), all.lib.dirs<-list.dirs(pkg.from, full.names=FALSE)) ans1<-setNames(file.copy(file.path(pkg.from, lib.dirs), file.path(lib.to, pkg), recursive=TRUE), lib.dirs) lib.files<-setdiff(list.files(pkg.from), all.lib.dirs) ans2<-setNames(file.copy(file.path(pkg.from, lib.files), file.path(lib.to, pkg)), lib.files) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0c403793..18f821b43 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,9 +6,9 @@ variables: TZ: "UTC" ## to avoid 'Failed to create bus connection' from timedatectl via Sys.timezone() on Docker with R 3.4. ## Setting TZ for all GLCI jobs to isolate them from timezone. We could have a new GLCI job to test under ## a non-UTC timezone, although, that's what we do routinely in dev. - R_REL_VERSION: "4.2" - R_DEVEL_VERSION: "4.3" - R_OLDREL_VERSION: "4.1" + R_REL_VERSION: "4.3" + R_DEVEL_VERSION: "4.4" + R_OLDREL_VERSION: "4.2" stages: - dependencies @@ -24,83 +24,87 @@ stages: paths: - bus -mirror-packages: ## mirror all recursive dependencies, source and win.binary of data.table suggests from DESCRIPTION +## mirror packages +# download all recursive dependencies once to be used across multiple test jobs +# sources and binaries for r-release, r-devel and r-oldrel +# cache between runs +mirror-packages: stage: dependencies tags: - linux - image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev + image: registry.gitlab.com/jangorecki/dockerfiles/r-base-minimal cache: paths: - - bus/$CI_BUILD_NAME/cran + - bus/$CI_JOB_NAME/cran script: - echo 'source(".ci/ci.R")' >> .Rprofile - - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib + - mkdir -p bus/$CI_JOB_NAME/cran/src/contrib - Rscript -e 'mirror.packages(dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran")' - - rm bus/$CI_BUILD_NAME/cran/src/contrib/PACKAGES.rds ## fallback to PACKAGES dcf so available.packages:3.4.4 works - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_REL_VERSION","R_DEVEL_VERSION","R_OLDREL_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))' <<: *artifacts -# mirror-other-packages: ## mirror integration suggests from pkgs at the top of inst/tests/other.Rraw; off now #5274 -# stage: dependencies -# tags: -# - linux -# image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev -# cache: -# paths: -# - bus/$CI_BUILD_NAME/cran -# script: -# - echo 'source(".ci/ci.R")' >> .Rprofile -# - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib -# - Rscript -e 'eval(parse("inst/tests/other.Rraw", n=1L)); mirror.packages(pkgs, repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-other-packages/cran")' -# <<: *artifacts - -build: ## build data.table sources as tar.gz archive +## build +# sources as tar.gz archive +# build vignettes +build: stage: build tags: - linux - image: registry.gitlab.com/jangorecki/dockerfiles/r-builder + image: registry.gitlab.com/jangorecki/dockerfiles/r-base ## r-base-gcc after rstudio/markdown#108 needs: ["mirror-packages"] before_script: - Rscript -e 'install.packages(c("knitr","rmarkdown"), repos=file.path("file:",normalizePath("bus/mirror-packages/cran")), quiet=TRUE)' - rm -r bus - sed -i '/^[[:space:]]*$/d' ./DESCRIPTION ## make last line end abruptly; i.e. without a final \n - - echo "Revision:" $CI_BUILD_REF >> ./DESCRIPTION + - echo "Revision:" $CI_COMMIT_SHA >> ./DESCRIPTION script: - R CMD build . - - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib - - mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib/. + - mkdir -p bus/$CI_JOB_NAME/cran/src/contrib + - mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME/cran/src/contrib/. - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/build/cran"), fields="Revision", addFiles=TRUE)' - - rm bus/$CI_BUILD_NAME/cran/src/contrib/PACKAGES.rds ## fallback to PACKAGES dcf so available.packages:3.4.4 works <<: *artifacts +## install deps aliases .test-install-deps: &install-deps - - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="most"), quiet=TRUE)' + - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), repos=file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE)), quiet=TRUE)' +.test-install-deps-win: &install-deps-win + - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'), repos=file.path('file://',getwd(),'bus/mirror-packages/cran'), quiet=TRUE)" +## copy data.table tar.gz from bus R repo to current directory .test-cp-src: &cp-src - cp $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) . .test-cp-src-win: &cp-src-win - cp.exe $(ls.exe -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head.exe -n 1) . +## move data.table tar.gz to bus .test-mv-src: &mv-src - - mkdir -p bus/$CI_BUILD_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME + - mkdir -p bus/$CI_JOB_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME .test-mv-src-win: &mv-src-win - - mkdir.exe -p bus/$CI_BUILD_NAME; mv.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) bus/$CI_BUILD_NAME + - mkdir.exe -p bus/$CI_JOB_NAME; mv.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) bus/$CI_JOB_NAME + +## move data.table binaries to bus R repo +.test-mv-bin-win: &mv-bin-win + - mkdir.exe -p cran/bin/windows/contrib/$R_VERSION; mv.exe $(ls.exe -1t data.table_*.zip | head.exe -n 1) cran/bin/windows/contrib/$R_VERSION +## remove data.table tar.gz .test-rm-src: &rm-src - rm $(ls -1t data.table_*.tar.gz | head -n 1) .test-rm-src-win: &rm-src-win - rm.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) -.test-mv-bin-win: &mv-bin-win - - mkdir.exe -p cran/bin/windows/contrib/$R_VERSION; mv.exe $(ls.exe -1t data.table_*.zip | head.exe -n 1) cran/bin/windows/contrib/$R_VERSION - +## install R on windows .test-install-r-rel-win: &install-r-rel-win - - curl.exe -s -o ../R-rel.exe https://cloud.r-project.org/bin/windows/base/R-4.2.2-win.exe; Start-Process -FilePath ..\R-rel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait - # see #5198 for discussion about the https link used above; it will break each time R is released and the version number will need to be updated + - curl.exe -s -o ../R-rel.exe https://cloud.r-project.org/bin/windows/base/old/4.3.2/R-4.3.2-win.exe; Start-Process -FilePath ..\R-rel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait .test-install-r-devel-win: &install-r-devel-win - curl.exe -s -o ../R-devel.exe https://cloud.r-project.org/bin/windows/base/R-devel-win.exe; Start-Process -FilePath ..\R-devel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait .test-install-r-oldrel-win: &install-r-oldrel-win - - curl.exe -s -o ../R-oldrel.exe https://cloud.r-project.org/bin/windows/base/old/4.1.3/R-4.1.3-win.exe; Start-Process -FilePath ..\R-oldrel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait + - curl.exe -s -o ../R-oldrel.exe https://cloud.r-project.org/bin/windows/base/old/4.2.3/R-4.2.3-win.exe; Start-Process -FilePath ..\R-oldrel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait + +## install Rtools on windows +.test-install-rtools42-win: &install-rtools42-win + - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools42/files/rtools42-5355-5357.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools42" -NoNewWindow -Wait +.test-install-rtools43-win: &install-rtools43-win + - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait .test-template: &test stage: test @@ -112,21 +116,6 @@ build: ## build data.table sources as tar.gz archive tags: - linux -.test-cran-lin-template: &test-cran-lin - <<: *test-lin - variables: - _R_CHECK_CRAN_INCOMING_: "TRUE" - _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE" - before_script: - - *install-deps - - *cp-src - - rm -r bus - script: - - *mv-src - - cd bus/$CI_BUILD_NAME - - R CMD check --as-cran --no-manual $(ls -1t data.table_*.tar.gz | head -n 1) - - *rm-src - .test-win-template: &test-win <<: *test tags: @@ -138,20 +127,26 @@ build: ## build data.table sources as tar.gz archive # tags: # - macosx -test-rel-lin: ## most comprehensive tests, force all suggests, also integration tests, using gcc -O3 -flto -fno-common -Wunused-result +## most comprehensive tests +# force all suggests +# flags: gcc -O3 -flto -fno-common -Wunused-result +# tests for compilation warnings +# measure memory usage during tests +test-rel-lin: <<: *test-lin - image: registry.gitlab.com/jangorecki/dockerfiles/r-builder - needs: ["mirror-packages","build"] # "mirror-other-packages" + image: registry.gitlab.com/jangorecki/dockerfiles/r-data.table + needs: ["mirror-packages","build"] + allow_failure: true ## temp workaround #5760 variables: _R_CHECK_CRAN_INCOMING_: "FALSE" _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE" _R_CHECK_FORCE_SUGGESTS_: "TRUE" _R_CHECK_TESTS_NLINES_: "0" OPENBLAS_MAIN_FREE: "1" - TEST_DATA_TABLE_WITH_OTHER_PACKAGES: "FALSE" #5274 + TEST_DATA_TABLE_MEMTEST: "1" before_script: - - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), quiet=TRUE)' ## does seem to be needed despite 'needs mirror-packages' - ## - Rscript -e 'eval(parse("inst/tests/other.Rraw", n=1L)); install.packages(pkgs, quiet=TRUE, repos=c(getOption("repos"), file.path("file:", normalizePath("bus/mirror-other-packages/cran", mustWork=FALSE))))' + - apt-get update -q && apt-get install -y --no-install-recommends libcurl4-openssl-dev ## remove after #5749 + - *install-deps - *cp-src - rm -r bus - mkdir -p ~/.R @@ -159,16 +154,20 @@ test-rel-lin: ## most comprehensive tests, force all suggests, also integration - echo 'CXXFLAGS=-g -O3 -flto -fno-common -Wunused-result -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars script: - *mv-src - - cd bus/$CI_BUILD_NAME + - cd bus/$CI_JOB_NAME - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1) - *rm-src - (! grep "warning:" data.table.Rcheck/00install.out) -test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, measure memory, using gcc -O0 -fno-openmp +## vanilla minimal +# no suggested deps +# no vignettes or manuals +# no openmp +# flags: gcc -O0 -fno-openmp +test-rel-vanilla-lin: <<: *test-lin - image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev - variables: - TEST_DATA_TABLE_MEMTEST: "1" + image: registry.gitlab.com/jangorecki/dockerfiles/r-base-gcc + allow_failure: true ## temp workaround #5484 before_script: - *cp-src - rm -r bus @@ -177,36 +176,46 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me - echo 'CXXFLAGS=-g -O0 -fno-openmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars script: - *mv-src - - cd bus/$CI_BUILD_NAME + - cd bus/$CI_JOB_NAME - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1) - *rm-src -test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual thus not from cran-lin template +## R-release on Linux +# strict checks for 0 NOTEs +# extra NOTEs check and build pdf manual thus not from cran-lin template +test-rel-cran-lin: <<: *test-lin - image: registry.gitlab.com/jangorecki/dockerfiles/r-builder + image: registry.gitlab.com/jangorecki/dockerfiles/r-base variables: _R_CHECK_CRAN_INCOMING_: "TRUE" ## stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though) _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE" ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284 - _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## effective from R 4.1.0 + _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## bytes + _R_CHECK_PKG_SIZES_THRESHOLD_: "7" ## MB 'checking installed package size' NOTE before_script: + - apt-get update -q && apt-get install -y --no-install-recommends libcurl4-openssl-dev ## temp workaround #5749 - *install-deps - *cp-src - rm -r bus - mkdir -p ~/.R - - echo 'CFLAGS=-g0 -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2'> ~/.R/Makevars ## -g0 because -g increases datatable.so size from 0.5MB to 1.5MB and breaches 'installed package size <= 5MB' note - - echo 'CXXFLAGS=-g0 -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars + - echo 'CFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars + - echo 'CXXFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars script: - *mv-src - - cd bus/$CI_BUILD_NAME + - cd bus/$CI_JOB_NAME - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1) - *rm-src - >- Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(l)) else q("no")' -test-dev-cran-lin: ## R-devel on Linux, --enable-strict-barrier --disable-long-double, check for new notes and compilation warnings, thus allow_failure +## R-devel on Linux +# TODO: --enable-strict-barrier --disable-long-double +# tests for compilation warnings +# tests for new notes +# thus allow_failure +test-dev-cran-lin: <<: *test-lin image: registry.gitlab.com/jangorecki/dockerfiles/r-devel - allow_failure: true + allow_failure: true ## to not be blocked by changes in r-devel variables: _R_CHECK_CRAN_INCOMING_: "TRUE" _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE" @@ -218,84 +227,95 @@ test-dev-cran-lin: ## R-devel on Linux, --enable-strict-barrier --disable-long-d - rm -r bus script: - *mv-src - - cd bus/$CI_BUILD_NAME + - cd bus/$CI_JOB_NAME - R CMD check --as-cran --no-manual $(ls -1t data.table_*.tar.gz | head -n 1) - *rm-src - (! grep "warning:" data.table.Rcheck/00install.out) - - >- + - >- ## this likely need an update but check fails now on complex NA so CI is not reaching here anyway Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 3 NOTEs")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 3 NOTEs"), " (size of tarball, installed package size, top-level files) but ", shQuote(l)) else q("no")' -test-310-cran-lin: ## R-3.1.0 on Linux, stated dependency of R - <<: *test-cran-lin +## R 3.1.0 +# stated dependency on R +test-310-cran-lin: image: registry.gitlab.com/jangorecki/dockerfiles/r-3.1.0 + <<: *test-lin + before_script: + - *install-deps + - *cp-src + - rm -r bus + script: + - *mv-src + - cd bus/$CI_JOB_NAME + - R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1) + - *rm-src -test-344-cran-lin: ## R-3.4.4 on Linux, last R non-altrep version - <<: *test-cran-lin - image: registry.gitlab.com/jangorecki/dockerfiles/r-3.4.4 - -test-350-cran-lin: ## R-3.5.0 on Linux, first R altrep version - <<: *test-cran-lin - image: registry.gitlab.com/jangorecki/dockerfiles/r-3.5.0 - -test-rel-win: ## R-release on Windows, test and build binaries +## R-release on Windows +# test and build binaries +test-rel-win: <<: *test-win variables: R_VERSION: "$R_REL_VERSION" before_script: - *install-r-rel-win - - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools42/files/rtools42-5355-5357.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools42" -NoNewWindow -Wait - - $ENV:PATH = "C:\R\bin;C:\rtools42\usr\bin;$ENV:PATH" - - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)" + - *install-rtools43-win + - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH" + - *install-deps-win - *cp-src-win - rm.exe -r bus script: - *mv-src-win - - cd bus/$CI_BUILD_NAME + - cd bus/$CI_JOB_NAME - R.exe CMD check --no-manual $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) - R.exe CMD INSTALL --build $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) - *rm-src-win - *mv-bin-win -test-dev-win: ## R-devel on Windows; see #5294 for changes in Dec 2021 related to UCRT and Rtools42 +## R-devel on Windows +# test and build binaries +test-dev-win: <<: *test-win variables: R_VERSION: "$R_DEVEL_VERSION" + allow_failure: true ## temp workaround #5748 before_script: - *install-r-devel-win - - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5493-5475.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait + - *install-rtools43-win - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH" - - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most', exclude=c('knitr','rmarkdown')), quiet=TRUE)" ## exclude= for #5294 + - *install-deps-win - *cp-src-win - rm.exe -r bus script: - *mv-src-win - - cd bus/$CI_BUILD_NAME + - cd bus/$CI_JOB_NAME - R.exe CMD check --no-manual --ignore-vignettes $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) - R.exe CMD INSTALL --build $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) - *rm-src-win - *mv-bin-win -test-old-win: ## R-oldrel on Windows +## R-oldrel on Windows +# test and build binaries +test-old-win: <<: *test-win variables: R_VERSION: "$R_OLDREL_VERSION" before_script: - *install-r-oldrel-win - - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools40-x86_64.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools40" -NoNewWindow -Wait - ## rtools42 doesn't support 32bit so oldrel-win (currently R 4.1) needs rtools40 - - $ENV:PATH = "C:\R\bin;C:\rtools40\usr\bin;$ENV:PATH" - - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most', exclude=c('knitr','rmarkdown')), quiet=TRUE)" ## exclude= for #5294 + - *install-rtools42-win + - $ENV:PATH = "C:\R\bin;C:\rtools42\usr\bin;$ENV:PATH" + - *install-deps-win - *cp-src-win - rm.exe -r bus script: - *mv-src-win - - cd bus/$CI_BUILD_NAME + - cd bus/$CI_JOB_NAME - R.exe CMD check --no-manual --ignore-vignettes $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) - R.exe CMD INSTALL --build $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) - *rm-src-win - *mv-bin-win -#test-rel-mac: ## R-release on MacOS, no macosx runner yet +## R-release on MacOS +# no macosx runner set yet +#test-rel-mac: # <<: *test-mac # variables: # R_VERSION: "$R_REL_VERSION" @@ -305,7 +325,7 @@ test-old-win: ## R-oldrel on Windows # - rm -r bus # script: # - *mv-src -# - cd bus/$CI_BUILD_NAME +# - cd bus/$CI_JOB_NAME # - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1) # - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1) # - mkdir -p cran/bin/macosx/el-capitan/contrib/$R_VERSION @@ -313,23 +333,28 @@ test-old-win: ## R-oldrel on Windows # - *rm-src # - *mv-bin-mac -integration: ## merging all artifacts to produce single R repository, documentation and website +## integrate artifacts +# merging package tarballs and binaries into single R repository +# rendering documentation +# setting up CRAN-like structure +# generating pkgdown website +integration: stage: integration image: registry.gitlab.com/jangorecki/dockerfiles/r-pkgdown tags: - linux only: - master - - tags - needs: ["mirror-packages","build","test-rel-lin","test-rel-cran-lin","test-dev-cran-lin","test-rel-vanilla-lin","test-310-cran-lin","test-344-cran-lin","test-350-cran-lin","test-rel-win","test-dev-win","test-old-win"] + needs: ["mirror-packages","build","test-rel-lin","test-rel-cran-lin","test-dev-cran-lin","test-rel-vanilla-lin","test-310-cran-lin","test-rel-win","test-dev-win","test-old-win"] script: + - R --version - Rscript -e 'pkgdown::build_site(override=list(destination="./pkgdown"))' ## html manual, vignettes, repos, cran_web, cran_checks - echo 'source(".ci/ci.R"); source(".ci/publish.R")' >> .Rprofile ## list of available test-* jobs dynamically based on bus/test-* directories - Rscript -e 'cat("\ntest.jobs <- c(\n"); cat(paste0(" \"",list.files("bus",pattern="^test-"),"\" = \"data.table\""), sep=",\n"); cat(")\n")' >> .Rprofile - Rscript -e 'sapply(names(test.jobs), check.test, pkg="data.table", simplify=FALSE)' - - mkdir -p bus/$CI_BUILD_NAME + - mkdir -p bus/$CI_JOB_NAME ## delete any existing non-dev version of data.table - rm -f bus/mirror-packages/cran/src/contrib/data.table_*.tar.gz - rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_REL_VERSION/data.table_*.zip @@ -339,14 +364,14 @@ integration: ## merging all artifacts to produce single R repository, documentat #- rm -f bus/mirror-packages/cran/bin/macosx/el-capitan/contrib/$R_DEVEL_VERSION/data.table_*.tgz #- rm -f bus/mirror-packages/cran/bin/macosx/el-capitan/contrib/$R_OLDREL_VERSION/data.table_*.tgz ## merge mirror-packages and R devel packages - - mv bus/mirror-packages/cran bus/$CI_BUILD_NAME/ + - mv bus/mirror-packages/cran bus/$CI_JOB_NAME/ ## publish package sources - - mkdir -p bus/$CI_BUILD_NAME/cran/library bus/$CI_BUILD_NAME/cran/doc - - mv $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib + - mkdir -p bus/$CI_JOB_NAME/cran/library bus/$CI_JOB_NAME/cran/doc + - mv $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME/cran/src/contrib - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="source"), type="source", fields="Revision", addFiles=TRUE)' ## publish binaries - Rscript -e 'move.bin("test-rel-win", Sys.getenv("R_REL_VERSION"), os.type="windows")' - - Rscript -e 'move.bin("test-dev-win", Sys.getenv("R_DEVEL_VERSION"), os.type="windows")' + - Rscript -e 'move.bin("test-dev-win", Sys.getenv("R_DEVEL_VERSION"), os.type="windows", silent=TRUE)' - Rscript -e 'move.bin("test-old-win", Sys.getenv("R_OLDREL_VERSION"), os.type="windows")' - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_REL_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)' - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_DEVEL_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)' @@ -385,67 +410,17 @@ integration: ## merging all artifacts to produce single R repository, documentat - mv pkgdown/* bus/integration/cran/ ## cleanup artifacts from other jobs - mkdir tmpbus - - mv bus/$CI_BUILD_NAME tmpbus + - mv bus/$CI_JOB_NAME tmpbus - rm -r bus - mv tmpbus bus <<: *artifacts -.docker-template: &docker - stage: deploy - tags: - - linux - image: docker - services: - - docker:dind - needs: - - job: build - - job: integration - artifacts: false - before_script: - - sed "s/SRC_IMAGE_NAME/$SRC_IMAGE_NAME/" < .ci/Dockerfile.in > Dockerfile - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - script: - - docker build --pull -t "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" -f Dockerfile . - - docker run --rm "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" Rscript -e 'cat(R.version.string, "\ndata.table revision", read.dcf(system.file("DESCRIPTION", package="data.table"), fields="Revision")[[1L]], "\n"); require(data.table); test.data.table()' - - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" - -docker-r-release: ## data.table on R-release - only: - - master - variables: - SRC_IMAGE_NAME: "r-base-dev" - IMAGE_NAME: "r-release" - IMAGE_TAG: "latest" - <<: *docker - -docker-r-release-builder: ## data.table on R-release extended for Rmd vignettes build dependencies - only: - - master - variables: - SRC_IMAGE_NAME: "r-builder" - IMAGE_NAME: "r-release-builder" - IMAGE_TAG: "latest" - <<: *docker - -docker-r-devel: ## data.table on R-devel - only: - - master - variables: - SRC_IMAGE_NAME: "r-devel" - IMAGE_NAME: "r-devel" - IMAGE_TAG: "latest" - <<: *docker - -docker-tags: ## data.table on R-release fixed version images - only: - - tags - variables: - SRC_IMAGE_NAME: "r-base-dev" - IMAGE_NAME: "r-release" - IMAGE_TAG: $CI_COMMIT_TAG - <<: *docker - -pages: ## publish R repository, test jobs summaries, html documentation of all packages in repo, pkgdown +## publish +# R repository +# test jobs summaries +# html documentation of all packages in repo +# pkgdown website +pages: stage: deploy environment: production tags: @@ -458,7 +433,7 @@ pages: ## publish R repository, test jobs summaries, html documentation of all p - mkdir -p public - cp -r bus/integration/cran/* public - cat public/src/contrib/PACKAGES - artifacts: ## publish only when no failure + artifacts: expire_in: 2 weeks paths: - public