mostly comments and refactoring

Rdatatable · Nov 24, 2023 · f7cd646 · f7cd646
1 parent bf0e5d3
commit f7cd646
Show file tree

Hide file tree

Showing 2 changed files with 82 additions and 43 deletions.
diff --git a/.ci/ci.R b/.ci/ci.R
@@ -185,7 +185,3 @@ function(pkgs,
   dp
 }
 
-## set repositories for CI tests
-if (as.logical(Sys.getenv("GITLAB_CI","false")) && identical(Sys.getenv("CI_PROJECT_NAME"), "data.table")) {
-  options("repos" = if (.Platform$OS.type == "windows") file.path("file://",getwd(),"bus/mirror-packages/cran") else file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE)))
-}
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -24,7 +24,11 @@ stages:
     paths:
       - bus
 
-mirror-packages: ## mirror all recursive dependencies, source and win.binary of data.table suggests from DESCRIPTION
+## mirror packages
+# download all recursive dependencies once to be used across multiple test jobs
+# sources and binaries for r-release, r-devel and r-oldrel
+# cache between runs
+mirror-packages:
   stage: dependencies
   tags:
     - linux
@@ -39,7 +43,10 @@ mirror-packages: ## mirror all recursive dependencies, source and win.binary of
     - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_REL_VERSION","R_DEVEL_VERSION","R_OLDREL_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'
   <<: *artifacts
 
-build: ## build data.table sources as tar.gz archive
+## build
+# sources as tar.gz archive
+# build vignettes
+build:
   stage: build
   tags:
     - linux
@@ -57,27 +64,35 @@ build: ## build data.table sources as tar.gz archive
     - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/build/cran"), fields="Revision", addFiles=TRUE)'
   <<: *artifacts
 
+## install deps aliases
 .test-install-deps: &install-deps
-  - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="most"), quiet=TRUE)'
+  - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), repos=file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE)), quiet=TRUE)'
+.test-install-deps-win: &install-deps-win
+  - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'), repos=file.path("file://",getwd(),"bus/mirror-packages/cran"), quiet=TRUE)"
 
+## copy data.table tar.gz from bus R repo to current directory
 .test-cp-src: &cp-src
   - cp $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) .
 .test-cp-src-win: &cp-src-win
   - cp.exe $(ls.exe -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head.exe -n 1) .
 
+## move data.table tar.gz to bus
 .test-mv-src: &mv-src
   - mkdir -p bus/$CI_JOB_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME
 .test-mv-src-win: &mv-src-win
   - mkdir.exe -p bus/$CI_JOB_NAME; mv.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) bus/$CI_JOB_NAME
 
+## move data.table binaries to bus R repo
+.test-mv-bin-win: &mv-bin-win
+  - mkdir.exe -p cran/bin/windows/contrib/$R_VERSION; mv.exe $(ls.exe -1t data.table_*.zip | head.exe -n 1) cran/bin/windows/contrib/$R_VERSION
+
+## remove data.table tar.gz
 .test-rm-src: &rm-src
   - rm $(ls -1t data.table_*.tar.gz | head -n 1)
 .test-rm-src-win: &rm-src-win
   - rm.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
 
-.test-mv-bin-win: &mv-bin-win
-  - mkdir.exe -p cran/bin/windows/contrib/$R_VERSION; mv.exe $(ls.exe -1t data.table_*.zip | head.exe -n 1) cran/bin/windows/contrib/$R_VERSION
-
+## install R and Rtools on windows
 .test-install-r-rel-win: &install-r-rel-win
   - curl.exe -s -o ../R-rel.exe https://cloud.r-project.org/bin/windows/base/old/4.3.2/R-4.3.2-win.exe; Start-Process -FilePath ..\R-rel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
 .test-install-r-devel-win: &install-r-devel-win
@@ -95,21 +110,6 @@ build: ## build data.table sources as tar.gz archive
   tags:
     - linux
 
-.test-cran-lin-template: &test-cran-lin
-  <<: *test-lin
-  variables:
-    _R_CHECK_CRAN_INCOMING_: "TRUE"
-    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
-  before_script:
-    - *install-deps
-    - *cp-src
-    - rm -r bus
-  script:
-    - *mv-src
-    - cd bus/$CI_JOB_NAME
-    - R CMD check --as-cran --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
-    - *rm-src
-
 .test-win-template: &test-win
   <<: *test
   tags:
@@ -121,18 +121,22 @@ build: ## build data.table sources as tar.gz archive
 #  tags:
 #    - macosx
 
-test-rel-lin: ## most comprehensive tests, force all suggests, also integration tests, using gcc -O3 -flto -fno-common -Wunused-result
+## most comprehensive tests
+# force all suggests
+# flags: gcc -O3 -flto -fno-common -Wunused-result
+# tests for compilation warnings
+test-rel-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
-  needs: ["mirror-packages","build"]  # "mirror-other-packages"
+  needs: ["mirror-packages","build"]
   variables:
     _R_CHECK_CRAN_INCOMING_: "FALSE"
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
     _R_CHECK_FORCE_SUGGESTS_: "TRUE"
     _R_CHECK_TESTS_NLINES_: "0"
     OPENBLAS_MAIN_FREE: "1"
   before_script:
-    - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), quiet=TRUE)'
+    - *install-deps
     - *cp-src
     - rm -r bus
     - mkdir -p ~/.R
@@ -145,7 +149,14 @@ test-rel-lin: ## most comprehensive tests, force all suggests, also integration
     - *rm-src
     - (! grep "warning:" data.table.Rcheck/00install.out)
 
-test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, measure memory, using gcc -O0 -fno-openmp
+## vanilla minimal
+# TODO: no OS deps: libcurl zlib
+# no suggested deps
+# no vignettes or manuals
+# no openmp
+# measure memory
+# flags: gcc -O0 -fno-openmp
+test-rel-vanilla-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
   variables:
@@ -162,14 +173,17 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
     - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)
     - *rm-src
 
-test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual thus not from cran-lin template
+## R-release on Linux
+# strict checks for 0 NOTEs
+# extra NOTEs check and build pdf manual thus not from cran-lin template
+test-rel-cran-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
   variables:
     _R_CHECK_CRAN_INCOMING_: "TRUE"           ## stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though)
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
-    _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## effective from R 4.1.0
-    #_R_CHECK_PKG_SIZES_THRESHOLD_: "7"## 'checking installed package size' NOTE - check if really needed before adding
+    _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000"
+    #_R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## 'checking installed package size' NOTE - possibly not yet needed bc -g0 (see flags below)
   before_script:
     - *install-deps
     - *cp-src
@@ -185,7 +199,12 @@ test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual
     - >-
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(l)) else q("no")'
 
-test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier --disable-long-double, check for new notes and compilation warnings, thus allow_failure
+## R-devel on Linux
+# TODO: --enable-strict-barrier --disable-long-double
+# tests for compilation warnings
+# tests for new notes
+# thus allow_failure
+test-dev-cran-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-devel
   allow_failure: true
@@ -207,19 +226,32 @@ test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier -
     - >-
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 3 NOTEs")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 3 NOTEs"), " (size of tarball, installed package size, top-level files) but ", shQuote(l)) else q("no")'
 
-test-310-cran-lin: ## R-3.1.0 on Linux, stated dependency of R
-  <<: *test-cran-lin
+## R 3.1.0
+# stated dependency on R
+test-310-cran-lin:
   image: registry.gitlab.com/jangorecki/dockerfiles/r-3.1.0
+  <<: *test-lin
+  before_script:
+    - *install-deps
+    - *cp-src
+    - rm -r bus
+  script:
+    - *mv-src
+    - cd bus/$CI_JOB_NAME
+    - R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
+    - *rm-src
 
-test-rel-win: ## R-release on Windows, test and build binaries
+## R-release on Windows
+# test and build binaries
+test-rel-win:
   <<: *test-win
   variables:
     R_VERSION: "$R_REL_VERSION"
   before_script:
     - *install-r-rel-win
     - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
     - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH"
-    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)"
+    - *install-deps-win
     - *cp-src-win
     - rm.exe -r bus
   script:
@@ -230,15 +262,17 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - *rm-src-win
     - *mv-bin-win
 
-test-dev-win: ## R-devel on Windows
+## R-devel on Windows
+# test and build binaries
+test-dev-win:
   <<: *test-win
   variables:
     R_VERSION: "$R_DEVEL_VERSION"
   before_script:
     - *install-r-devel-win
     - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
     - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH"
-    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)"
+    - *install-deps-win
     - *cp-src-win
     - rm.exe -r bus
   script:
@@ -249,15 +283,17 @@ test-dev-win: ## R-devel on Windows
     - *rm-src-win
     - *mv-bin-win
 
-test-old-win: ## R-oldrel on Windows
+## R-oldrel on Windows
+# test and build binaries
+test-old-win:
   <<: *test-win
   variables:
     R_VERSION: "$R_OLDREL_VERSION"
   before_script:
     - *install-r-oldrel-win
     - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools42/files/rtools42-5355-5357.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools42" -NoNewWindow -Wait
     - $ENV:PATH = "C:\R\bin;C:\rtools42\usr\bin;$ENV:PATH"
-    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)"
+    - *install-deps-win
     - *cp-src-win
     - rm.exe -r bus
   script:
@@ -268,7 +304,9 @@ test-old-win: ## R-oldrel on Windows
     - *rm-src-win
     - *mv-bin-win
 
-#test-rel-mac: ## R-release on MacOS, no macosx runner yet
+## R-release on MacOS
+# no macosx runner set yet
+#test-rel-mac:
 #  <<: *test-mac
 #  variables:
 #    R_VERSION: "$R_REL_VERSION"
@@ -286,7 +324,12 @@ test-old-win: ## R-oldrel on Windows
 #    - *rm-src
 #    - *mv-bin-mac
 
-integration: ## merging all artifacts to produce single R repository, documentation and website
+## integrate artifacts
+# merging package tarballs and binaries into single R repository
+# rendering documentation
+# setting up CRAN-like structure
+# generating pkgdown website
+integration:
   stage: integration
   image: registry.gitlab.com/jangorecki/dockerfiles/r-pkgdown
   tags: