From 63aca8f74f9fe57b383ce934a5eff3c9029d6754 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 14:49:00 +0100
Subject: [PATCH 01/20] bringing back GLCI

---
 .gitlab-ci.yml | 116 ++++++++-----------------------------------------
 1 file changed, 17 insertions(+), 99 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f0c4037933..6edce1efba 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,9 +6,9 @@ variables:
   TZ: "UTC"  ## to avoid 'Failed to create bus connection' from timedatectl via Sys.timezone() on Docker with R 3.4.
              ## Setting TZ for all GLCI jobs to isolate them from timezone. We could have a new GLCI job to test under
              ## a non-UTC timezone, although, that's what we do routinely in dev.
-  R_REL_VERSION: "4.2"
-  R_DEVEL_VERSION: "4.3"
-  R_OLDREL_VERSION: "4.1"
+  R_REL_VERSION: "4.3"
+  R_DEVEL_VERSION: "4.4"
+  R_OLDREL_VERSION: "4.2"
 
 stages:
   - dependencies
@@ -36,24 +36,9 @@ mirror-packages: ## mirror all recursive dependencies, source and win.binary of
     - echo 'source(".ci/ci.R")' >> .Rprofile
     - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
     - Rscript -e 'mirror.packages(dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran")'
-    - rm bus/$CI_BUILD_NAME/cran/src/contrib/PACKAGES.rds ## fallback to PACKAGES dcf so available.packages:3.4.4 works
     - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_REL_VERSION","R_DEVEL_VERSION","R_OLDREL_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'
   <<: *artifacts
 
-# mirror-other-packages: ## mirror integration suggests from pkgs at the top of inst/tests/other.Rraw;  off now #5274
-#  stage: dependencies
-#  tags:
-#    - linux
-#  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
-#  cache:
-#    paths:
-#      - bus/$CI_BUILD_NAME/cran
-#  script:
-#    - echo 'source(".ci/ci.R")' >> .Rprofile
-#    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
-#    - Rscript -e 'eval(parse("inst/tests/other.Rraw", n=1L)); mirror.packages(pkgs, repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-other-packages/cran")'
-#  <<: *artifacts
-
 build: ## build data.table sources as tar.gz archive
   stage: build
   tags:
@@ -70,7 +55,6 @@ build: ## build data.table sources as tar.gz archive
     - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
     - mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib/.
     - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/build/cran"), fields="Revision", addFiles=TRUE)'
-    - rm bus/$CI_BUILD_NAME/cran/src/contrib/PACKAGES.rds ## fallback to PACKAGES dcf so available.packages:3.4.4 works
   <<: *artifacts
 
 .test-install-deps: &install-deps
@@ -95,12 +79,11 @@ build: ## build data.table sources as tar.gz archive
   - mkdir.exe -p cran/bin/windows/contrib/$R_VERSION; mv.exe $(ls.exe -1t data.table_*.zip | head.exe -n 1) cran/bin/windows/contrib/$R_VERSION
 
 .test-install-r-rel-win: &install-r-rel-win
-  - curl.exe -s -o ../R-rel.exe https://cloud.r-project.org/bin/windows/base/R-4.2.2-win.exe; Start-Process -FilePath ..\R-rel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
-  # see #5198 for discussion about the https link used above; it will break each time R is released and the version number will need to be updated
+  - curl.exe -s -o ../R-rel.exe https://cloud.r-project.org/bin/windows/base/old/4.3.2/R-4.3.2-win.exe; Start-Process -FilePath ..\R-rel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
 .test-install-r-devel-win: &install-r-devel-win
   - curl.exe -s -o ../R-devel.exe https://cloud.r-project.org/bin/windows/base/R-devel-win.exe; Start-Process -FilePath ..\R-devel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
 .test-install-r-oldrel-win: &install-r-oldrel-win
-  - curl.exe -s -o ../R-oldrel.exe https://cloud.r-project.org/bin/windows/base/old/4.1.3/R-4.1.3-win.exe; Start-Process -FilePath ..\R-oldrel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
+  - curl.exe -s -o ../R-oldrel.exe https://cloud.r-project.org/bin/windows/base/old/4.2.3/R-4.2.3-win.exe; Start-Process -FilePath ..\R-oldrel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
 
 .test-template: &test
   stage: test
@@ -148,10 +131,8 @@ test-rel-lin: ## most comprehensive tests, force all suggests, also integration
     _R_CHECK_FORCE_SUGGESTS_: "TRUE"
     _R_CHECK_TESTS_NLINES_: "0"
     OPENBLAS_MAIN_FREE: "1"
-    TEST_DATA_TABLE_WITH_OTHER_PACKAGES: "FALSE"  #5274
   before_script:
-    - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), quiet=TRUE)'  ## does seem to be needed despite 'needs mirror-packages'
-    ## - Rscript -e 'eval(parse("inst/tests/other.Rraw", n=1L)); install.packages(pkgs, quiet=TRUE, repos=c(getOption("repos"), file.path("file:", normalizePath("bus/mirror-other-packages/cran", mustWork=FALSE))))'
+    - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), quiet=TRUE)'
     - *cp-src
     - rm -r bus
     - mkdir -p ~/.R
@@ -188,6 +169,7 @@ test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual
     _R_CHECK_CRAN_INCOMING_: "TRUE"           ## stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though)
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
     _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## effective from R 4.1.0
+    #_R_CHECK_PKG_SIZES_THRESHOLD_: "7"## 'checking installed package size' NOTE - check if really needed before adding
   before_script:
     - *install-deps
     - *cp-src
@@ -203,7 +185,7 @@ test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual
     - >-
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(l)) else q("no")'
 
-test-dev-cran-lin: ## R-devel on Linux, --enable-strict-barrier --disable-long-double, check for new notes and compilation warnings, thus allow_failure
+test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier --disable-long-double, check for new notes and compilation warnings, thus allow_failure
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-devel
   allow_failure: true
@@ -229,22 +211,14 @@ test-310-cran-lin: ## R-3.1.0 on Linux, stated dependency of R
   <<: *test-cran-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-3.1.0
 
-test-344-cran-lin: ## R-3.4.4 on Linux, last R non-altrep version
-  <<: *test-cran-lin
-  image: registry.gitlab.com/jangorecki/dockerfiles/r-3.4.4
-
-test-350-cran-lin: ## R-3.5.0 on Linux, first R altrep version
-  <<: *test-cran-lin
-  image: registry.gitlab.com/jangorecki/dockerfiles/r-3.5.0
-
 test-rel-win: ## R-release on Windows, test and build binaries
   <<: *test-win
   variables:
     R_VERSION: "$R_REL_VERSION"
   before_script:
     - *install-r-rel-win
-    - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools42/files/rtools42-5355-5357.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools42" -NoNewWindow -Wait
-    - $ENV:PATH = "C:\R\bin;C:\rtools42\usr\bin;$ENV:PATH"
+    - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
+    - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH"
     - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)"
     - *cp-src-win
     - rm.exe -r bus
@@ -256,15 +230,15 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - *rm-src-win
     - *mv-bin-win
 
-test-dev-win: ## R-devel on Windows; see #5294 for changes in Dec 2021 related to UCRT and Rtools42
+test-dev-win: ## R-devel on Windows
   <<: *test-win
   variables:
     R_VERSION: "$R_DEVEL_VERSION"
   before_script:
     - *install-r-devel-win
-    - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5493-5475.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
+    - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
     - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH"
-    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most', exclude=c('knitr','rmarkdown')), quiet=TRUE)"  ## exclude= for #5294
+    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)"
     - *cp-src-win
     - rm.exe -r bus
   script:
@@ -281,10 +255,9 @@ test-old-win: ## R-oldrel on Windows
     R_VERSION: "$R_OLDREL_VERSION"
   before_script:
     - *install-r-oldrel-win
-    - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools40-x86_64.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools40" -NoNewWindow -Wait
-    ## rtools42 doesn't support 32bit so oldrel-win (currently R 4.1) needs rtools40
-    - $ENV:PATH = "C:\R\bin;C:\rtools40\usr\bin;$ENV:PATH"
-    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most', exclude=c('knitr','rmarkdown')), quiet=TRUE)"  ## exclude= for #5294
+    - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools42/files/rtools42-5355-5357.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools42" -NoNewWindow -Wait
+    - $ENV:PATH = "C:\R\bin;C:\rtools42\usr\bin;$ENV:PATH"
+    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)"
     - *cp-src-win
     - rm.exe -r bus
   script:
@@ -321,7 +294,7 @@ integration: ## merging all artifacts to produce single R repository, documentat
   only:
     - master
     - tags
-  needs: ["mirror-packages","build","test-rel-lin","test-rel-cran-lin","test-dev-cran-lin","test-rel-vanilla-lin","test-310-cran-lin","test-344-cran-lin","test-350-cran-lin","test-rel-win","test-dev-win","test-old-win"]
+  needs: ["mirror-packages","build","test-rel-lin","test-rel-cran-lin","test-dev-cran-lin","test-rel-vanilla-lin","test-310-cran-lin","test-rel-win","test-dev-win","test-old-win"]
   script:
     - Rscript -e 'pkgdown::build_site(override=list(destination="./pkgdown"))'
     ## html manual, vignettes, repos, cran_web, cran_checks
@@ -390,61 +363,6 @@ integration: ## merging all artifacts to produce single R repository, documentat
     - mv tmpbus bus
   <<: *artifacts
 
-.docker-template: &docker
-  stage: deploy
-  tags:
-    - linux
-  image: docker
-  services:
-    - docker:dind
-  needs:
-    - job: build
-    - job: integration
-      artifacts: false
-  before_script:
-    - sed "s/SRC_IMAGE_NAME/$SRC_IMAGE_NAME/" < .ci/Dockerfile.in > Dockerfile
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-  script:
-    - docker build --pull -t "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" -f Dockerfile .
-    - docker run --rm "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" Rscript -e 'cat(R.version.string, "\ndata.table revision", read.dcf(system.file("DESCRIPTION", package="data.table"), fields="Revision")[[1L]], "\n"); require(data.table); test.data.table()'
-    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"
-
-docker-r-release: ## data.table on R-release
-  only:
-    - master
-  variables:
-    SRC_IMAGE_NAME: "r-base-dev"
-    IMAGE_NAME: "r-release"
-    IMAGE_TAG: "latest"
-  <<: *docker
-
-docker-r-release-builder: ## data.table on R-release extended for Rmd vignettes build dependencies
-  only:
-    - master
-  variables:
-    SRC_IMAGE_NAME: "r-builder"
-    IMAGE_NAME: "r-release-builder"
-    IMAGE_TAG: "latest"
-  <<: *docker
-
-docker-r-devel: ## data.table on R-devel
-  only:
-    - master
-  variables:
-    SRC_IMAGE_NAME: "r-devel"
-    IMAGE_NAME: "r-devel"
-    IMAGE_TAG: "latest"
-  <<: *docker
-
-docker-tags: ## data.table on R-release fixed version images
-  only:
-    - tags
-  variables:
-    SRC_IMAGE_NAME: "r-base-dev"
-    IMAGE_NAME: "r-release"
-    IMAGE_TAG: $CI_COMMIT_TAG
-  <<: *docker
-
 pages: ## publish R repository, test jobs summaries, html documentation of all packages in repo, pkgdown
   stage: deploy
   environment: production

From a8e033398ecd11f5f9805266de1d1a4b0cfa253b Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 14:50:06 +0100
Subject: [PATCH 02/20] enable testing on branch

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6edce1efba..7c532ff59e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -293,7 +293,7 @@ integration: ## merging all artifacts to produce single R repository, documentat
     - linux
   only:
     - master
-    - tags
+    - glci5 ## testing before merge
   needs: ["mirror-packages","build","test-rel-lin","test-rel-cran-lin","test-dev-cran-lin","test-rel-vanilla-lin","test-310-cran-lin","test-rel-win","test-dev-win","test-old-win"]
   script:
     - Rscript -e 'pkgdown::build_site(override=list(destination="./pkgdown"))'

From ef283636fee5d3dc621724f49c8ea967c19360de Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 15:35:42 +0100
Subject: [PATCH 03/20] debugging

---
 .gitlab-ci.yml | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7c532ff59e..651ea17515 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -121,7 +121,7 @@ build: ## build data.table sources as tar.gz archive
 #  tags:
 #    - macosx
 
-test-rel-lin: ## most comprehensive tests, force all suggests, also integration tests, using gcc -O3 -flto -fno-common -Wunused-result
+.test-rel-lin: ## most comprehensive tests, force all suggests, also integration tests, using gcc -O3 -flto -fno-common -Wunused-result
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
   needs: ["mirror-packages","build"]  # "mirror-other-packages"
@@ -151,6 +151,11 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
   variables:
     TEST_DATA_TABLE_MEMTEST: "1"
   before_script:
+    - ls bus/
+    - ls bus/build/
+    - ls bus/build/cran/
+    - ls bus/build/cran/src/
+    - ls bus/build/cran/src/contrib/
     - *cp-src
     - rm -r bus
     - mkdir -p ~/.R
@@ -162,7 +167,7 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
     - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)
     - *rm-src
 
-test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual thus not from cran-lin template
+.test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual thus not from cran-lin template
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
   variables:
@@ -185,7 +190,7 @@ test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual
     - >-
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(l)) else q("no")'
 
-test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier --disable-long-double, check for new notes and compilation warnings, thus allow_failure
+.test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier --disable-long-double, check for new notes and compilation warnings, thus allow_failure
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-devel
   allow_failure: true
@@ -207,7 +212,7 @@ test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier -
     - >-
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 3 NOTEs")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 3 NOTEs"), " (size of tarball, installed package size, top-level files) but ", shQuote(l)) else q("no")'
 
-test-310-cran-lin: ## R-3.1.0 on Linux, stated dependency of R
+.test-310-cran-lin: ## R-3.1.0 on Linux, stated dependency of R
   <<: *test-cran-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-3.1.0
 
@@ -230,7 +235,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - *rm-src-win
     - *mv-bin-win
 
-test-dev-win: ## R-devel on Windows
+.test-dev-win: ## R-devel on Windows
   <<: *test-win
   variables:
     R_VERSION: "$R_DEVEL_VERSION"
@@ -249,7 +254,7 @@ test-dev-win: ## R-devel on Windows
     - *rm-src-win
     - *mv-bin-win
 
-test-old-win: ## R-oldrel on Windows
+.test-old-win: ## R-oldrel on Windows
   <<: *test-win
   variables:
     R_VERSION: "$R_OLDREL_VERSION"
@@ -286,7 +291,7 @@ test-old-win: ## R-oldrel on Windows
 #    - *rm-src
 #    - *mv-bin-mac
 
-integration: ## merging all artifacts to produce single R repository, documentation and website
+.integration: ## merging all artifacts to produce single R repository, documentation and website
   stage: integration
   image: registry.gitlab.com/jangorecki/dockerfiles/r-pkgdown
   tags:
@@ -363,7 +368,7 @@ integration: ## merging all artifacts to produce single R repository, documentat
     - mv tmpbus bus
   <<: *artifacts
 
-pages: ## publish R repository, test jobs summaries, html documentation of all packages in repo, pkgdown
+.pages: ## publish R repository, test jobs summaries, html documentation of all packages in repo, pkgdown
   stage: deploy
   environment: production
   tags:

From afbf7da065ae0b82e5123137b6d3b1afd931638f Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 15:48:14 +0100
Subject: [PATCH 04/20] env vars rename

---
 .gitlab-ci.yml | 42 +++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 651ea17515..a4ed3bba59 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -31,10 +31,10 @@ mirror-packages: ## mirror all recursive dependencies, source and win.binary of
   image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
   cache:
     paths:
-      - bus/$CI_BUILD_NAME/cran
+      - bus/$CI_JOB_NAME/cran
   script:
     - echo 'source(".ci/ci.R")' >> .Rprofile
-    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
+    - mkdir -p bus/$CI_JOB_NAME/cran/src/contrib
     - Rscript -e 'mirror.packages(dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran")'
     - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_REL_VERSION","R_DEVEL_VERSION","R_OLDREL_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'
   <<: *artifacts
@@ -49,11 +49,11 @@ build: ## build data.table sources as tar.gz archive
     - Rscript -e 'install.packages(c("knitr","rmarkdown"), repos=file.path("file:",normalizePath("bus/mirror-packages/cran")), quiet=TRUE)'
     - rm -r bus
     - sed -i '/^[[:space:]]*$/d' ./DESCRIPTION ## make last line end abruptly; i.e. without a final \n
-    - echo "Revision:" $CI_BUILD_REF >> ./DESCRIPTION
+    - echo "Revision:" $CI_COMMIT_SHA >> ./DESCRIPTION
   script:
     - R CMD build .
-    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
-    - mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib/.
+    - mkdir -p bus/$CI_JOB_NAME/cran/src/contrib
+    - mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME/cran/src/contrib/.
     - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/build/cran"), fields="Revision", addFiles=TRUE)'
   <<: *artifacts
 
@@ -66,9 +66,9 @@ build: ## build data.table sources as tar.gz archive
   - cp.exe $(ls.exe -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head.exe -n 1) .
 
 .test-mv-src: &mv-src
-  - mkdir -p bus/$CI_BUILD_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME
+  - mkdir -p bus/$CI_JOB_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME
 .test-mv-src-win: &mv-src-win
-  - mkdir.exe -p bus/$CI_BUILD_NAME; mv.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) bus/$CI_BUILD_NAME
+  - mkdir.exe -p bus/$CI_JOB_NAME; mv.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) bus/$CI_JOB_NAME
 
 .test-rm-src: &rm-src
   - rm $(ls -1t data.table_*.tar.gz | head -n 1)
@@ -106,7 +106,7 @@ build: ## build data.table sources as tar.gz archive
     - rm -r bus
   script:
     - *mv-src
-    - cd bus/$CI_BUILD_NAME
+    - cd bus/$CI_JOB_NAME
     - R CMD check --as-cran --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
     - *rm-src
 
@@ -140,7 +140,7 @@ build: ## build data.table sources as tar.gz archive
     - echo 'CXXFLAGS=-g -O3 -flto -fno-common -Wunused-result -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
   script:
     - *mv-src
-    - cd bus/$CI_BUILD_NAME
+    - cd bus/$CI_JOB_NAME
     - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
     - *rm-src
     - (! grep "warning:" data.table.Rcheck/00install.out)
@@ -163,7 +163,7 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
     - echo 'CXXFLAGS=-g -O0 -fno-openmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
   script:
     - *mv-src
-    - cd bus/$CI_BUILD_NAME
+    - cd bus/$CI_JOB_NAME
     - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)
     - *rm-src
 
@@ -184,7 +184,7 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
     - echo 'CXXFLAGS=-g0 -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
   script:
     - *mv-src
-    - cd bus/$CI_BUILD_NAME
+    - cd bus/$CI_JOB_NAME
     - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
     - *rm-src
     - >-
@@ -205,7 +205,7 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
     - rm -r bus
   script:
     - *mv-src
-    - cd bus/$CI_BUILD_NAME
+    - cd bus/$CI_JOB_NAME
     - R CMD check --as-cran --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
     - *rm-src
     - (! grep "warning:" data.table.Rcheck/00install.out)
@@ -229,7 +229,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - rm.exe -r bus
   script:
     - *mv-src-win
-    - cd bus/$CI_BUILD_NAME
+    - cd bus/$CI_JOB_NAME
     - R.exe CMD check --no-manual $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
     - R.exe CMD INSTALL --build $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
     - *rm-src-win
@@ -248,7 +248,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - rm.exe -r bus
   script:
     - *mv-src-win
-    - cd bus/$CI_BUILD_NAME
+    - cd bus/$CI_JOB_NAME
     - R.exe CMD check --no-manual --ignore-vignettes $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
     - R.exe CMD INSTALL --build $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
     - *rm-src-win
@@ -267,7 +267,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - rm.exe -r bus
   script:
     - *mv-src-win
-    - cd bus/$CI_BUILD_NAME
+    - cd bus/$CI_JOB_NAME
     - R.exe CMD check --no-manual --ignore-vignettes $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
     - R.exe CMD INSTALL --build $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
     - *rm-src-win
@@ -283,7 +283,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
 #    - rm -r bus
 #  script:
 #    - *mv-src
-#    - cd bus/$CI_BUILD_NAME
+#    - cd bus/$CI_JOB_NAME
 #    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
 #    - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
 #    - mkdir -p cran/bin/macosx/el-capitan/contrib/$R_VERSION
@@ -307,7 +307,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
     ## list of available test-* jobs dynamically based on bus/test-* directories
     - Rscript -e 'cat("\ntest.jobs <- c(\n"); cat(paste0("  \"",list.files("bus",pattern="^test-"),"\" = \"data.table\""), sep=",\n"); cat(")\n")' >> .Rprofile
     - Rscript -e 'sapply(names(test.jobs), check.test, pkg="data.table", simplify=FALSE)'
-    - mkdir -p bus/$CI_BUILD_NAME
+    - mkdir -p bus/$CI_JOB_NAME
     ## delete any existing non-dev version of data.table
     - rm -f bus/mirror-packages/cran/src/contrib/data.table_*.tar.gz
     - rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_REL_VERSION/data.table_*.zip
@@ -317,10 +317,10 @@ test-rel-win: ## R-release on Windows, test and build binaries
     #- rm -f bus/mirror-packages/cran/bin/macosx/el-capitan/contrib/$R_DEVEL_VERSION/data.table_*.tgz
     #- rm -f bus/mirror-packages/cran/bin/macosx/el-capitan/contrib/$R_OLDREL_VERSION/data.table_*.tgz
     ## merge mirror-packages and R devel packages
-    - mv bus/mirror-packages/cran bus/$CI_BUILD_NAME/
+    - mv bus/mirror-packages/cran bus/$CI_JOB_NAME/
     ## publish package sources
-    - mkdir -p bus/$CI_BUILD_NAME/cran/library bus/$CI_BUILD_NAME/cran/doc
-    - mv $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib
+    - mkdir -p bus/$CI_JOB_NAME/cran/library bus/$CI_JOB_NAME/cran/doc
+    - mv $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME/cran/src/contrib
     - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="source"), type="source", fields="Revision", addFiles=TRUE)'
     ## publish binaries
     - Rscript -e 'move.bin("test-rel-win", Sys.getenv("R_REL_VERSION"), os.type="windows")'
@@ -363,7 +363,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - mv pkgdown/* bus/integration/cran/
     ## cleanup artifacts from other jobs
     - mkdir tmpbus
-    - mv bus/$CI_BUILD_NAME tmpbus
+    - mv bus/$CI_JOB_NAME tmpbus
     - rm -r bus
     - mv tmpbus bus
   <<: *artifacts

From bf0e5d30da02a888a0434174892b939d43c4aea0 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 15:53:24 +0100
Subject: [PATCH 05/20] enable all jobs

---
 .gitlab-ci.yml | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a4ed3bba59..469d5ea768 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -121,7 +121,7 @@ build: ## build data.table sources as tar.gz archive
 #  tags:
 #    - macosx
 
-.test-rel-lin: ## most comprehensive tests, force all suggests, also integration tests, using gcc -O3 -flto -fno-common -Wunused-result
+test-rel-lin: ## most comprehensive tests, force all suggests, also integration tests, using gcc -O3 -flto -fno-common -Wunused-result
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
   needs: ["mirror-packages","build"]  # "mirror-other-packages"
@@ -151,11 +151,6 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
   variables:
     TEST_DATA_TABLE_MEMTEST: "1"
   before_script:
-    - ls bus/
-    - ls bus/build/
-    - ls bus/build/cran/
-    - ls bus/build/cran/src/
-    - ls bus/build/cran/src/contrib/
     - *cp-src
     - rm -r bus
     - mkdir -p ~/.R
@@ -167,7 +162,7 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
     - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)
     - *rm-src
 
-.test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual thus not from cran-lin template
+test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual thus not from cran-lin template
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
   variables:
@@ -190,7 +185,7 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
     - >-
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(l)) else q("no")'
 
-.test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier --disable-long-double, check for new notes and compilation warnings, thus allow_failure
+test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier --disable-long-double, check for new notes and compilation warnings, thus allow_failure
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-devel
   allow_failure: true
@@ -212,7 +207,7 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
     - >-
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 3 NOTEs")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 3 NOTEs"), " (size of tarball, installed package size, top-level files) but ", shQuote(l)) else q("no")'
 
-.test-310-cran-lin: ## R-3.1.0 on Linux, stated dependency of R
+test-310-cran-lin: ## R-3.1.0 on Linux, stated dependency of R
   <<: *test-cran-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-3.1.0
 
@@ -235,7 +230,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - *rm-src-win
     - *mv-bin-win
 
-.test-dev-win: ## R-devel on Windows
+test-dev-win: ## R-devel on Windows
   <<: *test-win
   variables:
     R_VERSION: "$R_DEVEL_VERSION"
@@ -254,7 +249,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - *rm-src-win
     - *mv-bin-win
 
-.test-old-win: ## R-oldrel on Windows
+test-old-win: ## R-oldrel on Windows
   <<: *test-win
   variables:
     R_VERSION: "$R_OLDREL_VERSION"
@@ -291,7 +286,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
 #    - *rm-src
 #    - *mv-bin-mac
 
-.integration: ## merging all artifacts to produce single R repository, documentation and website
+integration: ## merging all artifacts to produce single R repository, documentation and website
   stage: integration
   image: registry.gitlab.com/jangorecki/dockerfiles/r-pkgdown
   tags:
@@ -368,7 +363,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - mv tmpbus bus
   <<: *artifacts
 
-.pages: ## publish R repository, test jobs summaries, html documentation of all packages in repo, pkgdown
+pages: ## publish R repository, test jobs summaries, html documentation of all packages in repo, pkgdown
   stage: deploy
   environment: production
   tags:

From f7cd646ac376ce647fa0ca0394b653e2298da4ef Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 16:59:49 +0100
Subject: [PATCH 06/20] mostly comments and refactoring

---
 .ci/ci.R       |   4 --
 .gitlab-ci.yml | 121 +++++++++++++++++++++++++++++++++----------------
 2 files changed, 82 insertions(+), 43 deletions(-)

diff --git a/.ci/ci.R b/.ci/ci.R
index a165de8189..f3a4285660 100644
--- a/.ci/ci.R
+++ b/.ci/ci.R
@@ -185,7 +185,3 @@ function(pkgs,
   dp
 }
 
-## set repositories for CI tests
-if (as.logical(Sys.getenv("GITLAB_CI","false")) && identical(Sys.getenv("CI_PROJECT_NAME"), "data.table")) {
-  options("repos" = if (.Platform$OS.type == "windows") file.path("file://",getwd(),"bus/mirror-packages/cran") else file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE)))
-}
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 469d5ea768..e866d82db8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,7 +24,11 @@ stages:
     paths:
       - bus
 
-mirror-packages: ## mirror all recursive dependencies, source and win.binary of data.table suggests from DESCRIPTION
+## mirror packages
+# download all recursive dependencies once to be used across multiple test jobs
+# sources and binaries for r-release, r-devel and r-oldrel
+# cache between runs
+mirror-packages:
   stage: dependencies
   tags:
     - linux
@@ -39,7 +43,10 @@ mirror-packages: ## mirror all recursive dependencies, source and win.binary of
     - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_REL_VERSION","R_DEVEL_VERSION","R_OLDREL_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'
   <<: *artifacts
 
-build: ## build data.table sources as tar.gz archive
+## build
+# sources as tar.gz archive
+# build vignettes
+build:
   stage: build
   tags:
     - linux
@@ -57,27 +64,35 @@ build: ## build data.table sources as tar.gz archive
     - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/build/cran"), fields="Revision", addFiles=TRUE)'
   <<: *artifacts
 
+## install deps aliases
 .test-install-deps: &install-deps
-  - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="most"), quiet=TRUE)'
+  - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), repos=file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE)), quiet=TRUE)'
+.test-install-deps-win: &install-deps-win
+  - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'), repos=file.path("file://",getwd(),"bus/mirror-packages/cran"), quiet=TRUE)"
 
+## copy data.table tar.gz from bus R repo to current directory
 .test-cp-src: &cp-src
   - cp $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) .
 .test-cp-src-win: &cp-src-win
   - cp.exe $(ls.exe -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head.exe -n 1) .
 
+## move data.table tar.gz to bus
 .test-mv-src: &mv-src
   - mkdir -p bus/$CI_JOB_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME
 .test-mv-src-win: &mv-src-win
   - mkdir.exe -p bus/$CI_JOB_NAME; mv.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1) bus/$CI_JOB_NAME
 
+## move data.table binaries to bus R repo
+.test-mv-bin-win: &mv-bin-win
+  - mkdir.exe -p cran/bin/windows/contrib/$R_VERSION; mv.exe $(ls.exe -1t data.table_*.zip | head.exe -n 1) cran/bin/windows/contrib/$R_VERSION
+
+## remove data.table tar.gz
 .test-rm-src: &rm-src
   - rm $(ls -1t data.table_*.tar.gz | head -n 1)
 .test-rm-src-win: &rm-src-win
   - rm.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
 
-.test-mv-bin-win: &mv-bin-win
-  - mkdir.exe -p cran/bin/windows/contrib/$R_VERSION; mv.exe $(ls.exe -1t data.table_*.zip | head.exe -n 1) cran/bin/windows/contrib/$R_VERSION
-
+## install R and Rtools on windows
 .test-install-r-rel-win: &install-r-rel-win
   - curl.exe -s -o ../R-rel.exe https://cloud.r-project.org/bin/windows/base/old/4.3.2/R-4.3.2-win.exe; Start-Process -FilePath ..\R-rel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
 .test-install-r-devel-win: &install-r-devel-win
@@ -95,21 +110,6 @@ build: ## build data.table sources as tar.gz archive
   tags:
     - linux
 
-.test-cran-lin-template: &test-cran-lin
-  <<: *test-lin
-  variables:
-    _R_CHECK_CRAN_INCOMING_: "TRUE"
-    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
-  before_script:
-    - *install-deps
-    - *cp-src
-    - rm -r bus
-  script:
-    - *mv-src
-    - cd bus/$CI_JOB_NAME
-    - R CMD check --as-cran --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
-    - *rm-src
-
 .test-win-template: &test-win
   <<: *test
   tags:
@@ -121,10 +121,14 @@ build: ## build data.table sources as tar.gz archive
 #  tags:
 #    - macosx
 
-test-rel-lin: ## most comprehensive tests, force all suggests, also integration tests, using gcc -O3 -flto -fno-common -Wunused-result
+## most comprehensive tests
+# force all suggests
+# flags: gcc -O3 -flto -fno-common -Wunused-result
+# tests for compilation warnings
+test-rel-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
-  needs: ["mirror-packages","build"]  # "mirror-other-packages"
+  needs: ["mirror-packages","build"]
   variables:
     _R_CHECK_CRAN_INCOMING_: "FALSE"
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
@@ -132,7 +136,7 @@ test-rel-lin: ## most comprehensive tests, force all suggests, also integration
     _R_CHECK_TESTS_NLINES_: "0"
     OPENBLAS_MAIN_FREE: "1"
   before_script:
-    - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), quiet=TRUE)'
+    - *install-deps
     - *cp-src
     - rm -r bus
     - mkdir -p ~/.R
@@ -145,7 +149,14 @@ test-rel-lin: ## most comprehensive tests, force all suggests, also integration
     - *rm-src
     - (! grep "warning:" data.table.Rcheck/00install.out)
 
-test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, measure memory, using gcc -O0 -fno-openmp
+## vanilla minimal
+# TODO: no OS deps: libcurl zlib
+# no suggested deps
+# no vignettes or manuals
+# no openmp
+# measure memory
+# flags: gcc -O0 -fno-openmp
+test-rel-vanilla-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
   variables:
@@ -162,14 +173,17 @@ test-rel-vanilla-lin: ## minimal, no suggested deps, no vignettes or manuals, me
     - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)
     - *rm-src
 
-test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual thus not from cran-lin template
+## R-release on Linux
+# strict checks for 0 NOTEs
+# extra NOTEs check and build pdf manual thus not from cran-lin template
+test-rel-cran-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
   variables:
     _R_CHECK_CRAN_INCOMING_: "TRUE"           ## stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though)
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
-    _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## effective from R 4.1.0
-    #_R_CHECK_PKG_SIZES_THRESHOLD_: "7"## 'checking installed package size' NOTE - check if really needed before adding
+    _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000"
+    #_R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## 'checking installed package size' NOTE - possibly not yet needed bc -g0 (see flags below)
   before_script:
     - *install-deps
     - *cp-src
@@ -185,7 +199,12 @@ test-rel-cran-lin: ## R-release on Linux, extra NOTEs check and build pdf manual
     - >-
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: OK")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: OK"), " but ", shQuote(l)) else q("no")'
 
-test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier --disable-long-double, check for new notes and compilation warnings, thus allow_failure
+## R-devel on Linux
+# TODO: --enable-strict-barrier --disable-long-double
+# tests for compilation warnings
+# tests for new notes
+# thus allow_failure
+test-dev-cran-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-devel
   allow_failure: true
@@ -207,11 +226,24 @@ test-dev-cran-lin: ## R-devel on Linux, currently NOT: --enable-strict-barrier -
     - >-
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 3 NOTEs")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 3 NOTEs"), " (size of tarball, installed package size, top-level files) but ", shQuote(l)) else q("no")'
 
-test-310-cran-lin: ## R-3.1.0 on Linux, stated dependency of R
-  <<: *test-cran-lin
+## R 3.1.0
+# stated dependency on R
+test-310-cran-lin:
   image: registry.gitlab.com/jangorecki/dockerfiles/r-3.1.0
+  <<: *test-lin
+  before_script:
+    - *install-deps
+    - *cp-src
+    - rm -r bus
+  script:
+    - *mv-src
+    - cd bus/$CI_JOB_NAME
+    - R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
+    - *rm-src
 
-test-rel-win: ## R-release on Windows, test and build binaries
+## R-release on Windows
+# test and build binaries
+test-rel-win:
   <<: *test-win
   variables:
     R_VERSION: "$R_REL_VERSION"
@@ -219,7 +251,7 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - *install-r-rel-win
     - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
     - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH"
-    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)"
+    - *install-deps-win
     - *cp-src-win
     - rm.exe -r bus
   script:
@@ -230,7 +262,9 @@ test-rel-win: ## R-release on Windows, test and build binaries
     - *rm-src-win
     - *mv-bin-win
 
-test-dev-win: ## R-devel on Windows
+## R-devel on Windows
+# test and build binaries
+test-dev-win:
   <<: *test-win
   variables:
     R_VERSION: "$R_DEVEL_VERSION"
@@ -238,7 +272,7 @@ test-dev-win: ## R-devel on Windows
     - *install-r-devel-win
     - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
     - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH"
-    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)"
+    - *install-deps-win
     - *cp-src-win
     - rm.exe -r bus
   script:
@@ -249,7 +283,9 @@ test-dev-win: ## R-devel on Windows
     - *rm-src-win
     - *mv-bin-win
 
-test-old-win: ## R-oldrel on Windows
+## R-oldrel on Windows
+# test and build binaries
+test-old-win:
   <<: *test-win
   variables:
     R_VERSION: "$R_OLDREL_VERSION"
@@ -257,7 +293,7 @@ test-old-win: ## R-oldrel on Windows
     - *install-r-oldrel-win
     - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools42/files/rtools42-5355-5357.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools42" -NoNewWindow -Wait
     - $ENV:PATH = "C:\R\bin;C:\rtools42\usr\bin;$ENV:PATH"
-    - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='most'), quiet=TRUE)"
+    - *install-deps-win
     - *cp-src-win
     - rm.exe -r bus
   script:
@@ -268,7 +304,9 @@ test-old-win: ## R-oldrel on Windows
     - *rm-src-win
     - *mv-bin-win
 
-#test-rel-mac: ## R-release on MacOS, no macosx runner yet
+## R-release on MacOS
+# no macosx runner set yet
+#test-rel-mac:
 #  <<: *test-mac
 #  variables:
 #    R_VERSION: "$R_REL_VERSION"
@@ -286,7 +324,12 @@ test-old-win: ## R-oldrel on Windows
 #    - *rm-src
 #    - *mv-bin-mac
 
-integration: ## merging all artifacts to produce single R repository, documentation and website
+## integrate artifacts
+# merging package tarballs and binaries into single R repository
+# rendering documentation
+# setting up CRAN-like structure
+# generating pkgdown website
+integration:
   stage: integration
   image: registry.gitlab.com/jangorecki/dockerfiles/r-pkgdown
   tags:

From d7bde7021f1c475dcad8f9a0f010abeb74d56552 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 17:07:39 +0100
Subject: [PATCH 07/20] upgrade for structure from newer R version

---
 .ci/publish.R  | 2 +-
 .gitlab-ci.yml | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.ci/publish.R b/.ci/publish.R
index 526d9bd80d..16521fc0e6 100644
--- a/.ci/publish.R
+++ b/.ci/publish.R
@@ -148,7 +148,7 @@ lib.copy <- function(lib.from, repodir="bus/integration/cran"){
   pkg.copy <- function(pkg.from, lib.to) {
     pkg<-basename(pkg.from);
     dir.create(file.path(lib.to, pkg), recursive=TRUE)
-    lib.dirs<-intersect(c("html","doc"), all.lib.dirs<-list.dirs(pkg.from, full.names=FALSE))
+    lib.dirs<-intersect(c("help","html","doc"), all.lib.dirs<-list.dirs(pkg.from, full.names=FALSE))
     ans1<-setNames(file.copy(file.path(pkg.from, lib.dirs), file.path(lib.to, pkg), recursive=TRUE), lib.dirs)
     lib.files<-setdiff(list.files(pkg.from), all.lib.dirs)
     ans2<-setNames(file.copy(file.path(pkg.from, lib.files), file.path(lib.to, pkg)), lib.files)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e866d82db8..90b4184ed5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -339,6 +339,7 @@ integration:
     - glci5 ## testing before merge
   needs: ["mirror-packages","build","test-rel-lin","test-rel-cran-lin","test-dev-cran-lin","test-rel-vanilla-lin","test-310-cran-lin","test-rel-win","test-dev-win","test-old-win"]
   script:
+    - R --version
     - Rscript -e 'pkgdown::build_site(override=list(destination="./pkgdown"))'
     ## html manual, vignettes, repos, cran_web, cran_checks
     - echo 'source(".ci/ci.R"); source(".ci/publish.R")' >> .Rprofile

From ec80ed4142fde554b17f9cd4adcaf12231ffe2cf Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 17:25:24 +0100
Subject: [PATCH 08/20] fix windows quotes

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 90b4184ed5..20947aa000 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -68,7 +68,7 @@ build:
 .test-install-deps: &install-deps
   - Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), repos=file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE)), quiet=TRUE)'
 .test-install-deps-win: &install-deps-win
-  - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'), repos=file.path("file://",getwd(),"bus/mirror-packages/cran"), quiet=TRUE)"
+  - Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'), repos=file.path('file://',getwd(),'bus/mirror-packages/cran'), quiet=TRUE)"
 
 ## copy data.table tar.gz from bus R repo to current directory
 .test-cp-src: &cp-src

From 4611c02c288807e7337e556e006ab8c6741cdf6d Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 17:32:22 +0100
Subject: [PATCH 09/20] fix tidy html NOTE

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 20947aa000..c2fa53c061 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -185,6 +185,7 @@ test-rel-cran-lin:
     _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000"
     #_R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## 'checking installed package size' NOTE - possibly not yet needed bc -g0 (see flags below)
   before_script:
+    - apt-get update -q && apt-get install -y --no-install-recommends tidy ## TODO this should go inside r-builder image but tidying up images is not work so we keep it here for the moment
     - *install-deps
     - *cp-src
     - rm -r bus

From fa4d1e2f2578e05bf0363a8a333be4023e0f62a4 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 18:20:20 +0100
Subject: [PATCH 10/20] clean unused stuff

---
 .ci/Dockerfile.in | 9 ---------
 1 file changed, 9 deletions(-)
 delete mode 100644 .ci/Dockerfile.in

diff --git a/.ci/Dockerfile.in b/.ci/Dockerfile.in
deleted file mode 100644
index 559bb9a40a..0000000000
--- a/.ci/Dockerfile.in
+++ /dev/null
@@ -1,9 +0,0 @@
-FROM registry.gitlab.com/jangorecki/dockerfiles/SRC_IMAGE_NAME
-
-MAINTAINER Jan Gorecki j.gorecki@wit.edu.pl
-
-COPY bus/build/cran/ /cran/
-
-RUN Rscript -e 'install.packages("data.table", repos=file.path("file:","cran"))'
-
-CMD ["R"]

From 63de8ba54d25e6b9fdae99ba94ded2497343cfcf Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 20:20:37 +0100
Subject: [PATCH 11/20] publish

---
 .gitlab-ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c2fa53c061..4b0d93d42d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -183,15 +183,15 @@ test-rel-cran-lin:
     _R_CHECK_CRAN_INCOMING_: "TRUE"           ## stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though)
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
     _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000"
-    #_R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## 'checking installed package size' NOTE - possibly not yet needed bc -g0 (see flags below)
+    _R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## 'checking installed package size' NOTE
   before_script:
     - apt-get update -q && apt-get install -y --no-install-recommends tidy ## TODO this should go inside r-builder image but tidying up images is not work so we keep it here for the moment
     - *install-deps
     - *cp-src
     - rm -r bus
     - mkdir -p ~/.R
-    - echo 'CFLAGS=-g0 -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2'> ~/.R/Makevars ## -g0 because -g increases datatable.so size from 0.5MB to 1.5MB and breaches 'installed package size <= 5MB' note
-    - echo 'CXXFLAGS=-g0 -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
+    - echo 'CFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2'> ~/.R/Makevars
+    - echo 'CXXFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
   script:
     - *mv-src
     - cd bus/$CI_JOB_NAME
@@ -269,6 +269,7 @@ test-dev-win:
   <<: *test-win
   variables:
     R_VERSION: "$R_DEVEL_VERSION"
+  allow_failure: true ## temporarily as we want to publish to GLCI
   before_script:
     - *install-r-devel-win
     - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
@@ -337,7 +338,6 @@ integration:
     - linux
   only:
     - master
-    - glci5 ## testing before merge
   needs: ["mirror-packages","build","test-rel-lin","test-rel-cran-lin","test-dev-cran-lin","test-rel-vanilla-lin","test-310-cran-lin","test-rel-win","test-dev-win","test-old-win"]
   script:
     - R --version

From 67f0985bb2be26a9c0c05eb1614cd4c3bfe91d77 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Fri, 24 Nov 2023 21:10:24 +0100
Subject: [PATCH 12/20] move to new images

---
 .gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4b0d93d42d..480a8c253c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -32,7 +32,7 @@ mirror-packages:
   stage: dependencies
   tags:
     - linux
-  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
+  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-minimal
   cache:
     paths:
       - bus/$CI_JOB_NAME/cran
@@ -50,7 +50,7 @@ build:
   stage: build
   tags:
     - linux
-  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
+  image: registry.gitlab.com/jangorecki/dockerfiles/r-base
   needs: ["mirror-packages"]
   before_script:
     - Rscript -e 'install.packages(c("knitr","rmarkdown"), repos=file.path("file:",normalizePath("bus/mirror-packages/cran")), quiet=TRUE)'

From 2db6ea7044604912aaad68759a5535ed5bfb2063 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Sat, 25 Nov 2023 10:37:09 +0100
Subject: [PATCH 13/20] moving more to new images

---
 .gitlab-ci.yml | 47 +++++++++++++++++++++++++++++++----------------
 1 file changed, 31 insertions(+), 16 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 480a8c253c..5456b979a4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -50,7 +50,7 @@ build:
   stage: build
   tags:
     - linux
-  image: registry.gitlab.com/jangorecki/dockerfiles/r-base
+  image: registry.gitlab.com/jangorecki/dockerfiles/r-base ## r-base-gcc after rstudio/markdown#108
   needs: ["mirror-packages"]
   before_script:
     - Rscript -e 'install.packages(c("knitr","rmarkdown"), repos=file.path("file:",normalizePath("bus/mirror-packages/cran")), quiet=TRUE)'
@@ -92,7 +92,7 @@ build:
 .test-rm-src-win: &rm-src-win
   - rm.exe $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)
 
-## install R and Rtools on windows
+## install R on windows
 .test-install-r-rel-win: &install-r-rel-win
   - curl.exe -s -o ../R-rel.exe https://cloud.r-project.org/bin/windows/base/old/4.3.2/R-4.3.2-win.exe; Start-Process -FilePath ..\R-rel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
 .test-install-r-devel-win: &install-r-devel-win
@@ -100,6 +100,12 @@ build:
 .test-install-r-oldrel-win: &install-r-oldrel-win
   - curl.exe -s -o ../R-oldrel.exe https://cloud.r-project.org/bin/windows/base/old/4.2.3/R-4.2.3-win.exe; Start-Process -FilePath ..\R-oldrel.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait
 
+## install Rtools on windows
+.test-install-rtools42-win: &install-rtools42-win
+  - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools42/files/rtools42-5355-5357.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools42" -NoNewWindow -Wait
+.test-install-rtools43-win: &install-rtools43-win
+  - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
+
 .test-template: &test
   stage: test
   needs: ["mirror-packages","build"]
@@ -127,7 +133,7 @@ build:
 # tests for compilation warnings
 test-rel-lin:
   <<: *test-lin
-  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
+  image: registry.gitlab.com/jangorecki/dockerfiles/r-base
   needs: ["mirror-packages","build"]
   variables:
     _R_CHECK_CRAN_INCOMING_: "FALSE"
@@ -136,6 +142,7 @@ test-rel-lin:
     _R_CHECK_TESTS_NLINES_: "0"
     OPENBLAS_MAIN_FREE: "1"
   before_script:
+    - apt-get update -q && apt-get install -y --no-install-recommends zlib1g-dev ## remove after #5753
     - *install-deps
     - *cp-src
     - rm -r bus
@@ -150,7 +157,7 @@ test-rel-lin:
     - (! grep "warning:" data.table.Rcheck/00install.out)
 
 ## vanilla minimal
-# TODO: no OS deps: libcurl zlib
+# TODO: remove install zlib
 # no suggested deps
 # no vignettes or manuals
 # no openmp
@@ -158,10 +165,11 @@ test-rel-lin:
 # flags: gcc -O0 -fno-openmp
 test-rel-vanilla-lin:
   <<: *test-lin
-  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
+  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-gcc
   variables:
     TEST_DATA_TABLE_MEMTEST: "1"
   before_script:
+    - apt-get update -q && apt-get install -y --no-install-recommends zlib1g-dev ## remove after #5753
     - *cp-src
     - rm -r bus
     - mkdir -p ~/.R
@@ -178,19 +186,19 @@ test-rel-vanilla-lin:
 # extra NOTEs check and build pdf manual thus not from cran-lin template
 test-rel-cran-lin:
   <<: *test-lin
-  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
+  image: registry.gitlab.com/jangorecki/dockerfiles/r-base
   variables:
     _R_CHECK_CRAN_INCOMING_: "TRUE"           ## stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though)
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
-    _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000"
-    _R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## 'checking installed package size' NOTE
+    _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## bytes
+    _R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## MB 'checking installed package size' NOTE
   before_script:
-    - apt-get update -q && apt-get install -y --no-install-recommends tidy ## TODO this should go inside r-builder image but tidying up images is not work so we keep it here for the moment
+    - apt-get update -q && apt-get install -y --no-install-recommends zlib1g-dev ## remove after #5753
     - *install-deps
     - *cp-src
     - rm -r bus
     - mkdir -p ~/.R
-    - echo 'CFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2'> ~/.R/Makevars
+    - echo 'CFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars
     - echo 'CXXFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars
   script:
     - *mv-src
@@ -215,6 +223,7 @@ test-dev-cran-lin:
     _R_S3_METHOD_LOOKUP_BASEENV_AFTER_GLOBALENV_: "FALSE" ## detects S3 method lookup found on search path #4777
     _R_S3_METHOD_LOOKUP_REPORT_SEARCH_PATH_USES_: "TRUE"
   before_script:
+    - apt-get update -q && apt-get install -y --no-install-recommends zlib1g-dev ## remove after #5753
     - *install-deps
     - *cp-src
     - rm -r bus
@@ -224,7 +233,7 @@ test-dev-cran-lin:
     - R CMD check --as-cran --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
     - *rm-src
     - (! grep "warning:" data.table.Rcheck/00install.out)
-    - >-
+    - >- ## this likely need an update but check fails now on complex NA so CI is not reaching here anyway
         Rscript -e 'l=tail(readLines("data.table.Rcheck/00check.log"), 1L); if (!identical(l, "Status: 3 NOTEs")) stop("Last line of ", shQuote("00check.log"), " is not ", shQuote("Status: 3 NOTEs"), " (size of tarball, installed package size, top-level files) but ", shQuote(l)) else q("no")'
 
 ## R 3.1.0
@@ -250,7 +259,7 @@ test-rel-win:
     R_VERSION: "$R_REL_VERSION"
   before_script:
     - *install-r-rel-win
-    - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
+    - *install-rtools43-win
     - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH"
     - *install-deps-win
     - *cp-src-win
@@ -272,7 +281,7 @@ test-dev-win:
   allow_failure: true ## temporarily as we want to publish to GLCI
   before_script:
     - *install-r-devel-win
-    - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools43/files/rtools43-5863-5818.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools43" -NoNewWindow -Wait
+    - *install-rtools43-win
     - $ENV:PATH = "C:\R\bin;C:\rtools43\usr\bin;$ENV:PATH"
     - *install-deps-win
     - *cp-src-win
@@ -293,7 +302,7 @@ test-old-win:
     R_VERSION: "$R_OLDREL_VERSION"
   before_script:
     - *install-r-oldrel-win
-    - curl.exe -s -o ../rtools.exe https://cloud.r-project.org/bin/windows/Rtools/rtools42/files/rtools42-5355-5357.exe; Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools42" -NoNewWindow -Wait
+    - *install-rtools42-win
     - $ENV:PATH = "C:\R\bin;C:\rtools42\usr\bin;$ENV:PATH"
     - *install-deps-win
     - *cp-src-win
@@ -338,6 +347,7 @@ integration:
     - linux
   only:
     - master
+    - glci5
   needs: ["mirror-packages","build","test-rel-lin","test-rel-cran-lin","test-dev-cran-lin","test-rel-vanilla-lin","test-310-cran-lin","test-rel-win","test-dev-win","test-old-win"]
   script:
     - R --version
@@ -408,7 +418,12 @@ integration:
     - mv tmpbus bus
   <<: *artifacts
 
-pages: ## publish R repository, test jobs summaries, html documentation of all packages in repo, pkgdown
+## publish
+# R repository
+# test jobs summaries
+# html documentation of all packages in repo
+# pkgdown website
+pages:
   stage: deploy
   environment: production
   tags:
@@ -421,7 +436,7 @@ pages: ## publish R repository, test jobs summaries, html documentation of all p
     - mkdir -p public
     - cp -r bus/integration/cran/* public
     - cat public/src/contrib/PACKAGES
-  artifacts: ## publish only when no failure
+  artifacts:
     expire_in: 2 weeks
     paths:
       - public

From f7cf939e2d9d32cd0983c0373e6bb991c246ecd4 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Sat, 25 Nov 2023 11:18:22 +0100
Subject: [PATCH 14/20] clean zlib optional, move memtest

---
 .gitlab-ci.yml | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5456b979a4..2256813843 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -131,9 +131,10 @@ build:
 # force all suggests
 # flags: gcc -O3 -flto -fno-common -Wunused-result
 # tests for compilation warnings
+# measure memory usage during tests
 test-rel-lin:
   <<: *test-lin
-  image: registry.gitlab.com/jangorecki/dockerfiles/r-base
+  image: registry.gitlab.com/jangorecki/dockerfiles/r-data.table
   needs: ["mirror-packages","build"]
   variables:
     _R_CHECK_CRAN_INCOMING_: "FALSE"
@@ -141,8 +142,9 @@ test-rel-lin:
     _R_CHECK_FORCE_SUGGESTS_: "TRUE"
     _R_CHECK_TESTS_NLINES_: "0"
     OPENBLAS_MAIN_FREE: "1"
+    TEST_DATA_TABLE_MEMTEST: "1"
   before_script:
-    - apt-get update -q && apt-get install -y --no-install-recommends zlib1g-dev ## remove after #5753
+    - apt-get update -q && apt-get install -y --no-install-recommends libcurl4-openssl-dev ## remove after #5749
     - *install-deps
     - *cp-src
     - rm -r bus
@@ -157,19 +159,14 @@ test-rel-lin:
     - (! grep "warning:" data.table.Rcheck/00install.out)
 
 ## vanilla minimal
-# TODO: remove install zlib
 # no suggested deps
 # no vignettes or manuals
 # no openmp
-# measure memory
 # flags: gcc -O0 -fno-openmp
 test-rel-vanilla-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-base-gcc
-  variables:
-    TEST_DATA_TABLE_MEMTEST: "1"
   before_script:
-    - apt-get update -q && apt-get install -y --no-install-recommends zlib1g-dev ## remove after #5753
     - *cp-src
     - rm -r bus
     - mkdir -p ~/.R
@@ -193,7 +190,6 @@ test-rel-cran-lin:
     _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## bytes
     _R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## MB 'checking installed package size' NOTE
   before_script:
-    - apt-get update -q && apt-get install -y --no-install-recommends zlib1g-dev ## remove after #5753
     - *install-deps
     - *cp-src
     - rm -r bus
@@ -223,7 +219,6 @@ test-dev-cran-lin:
     _R_S3_METHOD_LOOKUP_BASEENV_AFTER_GLOBALENV_: "FALSE" ## detects S3 method lookup found on search path #4777
     _R_S3_METHOD_LOOKUP_REPORT_SEARCH_PATH_USES_: "TRUE"
   before_script:
-    - apt-get update -q && apt-get install -y --no-install-recommends zlib1g-dev ## remove after #5753
     - *install-deps
     - *cp-src
     - rm -r bus

From fa1f1b57c209625a14e178488e72dcd6be94e3a2 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Sat, 25 Nov 2023 12:27:02 +0100
Subject: [PATCH 15/20] more allow failures for the moment

---
 .gitlab-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2256813843..4d7971b662 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -166,6 +166,7 @@ test-rel-lin:
 test-rel-vanilla-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-base-gcc
+  allow_failure: true ## temporarily as we want to publish to GLCI
   before_script:
     - *cp-src
     - rm -r bus
@@ -189,7 +190,9 @@ test-rel-cran-lin:
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
     _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## bytes
     _R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## MB 'checking installed package size' NOTE
+  allow_failure: true ## temporarily as we want to publish to GLCI
   before_script:
+    - apt-get update -q && apt-get install -y --no-install-recommends libcurl4-openssl-dev ## remove after #5749
     - *install-deps
     - *cp-src
     - rm -r bus

From aa83ec88e5643933d1d8d9126bdaf19f5024bfa6 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Sat, 25 Nov 2023 12:45:11 +0100
Subject: [PATCH 16/20] comments and remove one allow fail

---
 .gitlab-ci.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4d7971b662..ad24fcf047 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -166,7 +166,7 @@ test-rel-lin:
 test-rel-vanilla-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-base-gcc
-  allow_failure: true ## temporarily as we want to publish to GLCI
+  allow_failure: true ## temp workaround #5484
   before_script:
     - *cp-src
     - rm -r bus
@@ -190,9 +190,8 @@ test-rel-cran-lin:
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
     _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## bytes
     _R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## MB 'checking installed package size' NOTE
-  allow_failure: true ## temporarily as we want to publish to GLCI
   before_script:
-    - apt-get update -q && apt-get install -y --no-install-recommends libcurl4-openssl-dev ## remove after #5749
+    - apt-get update -q && apt-get install -y --no-install-recommends libcurl4-openssl-dev ## temp workaround #5749
     - *install-deps
     - *cp-src
     - rm -r bus
@@ -215,7 +214,7 @@ test-rel-cran-lin:
 test-dev-cran-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-devel
-  allow_failure: true
+  allow_failure: true ## to not be blocked by changes in r-devel
   variables:
     _R_CHECK_CRAN_INCOMING_: "TRUE"
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
@@ -276,7 +275,7 @@ test-dev-win:
   <<: *test-win
   variables:
     R_VERSION: "$R_DEVEL_VERSION"
-  allow_failure: true ## temporarily as we want to publish to GLCI
+  allow_failure: true ## temp workaround #5748
   before_script:
     - *install-r-devel-win
     - *install-rtools43-win

From fc4575e59d8b58e2fb489ad7e42f9ae681e745e5 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Sat, 25 Nov 2023 13:36:00 +0100
Subject: [PATCH 17/20] pass known issue

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ad24fcf047..48f075619c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -190,6 +190,7 @@ test-rel-cran-lin:
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
     _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## bytes
     _R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## MB 'checking installed package size' NOTE
+  allow_failure: true ## temp workaround #5760
   before_script:
     - apt-get update -q && apt-get install -y --no-install-recommends libcurl4-openssl-dev ## temp workaround #5749
     - *install-deps

From 4601fcac18d21d7ec987e9ec88772b833692ad6a Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Sat, 25 Nov 2023 14:20:07 +0100
Subject: [PATCH 18/20] allow correct job

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 48f075619c..ed317dca68 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -136,6 +136,7 @@ test-rel-lin:
   <<: *test-lin
   image: registry.gitlab.com/jangorecki/dockerfiles/r-data.table
   needs: ["mirror-packages","build"]
+  allow_failure: true ## temp workaround #5760
   variables:
     _R_CHECK_CRAN_INCOMING_: "FALSE"
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
@@ -190,7 +191,6 @@ test-rel-cran-lin:
     _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"   ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284
     _R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## bytes
     _R_CHECK_PKG_SIZES_THRESHOLD_: "7"        ## MB 'checking installed package size' NOTE
-  allow_failure: true ## temp workaround #5760
   before_script:
     - apt-get update -q && apt-get install -y --no-install-recommends libcurl4-openssl-dev ## temp workaround #5749
     - *install-deps

From 5b17bb59ac9f061c76db612f2232d3ff79b529b4 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Sat, 25 Nov 2023 14:20:41 +0100
Subject: [PATCH 19/20] ready to merge

---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ed317dca68..97b4306321 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -345,7 +345,6 @@ integration:
     - linux
   only:
     - master
-    - glci5
   needs: ["mirror-packages","build","test-rel-lin","test-rel-cran-lin","test-dev-cran-lin","test-rel-vanilla-lin","test-310-cran-lin","test-rel-win","test-dev-win","test-old-win"]
   script:
     - R --version

From 8609427fe1d03adcd08b83477da502dd36fce5e5 Mon Sep 17 00:00:00 2001
From: jangorecki <j.gorecki@wit.edu.pl>
Date: Sat, 25 Nov 2023 15:10:33 +0100
Subject: [PATCH 20/20] binaries may not be available

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 97b4306321..18f821b43d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -371,7 +371,7 @@ integration:
     - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="source"), type="source", fields="Revision", addFiles=TRUE)'
     ## publish binaries
     - Rscript -e 'move.bin("test-rel-win", Sys.getenv("R_REL_VERSION"), os.type="windows")'
-    - Rscript -e 'move.bin("test-dev-win", Sys.getenv("R_DEVEL_VERSION"), os.type="windows")'
+    - Rscript -e 'move.bin("test-dev-win", Sys.getenv("R_DEVEL_VERSION"), os.type="windows", silent=TRUE)'
     - Rscript -e 'move.bin("test-old-win", Sys.getenv("R_OLDREL_VERSION"), os.type="windows")'
     - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_REL_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'
     - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_DEVEL_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'