Merge branch 'master' into fix-measure-cols-doc

.appveyor.yml

@@ -16,18 +16,14 @@ environment:
   global:
     CRAN: http://cloud.r-project.org
     WARNINGS_ARE_ERRORS: 1
-    R_CHECK_ARGS: --no-manual --no-multiarch
-    R_ARCH: i386
-# R_CHECK_ARGS specified in order to turn off --as-cran (on by default) as that can be slow
-# multiarch is on by default which (when R_ARCH: x64) compiles and tests both 32bit and 64bit in one x64 job
-# --no-multiarch so as to not run both 32bit and 64bit on every commit in PRs to save dev cycle time; GLCI after merge is full-strength
-# GHA has MacOS 64bit (test-coverage) and Ubuntu 64bit, therefore picked 32bit for Windows
-    GCC_PATH: mingw_64
-# Default GCC_PATH appears to be gcc-4.6.3 which is now unsupported as from Rtools.exe v3.4.
+    R_CHECK_ARGS: --as-cran --no-manual
+# --no-manual to avoid error 'pdflatex is not available'
+# --as-cran no longer a lot slower (now takes under 6 mins with and without); logs show _R_CHECK_CRAN_INCOMING_=FALSE which could take 5+ mins
     _R_CHECK_NO_STOP_ON_TEST_ERROR_: true
 # continue tests even if some script failed
     _R_CHECK_TESTS_NLINES_: 0
 # Block truncation of any error messages in R CMD check
+# R is 64-bit only on Windows from 4.2.0 (prior default was build and test both 32bit and 64bit) so we no longer use R_ARCH to pick one to reduce CI time in PRs
 
   matrix:
 

.ci/ci.R

@@ -47,7 +47,8 @@ function (repos, type = getOption("pkgType"), ver)
 dcf.dependencies <-
 function(file = "DESCRIPTION",
          which = NA,
-         except.priority = "base") {
+         except.priority = "base",
+         exclude = NULL) {
   if (!is.character(file) || !length(file) || !all(file.exists(file)))
     stop("file argument must be character of filepath(s) to existing DESCRIPTION file(s)")
   if (!is.character(except.priority))
@@ -79,7 +80,13 @@ function(file = "DESCRIPTION",
   }
   x <- unlist(lapply(x, local.extract_dependency_package_names))
   except <- if (length(except.priority)) c("R", unlist(tools:::.get_standard_package_names()[except.priority], use.names = FALSE))
-  setdiff(x, except)
+  x = setdiff(x, except)
+  if (length(exclude)) {  # to exclude knitr/rmarkdown, 5294
+    if (!is.character(exclude) || anyDuplicated(exclude))
+      stop("exclude may be NULL or a character vector containing no duplicates")
+    x = setdiff(x, exclude)
+  }
+  x
 }
 
 ## returns additional repositories for dependency packages based on its DESCRIPTION file

.dev/.bash_aliases

@@ -8,14 +8,16 @@ alias gdm='git difftool master &> /dev/null'
 # If meld has scrolling issues, turn off GTK animation which I don't need:
 #   https://gitlab.gnome.org/GNOME/meld/-/issues/479#note_866040
 
+alias perfbar=~/build/gtk_perfbar/linux_perfbar   # revdep.R; https://github.com/tomkraljevic/gtk_perfbar
+
 alias Rdevel='~/build/R-devel/bin/R --vanilla'
 alias Rdevel-strict-gcc='~/build/R-devel-strict-gcc/bin/R --vanilla'
 alias Rdevel-strict-clang='~/build/R-devel-strict-clang/bin/R --vanilla'
 alias Rdevel-valgrind='~/build/R-devel-valgrind/bin/R --vanilla'
 alias Rdevel32='~/build/32bit/R-devel/bin/R --vanilla'
 alias R310='~/build/R-3.1.0/bin/R --vanilla'
 
-alias revdepsh='cd ~/build/revdeplib/ && export TZ=UTC && export R_LIBS_SITE=none && export R_LIBS=~/build/revdeplib/ && export _R_CHECK_FORCE_SUGGESTS_=true'
+alias revdepsh='cd ~/build/revdeplib/ && export TZ=UTC && export R_LIBS_SITE=NULL && export R_LIBS=~/build/revdeplib/ && export _R_CHECK_FORCE_SUGGESTS_=true'
 alias revdepr='revdepsh; R_PROFILE_USER=~/GitHub/data.table/.dev/revdep.R R'
 # use ~/build/R-devel/bin/R at the end of revdepr to use R-devel instead of R-release.
 # If so, doing a `rm -rf *` in revdeplib first to rebuild everything is easiest way to avoid potential problems later. A full rebuild is a good idea periodically anyway. Packages in

.dev/CRAN_Release.cmd

@@ -185,7 +185,7 @@ grep -En "for\s*[(]\s*[a-zA-Z0-9_]+\s*=" src/*.c | grep -Fv "#loop_counter_not_l
 
 cd ..
 R
-cc(test=TRUE, clean=TRUE, CC="gcc-10")  # to compile with -pedandic -Wall, latest gcc as CRAN: https://cran.r-project.org/web/checks/check_flavors.html
+cc(test=TRUE, clean=TRUE, CC="gcc-12")  # to compile with -pedandic -Wall, latest gcc as CRAN: https://cran.r-project.org/web/checks/check_flavors.html
 saf = options()$stringsAsFactors
 options(stringsAsFactors=!saf)    # check tests (that might be run by user) are insensitive to option, #2718
 test.data.table()
@@ -195,15 +195,15 @@ R CMD build .
 export GITHUB_PAT="f1c.. github personal access token ..7ad"
 # avoids many too-many-requests in --as-cran's ping-all-URLs step (20 mins) inside the `checking CRAN incoming feasibility...` step.
 # Many thanks to Dirk for the tipoff that setting this env variable solves the problem, #4832.
-R CMD check data.table_1.14.1.tar.gz --as-cran
-R CMD INSTALL data.table_1.14.1.tar.gz --html
+R CMD check data.table_1.14.9.tar.gz --as-cran
+R CMD INSTALL data.table_1.14.9.tar.gz --html
 
 # Test C locale doesn't break test suite (#2771)
 echo LC_ALL=C > ~/.Renviron
 R
 Sys.getlocale()=="C"
 q("no")
-R CMD check data.table_1.14.1.tar.gz
+R CMD check data.table_1.14.9.tar.gz
 rm ~/.Renviron
 
 # Test non-English does not break test.data.table() due to translation of messages; #3039, #630
@@ -220,16 +220,24 @@ q("no")
 
 # User supplied PKG_CFLAGS and PKG_LIBS passed through, #4664
 # Next line from https://mac.r-project.org/openmp/. Should see the arguments passed through and then fail with gcc on linux.
-PKG_CFLAGS='-Xclang -fopenmp' PKG_LIBS=-lomp R CMD INSTALL data.table_1.14.1.tar.gz
+PKG_CFLAGS='-Xclang -fopenmp' PKG_LIBS=-lomp R CMD INSTALL data.table_1.14.9.tar.gz
 # Next line should work on Linux, just using superfluous and duplicate but valid parameters here to see them retained and work 
-PKG_CFLAGS='-fopenmp' PKG_LIBS=-lz R CMD INSTALL data.table_1.14.1.tar.gz
+PKG_CFLAGS='-fopenmp' PKG_LIBS=-lz R CMD INSTALL data.table_1.14.9.tar.gz
 
 R
 remove.packages("xml2")    # we checked the URLs; don't need to do it again (many minutes)
 require(data.table)
+f1 = tempfile()
+f2 = tempfile()
+suppressWarnings(try(rm(list=c(".Last",".Random.seed"))))
+save.image(f1)
 test.data.table(script="other.Rraw")
 test.data.table(script="*.Rraw")
 test.data.table(verbose=TRUE)   # since main.R no longer tests verbose mode
+suppressWarnings(try(rm(list=c(".Last",".Random.seed"))))
+save.image(f2)
+system(paste("diff",f1,f2))  # to detect any changes to .GlobalEnv, #5514
+# print(load(f1)); print(load(f2)) # run if diff found any difference
 
 # check example() works on every exported function, with these sticter options too, and also that all help pages have examples
 options(warn=2, warnPartialMatchArgs=TRUE, warnPartialMatchAttr=TRUE, warnPartialMatchDollar=TRUE)
@@ -258,7 +266,7 @@ alias R310=~/build/R-3.1.0/bin/R
 ### END ONE TIME BUILD
 
 cd ~/GitHub/data.table
-R310 CMD INSTALL ./data.table_1.14.1.tar.gz
+R310 CMD INSTALL ./data.table_1.14.9.tar.gz
 R310
 require(data.table)
 test.data.table(script="*.Rraw")
@@ -270,15 +278,15 @@ test.data.table(script="*.Rraw")
 vi ~/.R/Makevars
 # Make line SHLIB_OPENMP_CFLAGS= active to remove -fopenmp
 R CMD build .
-R CMD INSTALL data.table_1.14.1.tar.gz   # ensure that -fopenmp is missing and there are no warnings
+R CMD INSTALL data.table_1.14.9.tar.gz   # ensure that -fopenmp is missing and there are no warnings
 R
 require(data.table)   # observe startup message about no OpenMP detected
 test.data.table()
 q("no")
 vi ~/.R/Makevars
 # revert change above
 R CMD build .
-R CMD check data.table_1.14.1.tar.gz
+R CMD check data.table_1.14.9.tar.gz
 
 
 #####################################################
@@ -289,25 +297,30 @@ cd ~/build
 wget -N https://stat.ethz.ch/R/daily/R-devel.tar.gz
 rm -rf R-devel
 rm -rf R-devel-strict-*
-tar xvf R-devel.tar.gz
+tar xf R-devel.tar.gz
 mv R-devel R-devel-strict-gcc
-tar xvf R-devel.tar.gz
+tar xf R-devel.tar.gz
 mv R-devel R-devel-strict-clang
-tar xvf R-devel.tar.gz
+tar xf R-devel.tar.gz
 
+sudo apt-get -y build-dep r-base
 cd R-devel  # may be used for revdep testing: .dev/revdep.R.
 # important to change directory name before building not after because the path is baked into the build, iiuc
 ./configure CFLAGS="-O0 -Wall -pedantic"
 make
 
 # use latest available `apt-cache search gcc-` or `clang-`
+# wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
+# sudo add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main'
+# sudo apt-get install clang-15
+
 cd ~/build/R-devel-strict-clang
-./configure --without-recommended-packages --disable-byte-compiled-packages --enable-strict-barrier --disable-long-double CC="clang-11 -fsanitize=undefined,address -fno-sanitize=float-divide-by-zero -fno-omit-frame-pointer"
+./configure --without-recommended-packages --disable-byte-compiled-packages --enable-strict-barrier --disable-long-double CC="clang-15 -fsanitize=undefined,address -fno-sanitize=float-divide-by-zero -fno-sanitize=alignment -fno-omit-frame-pointer" CFLAGS="-g -O3 -Wall -pedantic"
 make
 
 cd ~/build/R-devel-strict-gcc
-# gcc-10 (in dev currently) failed to build R, so using regular gcc-9 (9.3.0 as per focal/Pop!_OS 20.04)
-./configure --without-recommended-packages --disable-byte-compiled-packages --disable-openmp --enable-strict-barrier --disable-long-double CC="gcc-9 -fsanitize=undefined,address -fno-sanitize=float-divide-by-zero -fno-omit-frame-pointer"
+# gcc-10 failed to build R-devel at some point, so using regular gcc-9 (9.3.0 as per focal/Pop!_OS 20.04)
+./configure --without-recommended-packages --disable-byte-compiled-packages --disable-openmp --enable-strict-barrier --disable-long-double CC="gcc-11 -fsanitize=undefined,address -fno-sanitize=float-divide-by-zero -fno-omit-frame-pointer"
 make
 
 # See R-exts#4.3.3
@@ -328,15 +341,23 @@ alias Rdevel-strict-gcc='~/build/R-devel-strict-gcc/bin/R --vanilla'
 alias Rdevel-strict-clang='~/build/R-devel-strict-clang/bin/R --vanilla'
 
 cd ~/GitHub/data.table
-Rdevel-strict-gcc CMD INSTALL data.table_1.14.1.tar.gz
-Rdevel-strict-clang CMD INSTALL data.table_1.14.1.tar.gz
-# Check UBSAN and ASAN flags appear in compiler output above. Rdevel was compiled with them so should be passed through to here
-Rdevel-strict-gcc
-Rdevel-strict-clang  # repeat below with clang and gcc
+Rdevel-strict-[gcc|clang] CMD INSTALL data.table_1.14.9.tar.gz
+# Check UBSAN and ASAN flags appear in compiler output above. Rdevel was compiled with them so they should be
+# passed through to here. However, our configure script seems to get in the way and gets them from {R_HOME}/bin/R
+# So I needed to edit my ~/.R/Makevars to get CFLAGS the way I needed.
+Rdevel-strict-[gcc|clang] CMD check data.table_1.14.9.tar.gz
+# Use the (failed) output to get the list of currently needed packages and install them
+Rdevel-strict-[gcc|clang]
 isTRUE(.Machine$sizeof.longdouble==0)  # check noLD is being tested
 options(repos = "http://cloud.r-project.org")
-install.packages(c("bit64","xts","nanotime","R.utils","yaml")) # minimum packages needed to not skip any tests in test.data.table()
-# install.packages(c("curl","knitr"))                          # for `R CMD check` when not strict. Too slow to install when strict
+install.packages(c("bit64", "bit", "curl", "R.utils", "xts","nanotime", "zoo", "yaml", "knitr", "rmarkdown", "markdown"),
+                 Ncpus=4)
+# Issue #5491 showed that CRAN is running UBSAN on .Rd examples which found an error so we now run full R CMD check
+q("no")
+Rdevel-strict-[gcc|clang] CMD check data.table_1.14.9.tar.gz
+# UBSAN errors occur on stderr and don't affect R CMD check result. Made many failed attempts to capture them. So grep for them. 
+find data.table.Rcheck -name "*Rout*" -exec grep -H "runtime error" {} \;
+
 require(data.table)
 test.data.table(script="*.Rraw") # 7 mins (vs 1min normally) under UBSAN, ASAN and --strict-barrier
                                  # without the fix in PR#3515, the --disable-long-double lumped into this build does now work and correctly reproduces the noLD problem
@@ -370,7 +391,7 @@ cd R-devel-valgrind
 make
 cd ~/GitHub/data.table
 vi ~/.R/Makevars  # make the -O2 -g line active, for info on source lines with any problems
-Rdevel-valgrind CMD INSTALL data.table_1.14.1.tar.gz
+Rdevel-valgrind CMD INSTALL data.table_1.14.9.tar.gz
 R_DONT_USE_TK=true Rdevel-valgrind -d "valgrind --tool=memcheck --leak-check=full --track-origins=yes --show-leak-kinds=definite,possible --gen-suppressions=all --suppressions=./.dev/valgrind.supp -s"
 # the default for --show-leak-kinds is 'definite,possible' which we're setting explicitly here as a reminder. CRAN uses the default too.
 #   including 'reachable' (as 'all' does) generates too much output from R itself about by-design permanent blocks
@@ -408,7 +429,7 @@ cd ~/build/rchk/trunk
 . ../scripts/config.inc
 . ../scripts/cmpconfig.inc
 vi ~/.R/Makevars   # set CFLAGS=-O0 -g so that rchk can provide source line numbers
-echo 'install.packages("~/GitHub/data.table/data.table_1.14.1.tar.gz",repos=NULL)' | ./bin/R --slave
+echo 'install.packages("~/GitHub/data.table/data.table_1.14.9.tar.gz",repos=NULL)' | ./bin/R --slave
 # objcopy warnings (if any) can be ignored: https://github.com/kalibera/rchk/issues/17#issuecomment-497312504
 . ../scripts/check_package.sh data.table
 cat packages/lib/data.table/libs/*check
@@ -469,14 +490,15 @@ shutdown now   # doesn't return you to host prompt properly so just kill the win
 #  Downstream dependencies
 ###############################################
 
-# IF NOT ALREADY INSTALLED
+# IF NOT ALREADY INSTALLED, OR AFTER AN OS UPGRADE
+# No harm rerunning these commands; they do not reinstall if already latest version
 sudo apt-get update
 sudo apt-get -y install htop
 sudo apt-get -y install r-base r-base-dev
 sudo apt-get -y build-dep r-base-dev
 sudo apt-get -y build-dep qpdf
 sudo apt-get -y install aptitude
-sudo aptitude -y build-dep r-cran-rgl   # leads to libglu1-mesa-dev
+sudo apt-get -y build-dep r-cran-rgl   # leads to libglu1-mesa-dev
 sudo apt-get -y build-dep r-cran-rmpi
 sudo apt-get -y build-dep r-cran-cairodevice
 sudo apt-get -y build-dep r-cran-tkrplot
@@ -524,6 +546,8 @@ sudo apt-get -y install libgit2-dev  # for gert
 sudo apt-get -y install cmake  # for symengine for RxODE
 sudo apt-get -y install libxslt1-dev  # for xslt
 sudo apt-get -y install flex  # for RcppCWB
+sudo apt-get -y install libavfilter-dev libsodium-dev libgmp-dev libssh-dev librdf0-dev
+sudo apt-get -y install libmariadb-dev mariadb-client   # RMySQL for xQTLbiolinks
 sudo R CMD javareconf
 # ENDIF
 
@@ -532,6 +556,7 @@ inst()   # *** ensure latest dev version of data.table installed into revdeplib
 run()    # prints menu of options
 status() # includes timestamp of installed data.table that is being tested.
 log()    # cats all fail logs to ~/fail.log
+cran()   # compare packages with error or warning to their status on CRAN
 
 # Once all issues resolved with CRAN packages, tackle long-term unfixed bioconductor packages as follows.
 # 1. Note down all error and warning bioc packages
@@ -569,7 +594,7 @@ du -k inst/tests                # 0.75MB after
 R CMD build .
 export GITHUB_PAT="f1c.. github personal access token ..7ad"
 Rdevel -q -e "packageVersion('xml2')"   # ensure installed
-Rdevel CMD check data.table_1.14.0.tar.gz --as-cran  # use latest Rdevel as it may have extra checks
+Rdevel CMD check data.table_1.14.10.tar.gz --as-cran  # use latest Rdevel as it may have extra checks
 #
 bunzip2 inst/tests/*.Rraw.bz2  # decompress *.Rraw again so as not to commit compressed *.Rraw to git
 #
@@ -596,8 +621,8 @@ When CRAN's email contains "Pretest results OK pending a manual inspection" (or
 3. Add new heading in NEWS for the next dev version. Add "(submitted to CRAN on <today>)" on the released heading.
 4. Bump dllVersion() in init.c
 5. Bump 3 version numbers in Makefile
-6. Search and replace this .dev/CRAN_Release.cmd to update 1.13.7 to 1.14.1, and 1.13.6 to 1.14.0 (e.g. in step 8 and 9 below)
+6. Search and replace this .dev/CRAN_Release.cmd to update 1.14.9 to 1.14.11 inc below, 1.14.10 to 1.14.12 above, 1.14.8 to 1.14.10 below
 7. Another final gd to view all diffs using meld. (I have `alias gd='git difftool &> /dev/null'` and difftool meld: http://meldmerge.org/)
-8. Push to master with this consistent commit message: "1.14.0 on CRAN. Bump to 1.14.1"
-9. Take sha from step 8 and run `git tag 1.14.0 96c..sha..d77` then `git push origin 1.14.0` (not `git push --tags` according to https://stackoverflow.com/a/5195913/403310)
+8. Push to master with this consistent commit message: "1.14.8 on CRAN. Bump to 1.14.10"
+9. Take sha from step 8 and run `git tag 1.14.8 96c..sha..d77` then `git push origin 1.14.8` (not `git push --tags` according to https://stackoverflow.com/a/5195913/403310)
 ######

.dev/cc.R

@@ -22,8 +22,6 @@
 # c
 # test and step between R and C
 
-options(datatable.print.class = TRUE)
-
 sourceDir = function(path=getwd(), trace = TRUE, ...) {
   # copied verbatim from example(source) in base R
   for (nm in list.files(path, pattern = "\\.[RrSsQq]$")) {
@@ -50,7 +48,7 @@ cc = function(test=FALSE, clean=FALSE, debug=FALSE, omp=!debug, cc_dir, path=Sys
 
   # Make sure library .so is not loaded (neither installed package nor from dev)
   dll = unlist(do.call("rbind",getLoadedDLLs())[,"path"])
-  dll = grep("data_table.so",dll,value=TRUE)
+  dll = grep("data_table.so", dll, fixed=TRUE, value=TRUE)
   sapply(dll, dyn.unload)
   gc()
 
@@ -63,27 +61,31 @@ cc = function(test=FALSE, clean=FALSE, debug=FALSE, omp=!debug, cc_dir, path=Sys
   if (debug) {
     ret = system(sprintf("MAKEFLAGS='-j CC=%s PKG_CFLAGS=-f%sopenmp CFLAGS=-std=c99\\ -O0\\ -ggdb\\ -pedantic' R CMD SHLIB -d -o data_table.so *.c", CC, OMP))
   } else {
-    ret = system(sprintf("MAKEFLAGS='-j CC=%s CFLAGS=-f%sopenmp\\ -std=c99\\ -O3\\ -pipe\\ -Wall\\ -pedantic\\ -fno-common' R CMD SHLIB -o data_table.so *.c", CC, OMP))
+    ret = system(sprintf("MAKEFLAGS='-j CC=%s CFLAGS=-f%sopenmp\\ -std=c99\\ -O3\\ -pipe\\ -Wall\\ -pedantic\\ -Wstrict-prototypes\\ -isystem\\ /usr/share/R/include\\ -fno-common' R CMD SHLIB -o data_table.so *.c", CC, OMP))
+    # the -isystem suppresses strict-prototypes warnings from R's headers, #5477. Look at the output to see what -I is and pass the same path to -isystem.
     # TODO add -Wextra too?
   }
   if (ret) return()
   # clang -Weverything includes -pedantic and issues many more warnings than gcc
   # system("R CMD SHLIB -o data_table.so *.c")
-  if (any(sapply(objects(envir=.GlobalEnv),function(x){inherits(get(x,.GlobalEnv),"data.table")}))) {
-    cat("ABOUT TO RELOAD .SO BUT THERE ARE DATA.TABLE OBJECTS IN .GLOBALENV SO FINALIZER MIGHT CRASH\n")
+  for (obj in ls(.GlobalEnv)) {
+    if (inherits(.GlobalEnv[[obj]], "data.table")) {
+      cat("ABOUT TO RELOAD .SO BUT THERE ARE DATA.TABLE OBJECTS IN .GLOBALENV SO FINALIZER MIGHT CRASH\n")
+      break
+    }
   }
   dyn.load("data_table.so")
   setwd(old)
   xx = getDLLRegisteredRoutines("data_table",TRUE)
-  for (i in seq_along(xx$.Call))
-    assign(xx$.Call[[i]]$name,  xx$.Call[[i]]$address, envir=.GlobalEnv)
-  for (i in seq_along(xx$.External))
-    assign(xx$.External[[i]]$name,  xx$.External[[i]]$address, envir=.GlobalEnv)
-  sourceDir(paste0(path,"/R"))
-  if (base::getRversion()<"4.0.0") rm(list=c("rbind.data.table","cbind.data.table"), envir=.GlobalEnv) # 3968 follow up
-  assign("testDir", function(x)paste0(path,"/inst/tests/",x), envir=.GlobalEnv)
+  for (Call in xx$.Call)
+    .GlobalEnv[[Call$name]] = Call$address
+  for (Extern in xx$.External)
+    .GlobalEnv[[Extern$name]] = Extern$address
+  sourceDir(file.path(path, "R"))
+  if (base::getRversion()<"4.0.0") rm(list=c("rbind.data.table", "cbind.data.table"), envir=.GlobalEnv) # 3968 follow up
+  .GlobalEnv$testDir = function(x) file.path(path,"inst/tests",x)
   .onLoad()
-  if (is.logical(test) && isTRUE(test)) test.data.table() else if (is.character(test)) test.data.table(script=test)
+  if (isTRUE(test)) test.data.table() else if (is.character(test)) test.data.table(script=test)
   gc()
   invisible()
 }