analyze.Rmd

---
title: data.table reverse dependency checks
output: html_document
---

```{r Ropts, echo=FALSE}
options(width=200)
knitr::opts_chunk$set(
  echo=FALSE,
  results="asis")
```

```{r results=TRUE}
job.dir <- readLines("JOBDIR",n=1)

plural <- function(number, singular){
  s <- ifelse(number==1, "", "s")
  sprintf("%d %s%s", number, singular, s)
}
suppressMessages({
  if(!requireNamespace("nc"))install.packages("nc")
  if(!requireNamespace("slurm")){
    install.packages("~/R/slurm", repos=NULL)
  }
  library(data.table)
})
job.date <- basename(job.dir)
deps.dt <- data.table::fread(file.path(job.dir, "deps.csv"))
JOBID.file <- file.path(job.dir, "JOBID")
JOBID.vec <- gsub(" ", "", readLines(JOBID.file))
PARAMS_ID.file <- file.path(job.dir, "PARAMS_ID")
PARAMS_ID <- readLines(PARAMS_ID.file, n=1)
JOBID.str <- paste(JOBID.vec, collapse=",")
sacct.arg <- paste0("-j", JOBID.str)
sacct.dt <- slurm::sacct(sacct.arg)
some.sacct <- sacct.dt[, .(
  task,
  check.time=Elapsed,
  check.memory.MB=round(megabytes),
  State=State_blank
)]
deps.sacct <- some.sacct[deps.dt, on=.(task=task.id)]
dt.tar.gz.vec <- Sys.glob(file.path(job.dir, "*.tar.gz"))
dt.version.vec <- gsub(".tar.gz$|.*data.table_", "", dt.tar.gz.vec)

## Read all logs.
if(FALSE){
  log.file.glob <- "analyze/2024-02-20/*.txt"
  log.pattern <- list(
    log.txt=list(
      "analyze/",
      date=".*?",
      "/",
      checked=".*?",
      ".txt"))
}else{
  log.file.glob <- file.path(
    job.dir, "packages", "*", "log.txt")
  log.pattern <- list(
    log.txt=list(
      ".*/",
      checked=".*?",
      "/log.txt"))
}
get.dl.table <- function(f){
  also.dl <- nc::capture_all_str(
    f,
    "also installing the.*\n",
    downloads=list(
      "(?:.*\n)*?"
    ),
    "[*]")
  dl.text <- if(nrow(also.dl))also.dl[["downloads"]] else " "
  nc::capture_all_str(
    dl.text,
    "trying URL '",
    repos=".*",
    "/src/contrib/",
    file=list(
      Package=".*?",
      "_",
      version=".*?",
      "[.]tar[.]gz"
    ),
    "'\nContent type 'application/x-gzip' length ",
    bytes="[0-9]+", as.integer)
}  
also.dt <- nc::capture_first_glob(log.file.glob, log.pattern, READ=get.dl.table)
myRound <- function(x){
  prettyNum(ceiling(x), big.mark=",")
}
also.counts <- also.dt[!is.na(Package), .(
  downloads=.N
), by=.(
  Package, version,
  kilobytes=bytes/1024,
  repos
)][
, megabytes := downloads*kilobytes/1024
][
  order(-megabytes)
][,`:=`(
  KB.per.dl=myRound(kilobytes),
  MB.bandwidth=myRound(megabytes),
  kilobytes=NULL,
  megabytes=NULL
)][]
bioc.counts <- also.counts[grepl("bioconductor.org", repos)]
if(FALSE){
  data.table::fwrite(bioc.counts, "popular_deps.csv", append=TRUE)
}

## Read R versions, which should have been written to job.dir by params.R.
R.versions <- sapply(c("devel","release"), function(v.lo){
  readLines(file.path(job.dir, paste0("R_",toupper(v.lo))), warn=FALSE)
})
## Read all diffs.
sig.cols <- c(
  "first.bad.commit",
  "Package", "checking", "Rvers",
  "comments",
  "CRAN", "release", "master")
diff.file.vec <- Sys.glob(file.path(
  job.dir, "packages", "*", "*", "significant_differences.csv"))
sig.diff.dt <- if(length(diff.file.vec)){
  diff.file.cols <- names(fread(diff.file.vec[1], nrows=0))
  sig.diff.wide <- data.table(diff.csv=diff.file.vec)[, {
    fread(diff.csv, colClasses=list(character=diff.file.cols))
  }, by=diff.csv]
  Rversion.pattern <- list(
    job.dir,
    "/packages/",
    Package=".*?",
    "/",
    Rversion=".*?",
    "/")
  sig.diff.all <- nc::capture_first_df(
    sig.diff.wide, diff.csv=Rversion.pattern
  )
  sig.diff.all[
  , Rvers := ifelse(grepl("devel", Rversion), "devel", "release")
  ][, sig.cols, with=FALSE]
}else{
  data.table()[, (sig.cols) := character()]
}
setkeyv(sig.diff.dt, sig.cols)
deps.sacct[, sig.diffs := NA_integer_]
deps.sacct[sig.diff.dt, sig.diffs := .N, by=Package, on=.(Package)]
deps.sacct[is.na(sig.diffs), sig.diffs := 0]

if(FALSE){#TODO need to copy files from /tmp or do this in each task.
  ## add number of missing check logs.
  check.file.glob <- file.path(
    job.dir, "packages", "*", "*", "*", "00check.log")
  check.file.vec <- Sys.glob(check.file.glob)
  check.file.meta <- nc::capture_first_vec(
    check.file.vec,
    check.log=list(
      Rversion.pattern,
      dt.version=".*?",
      "/00check.log"))
  CJ.args <- lapply(check.file.meta[, .(Rversion, dt.version)], unique)
  CJ.args$Package <- deps.dt$Package
  select.dt <- do.call(CJ, CJ.args)
  expected.checks <- check.file.meta[select.dt, on=names(select.dt)]
  expected.checks[, dt.vers := sub("_.*", "", dt.version)]
  expected.checks[, log.txt := file.path(job.dir, "packages", Package, "log.txt")]
  missing.logs <- expected.checks[is.na(check.log)]
  deps.sacct[, no.log := NA_integer_]
  deps.sacct[missing.logs, no.log := .N, by=Package, on=.(Package)]
  deps.sacct[is.na(no.log), no.log := 0]
}

## report versions used.
pvers <- function(what, ver.vec){
  cat(what, "versions checked:\n")
  print(ver.vec)
}
pvers("R", R.versions)
pvers("data.table", dt.version.vec)
cat("data.table master is", readLines("data.table_master.git.log"), "\n")
revdep.version <- system("git log|head -1", intern=TRUE)
cat("using data.table-revdeps:", revdep.version, "\n")

## were deps not available during check?
grep.not.avail <- paste(
  "grep 'but not available'",
  log.file.glob)
not.avail.vec <- suppressWarnings({
  system(grep.not.avail, intern=TRUE)
})
deps.sacct[, not.avail := 0L]
not.avail.parsed <- if(length(not.avail.vec)){
  not.avail.uniq <- unique(not.avail.vec)
  not.avail.dt <- nc::capture_first_vec(
    not.avail.uniq,
    log.pattern)
  not.avail.pattern <- list(
    type='suggested|enhances|required',
    ' but not available.*?:',
    after.colon='(?:.*\n)+?',
    "[*|\n]")
  not.avail.deps <- not.avail.dt[, nc::capture_all_str(
    log.txt,
    not.avail.pattern),
    by=.(log.txt, checked)]
  not.avail.deps[after.colon==""]
  not.avail.deps$after.colon
  not.avail.deps[1]
  dep.dt <- unique(not.avail.deps)[, nc::capture_all_str(
    after.colon,
    "'",
    dep=".*?",
    "'"),
    by=.(Package=checked, type)]
  deps.sacct[dep.dt, not.avail := .N, by=Package, on="Package"]
  dep.dt[, .SD[1], keyby=.(type, dep)][, .(dep, type, Package)]
}else{
  data.table(dep=character(), type=character(), Package=character())
}

## add counts of download/installation failed.
grep.config.fail <- paste(
  "grep -E '(configuration|lazy loading) failed for package'",
  log.file.glob)
config.fail.vec <- suppressWarnings({
  system(grep.config.fail, intern=TRUE)
})
deps.sacct[, config.fail := 0L]
if(length(config.fail.vec)){
  config.fail.uniq <- unique(config.fail.vec)
  config.fail.dt <- nc::capture_first_vec(
    config.fail.uniq,
    log.pattern,
    ":ERROR: ",
    action=".*?",
    " failed for package '",
    failed=".*?",
    "'")
  config.fail.first <- config.fail.dt[, .SD[1, .(checked)], by=.(action, failed)]
  deps.sacct[
    config.fail.dt,
    config.fail := .N,
    by=Package,
    on=.(Package=checked)]
}else{
  config.fail.first <- data.table(
    action=character(), failed=character(), checked=character())
}
grep.dl.fail <- paste(
  "grep 'download of package.*failed'",
  log.file.glob)
dl.fail.vec <- suppressWarnings({
  system(grep.dl.fail, intern=TRUE)
})
deps.sacct[, dl.fail := 0L]
if(length(dl.fail.vec)){
  dl.fail.dt <- nc::capture_first_vec(
    dl.fail.vec,
    log.pattern,
    ".*?download of package '",
    failed=".*?",
    "' failed")
  deps.sacct[dl.fail.dt, dl.fail := .N, by=Package, on=.(Package=checked)]
}

## Add column to show if diff is new since last run.
sig_diff_files <- grep(
  job.date,
  Sys.glob("analyze/*/significant_differences.csv"),
  invert=TRUE, value=TRUE)
prev.sig_diff.csv <- sig_diff_files[length(sig_diff_files)]
prev.sig_diff.dt <- fread(prev.sig_diff.csv, colClasses="character")
diff.join.cols <- c(
  "Package", "checking", "Rvers", "release", "master")
newly.fixed <- prev.sig_diff.dt[!sig.diff.dt, on=diff.join.cols]
sig.diff.dt[, new := "NEW!"]
sig.diff.dt[prev.sig_diff.dt, new := "", on=diff.join.cols]

## copy log files from scratch to projects/analyze subdir.
log.file.vec <- Sys.glob(log.file.glob)
log.pkg <- basename(dirname(log.file.vec))
deps.sacct[
  log.pkg,
  log.disk.KB := as.integer(file.size(log.file.vec)/1024),
  on="Package"]
analyze.dir <- file.path("analyze", job.date)
dir.create(analyze.dir, showWarnings=FALSE)
pkg.txt.vec <- file.path(analyze.dir, paste0(log.pkg, ".txt"))
copied <- file.copy(log.file.vec, pkg.txt.vec)
ahref <- function(href, text){
  sprintf('<a href="%s">%s</a>', href, text)
}
mytab <- function(DT){
  if (!nrow(DT)) {
    cat("Empty table: nothing to print.\n")
    return(invisible())
  }
  copied <- data.table(DT)
  if("CRAN" %in% names(copied)){
    copied[, CRAN := ahref(sprintf(
      "https://cloud.r-project.org/web/checks/check_results_%s.html",
      Package),
      CRAN)]
  }
  if("first.bad.commit" %in% names(copied)){
    copied[, first.bad.commit := paste(ahref(paste0(
      "https://github.com/Rdatatable/data.table/commit/",
      first.bad.commit),
      first.bad.commit),
      comments)]
    copied[, comments := NULL]
  }
  show.cols <- c("sig.diffs","no.log","config.fail","dl.fail","not.avail")
  some.cols <- show.cols[show.cols %in% names(copied)]
  if(length(some.cols)){
    int.mat <- as.matrix(copied[, some.cols, with=FALSE])
    name.mat <- matrix(some.cols, nrow(int.mat), ncol(int.mat), byrow=TRUE)
    chr.mat <- ifelse(int.mat==0, "", sprintf("%s=%d", name.mat, int.mat))
    copied[, warnings := apply(chr.mat, 1, paste, collapse=" ")]
    set(copied, j=some.cols, value=NULL)
  }
  copied[, Package := ahref(paste0(
    Package, ".txt"),
    Package)]
  knitr::kable(copied)
}

count.dt <- deps.sacct[, .(
  dep.pkgs=.N
), keyby=.(
  COMPLETED=State=="COMPLETED",
  diffs=sig.diffs>0,
  fail=config.fail|dl.fail
)]
digits.pattern <- list("[0-9]{2}", as.integer)
time.dt <- nc::capture_first_vec(
  deps.sacct[["check.time"]],
  hours.only=digits.pattern, ":",
  minutes.only=digits.pattern, ":",
  seconds.only=digits.pattern
)[,
  minutes := hours.only*60 + minutes.only + seconds.only/60
  ]
time.wide <- dcast(
  time.dt,
  . ~ .,
  list(min, median, max, sum),
  value.var="minutes"
)
time.tall <- nc::capture_melt_single(
  time.wide[,-1],
  "minutes_",
  stat=".*",
  value.name="minutes"
)[, hours := minutes/60][, days := hours/24][]
source("myStatus.R")#for get_flavor
```

Time is now 
`r format(Sys.time(), "%F %T %Z")`, 
`r plural(nrow(deps.sacct), "revdep")` 
checked in parallel, time stats:

```{r}
knitr::kable(time.tall, digits=2)
```

## `r plural(nrow(sig.diff.dt), "Significant difference")` found

The table below has one row for each check found to have different
results when using two versions of `data.table` (master and
release). The current CRAN result for that check is also shown (using
linux platform with either `r get_flavor("r-devel")` or `r get_flavor("r-release")`).
Instructions about how to interpret and file issues based on these results are
explained [on a wiki page](https://github.com/Rdatatable/data.table/wiki/Release-management-and-revdep-checks).


```{r}
is.ignore <- sig.diff.dt[
, grepl("same as", comments) | (!is.na(master)&master=="OK")
]
mytab(sig.diff.dt[!is.ignore])
```

The table above contains likely issues that need to be fixed. 
The table below contains significant differences which probably do not need to be fixed (can be ignored), because either

* git bisect found trivial commit (spurious failure / not reproducible), or
* check using `data.table` master is OK (issue in CRAN `data.table` that has already been fixed).

```{r}
mytab(sig.diff.dt[is.ignore])
```

## `r plural(nrow(newly.fixed), "significant difference")` fixed since last check

The significant differences below were found in the last check, `r prev.sig_diff.csv` but are no longer present in the current check above. 
If they are really fixed, then please search for the corresponding [issue](https://github.com/Rdatatable/data.table/issues) and close it.

```{r}
mytab(newly.fixed)
```

## `r plural(nrow(config.fail.first), "Configuration failure")`

In the table below, the `failed` column shows which package failed to
configure/install, and the `Package` column shows the log file of the
package that was supposed to be checked. Each failure is only reported
once (only the first log with that configuration failure is
shown). Ideally this table should be empty, so if there are entries
here, they should be reported to the maintainer of this revdep check
system via [this
issue tracker](https://github.com/tdhock/data.table-revdeps/issues).

```{r}
mytab(config.fail.first[, .(action, failed, Package=checked)])
```

## `r plural(nrow(not.avail.parsed), "not available dependent package")`

In the table below, we list the packages which should have been
available during a check, but were not. Each dependent package and
type is only reported once (only the first log with that dependent
package missing, for that type, is shown). Ideally this table should
be empty, so if there are entries here, they should be reported to the
maintainer of this revdep check system via [this issue
tracker](https://github.com/tdhock/data.table-revdeps/issues).

```{r}
mytab(not.avail.parsed)
```

## Longest running jobs

These are the top 10 packages in terms of time to install, then
compute checks and git bisect. If any have `State=TIMEOUT` regularly,
then there are two options to fix:

* check the log, and if there is a lot of time spent installing packages, then this package should be added to [popular_deps.csv](https://github.com/tdhock/data.table-revdeps/blob/master/popular_deps.csv), which will increase time spent installing packages to the shared library prior to launching check jobs, but will decrease the time of each check job that would have had to install the popular packages to the job-specific library.
* otherwise, if not much time is spent installing packages, then increase the `--time` parameter of the `run_one.sh` script in [params.R](https://github.com/tdhock/data.table-revdeps/blob/master/params.R).

```{r}
mytab(deps.sacct[order(-check.time)][1:10])
```

## Largest memory / out of memory jobs

Below are the top 10 packages in terms of max memory usage in the job
which installs, then computes checks and git bisect. 

```{r}
mytab(deps.sacct[order(-check.memory.MB)][1:10])
```

Below packages ran out of memory during checks. 

```{r}
mytab(deps.sacct[State=="OUT_OF_MEMORY"][order(-check.memory.MB)])
```

If any have State `OUT_OF_MEMORY` then either

* add that package to `large_memory.csv` 
* or if it is already listed there, then increase the `gigabytes`
  value in `array.mem.dt` in
  [params.R](https://github.com/tdhock/data.table-revdeps/blob/master/params.R).

## Most installed packages

These are the top 10 packages which appeared in "also installing the
dependencies..." in the logs, in terms of total download bandwidth,
and number of times downloaded. Packages in this table should be
considered for inclusion in
[popular_deps.csv](https://github.com/tdhock/data.table-revdeps/blob/master/popular_deps.csv),
which is the list of packages that are installed to the shared
library, prior to launching the check jobs (each of which has its own
job-specific library where packages are installed if they are not
present in the shared library).

```{r}
knitr::kable(also.counts[1:min(10,.N)], align="r")
```

```{r}
knitr::kable(also.counts[order(-downloads)][1:min(10,.N)], align="r")
```

## Installed packages from Bioconductor

These are the Bioconductor packages which appeared in "also installing the
dependencies..." in the logs. Packages in this table should be
considered for inclusion in
[popular_deps.csv](https://github.com/tdhock/data.table-revdeps/blob/master/popular_deps.csv),
which is the list of packages that are installed to the shared
library, prior to launching the check jobs (each of which has its own
job-specific library where packages are installed if they are not
present in the shared library).

```{r}
knitr::kable(bioc.counts, align="r")
```

## Full list of jobs

The SLURM job IDs were

* `r PARAMS_ID` for `params.sh`, one task for building R and
  installing popular packages,
* `r JOBID.str` for `run_one.sh`, several tasks, one for each revdep,
  shown in table below, and
* `r Sys.getenv("SLURM_JOB_ID")` for `analyze.sh`, one task for
  creating this final report.

The table below has a row for every reverse dependency (SLURM task ID
is shown in first column). If any of the following are not zero, then
there will be an entry in the warnings column:

* `sig.diffs`: rows in the significant differences table.
* `no.log`: missing log files (there could be up to 4).
* `config.fail`: dependent packages which failed to configure/build or lazy load.
* `dl.fail`: packages which failed to download.
* `not.avail`: dependent packages which were not available for checking.

```{r}
mytab(deps.sacct)
```

## CSV data

```{r}
fsave <- function(DT, f){
  out.csv <- paste0(f, ".csv")
  out.path <- file.path(analyze.dir, out.csv)
  data.table::fwrite(DT, out.path)
  cat('<p>', ahref(out.csv, f), '</p>\n')
}
fsave(deps.sacct, "full_list_of_jobs")
fsave(sig.diff.dt, "significant_differences")
```

## Source code

Computed using [code](https://github.com/tdhock/data.table-revdeps)
written by Toby Dylan Hocking.