From 5537ba7193c987c189f9f8f9784078cd7cb23d68 Mon Sep 17 00:00:00 2001
From: Sebastian Krantz <sebastian.krantz@graduateinstitute.ch>
Date: Sat, 7 Dec 2024 23:12:39 +0100
Subject: [PATCH 1/4] Postpone the problem.

---
 src/R_Defn.h     | 86 ++++++++++++++++++++++++++++++++++++++++++++++++
 src/assign.c     | 70 +++++++++++++++++++--------------------
 src/chmatch.c    | 28 ++++++++--------
 src/data.table.h | 11 ++++---
 src/dogroups.c   | 40 +++++++++++-----------
 src/fmelt.c      |  8 ++---
 src/forder.c     | 26 +++++++--------
 src/freadR.c     | 12 +++----
 src/init.c       |  2 +-
 src/rbindlist.c  | 46 +++++++++++++-------------
 src/subset.c     |  8 ++---
 src/utils.c      | 10 +++---
 12 files changed, 217 insertions(+), 130 deletions(-)
 create mode 100644 src/R_Defn.h

diff --git a/src/R_Defn.h b/src/R_Defn.h
new file mode 100644
index 0000000000..41726c6b4e
--- /dev/null
+++ b/src/R_Defn.h
@@ -0,0 +1,86 @@
+#include <R.h>
+#include <Rinternals.h>
+
+// NOTE: All of this is copied from Defn.h: https://github.com/wch/r-source/blob/28de75af0541f93832c5899139b969d290bf422e/src/include/Defn.h
+// We intend to gradually remove the need for this header file
+
+// Writable vector/string pointer
+#define SEXPPTR(x) ((SEXP *)DATAPTR(x))
+
+#ifndef NAMED_BITS
+# define NAMED_BITS 16
+#endif
+
+#ifndef SEXPREC_HEADER
+
+// https://github.com/wch/r-source/blob/2640a203d13473f95c9c7508eb2976fefb5c931c/src/include/Defn.h#L123
+struct sxpinfo_struct {
+  SEXPTYPE type      :  TYPE_BITS;
+  /* ==> (FUNSXP == 99) %% 2^5 == 3 == CLOSXP
+   * -> warning: `type' is narrower than values
+   *              of its type
+   * when SEXPTYPE was an enum */
+  unsigned int scalar:  1;
+  unsigned int obj   :  1;
+  unsigned int alt   :  1;
+  unsigned int gp    : 16;
+  unsigned int mark  :  1;
+  unsigned int debug :  1;
+  unsigned int trace :  1;  /* functions and memory tracing */
+  unsigned int spare :  1;  /* used on closures and when REFCNT is defined */
+  unsigned int gcgen :  1;  /* old generation number */
+  unsigned int gccls :  3;  /* node class */
+  unsigned int named : NAMED_BITS;
+  unsigned int extra : 32 - NAMED_BITS; /* used for immediate bindings */
+}; /*		    Tot: 64 */
+  
+// https://github.com/wch/r-source/blob/2640a203d13473f95c9c7508eb2976fefb5c931c/src/include/Defn.h#L190  
+#define SEXPREC_HEADER           \
+  struct sxpinfo_struct sxpinfo; \
+  struct SEXPREC *attrib;        \
+  struct SEXPREC *gengc_next_node, *gengc_prev_node
+
+// https://github.com/wch/r-source/blob/2640a203d13473f95c9c7508eb2976fefb5c931c/src/include/Defn.h#L143
+struct vecsxp_struct {
+  R_xlen_t	length;
+  R_xlen_t	truelength;
+};
+  
+// https://github.com/wch/r-source/blob/2640a203d13473f95c9c7508eb2976fefb5c931c/src/include/Defn.h#L214
+typedef struct VECTOR_SEXPREC {
+  SEXPREC_HEADER;
+  struct vecsxp_struct vecsxp;
+} VECTOR_SEXPREC, *VECSEXP;
+  
+#endif
+  
+// https://github.com/wch/r-source/blob/2640a203d13473f95c9c7508eb2976fefb5c931c/src/include/Defn.h#L197  
+typedef struct {
+  SEXPREC_HEADER;
+} SEXPREC_partial;
+
+// (SET_)TRULENGTH: https://github.com/wch/r-source/blob/2640a203d13473f95c9c7508eb2976fefb5c931c/src/include/Defn.h#L386
+#define STDVEC_TRUELENGTH(x) (((VECSEXP) (x))->vecsxp.truelength)
+// No method to set ALTREP_TRUELENGTH (gives error): https://github.com/wch/r-source/blob/2640a203d13473f95c9c7508eb2976fefb5c931c/src/include/Defn.h#L388
+#define SET_TRULEN(x, v) (STDVEC_TRUELENGTH(x)=(v))
+// ALTREP_TRUELENGTH is 0: https://github.com/wch/r-source/blob/48f06c1071fea6a6e7e365ad3d745217268e2175/src/main/altrep.c#L345
+#define TRULEN(x) (ALTREP(x) ? 0 : STDVEC_TRUELENGTH(x))
+  
+// SETLENGTH: https://github.com/wch/r-source/blob/05bb18266d49e87f2477120ecb0ab1440f4e9b40/src/include/Defn.h#L385
+#define STDVEC_LENGTH(x) (((VECSEXP) (x))->vecsxp.length)
+#define SETSCAL(x, v) ((((SEXPREC_partial *)(x))->sxpinfo.scalar) = (v))
+#define SET_STDVEC_LENGTH(x,v) do {		     \
+  SEXP __x__ = (x);			                   \
+  R_xlen_t __v__ = (v);			               \
+  STDVEC_LENGTH(__x__) = __v__;		         \
+  SETSCAL(__x__, __v__ == 1 ? 1 : 0);	     \
+} while (0)                                
+// https://github.com/wch/r-source/blob/05bb18266d49e87f2477120ecb0ab1440f4e9b40/src/main/memory.c#L4072  
+#define SET_LEN(x, v) SET_STDVEC_LENGTH((x), (v))
+
+// LEVELS: https://github.com/wch/r-source/blob/2640a203d13473f95c9c7508eb2976fefb5c931c/src/include/Defn.h#L228    
+#define LEVLS(x)	(((SEXPREC_partial *)(x))->sxpinfo.gp)
+
+// SET_GROWABLE_BIT: https://github.com/wch/r-source/blob/2640a203d13473f95c9c7508eb2976fefb5c931c/src/include/Defn.h#L374
+#define GROWBLE_MASK ((unsigned short)(1<<5))
+#define SET_GROWBLE_BIT(x) (LEVLS(x) |= GROWBLE_MASK)
diff --git a/src/assign.c b/src/assign.c
index b280c2259f..2009dde346 100644
--- a/src/assign.c
+++ b/src/assign.c
@@ -8,7 +8,7 @@ static void finalizer(SEXP p)
   p = R_ExternalPtrTag(p);
   if (!isString(p)) internal_error(__func__, "ExternalPtr doesn't see names in tag"); // # nocov
   l = LENGTH(p);
-  tl = TRUELENGTH(p);
+  tl = TRULEN(p);
   if (l<0 || tl<l) internal_error(__func__, "l=%d, tl=%d", l, tl); // # nocov
   n = tl-l;
   if (n==0) {
@@ -18,7 +18,7 @@ static void finalizer(SEXP p)
   }
   x = PROTECT(allocVector(INTSXP, 50));  // 50 so it's big enough to be on LargeVector heap. See NodeClassSize in memory.c:allocVector
                                          // INTSXP rather than VECSXP so that GC doesn't inspect contents after LENGTH (thanks to Karl Miller, Jul 2015)
-  SETLENGTH(x,50+n*2*sizeof(void *)/4);  // 1*n for the names, 1*n for the VECSXP itself (both are over allocated).
+  SET_LEN(x,50+n*2*sizeof(void *)/4);  // 1*n for the names, 1*n for the VECSXP itself (both are over allocated).
   UNPROTECT(1);
   return;
 }
@@ -86,8 +86,8 @@ closest I got to getting it to pass all tests :
   UNPROTECT(2);
 
 Then in finalizer:
-  SETLENGTH(names, tl)
-  SETLENGTH(dt, tl)
+  SET_LEN(names, tl)
+  SET_LEN(dt, tl)
 
 and that finalizer indeed now happens before the GC releases memory (thanks to the env wrapper).
 
@@ -127,14 +127,14 @@ static int _selfrefok(SEXP x, Rboolean checkNames, Rboolean verbose) {
   if (!(isNull(tag) || isString(tag))) internal_error(__func__, ".internal.selfref tag is neither NULL nor a character vector"); // # nocov
   names = getAttrib(x, R_NamesSymbol);
   if (names!=tag && isString(names) && !ALTREP(names))  // !ALTREP for #4734
-    SET_TRUELENGTH(names, LENGTH(names));
+    SET_TRULEN(names, LENGTH(names));
     // R copied this vector not data.table; it's not actually over-allocated. It looks over-allocated
     // because R copies the original vector's tl over despite allocating length.
   prot = R_ExternalPtrProtected(v);
   if (TYPEOF(prot) != EXTPTRSXP)   // Very rare. Was error(_(".internal.selfref prot is not itself an extptr")).
     return 0;                      // # nocov ; see http://stackoverflow.com/questions/15342227/getting-a-random-internal-selfref-error-in-data-table-for-r
   if (x!=R_ExternalPtrAddr(prot) && !ALTREP(x))
-    SET_TRUELENGTH(x, LENGTH(x));  // R copied this vector not data.table, it's not actually over-allocated
+    SET_TRULEN(x, LENGTH(x));  // R copied this vector not data.table, it's not actually over-allocated
   return checkNames ? names==tag : x==R_ExternalPtrAddr(prot);
 }
 
@@ -189,10 +189,10 @@ static SEXP shallow(SEXP dt, SEXP cols, R_len_t n)
   setAttrib(newdt, R_NamesSymbol, newnames);
   // setAttrib appears to change length and truelength, so need to do that first _then_ SET next,
   // otherwise (if the SET were were first) the 100 tl is assigned to length.
-  SETLENGTH(newnames,l);
-  SET_TRUELENGTH(newnames,n);
-  SETLENGTH(newdt,l);
-  SET_TRUELENGTH(newdt,n);
+  SET_LEN(newnames,l);
+  SET_TRULEN(newnames,n);
+  SET_LEN(newdt,l);
+  SET_TRULEN(newdt,n);
   setselfref(newdt);
   UNPROTECT(protecti);
   return(newdt);
@@ -260,10 +260,10 @@ SEXP alloccol(SEXP dt, R_len_t n, Rboolean verbose)
     return shallow(dt,R_NilValue,(n>l) ? n : l);  // e.g. test 848 and 851 in R > 3.0.2
     // added (n>l) ? ... for #970, see test 1481.
   // TO DO:  test realloc names if selfrefnamesok (users can setattr(x,"name") themselves for example.
-  // if (TRUELENGTH(getAttrib(dt,R_NamesSymbol))!=tl)
-  //    internal_error(__func__, "tl of dt passes checks, but tl of names (%d) != tl of dt (%d)", tl, TRUELENGTH(getAttrib(dt,R_NamesSymbol))); // # nocov
+  // if (TRULEN(getAttrib(dt,R_NamesSymbol))!=tl)
+  //    internal_error(__func__, "tl of dt passes checks, but tl of names (%d) != tl of dt (%d)", tl, TRULEN(getAttrib(dt,R_NamesSymbol))); // # nocov
 
-  tl = TRUELENGTH(dt);
+  tl = TRULEN(dt);
   // R <= 2.13.2 and we didn't catch uninitialized tl somehow
   if (tl<0) internal_error(__func__, "tl of class is marked but tl<0"); // # nocov
   if (tl>0 && tl<l) internal_error(__func__, "tl (%d) < l (%d) but tl of class is marked", tl, l); // # nocov
@@ -313,11 +313,11 @@ SEXP shallowwrapper(SEXP dt, SEXP cols) {
   if (!selfrefok(dt, FALSE)) {
     int n = isNull(cols) ? length(dt) : length(cols);
     return(shallow(dt, cols, n));
-  } else return(shallow(dt, cols, TRUELENGTH(dt)));
+  } else return(shallow(dt, cols, TRULEN(dt)));
 }
 
 SEXP truelength(SEXP x) {
-  return ScalarInteger(isNull(x) ? 0 : TRUELENGTH(x));
+  return ScalarInteger(isNull(x) ? 0 : TRULEN(x));
 }
 
 SEXP selfrefokwrapper(SEXP x, SEXP verbose) {
@@ -514,7 +514,7 @@ SEXP assign(SEXP dt, SEXP rows, SEXP cols, SEXP newcolnames, SEXP values)
   // modify DT by reference. Other than if new columns are being added and the allocVec() fails with
   // out-of-memory. In that case the user will receive hard halt and know to rerun.
   if (length(newcolnames)) {
-    oldtncol = TRUELENGTH(dt);   // TO DO: oldtncol can be just called tl now, as we won't realloc here any more.
+    oldtncol = TRULEN(dt);   // TO DO: oldtncol can be just called tl now, as we won't realloc here any more.
 
     if (oldtncol<oldncol) {
       if (oldtncol==0) error(_("This data.table has either been loaded from disk (e.g. using readRDS()/load()) or constructed manually (e.g. using structure()). Please run setDT() or setalloccol() on it first (to pre-allocate space for new columns) before assigning by reference to it."));   // #2996
@@ -527,13 +527,13 @@ SEXP assign(SEXP dt, SEXP rows, SEXP cols, SEXP newcolnames, SEXP values)
       error(_("It appears that at some earlier point, names of this data.table have been reassigned. Please ensure to use setnames() rather than names<- or colnames<-. Otherwise, please report to data.table issue tracker."));  // # nocov
       // Can growVector at this point easily enough, but it shouldn't happen in first place so leave it as
       // strong error message for now.
-    else if (TRUELENGTH(names) != oldtncol)
+    else if (TRULEN(names) != oldtncol)
       // Use (long long) to cast R_xlen_t to a fixed type to robustly avoid -Wformat compiler warnings, see #5768, PRId64 didn't work
-      internal_error(__func__, "selfrefnames is ok but tl names [%lld] != tl [%d]", (long long)TRUELENGTH(names), oldtncol);  // # nocov
+      internal_error(__func__, "selfrefnames is ok but tl names [%lld] != tl [%d]", (long long)TRULEN(names), oldtncol);  // # nocov
     if (!selfrefok(dt, verbose)) // #6410 setDT(dt) and subsequent attr<- can lead to invalid selfref
       error(_("It appears that at some earlier point, attributes of this data.table have been reassigned. Please use setattr(DT, name, value) rather than attr(DT, name) <- value. If that doesn't apply to you, please report your case to the data.table issue tracker."));
-    SETLENGTH(dt, oldncol+LENGTH(newcolnames));
-    SETLENGTH(names, oldncol+LENGTH(newcolnames));
+    SET_LEN(dt, oldncol+LENGTH(newcolnames));
+    SET_LEN(names, oldncol+LENGTH(newcolnames));
     for (int i=0; i<LENGTH(newcolnames); ++i)
       SET_STRING_ELT(names,oldncol+i,STRING_ELT(newcolnames,i));
     // truelengths of both already set by alloccol
@@ -730,8 +730,8 @@ SEXP assign(SEXP dt, SEXP rows, SEXP cols, SEXP newcolnames, SEXP values)
       SET_VECTOR_ELT(dt, i, R_NilValue);
       SET_STRING_ELT(names, i, NA_STRING);  // release reference to the CHARSXP
     }
-    SETLENGTH(dt,    ndt-ndelete);
-    SETLENGTH(names, ndt-ndelete);
+    SET_LEN(dt,    ndt-ndelete);
+    SET_LEN(names, ndt-ndelete);
     if (LENGTH(names)==0) {
       // That was last column deleted, leaving NULL data.table, so we need to reset .row_names, so that it really is the NULL data.table.
       PROTECT(nullint=allocVector(INTSXP, 0)); protecti++;
@@ -830,26 +830,26 @@ const char *memrecycle(const SEXP target, const SEXP where, const int start, con
         savetl_init();
         for (int k=0; k<nTargetLevels; ++k) {
           const SEXP s = targetLevelsD[k];
-          const int tl = TRUELENGTH(s);
+          const int tl = TRULEN(s);
           if (tl>0) {
             savetl(s);
           } else if (tl<0) {
             // # nocov start
-            for (int j=0; j<k; ++j) SET_TRUELENGTH(s, 0);  // wipe our negative usage and restore 0
+            for (int j=0; j<k; ++j) SET_TRULEN(s, 0);  // wipe our negative usage and restore 0
             savetl_end();                                  // then restore R's own usage (if any)
             internal_error(__func__, "levels of target are either not unique or have truelength<0"); // # nocov
             // # nocov end
           }
-          SET_TRUELENGTH(s, -k-1);
+          SET_TRULEN(s, -k-1);
         }
         int nAdd = 0;
         for (int k=0; k<nSourceLevels; ++k) {
           const SEXP s = sourceLevelsD[k];
-          const int tl = TRUELENGTH(s);
+          const int tl = TRULEN(s);
           if (tl>=0) {
             if (!sourceIsFactor && s==NA_STRING) continue; // don't create NA factor level when assigning character to factor; test 2117
             if (tl>0) savetl(s);
-            SET_TRUELENGTH(s, -nTargetLevels-(++nAdd));
+            SET_TRULEN(s, -nTargetLevels-(++nAdd));
           } // else, when sourceIsString, it's normal for there to be duplicates here
         }
         const int nSource = length(source);
@@ -858,34 +858,34 @@ const char *memrecycle(const SEXP target, const SEXP where, const int start, con
           const int *sourceD = INTEGER(source);
           for (int i=0; i<nSource; ++i) {  // convert source integers to refer to target levels
             const int val = sourceD[i];
-            newSourceD[i] = val==NA_INTEGER ? NA_INTEGER : -TRUELENGTH(sourceLevelsD[val-1]); // retains NA factor levels here via TL(NA_STRING); e.g. ordered factor
+            newSourceD[i] = val==NA_INTEGER ? NA_INTEGER : -TRULEN(sourceLevelsD[val-1]); // retains NA factor levels here via TL(NA_STRING); e.g. ordered factor
           }
         } else {
           const SEXP *sourceD = STRING_PTR_RO(source);
           for (int i=0; i<nSource; ++i) {  // convert source integers to refer to target levels
             const SEXP val = sourceD[i];
-            newSourceD[i] = val==NA_STRING ? NA_INTEGER : -TRUELENGTH(val);
+            newSourceD[i] = val==NA_STRING ? NA_INTEGER : -TRULEN(val);
           }
         }
         source = newSource;
-        for (int k=0; k<nTargetLevels; ++k) SET_TRUELENGTH(targetLevelsD[k], 0);  // don't need those anymore
+        for (int k=0; k<nTargetLevels; ++k) SET_TRULEN(targetLevelsD[k], 0);  // don't need those anymore
         if (nAdd) {
           // cannot grow the levels yet as that would be R call which could fail to alloc and we have no hook to clear up
           SEXP *temp = (SEXP *)malloc(nAdd * sizeof(SEXP *));
           if (!temp) {
             // # nocov start
-            for (int k=0; k<nSourceLevels; ++k) SET_TRUELENGTH(sourceLevelsD[k], 0);
+            for (int k=0; k<nSourceLevels; ++k) SET_TRULEN(sourceLevelsD[k], 0);
             savetl_end();
             error(_("Unable to allocate working memory of %zu bytes to combine factor levels"), nAdd*sizeof(SEXP *));
             // # nocov end
           }
           for (int k=0, thisAdd=0; thisAdd<nAdd; ++k) {   // thisAdd<nAdd to stop early when the added ones are all reached
             SEXP s = sourceLevelsD[k];
-            int tl = TRUELENGTH(s);
+            int tl = TRULEN(s);
             if (tl) {  // tl negative here
               if (tl != -nTargetLevels-thisAdd-1) internal_error(__func__, "extra level check sum failed"); // # nocov
               temp[thisAdd++] = s;
-              SET_TRUELENGTH(s,0);
+              SET_TRULEN(s,0);
             }
           }
           savetl_end();
@@ -1312,14 +1312,14 @@ void savetl(SEXP s)
     savedtl = (R_len_t *)tmp;
   }
   saveds[nsaved] = s;
-  savedtl[nsaved] = TRUELENGTH(s);
+  savedtl[nsaved] = TRULEN(s);
   nsaved++;
 }
 
 void savetl_end(void) {
   // Can get called if nothing has been saved yet (nsaved==0), or even if _init() hasn't been called yet (pointers NULL). Such
   // as to clear up before error. Also, it might be that nothing needed to be saved anyway.
-  for (int i=0; i<nsaved; i++) SET_TRUELENGTH(saveds[i],savedtl[i]);
+  for (int i=0; i<nsaved; i++) SET_TRULEN(saveds[i],savedtl[i]);
   free(saveds);  // possible free(NULL) which is safe no-op
   saveds = NULL;
   free(savedtl);
diff --git a/src/chmatch.c b/src/chmatch.c
index 2bad963f94..6c7fc8a813 100644
--- a/src/chmatch.c
+++ b/src/chmatch.c
@@ -58,10 +58,10 @@ static SEXP chmatchMain(SEXP x, SEXP table, int nomatch, bool chin, bool chmatch
   savetl_init();
   for (int i=0; i<xlen; i++) {
     SEXP s = xd[i];
-    const int tl = TRUELENGTH(s);
+    const int tl = TRULEN(s);
     if (tl>0) {
       savetl(s);  // R's internal hash (which is positive); save it
-      SET_TRUELENGTH(s,0);
+      SET_TRULEN(s,0);
     } else if (tl<0) {
       // R 2.14.0+ initializes truelength to 0 (before that it was uninitialized/random).
       // Now that data.table depends on R 3.1.0+, that is after 2.14.0 too.
@@ -75,13 +75,13 @@ static SEXP chmatchMain(SEXP x, SEXP table, int nomatch, bool chin, bool chmatch
   int nuniq=0;
   for (int i=0; i<tablelen; ++i) {
     const SEXP s = td[i];
-    int tl = TRUELENGTH(s);
+    int tl = TRULEN(s);
     if (tl>0) { savetl(s); tl=0; }
-    if (tl==0) SET_TRUELENGTH(s, chmatchdup ? -(++nuniq) : -i-1); // first time seen this string in table
+    if (tl==0) SET_TRULEN(s, chmatchdup ? -(++nuniq) : -i-1); // first time seen this string in table
   }
   // in future if we need NAs in x not to be matched to NAs in table ...
-  // if (!matchNAtoNA && TRUELENGTH(NA_STRING)<0)
-  //   SET_TRUELENGTH(NA_STRING, 0);
+  // if (!matchNAtoNA && TRULEN(NA_STRING)<0)
+  //   SET_TRULEN(NA_STRING, 0);
   if (chmatchdup) {
     // chmatchdup() is basically base::pmatch() but without the partial matching part. For example :
     //   chmatchdup(c("a", "a"), c("a", "a"))   # 1,2  - the second 'a' in 'x' has a 2nd match in 'table'
@@ -100,21 +100,21 @@ static SEXP chmatchMain(SEXP x, SEXP table, int nomatch, bool chin, bool chmatch
     if (!counts || !map) {
       // # nocov start
       free(counts); free(map);
-      for (int i=0; i<tablelen; i++) SET_TRUELENGTH(td[i], 0);
+      for (int i=0; i<tablelen; i++) SET_TRULEN(td[i], 0);
       savetl_end();
       error(_("Failed to allocate %"PRIu64" bytes working memory in chmatchdup: length(table)=%d length(unique(table))=%d"), ((uint64_t)tablelen*2+nuniq)*sizeof(int), tablelen, nuniq);
       // # nocov end
     }
-    for (int i=0; i<tablelen; ++i) counts[-TRUELENGTH(td[i])-1]++;
+    for (int i=0; i<tablelen; ++i) counts[-TRULEN(td[i])-1]++;
     for (int i=0, sum=0; i<nuniq; ++i) { int tt=counts[i]; counts[i]=sum; sum+=tt+1; }
-    for (int i=0; i<tablelen; ++i) map[counts[-TRUELENGTH(td[i])-1]++] = i+1;           // 0 is left ending each group thanks to the calloc
+    for (int i=0; i<tablelen; ++i) map[counts[-TRULEN(td[i])-1]++] = i+1;           // 0 is left ending each group thanks to the calloc
     for (int i=0, last=0; i<nuniq; ++i) {int tt=counts[i]+1; counts[i]=last; last=tt;}  // rewind counts to the beginning of each group
     for (int i=0; i<xlen; ++i) {
-      int u = TRUELENGTH(xd[i]);
+      int u = TRULEN(xd[i]);
       if (u<0) {
         const int w = counts[-u-1]++;
         if (map[w]) { ansd[i]=map[w]; continue; }
-        SET_TRUELENGTH(xd[i],0); // w falls on ending 0 marker: dups used up; any more dups should return nomatch
+        SET_TRULEN(xd[i],0); // w falls on ending 0 marker: dups used up; any more dups should return nomatch
         // we still need the 0-setting loop at the end of this function because often there will be some values in table that are not matched to at all.
       }
       ansd[i] = nomatch;
@@ -123,16 +123,16 @@ static SEXP chmatchMain(SEXP x, SEXP table, int nomatch, bool chin, bool chmatch
     free(map);
   } else if (chin) {
     for (int i=0; i<xlen; i++) {
-      ansd[i] = TRUELENGTH(xd[i])<0;
+      ansd[i] = TRULEN(xd[i])<0;
     }
   } else {
     for (int i=0; i<xlen; i++) {
-      const int m = TRUELENGTH(xd[i]);
+      const int m = TRULEN(xd[i]);
       ansd[i] = (m<0) ? -m : nomatch;
     }
   }
   for (int i=0; i<tablelen; i++)
-    SET_TRUELENGTH(td[i], 0);  // reinstate 0 rather than leave the -i-1
+    SET_TRULEN(td[i], 0);  // reinstate 0 rather than leave the -i-1
   savetl_end();
   UNPROTECT(nprotect);  // ans, xd, td
   return ans;
diff --git a/src/data.table.h b/src/data.table.h
index f2c1874e19..63cb6b5453 100644
--- a/src/data.table.h
+++ b/src/data.table.h
@@ -10,19 +10,20 @@
 #  define R_Realloc(x, y, z) Realloc(x, y, z)
 #  define R_Free(x) Free(x)
 #endif
-#if R_VERSION < R_Version(3, 4, 0)
-#  define SET_GROWABLE_BIT(x)  // #3292
-#endif
+// #if R_VERSION < R_Version(3, 4, 0)
+// #  define SET_GROWABLE_BIT(x)  // #3292
+// #endif
 #include <Rinternals.h>
 #define SEXPPTR_RO(x) ((const SEXP *)DATAPTR_RO(x))  // to avoid overhead of looped STRING_ELT and VECTOR_ELT
 #ifndef STRING_PTR_RO
-#define STRING_PTR_RO STRING_PTR
+#define STRING_PTR_RO SEXPPTR
 #endif
 #include <stdint.h>    // for uint64_t rather than unsigned long long
 #include <stdarg.h>    // for va_list, va_start
 #include <stdbool.h>
 #include "types.h"
 #include "po.h"
+#include "R_Defn.h"
 #ifdef WIN32  // positional specifiers (%n$) used in translations; #4402
 #  define snprintf dt_win_snprintf  // see our snprintf.c; tried and failed to link to _sprintf_p on Windows
 #endif
@@ -37,7 +38,7 @@
 /* we mean the encoding bits, not CE_NATIVE in a UTF-8 locale */
 #define IS_UTF8(x)  (getCharCE(x) == CE_UTF8)
 #define IS_LATIN(x) (getCharCE(x) == CE_LATIN1)
-#define IS_ASCII(x) (LEVELS(x) & 64) // API expected in R >= 4.5
+#define IS_ASCII(x) (LEVLS(x) & 64) // API expected in R >= 4.5
 #define IS_TRUE(x)  (TYPEOF(x)==LGLSXP && LENGTH(x)==1 && LOGICAL(x)[0]==TRUE)
 #define IS_FALSE(x) (TYPEOF(x)==LGLSXP && LENGTH(x)==1 && LOGICAL(x)[0]==FALSE)
 #define IS_TRUE_OR_FALSE(x) (TYPEOF(x)==LGLSXP && LENGTH(x)==1 && LOGICAL(x)[0]!=NA_LOGICAL)
diff --git a/src/dogroups.c b/src/dogroups.c
index cc7554c3e3..e1ef81eed6 100644
--- a/src/dogroups.c
+++ b/src/dogroups.c
@@ -46,9 +46,9 @@ static bool anySpecialStatic(SEXP x) {
   if (n==0)
     return false;
   if (isVectorAtomic(x))
-    return ALTREP(x) || TRUELENGTH(x)<0;
+    return ALTREP(x) || TRULEN(x)<0;
   if (isNewList(x)) {
-    if (TRUELENGTH(x)<0)
+    if (TRULEN(x)<0)
       return true;  // test 2158
     for (int i=0; i<n; ++i) {
       list_el = VECTOR_ELT(x,i);
@@ -102,7 +102,7 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
     defineVar(install(CHAR(STRING_ELT(bynames,i))), VECTOR_ELT(BY,i), env);      // by vars can be used by name in j as well as via .BY
     if (SIZEOF(VECTOR_ELT(BY,i))==0)
       internal_error(__func__, "unsupported size-0 type '%s' in column %d of 'by' should have been caught earlier", type2char(TYPEOF(VECTOR_ELT(BY, i))), i+1); // # nocov
-    SET_TRUELENGTH(VECTOR_ELT(BY,i), -1); // marker for anySpecialStatic(); see its comments
+    SET_TRULEN(VECTOR_ELT(BY,i), -1); // marker for anySpecialStatic(); see its comments
   }
   setAttrib(BY, R_NamesSymbol, bynames); // Fix for #42 - BY doesn't retain names anymore
   R_LockBinding(sym_BY, env);
@@ -111,9 +111,9 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
   // TO DO: check this check above.
 
   N =   PROTECT(findVar(install(".N"), env));   nprotect++; // PROTECT for rchk
-  SET_TRUELENGTH(N, -1);  // marker for anySpecialStatic(); see its comments
+  SET_TRULEN(N, -1);  // marker for anySpecialStatic(); see its comments
   GRP = PROTECT(findVar(install(".GRP"), env)); nprotect++;
-  SET_TRUELENGTH(GRP, -1);  // marker for anySpecialStatic(); see its comments
+  SET_TRULEN(GRP, -1);  // marker for anySpecialStatic(); see its comments
   iSD = PROTECT(findVar(install(".iSD"), env)); nprotect++; // 1-row and possibly no cols (if no i variables are used via JIS)
   xSD = PROTECT(findVar(install(".xSD"), env)); nprotect++;
   R_len_t maxGrpSize = 0;
@@ -122,7 +122,7 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
     if (ilens[i] > maxGrpSize) maxGrpSize = ilens[i];
   }
   defineVar(install(".I"), I = PROTECT(allocVector(INTSXP, maxGrpSize)), env); nprotect++;
-  SET_TRUELENGTH(I, -maxGrpSize);  // marker for anySpecialStatic(); see its comments
+  SET_TRULEN(I, -maxGrpSize);  // marker for anySpecialStatic(); see its comments
   R_LockBinding(install(".I"), env);
 
   SEXP dtnames = PROTECT(getAttrib(dt, R_NamesSymbol)); nprotect++; // added here to fix #91 - `:=` did not issue recycling warning during "by"
@@ -149,7 +149,7 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
     nameSyms[i] = install(CHAR(STRING_ELT(names, i)));
     // fixes http://stackoverflow.com/questions/14753411/why-does-data-table-lose-class-definition-in-sd-after-group-by
     copyMostAttrib(VECTOR_ELT(dt,INTEGER(dtcols)[i]-1), this);  // not names, otherwise test 778 would fail
-    SET_TRUELENGTH(this, -maxGrpSize);  // marker for anySpecialStatic(); see its comments
+    SET_TRULEN(this, -maxGrpSize);  // marker for anySpecialStatic(); see its comments
   }
 
   SEXP xknames = PROTECT(getAttrib(xSD, R_NamesSymbol)); nprotect++;
@@ -194,7 +194,7 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
     INTEGER(GRP)[0] = i+1;  // group counter exposed as .GRP
     INTEGER(rownames)[1] = -grpn;  // the .set_row_names() of .SD. Not .N when nomatch=NA and this is a nomatch
     for (int j=0; j<length(SDall); ++j) {
-      SETLENGTH(VECTOR_ELT(SDall,j), grpn);  // before copying data in otherwise assigning after the end could error R API checks
+      SET_LEN(VECTOR_ELT(SDall,j), grpn);  // before copying data in otherwise assigning after the end could error R API checks
       defineVar(nameSyms[j], VECTOR_ELT(SDall, j), env);
       // Redo this defineVar for each group in case user's j assigned to the column names (env is static) (tests 387 and 388)
       // nameSyms pre-stored to save repeated install() for efficiency, though.
@@ -227,14 +227,14 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
         // and we hope that reference counting on by default from R 4.0 will avoid costly gc()s.
       }
       grpn = 1;  // it may not be 1 e.g. test 722. TODO: revisit.
-      SETLENGTH(I, grpn);
+      SET_LEN(I, grpn);
       INTEGER(I)[0] = 0;
       for (int j=0; j<length(xSD); ++j) {
         writeNA(VECTOR_ELT(xSD, j), 0, 1, false);
       }
     } else {
       if (verbose) tstart = wallclock();
-      SETLENGTH(I, grpn);
+      SET_LEN(I, grpn);
       int *iI = INTEGER(I);
       if (LENGTH(order)==0) {
         const int rownum = grpn ? istarts[i]-1 : -1;
@@ -315,13 +315,13 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
         RHS = VECTOR_ELT(jval,j%LENGTH(jval));
         if (isNull(target)) {
           // first time adding to new column
-          if (TRUELENGTH(dt) < colj+1) internal_error(__func__, "Trying to add new column by reference but tl is full; setalloccol should have run first at R level before getting to this point"); // # nocov
+          if (TRULEN(dt) < colj+1) internal_error(__func__, "Trying to add new column by reference but tl is full; setalloccol should have run first at R level before getting to this point"); // # nocov
           target = PROTECT(allocNAVectorLike(RHS, n));
           // Even if we could know reliably to switch from allocNAVectorLike to allocVector for slight speedup, user code could still
           // contain a switched halt, and in that case we'd want the groups not yet done to have NA rather than 0 or uninitialized.
           // Increment length only if the allocation passes, #1676. But before SET_VECTOR_ELT otherwise attempt-to-set-index-n/n R error
-          SETLENGTH(dtnames, LENGTH(dtnames)+1);
-          SETLENGTH(dt, LENGTH(dt)+1);
+          SET_LEN(dtnames, LENGTH(dtnames)+1);
+          SET_LEN(dt, LENGTH(dt)+1);
           SET_VECTOR_ELT(dt, colj, target);
           UNPROTECT(1);
           SET_STRING_ELT(dtnames, colj, STRING_ELT(newnames, colj-origncol));
@@ -479,17 +479,17 @@ SEXP dogroups(SEXP dt, SEXP dtcols, SEXP groups, SEXP grpcols, SEXP jiscols, SEX
   // Also reset truelength on specials; see comments in anySpecialStatic().
   for (int j=0; j<length(SDall); ++j) {
     SEXP this = VECTOR_ELT(SDall,j);
-    SETLENGTH(this, maxGrpSize);
-    SET_TRUELENGTH(this, maxGrpSize);
+    SET_LEN(this, maxGrpSize);
+    SET_TRULEN(this, maxGrpSize);
   }
-  SETLENGTH(I, maxGrpSize);
-  SET_TRUELENGTH(I, maxGrpSize);
+  SET_LEN(I, maxGrpSize);
+  SET_TRULEN(I, maxGrpSize);
   for (int i=0; i<length(BY); ++i) {
     SEXP this = VECTOR_ELT(BY, i);
-    SET_TRUELENGTH(this, length(this)); // might be 0 or 1; see its allocVector above
+    SET_TRULEN(this, length(this)); // might be 0 or 1; see its allocVector above
   }
-  SET_TRUELENGTH(N, 1);
-  SET_TRUELENGTH(GRP, 1);
+  SET_TRULEN(N, 1);
+  SET_TRULEN(GRP, 1);
   if (verbose) {
     if (nblock[0] && nblock[1]) internal_error(__func__, "block 0 [%d] and block 1 [%d] have both run", nblock[0], nblock[1]); // # nocov
     int w = nblock[1]>0;
diff --git a/src/fmelt.c b/src/fmelt.c
index 59e82455b4..deabafbbd1 100644
--- a/src/fmelt.c
+++ b/src/fmelt.c
@@ -414,10 +414,10 @@ static SEXP combineFactorLevels(SEXP factorLevels, SEXP target, int * factorType
     for (int k=0; k<thisn; ++k) {
       SEXP s = thisd[k];
       if (s==NA_STRING) continue;  // NA shouldn't be in levels but remove it just in case
-      int tl = TRUELENGTH(s);
+      int tl = TRULEN(s);
       if (tl<0) continue;  // seen this level before
       if (tl>0) savetl(s);
-      SET_TRUELENGTH(s,-(++nlevel));
+      SET_TRULEN(s,-(++nlevel));
       levelsRaw[nlevel-1] = s;
     }
   }
@@ -425,11 +425,11 @@ static SEXP combineFactorLevels(SEXP factorLevels, SEXP target, int * factorType
     if (targetd[i]==NA_STRING) {
       *ansd++ = NA_INTEGER;
     } else {
-      int tl = TRUELENGTH(targetd[i]);
+      int tl = TRULEN(targetd[i]);
       *ansd++ = tl<0 ? -tl : NA_INTEGER;
     }
   }
-  for (int i=0; i<nlevel; ++i) SET_TRUELENGTH(levelsRaw[i], 0);
+  for (int i=0; i<nlevel; ++i) SET_TRULEN(levelsRaw[i], 0);
   savetl_end();
   // now after savetl_end, we can alloc (which might fail)
   SEXP levelsSxp;
diff --git a/src/forder.c b/src/forder.c
index 8a735e0e8a..c1e001b062 100644
--- a/src/forder.c
+++ b/src/forder.c
@@ -72,7 +72,7 @@ static char msg[1001];
 
 static void free_ustr(void) {
   for(int i=0; i<ustr_n; i++)
-    SET_TRUELENGTH(ustr[i],0);
+    SET_TRULEN(ustr[i],0);
   free(ustr); ustr=NULL;
   ustr_alloc=0; ustr_n=0; ustr_maxlen=0;
 }
@@ -307,10 +307,10 @@ static void range_str(const SEXP *x, int n, uint64_t *out_min, uint64_t *out_max
       na_count++;
       continue;
     }
-    if (TRUELENGTH(s)<0) continue;  // seen this group before
+    if (TRULEN(s)<0) continue;  // seen this group before
     #pragma omp critical
-    if (TRUELENGTH(s)>=0) {  // another thread may have set it while I was waiting, so check it again
-      if (TRUELENGTH(s)>0)   // save any of R's own usage of tl (assumed positive, so we can both count and save in one scan), to restore
+    if (TRULEN(s)>=0) {  // another thread may have set it while I was waiting, so check it again
+      if (TRULEN(s)>0)   // save any of R's own usage of tl (assumed positive, so we can both count and save in one scan), to restore
         savetl(s);           // afterwards. From R 2.14.0, tl is initialized to 0, prior to that it was random so this step saved too much.
       // now save unique SEXP in ustr so i) we can loop through them afterwards and reset TRUELENGTH to 0 and ii) sort uniques when sorting too
       if (ustr_alloc<=ustr_n) {
@@ -320,7 +320,7 @@ static void range_str(const SEXP *x, int n, uint64_t *out_min, uint64_t *out_max
         if (ustr==NULL) STOP(_("Unable to realloc %d * %d bytes in range_str"), ustr_alloc, (int)sizeof(SEXP));  // # nocov
       }
       ustr[ustr_n++] = s;
-      SET_TRUELENGTH(s, -ustr_n);  // unique in any order is fine. first-appearance order is achieved later in count_group
+      SET_TRULEN(s, -ustr_n);  // unique in any order is fine. first-appearance order is achieved later in count_group
       if (LENGTH(s)>ustr_maxlen) ustr_maxlen=LENGTH(s);
       if (!anynotutf8 &&    // even if anynotascii we still want to know if anynotutf8, and anynotutf8 implies anynotascii already
             !IS_ASCII(s)) { // anynotutf8 implies anynotascii and IS_ASCII will be cheaper than IS_UTF8, so start with this one
@@ -351,23 +351,23 @@ static void range_str(const SEXP *x, int n, uint64_t *out_min, uint64_t *out_max
     for (int i=0; i<ustr_n; i++) {
       SEXP s = ustr3[i];
       if (LENGTH(s)>ustr_maxlen) ustr_maxlen=LENGTH(s);
-      if (TRUELENGTH(s)>0) savetl(s);
+      if (TRULEN(s)>0) savetl(s);
     }
     cradix(ustr3, ustr_n);  // sort to detect possible duplicates after converting; e.g. two different non-utf8 map to the same utf8
-    SET_TRUELENGTH(ustr3[0], -1);
+    SET_TRULEN(ustr3[0], -1);
     int o = -1;
     for (int i=1; i<ustr_n; i++) {
       if (ustr3[i] == ustr3[i-1]) continue;  // use the same o for duplicates
-      SET_TRUELENGTH(ustr3[i], --o);
+      SET_TRULEN(ustr3[i], --o);
     }
     // now use the 1-1 mapping from ustr to ustr2 to get the ordering back into original ustr, being careful to reset tl to 0
     int *tl = (int *)malloc(ustr_n * sizeof(int));
     if (!tl)
       STOP(_("Failed to alloc tl when converting strings to UTF8"));  // # nocov
     const SEXP *tt = STRING_PTR_RO(ustr2);
-    for (int i=0; i<ustr_n; i++) tl[i] = TRUELENGTH(tt[i]);   // fetches the o in ustr3 into tl which is ordered by ustr
-    for (int i=0; i<ustr_n; i++) SET_TRUELENGTH(ustr3[i], 0);    // reset to 0 tl of the UTF8 (and possibly non-UTF in ustr too)
-    for (int i=0; i<ustr_n; i++) SET_TRUELENGTH(ustr[i], tl[i]); // put back the o into ustr's tl
+    for (int i=0; i<ustr_n; i++) tl[i] = TRULEN(tt[i]);   // fetches the o in ustr3 into tl which is ordered by ustr
+    for (int i=0; i<ustr_n; i++) SET_TRULEN(ustr3[i], 0);    // reset to 0 tl of the UTF8 (and possibly non-UTF in ustr too)
+    for (int i=0; i<ustr_n; i++) SET_TRULEN(ustr[i], tl[i]); // put back the o into ustr's tl
     free(tl);
     free(ustr3);
     UNPROTECT(1);
@@ -380,7 +380,7 @@ static void range_str(const SEXP *x, int n, uint64_t *out_min, uint64_t *out_max
       // that this is always ascending; descending is done in WRITE_KEY using max-this
       cradix(ustr, ustr_n);  // sorts ustr in-place by reference. assumes NA_STRING not present.
       for(int i=0; i<ustr_n; i++)     // save ordering in the CHARSXP. negative so as to distinguish with R's own usage.
-        SET_TRUELENGTH(ustr[i], -i-1);
+        SET_TRULEN(ustr[i], -i-1);
     }
     // else group appearance order was already saved to tl in the first pass
   }
@@ -766,7 +766,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrpArg, SEXP retStatsArg, SEXP sortGroupsA
           if (nalast==-1) anso[i]=0;
           elem = naval;
         } else {
-          elem = -TRUELENGTH(xd[i]);
+          elem = -TRULEN(xd[i]);
         }
         WRITE_KEY
       }}
diff --git a/src/freadR.c b/src/freadR.c
index 05b8cd00e1..c0d81a176a 100644
--- a/src/freadR.c
+++ b/src/freadR.c
@@ -516,7 +516,7 @@ size_t allocateDT(int8_t *typeArg, int8_t *sizeArg, int ncolArg, int ndrop, size
 
         setAttrib(thiscol, sym_tzone, ScalarString(char_UTC)); // see news for v1.13.0
       }
-      SET_TRUELENGTH(thiscol, allocNrow);
+      SET_TRULEN(thiscol, allocNrow);
       DTbytes += SIZEOF(thiscol)*allocNrow;
     }
     resi++;
@@ -533,9 +533,9 @@ void setFinalNrow(size_t nrow) {
       return;
     const int ncol=LENGTH(DT);
     for (int i=0; i<ncol; i++) {
-      SETLENGTH(VECTOR_ELT(DT,i), nrow);
-      SET_TRUELENGTH(VECTOR_ELT(DT,i), dtnrows);
-      SET_GROWABLE_BIT(VECTOR_ELT(DT,i));  // #3292
+      SET_LEN(VECTOR_ELT(DT,i), nrow);
+      SET_TRULEN(VECTOR_ELT(DT,i), dtnrows);
+      SET_GROWBLE_BIT(VECTOR_ELT(DT,i));  // #3292
     }
   }
   R_FlushConsole(); // # 2481. Just a convenient place; nothing per se to do with setFinalNrow()
@@ -548,8 +548,8 @@ void dropFilledCols(int* dropArg, int ndelete) {
     SET_VECTOR_ELT(DT, dropFill[i], R_NilValue);
     SET_STRING_ELT(colNamesSxp, dropFill[i], NA_STRING);
   }
-  SETLENGTH(DT, ndt-ndelete);
-  SETLENGTH(colNamesSxp, ndt-ndelete);
+  SET_LEN(DT, ndt-ndelete);
+  SET_LEN(colNamesSxp, ndt-ndelete);
 }
 
 void pushBuffer(ThreadLocalFreadParsingContext *ctx)
diff --git a/src/init.c b/src/init.c
index 204dc1088d..9ade1cb266 100644
--- a/src/init.c
+++ b/src/init.c
@@ -203,7 +203,7 @@ void attribute_visible R_init_data_table(DllInfo *info)
   SEXP tmp = PROTECT(allocVector(INTSXP,2));
   if (LENGTH(tmp)!=2) error(_("Checking LENGTH(allocVector(INTSXP,2)) [%d] is 2 %s"), LENGTH(tmp), msg);
   // Use (long long) to cast R_xlen_t to a fixed type to robustly avoid -Wformat compiler warnings, see #5768
-  if (TRUELENGTH(tmp)!=0) error(_("Checking TRUELENGTH(allocVector(INTSXP,2)) [%lld] is 0 %s"), (long long)TRUELENGTH(tmp), msg);
+  if (TRULEN(tmp)!=0) error(_("Checking TRULEN(allocVector(INTSXP,2)) [%lld] is 0 %s"), (long long)TRULEN(tmp), msg);
   UNPROTECT(1);
 
   // According to IEEE (http://en.wikipedia.org/wiki/IEEE_754-1985#Zero) we can rely on 0.0 being all 0 bits.
diff --git a/src/rbindlist.c b/src/rbindlist.c
index 42ba9ad749..2e61b3a519 100644
--- a/src/rbindlist.c
+++ b/src/rbindlist.c
@@ -86,10 +86,10 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
       const SEXP *cnp = STRING_PTR_RO(cn);
       for (int j=0; j<thisncol; j++) {
         SEXP s = ENC2UTF8(cnp[j]); // convert different encodings for use.names #5452
-        if (TRUELENGTH(s)<0) continue;  // seen this name before
-        if (TRUELENGTH(s)>0) savetl(s);
+        if (TRULEN(s)<0) continue;  // seen this name before
+        if (TRULEN(s)>0) savetl(s);
         uniq[nuniq++] = s;
-        SET_TRUELENGTH(s,-nuniq);
+        SET_TRULEN(s,-nuniq);
       }
     }
     if (nuniq>0) uniq = realloc(uniq, nuniq*sizeof(SEXP));  // shrink to only what we need to release the spare
@@ -99,7 +99,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
     int *maxdup = (int *)calloc(nuniq, sizeof(int)); // the most number of dups for any name within one colname vector
     if (!counts || !maxdup) {
       // # nocov start
-      for (int i=0; i<nuniq; ++i) SET_TRUELENGTH(uniq[i], 0);
+      for (int i=0; i<nuniq; ++i) SET_TRULEN(uniq[i], 0);
       free(uniq); free(counts); free(maxdup);
       savetl_end();
       error(_("Failed to allocate nuniq=%d items working memory in rbindlist.c"), nuniq);
@@ -116,7 +116,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
       memset(counts, 0, nuniq*sizeof(int));
       for (int j=0; j<thisncol; j++) {
         SEXP s = ENC2UTF8(cnp[j]); // convert different encodings for use.names #5452
-        counts[ -TRUELENGTH(s)-1 ]++;
+        counts[ -TRULEN(s)-1 ]++;
       }
       for (int u=0; u<nuniq; u++) {
         if (counts[u] > maxdup[u]) maxdup[u] = counts[u];
@@ -134,7 +134,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
     int *dupLink = (int *)malloc(ncol * sizeof(int)); // if a colname has occurred before (a dup) links from the 1st to the 2nd time in the final result, 2nd to 3rd, etc
     if (!colMapRaw || !uniqMap || !dupLink) {
       // # nocov start
-      for (int i=0; i<nuniq; ++i) SET_TRUELENGTH(uniq[i], 0);
+      for (int i=0; i<nuniq; ++i) SET_TRULEN(uniq[i], 0);
       free(uniq); free(counts); free(colMapRaw); free(uniqMap); free(dupLink);
       savetl_end();
       error(_("Failed to allocate ncol=%d items working memory in rbindlist.c"), ncol);
@@ -157,7 +157,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
         memset(counts, 0, nuniq*sizeof(int));
         for (int j=0; j<thisncol; j++) {
           SEXP s = ENC2UTF8(cnp[j]); // convert different encodings for use.names #5452
-          int w = -TRUELENGTH(s)-1;
+          int w = -TRULEN(s)-1;
           int wi = counts[w]++; // how many dups have we seen before of this name within this item
           if (uniqMap[w]==-1) {
             // first time seen this name across all items
@@ -174,7 +174,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
         }
       }
     }
-    for (int i=0; i<nuniq; ++i) SET_TRUELENGTH(uniq[i], 0);  // zero out our usage of tl
+    for (int i=0; i<nuniq; ++i) SET_TRULEN(uniq[i], 0);  // zero out our usage of tl
     free(uniq); free(counts); free(uniqMap); free(dupLink);  // all local scope so no need to set to NULL
     savetl_end();  // restore R's usage
 
@@ -384,9 +384,9 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
         }
         for (int k=0; k<longestLen; ++k) {
           SEXP s = sd[k];
-          if (TRUELENGTH(s)>0) savetl(s);
+          if (TRULEN(s)>0) savetl(s);
           levelsRaw[k] = s;
-          SET_TRUELENGTH(s,-k-1);
+          SET_TRULEN(s,-k-1);
         }
         for (int i=0; i<LENGTH(l); ++i) {
           SEXP li = VECTOR_ELT(l, i);
@@ -399,7 +399,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
             const int n = length(levels);
             for (int k=0, last=0; k<n; ++k) {
               SEXP s = levelsD[k];
-              const int tl = TRUELENGTH(s);
+              const int tl = TRULEN(s);
               if (tl>=last) {  // if tl>=0 then also tl>=last because last<=0
                 if (tl>=0) {
                   snprintf(warnStr, 1000,   // not direct warning as we're inside tl region
@@ -438,8 +438,8 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
           for (int k=0; k<n; ++k) {
             SEXP s = thisColStrD[k];
             if (s==NA_STRING ||             // remove NA from levels; test 1979 found by package emil when revdep testing 1.12.2 (#3473)
-                TRUELENGTH(s)<0) continue;  // seen this level before; handles removing dups from levels as well as finding unique of character columns
-            if (TRUELENGTH(s)>0) savetl(s);
+                TRULEN(s)<0) continue;  // seen this level before; handles removing dups from levels as well as finding unique of character columns
+            if (TRULEN(s)>0) savetl(s);
             if (allocLevel==nLevel) {       // including initial time when allocLevel==nLevel==0
               SEXP *tt = NULL;
               if (allocLevel<INT_MAX) {
@@ -450,7 +450,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
               if (tt==NULL) {
                 // # nocov start
                 // C spec states that if realloc() fails (above) the original block (levelsRaw) is left untouched: it is not freed or moved. We ...
-                for (int k=0; k<nLevel; k++) SET_TRUELENGTH(levelsRaw[k], 0);   // ... rely on that in this loop which uses levelsRaw.
+                for (int k=0; k<nLevel; k++) SET_TRULEN(levelsRaw[k], 0);   // ... rely on that in this loop which uses levelsRaw.
                 free(levelsRaw);
                 savetl_end();
                 error(_("Failed to allocate working memory for %d factor levels of result column %d when reading item %d of item %d"), allocLevel, idcol+j+1, w+1, i+1);
@@ -458,7 +458,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
               }
               levelsRaw = tt;
             }
-            SET_TRUELENGTH(s,-(++nLevel));
+            SET_TRULEN(s,-(++nLevel));
             levelsRaw[nLevel-1] = s;
           }
           int *targetd = INTEGER(target);
@@ -467,7 +467,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
             if (length(thisCol)<=1) {
               // recycle length-1, or NA-fill length-0
               SEXP lev;
-              const int val = (length(thisCol)==1 && id[0]!=NA_INTEGER && (lev=thisColStrD[id[0]-1])!=NA_STRING) ? -TRUELENGTH(lev) : NA_INTEGER;
+              const int val = (length(thisCol)==1 && id[0]!=NA_INTEGER && (lev=thisColStrD[id[0]-1])!=NA_STRING) ? -TRULEN(lev) : NA_INTEGER;
               //                                                                                    ^^ #3915 and tests 2015.2-5
               for (int r=0; r<thisnrow; ++r) targetd[ansloc+r] = val;
             } else {
@@ -478,22 +478,22 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
                 // retain the position of NA level (if any) and the integer mappings to it
                 for (int k=0; k<n; ++k) {
                   SEXP s = thisColStrD[k];
-                  if (s!=NA_STRING && -TRUELENGTH(s)!=k+1) { hop=true; break; }
+                  if (s!=NA_STRING && -TRULEN(s)!=k+1) { hop=true; break; }
                 }
               } else {
                 for (int k=0; k<n; ++k) {
                   SEXP s = thisColStrD[k];
-                  if (s==NA_STRING || -TRUELENGTH(s)!=k+1) { hop=true; break; }
+                  if (s==NA_STRING || -TRULEN(s)!=k+1) { hop=true; break; }
                 }
               }
               if (hop) {
                 if (orderedFactor) {
                   for (int r=0; r<thisnrow; ++r)
-                    targetd[ansloc+r] = id[r]==NA_INTEGER ? NA_INTEGER : -TRUELENGTH(thisColStrD[id[r]-1]);
+                    targetd[ansloc+r] = id[r]==NA_INTEGER ? NA_INTEGER : -TRULEN(thisColStrD[id[r]-1]);
                 } else {
                   for (int r=0; r<thisnrow; ++r) {
                     SEXP lev;
-                    targetd[ansloc+r] = id[r]==NA_INTEGER || (lev=thisColStrD[id[r]-1])==NA_STRING ? NA_INTEGER : -TRUELENGTH(lev);
+                    targetd[ansloc+r] = id[r]==NA_INTEGER || (lev=thisColStrD[id[r]-1])==NA_STRING ? NA_INTEGER : -TRULEN(lev);
                   }
                 }
               } else {
@@ -503,16 +503,16 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg, SEXP ignor
           } else {
             const SEXP *sd = STRING_PTR_RO(thisColStr);
             if (length(thisCol)<=1) {
-              const int val = (length(thisCol)==1 && sd[0]!=NA_STRING) ? -TRUELENGTH(sd[0]) : NA_INTEGER;
+              const int val = (length(thisCol)==1 && sd[0]!=NA_STRING) ? -TRULEN(sd[0]) : NA_INTEGER;
               for (int r=0; r<thisnrow; ++r) targetd[ansloc+r] = val;
             } else {
-              for (int r=0; r<thisnrow; ++r) targetd[ansloc+r] = sd[r]==NA_STRING ? NA_INTEGER : -TRUELENGTH(sd[r]);
+              for (int r=0; r<thisnrow; ++r) targetd[ansloc+r] = sd[r]==NA_STRING ? NA_INTEGER : -TRULEN(sd[r]);
             }
           }
         }
         ansloc += thisnrow;
       }
-      for (int k=0; k<nLevel; ++k) SET_TRUELENGTH(levelsRaw[k], 0);
+      for (int k=0; k<nLevel; ++k) SET_TRULEN(levelsRaw[k], 0);
       savetl_end();
       if (warnStr[0]) warning("%s", warnStr);  // now savetl_end() has happened it's safe to call warning (could error if options(warn=2))
       SEXP levelsSxp;
diff --git a/src/subset.c b/src/subset.c
index c7653b893e..4da2a6416f 100644
--- a/src/subset.c
+++ b/src/subset.c
@@ -305,8 +305,8 @@ SEXP subsetDT(SEXP x, SEXP rows, SEXP cols) { // API change needs update NEWS.md
   // includes row.names (oddly, given other dims aren't) and "sorted" dealt with below
   // class is also copied here which retains superclass name in class vector as has been the case for many years; e.g. tests 1228.* for #64
 
-  SET_TRUELENGTH(ans, LENGTH(ans));
-  SETLENGTH(ans, LENGTH(cols));
+  SET_TRULEN(ans, LENGTH(ans));
+  SET_LEN(ans, LENGTH(cols));
   int ansn;
   if (isNull(rows)) {
     ansn = nrow;
@@ -330,8 +330,8 @@ SEXP subsetDT(SEXP x, SEXP rows, SEXP cols) { // API change needs update NEWS.md
     }
   }
   SEXP tmp = PROTECT(allocVector(STRSXP, LENGTH(cols)+overAlloc)); nprotect++;
-  SET_TRUELENGTH(tmp, LENGTH(tmp));
-  SETLENGTH(tmp, LENGTH(cols));
+  SET_TRULEN(tmp, LENGTH(tmp));
+  SET_LEN(tmp, LENGTH(cols));
   setAttrib(ans, R_NamesSymbol, tmp);
   subsetVectorRaw(tmp, getAttrib(x, R_NamesSymbol), cols, /*anyNA=*/false);
 
diff --git a/src/utils.c b/src/utils.c
index 7f529e943e..7d9dde3c9f 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -267,27 +267,27 @@ void copySharedColumns(SEXP x) {
   // the savetl() function elsewhere is for CHARSXP. Here, we are using truelength on atomic vectors.
   for (int i=0; i<ncol; ++i) {
     const SEXP thiscol = xp[i];
-    savetl[i] = ALTREP(thiscol) ? 0 : TRUELENGTH(thiscol);
-    SET_TRUELENGTH(thiscol, 0);
+    savetl[i] = TRULEN(thiscol);
+    SET_TRULEN(thiscol, 0);
   }
   int nShared=0;
   for (int i=0; i<ncol; ++i) {
     SEXP thiscol = xp[i];
-    if (ALTREP(thiscol) || TRUELENGTH(thiscol)<0) {
+    if (ALTREP(thiscol) || TRULEN(thiscol)<0) {
       shared[i] = true;  // we mark ALTREP as 'shared' too, whereas 'tocopy' would be better word to use for ALTREP
       nShared++;
       // do not copyAsPlain() here yet, as its alloc might fail. Must restore tl first to all columns before attempting any copies.
     } else {
       shared[i] = false;              // so the first column will never be shared (unless it is an altrep) even it is shared
                                       // 'shared' means a later column shares an earlier column
-      SET_TRUELENGTH(thiscol, -i-1);  // -i-1 so that if, for example, column 3 shares column 1, in iteration 3 we'll know not
+      SET_TRULEN(thiscol, -i-1);  // -i-1 so that if, for example, column 3 shares column 1, in iteration 3 we'll know not
                                       // only that the 3rd column is shared with an earlier column, but which one too. Although
                                       // we don't use that information currently, we could do in future.
     }
   }
   // now we know nShared and which ones they are (if any), restore original tl back to the unique set of columns
   for (int i=0; i<ncol; ++i) {
-    if (!shared[i]) SET_TRUELENGTH(xp[i], savetl[i]);
+    if (!shared[i]) SET_TRULEN(xp[i], savetl[i]);
     //  ^^^^^^^^^^ important because if there are shared columns, the dup will have savetl==0 but we want the first restore to stand
   }
   // now that truelength has been restored for all columns, we can finally call copyAsPlain()

From 03fc49d5689bd7c179d0d17d2098269cfcae0c3e Mon Sep 17 00:00:00 2001
From: Sebastian Krantz <sebastian.krantz@graduateinstitute.ch>
Date: Sun, 8 Dec 2024 00:39:15 +0100
Subject: [PATCH 2/4] Performance boost by replacing TRUELENGTH ->
 STDVEC_TRUELENGTH for strings (no ALTREP strings + algorithms wouldn't work
 anymore).

---
 src/chmatch.c | 14 +++++++-------
 src/forder.c  | 12 ++++++------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/chmatch.c b/src/chmatch.c
index 6c7fc8a813..99c70d3e71 100644
--- a/src/chmatch.c
+++ b/src/chmatch.c
@@ -58,7 +58,7 @@ static SEXP chmatchMain(SEXP x, SEXP table, int nomatch, bool chin, bool chmatch
   savetl_init();
   for (int i=0; i<xlen; i++) {
     SEXP s = xd[i];
-    const int tl = TRULEN(s);
+    const int tl = STDVEC_TRUELENGTH(s);
     if (tl>0) {
       savetl(s);  // R's internal hash (which is positive); save it
       SET_TRULEN(s,0);
@@ -75,7 +75,7 @@ static SEXP chmatchMain(SEXP x, SEXP table, int nomatch, bool chin, bool chmatch
   int nuniq=0;
   for (int i=0; i<tablelen; ++i) {
     const SEXP s = td[i];
-    int tl = TRULEN(s);
+    int tl = STDVEC_TRUELENGTH(s);
     if (tl>0) { savetl(s); tl=0; }
     if (tl==0) SET_TRULEN(s, chmatchdup ? -(++nuniq) : -i-1); // first time seen this string in table
   }
@@ -105,12 +105,12 @@ static SEXP chmatchMain(SEXP x, SEXP table, int nomatch, bool chin, bool chmatch
       error(_("Failed to allocate %"PRIu64" bytes working memory in chmatchdup: length(table)=%d length(unique(table))=%d"), ((uint64_t)tablelen*2+nuniq)*sizeof(int), tablelen, nuniq);
       // # nocov end
     }
-    for (int i=0; i<tablelen; ++i) counts[-TRULEN(td[i])-1]++;
+    for (int i=0; i<tablelen; ++i) counts[-STDVEC_TRUELENGTH(td[i])-1]++;
     for (int i=0, sum=0; i<nuniq; ++i) { int tt=counts[i]; counts[i]=sum; sum+=tt+1; }
-    for (int i=0; i<tablelen; ++i) map[counts[-TRULEN(td[i])-1]++] = i+1;           // 0 is left ending each group thanks to the calloc
+    for (int i=0; i<tablelen; ++i) map[counts[-STDVEC_TRUELENGTH(td[i])-1]++] = i+1;           // 0 is left ending each group thanks to the calloc
     for (int i=0, last=0; i<nuniq; ++i) {int tt=counts[i]+1; counts[i]=last; last=tt;}  // rewind counts to the beginning of each group
     for (int i=0; i<xlen; ++i) {
-      int u = TRULEN(xd[i]);
+      int u = STDVEC_TRUELENGTH(xd[i]);
       if (u<0) {
         const int w = counts[-u-1]++;
         if (map[w]) { ansd[i]=map[w]; continue; }
@@ -123,11 +123,11 @@ static SEXP chmatchMain(SEXP x, SEXP table, int nomatch, bool chin, bool chmatch
     free(map);
   } else if (chin) {
     for (int i=0; i<xlen; i++) {
-      ansd[i] = TRULEN(xd[i])<0;
+      ansd[i] = STDVEC_TRUELENGTH(xd[i])<0;
     }
   } else {
     for (int i=0; i<xlen; i++) {
-      const int m = TRULEN(xd[i]);
+      const int m = STDVEC_TRUELENGTH(xd[i]);
       ansd[i] = (m<0) ? -m : nomatch;
     }
   }
diff --git a/src/forder.c b/src/forder.c
index c1e001b062..2ca68f3397 100644
--- a/src/forder.c
+++ b/src/forder.c
@@ -307,10 +307,10 @@ static void range_str(const SEXP *x, int n, uint64_t *out_min, uint64_t *out_max
       na_count++;
       continue;
     }
-    if (TRULEN(s)<0) continue;  // seen this group before
+    if (STDVEC_TRUELENGTH(s)<0) continue;  // seen this group before
     #pragma omp critical
-    if (TRULEN(s)>=0) {  // another thread may have set it while I was waiting, so check it again
-      if (TRULEN(s)>0)   // save any of R's own usage of tl (assumed positive, so we can both count and save in one scan), to restore
+    if (STDVEC_TRUELENGTH(s)>=0) {  // another thread may have set it while I was waiting, so check it again
+      if (STDVEC_TRUELENGTH(s)>0)   // save any of R's own usage of tl (assumed positive, so we can both count and save in one scan), to restore
         savetl(s);           // afterwards. From R 2.14.0, tl is initialized to 0, prior to that it was random so this step saved too much.
       // now save unique SEXP in ustr so i) we can loop through them afterwards and reset TRUELENGTH to 0 and ii) sort uniques when sorting too
       if (ustr_alloc<=ustr_n) {
@@ -351,7 +351,7 @@ static void range_str(const SEXP *x, int n, uint64_t *out_min, uint64_t *out_max
     for (int i=0; i<ustr_n; i++) {
       SEXP s = ustr3[i];
       if (LENGTH(s)>ustr_maxlen) ustr_maxlen=LENGTH(s);
-      if (TRULEN(s)>0) savetl(s);
+      if (STDVEC_TRUELENGTH(s)>0) savetl(s);
     }
     cradix(ustr3, ustr_n);  // sort to detect possible duplicates after converting; e.g. two different non-utf8 map to the same utf8
     SET_TRULEN(ustr3[0], -1);
@@ -365,7 +365,7 @@ static void range_str(const SEXP *x, int n, uint64_t *out_min, uint64_t *out_max
     if (!tl)
       STOP(_("Failed to alloc tl when converting strings to UTF8"));  // # nocov
     const SEXP *tt = STRING_PTR_RO(ustr2);
-    for (int i=0; i<ustr_n; i++) tl[i] = TRULEN(tt[i]);   // fetches the o in ustr3 into tl which is ordered by ustr
+    for (int i=0; i<ustr_n; i++) tl[i] = STDVEC_TRUELENGTH(tt[i]);   // fetches the o in ustr3 into tl which is ordered by ustr
     for (int i=0; i<ustr_n; i++) SET_TRULEN(ustr3[i], 0);    // reset to 0 tl of the UTF8 (and possibly non-UTF in ustr too)
     for (int i=0; i<ustr_n; i++) SET_TRULEN(ustr[i], tl[i]); // put back the o into ustr's tl
     free(tl);
@@ -766,7 +766,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrpArg, SEXP retStatsArg, SEXP sortGroupsA
           if (nalast==-1) anso[i]=0;
           elem = naval;
         } else {
-          elem = -TRULEN(xd[i]);
+          elem = -STDVEC_TRUELENGTH(xd[i]);
         }
         WRITE_KEY
       }}

From 75f2e159092525d5bfa6d95d44107d32c5bc5776 Mon Sep 17 00:00:00 2001
From: Sebastian Krantz <sebastian.krantz@graduateinstitute.ch>
Date: Sun, 8 Dec 2024 00:51:46 +0100
Subject: [PATCH 3/4] Hard to resist that one...

---
 DESCRIPTION | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/DESCRIPTION b/DESCRIPTION
index 7447fed2a2..17ff723b4d 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -100,5 +100,6 @@ Authors@R: c(
   person("Vincent", "Rocher",      role="ctb"),
   person("Vijay", "Lulla",         role="ctb"),
   person("Aljaž", "Sluga",         role="ctb"),
-  person("Bill", "Evans",          role="ctb")
+  person("Bill", "Evans",          role="ctb"),
+  person("Sebastian", "Krantz",    role="ctb")
   )

From 9cecde1deba67769c57af5c31846a402cc6f5317 Mon Sep 17 00:00:00 2001
From: Sebastian Krantz <sebastian.krantz@graduateinstitute.ch>
Date: Sun, 8 Dec 2024 19:11:27 +0100
Subject: [PATCH 4/4] Better.

---
 src/R_Defn.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/R_Defn.h b/src/R_Defn.h
index 41726c6b4e..111c6d04a4 100644
--- a/src/R_Defn.h
+++ b/src/R_Defn.h
@@ -76,7 +76,7 @@ typedef struct {
   SETSCAL(__x__, __v__ == 1 ? 1 : 0);	     \
 } while (0)                                
 // https://github.com/wch/r-source/blob/05bb18266d49e87f2477120ecb0ab1440f4e9b40/src/main/memory.c#L4072  
-#define SET_LEN(x, v) SET_STDVEC_LENGTH((x), (v))
+#define SET_LEN SET_STDVEC_LENGTH
 
 // LEVELS: https://github.com/wch/r-source/blob/2640a203d13473f95c9c7508eb2976fefb5c931c/src/include/Defn.h#L228    
 #define LEVLS(x)	(((SEXPREC_partial *)(x))->sxpinfo.gp)