Skip to content

Commit ccc36c1

Browse files
MichaelChiricoMichaelChirico
committed
cbindlist
add cbind by reference, timing R prototype of mergelist wording use lower overhead funs stick to int32 for now, correct R_alloc bmerge C refactor for codecov and one loop for speed address revealed codecov gaps refactor vecseq for codecov seqexp helper, some alloccol export on C bmerge codecov, types handled in R bmerge already better comment seqexp bmerge mult=error #655 multiple new C utils swap if branches explain new C utils comments mostly reduce conflicts to PR #4386 comment C code address multiple matches during update-on-join #3747 Revert "address multiple matches during update-on-join #3747" This reverts commit b64c0c3. merge.dt has temporarily mult arg, for testing minor changes to cbindlist c dev mergelist, for single pair now add quiet option to cc() mergelist tests add check for names to perhaps.dt rm mult from merge.dt method rework, clean, polish multer, fix righ and full joins make full join symmetric mergepair inner function to loop on extra check for symmetric mergelist manual ensure no df-dt passed where list expected comments and manual handle 0 cols tables more tests more tests and debugging move more logic closer to bmerge, simplify mergepair more tests revert not used changes reduce not needed checks, cleanup copy arg behavior, manual, no tests yet cbindlist manual, export both cleanup processing bmerge to dtmatch test function match order for easier preview vecseq gets short-circuit batch test allow browser big cleanup remmove unneeded stuff, reduce diff more cleanup, minor manual fixes add proper test scripts Merge branch 'master' into cbind-merge-list comment out not used code for coverage more tests, some nocopy opts rename sql test script, should fix codecov simplify dtmatch inner branch more precise copy, now copy only T or F unused arg not yet in api, wording comments and refer issues codecov hasindex coverage codecov gap tests for join using key, cols argument fix missing import forderv more tests, improve missing on handling more tests for order of inner and full join for long keys new allow.cartesian option, #4383, #914 reduce diff, improve codecov reduce diff, comments need more DT, not lists, mergelist 3+ tbls proper escape heavy check unit tests more tests, address overalloc failure mergelist and cbindlist retain index manual, examples fix manual minor clarify in manual retain keys, right outer join for snowflake schema joins duplicates in cbindlist recycling in cbindlist escape 0 input in copyCols empty input handling closing cbindlist vectorized _on_ and _join.many_ arg rename dtmatch to dtmerge vectorized args: how, mult push down input validation add support for cross join, semi join, anti join full join, reduce overhead for mult=error mult default value dynamic fix manual add "see details" to Rd mention shared on in arg description amend feedback from Michael semi and anti joins will not reorder x columns Merge branch 'master' into cbind-merge-list spelling, thx to @jan-glx check all new funs used and add comments bugfix, sort=T needed for now Merge branch 'master' into cbind-merge-list Update NEWS.md Merge branch 'master' into cbind-merge-list Merge branch 'master' into cbind-merge-list NEWS placement numbering ascArg->order Merge remote-tracking branch 'origin/cbind-merge-list' into cbind-merge-list attempt to restore from master Update to stopf() error style Need isFrame for now More quality checks: any(!x)->!all(x); use vapply_1{b,c,i} really restore from master try to PROTECT() before duplicate() update error message in test appease the rchk gods extraneous space missing ';' use catf simplify perhapsDataTableR move sqlite.Rraw.manual into other.Rraw simplify for loop Merge remote-tracking branch 'origin/cbind-merge-list' into cbind-merge-list
1 parent cbcf3ed commit ccc36c1

File tree

3 files changed

+305
-0
lines changed

3 files changed

+305
-0
lines changed

‎R/mergelist.R

+110
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,116 @@ dtmerge = function(x, i, on, how, mult, join.many, void=FALSE, verbose) {
125125
return(list(ans=ans, irows=irows, xrows=xrows))
126126
}
127127

128+
# atomic join between two tables
129+
mergepair = function(lhs, rhs, on, how, mult, lhs.cols=names(lhs), rhs.cols=names(rhs), copy=TRUE, join.many=TRUE, verbose=FALSE) {
130+
semianti = how=="semi" || how=="anti"
131+
innerfull = how=="inner" || how=="full"
132+
{
133+
if (how!="cross") {
134+
if (is.null(on)) {
135+
if (how=="left" || semianti) on = key(rhs)
136+
else if (how=="right") on = key(lhs)
137+
else if (innerfull) on = onkeys(key(lhs), key(rhs))
138+
if (is.null(on))
139+
stopf("'on' is missing and necessary key is not present")
140+
}
141+
if (any(bad.on <- !on %chin% names(lhs)))
142+
stopf("'on' argument specify columns to join [%s] that are not present in LHS table [%s]", brackify(on[bad.on]), brackify(names(lhs)))
143+
if (any(bad.on <- !on %chin% names(rhs)))
144+
stopf("'on' argument specify columns to join [%s] that are not present in RHS table [%s]", brackify(on[bad.on]), brackify(names(rhs)))
145+
} else if (is.null(on)) {
146+
on = character() ## cross join only
147+
}
148+
} ## on
149+
{
150+
if (how!="right") {
151+
jnfm = lhs; fm.cols = lhs.cols; jnto = rhs; to.cols = rhs.cols
152+
} else {
153+
jnfm = rhs; fm.cols = rhs.cols; jnto = lhs; to.cols = lhs.cols
154+
}
155+
} ## join-to and join-from tables and columns (right outer join swap)
156+
157+
## ensure symmetric join for inner|full join, apply mult on both tables, bmerge do only 'x' table
158+
cp.i = FALSE ## copy marker of out.i
159+
if ((innerfull) && !is.null(mult) && (mult=="first" || mult=="last")) {
160+
jnfm = fdistinct(jnfm, on=on, mult=mult, cols=fm.cols, copy=FALSE) ## might not copy when already unique by 'on'
161+
cp.i = nrow(jnfm)!=nrow(lhs) ## nrow(lhs) bc how='inner|full' so jnfm=lhs
162+
} else if (how=="inner" && (is.null(mult) || mult=="error")) { ## we do this branch only to raise error from bmerge, we cannot use forder to just find duplicates because those duplicates might not have matching rows in another table, full join checks mult='error' during two non-void bmerges
163+
dtmerge(x=jnfm, i=jnto, on=on, how=how, mult=mult, verbose=verbose, join.many=join.many, void=TRUE)
164+
}
165+
166+
## binary merge
167+
ans = dtmerge(x=jnto, i=jnfm, on=on, how=how, mult=mult, verbose=verbose, join.many=join.many)
168+
169+
## make i side
170+
out.i = if (is.null(ans$irows))
171+
.shallow(jnfm, cols=someCols(jnfm, fm.cols, keep=on, retain.order=semianti), retain.key=TRUE)
172+
else
173+
.Call(CsubsetDT, jnfm, ans$irows, someCols(jnfm, fm.cols, keep=on, retain.order=semianti))
174+
cp.i = cp.i || !is.null(ans$irows)
175+
176+
## make x side
177+
if (semianti) {
178+
out.x = list(); cp.x = TRUE
179+
} else {
180+
out.x = if (is.null(ans$xrows)) ## as of now xrows cannot be NULL #4409 thus nocov below
181+
internal_error("dtmerge()$xrows returned NULL, #4409 been resolved but related code has not been updated?") #.shallow(jnto, cols=someCols(jnto, to.cols, drop=on), retain.key=TRUE) # nocov ## as of now nocov does not make difference r-lib/covr#279
182+
else
183+
.Call(CsubsetDT, jnto, ans$xrows, someCols(jnto, to.cols, drop=on))
184+
cp.x = !is.null(ans$xrows)
185+
## ensure no duplicated column names in merge results
186+
if (any(dup.i<-names(out.i) %chin% names(out.x)))
187+
stopf("merge result has duplicated column names, use 'cols' argument or rename columns in 'l' tables, duplicated column(s): %s", brackify(names(out.i)[dup.i]))
188+
}
189+
190+
## stack i and x
191+
if (how!="full") {
192+
if (!cp.i && copy) out.i = copy(out.i)
193+
#if (!cp.x && copy) out.x = copy(out.x) ## as of now cp.x always TRUE, search for #4409 here
194+
out = .Call(Ccbindlist, list(out.i, out.x), FALSE)
195+
if (how=="right") setcolorder(out, neworder=c(on, names(out.x))) ## arrange columns: i.on, x.cols, i.cols
196+
} else { # how=="full"
197+
## we made left join side above, proceed to right join side, so swap tbls
198+
jnfm = rhs; fm.cols = rhs.cols; jnto = lhs; to.cols = lhs.cols
199+
200+
cp.r = FALSE
201+
if (!is.null(mult) && (mult=="first" || mult=="last")) {
202+
jnfm = fdistinct(jnfm, on=on, mult=mult, cols=fm.cols, copy=FALSE)
203+
cp.r = nrow(jnfm)!=nrow(rhs) ## nrow(rhs) bc jnfm=rhs
204+
} ## mult=="error" check was made on one side already, below we do on the second side, test 101.43
205+
206+
## binary merge anti join
207+
bns = dtmerge(x=jnto, i=jnfm, on=on, how="anti", mult=if (!is.null(mult) && mult!="all") mult, verbose=verbose, join.many=join.many)
208+
209+
## make anti join side
210+
out.r = if (is.null(bns$irows))
211+
.shallow(jnfm, cols=someCols(jnfm, fm.cols, keep=on), retain.key=TRUE) ## retain.key is used only in the edge case when !nrow(out.i)
212+
else
213+
.Call(CsubsetDT, jnfm, bns$irows, someCols(jnfm, fm.cols, keep=on))
214+
cp.r = cp.r || !is.null(bns$irows)
215+
216+
## short circuit to avoid rbindlist to empty sets and retains keys
217+
if (!nrow(out.r)) { ## possibly also !nrow(out.i)
218+
if (!cp.i && copy) out.i = copy(out.i)
219+
#if (!cp.x && copy) out.x = copy(out.x) ## as of now cp.x always TRUE, search for #4409 here
220+
out = .Call(Ccbindlist, list(out.i, out.x), FALSE)
221+
} else if (!nrow(out.i)) { ## but not !nrow(out.r)
222+
if (!cp.r && copy) out.r = copy(out.r)
223+
if (length(add<-setdiff(names(out.i), names(out.r)))) { ## add missing columns of proper types NA
224+
neworder = copy(names(out.i)) #set(out.r, NULL, add, lapply(unclass(out.i)[add], `[`, 1L)) ## 291.04 overalloc exceed fail during set()
225+
out.i = lapply(unclass(out.i)[add], `[`, seq_len(nrow(out.r))) ## could eventually remove this when cbindlist recycle 0 rows up, note that we need out.r not to be copied
226+
out.r = .Call(Ccbindlist, list(out.r, out.i), FALSE)
227+
setcolorder(out.r, neworder=neworder)
228+
}
229+
out = out.r
230+
} else { ## all might have not been copied yet, rbindlist will copy
231+
out.l = .Call(Ccbindlist, list(out.i, out.x), FALSE)
232+
out = rbindlist(list(out.l, out.r), use.names=TRUE, fill=TRUE)
233+
}
234+
}
235+
setDT(out)
236+
}
237+
128238
# Previously, we had a custom C implementation here, which is ~2x faster,
129239
# but this is fast enough we don't bother maintaining a new routine.
130240
# Hopefully in the future rep() can recognize the ALTREP and use that, too.

‎R/onLoad.R

+1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
"datatable.print.trunc.cols"="FALSE", # for print.data.table
8282
"datatable.show.indices"="FALSE", # for print.data.table
8383
"datatable.allow.cartesian"="FALSE", # datatable.<argument name>
84+
"datatable.join.many"="TRUE", # mergelist, [.data.table #4383 #914
8485
"datatable.dfdispatchwarn"="TRUE", # not a function argument
8586
"datatable.warnredundantby"="TRUE", # not a function argument
8687
"datatable.alloccol"="1024L", # argument 'n' of alloc.col. Over-allocate 1024 spare column slots

‎inst/tests/mergelist.Rraw

+194
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,24 @@ if (exists("test.data.table", .GlobalEnv, inherits=FALSE)) {
66
} else {
77
require(data.table)
88
test = data.table:::test
9+
mergepair = data.table:::mergepair
910
perhaps.data.table = data.table:::perhaps.data.table
1011
hasindex = data.table:::hasindex
1112
fdistinct = data.table:::fdistinct
1213
forderv = data.table:::forderv
1314
}
1415

1516
addresses = function(x) vapply(x, address, "")
17+
copied = function(ans, l) {
18+
all(!addresses(ans) %chin% unlist(recursive=FALSE, lapply(l, addresses)))
19+
}
20+
notcopied = function(ans, l, how="left", unless=character()) {
21+
if (how %chin% unless) return(copied(ans, l)) ## used during looping tests for easier escape
22+
if (how=="full") return( ## either side, left|right, notcopied is fine
23+
all(addresses(l[[1L]]) %chin% addresses(ans)) || all(addresses(l[[length(l)]]) %chin% addresses(ans))
24+
)
25+
all(addresses(if (how=="right") l[[length(l)]] else l[[1L]]) %chin% addresses(ans))
26+
}
1627

1728
# internal helpers
1829

@@ -110,6 +121,189 @@ test(13.04, key(ans), "id1")
110121
test(13.05, indices(ans), c("id1","id2","id3","id1__id2__id3","id6","id7","id9"))
111122
test(13.06, ii, lapply(l, indices)) ## this tests that original indices have not been touched, shallow_duplicate in mergeIndexAttrib
112123

124+
# mergepair
125+
126+
## test copy-ness argument in mergepair
127+
128+
### LHS equal to RHS: no copy in all cases
129+
num = 21.000
130+
l = list(
131+
lhs = data.table(id1=1:2, v1=1:2),
132+
rhs = data.table(id1=1:2, v2=1:2)
133+
)
134+
expected = data.table(id1=1:2, v1=1:2, v2=1:2)
135+
for (how in c("inner","left","right","full")) {
136+
num = trunc(num*10)/10 + 0.1
137+
for (mult in c("all","first","last","error")) {
138+
num = trunc(num*100)/100 + 0.01
139+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=TRUE), expected) ## copy=TRUE: no shared columns
140+
test(num<-num+0.001, copied(ans, l))
141+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=FALSE), expected) ## copy=FALSE: LHS shared but no RHS
142+
test(num<-num+0.001, notcopied(ans, l, how=how))
143+
}
144+
}
145+
### RHS includes LHS: no copy in inner, left, right
146+
num = 22.000
147+
unless = "full"
148+
l = list(
149+
lhs = data.table(id1=1:2, v1=1:2),
150+
rhs = data.table(id1=1:3, v2=1:3)
151+
)
152+
expected = list(
153+
inner = data.table(id1=1:2, v1=1:2, v2=1:2),
154+
left = data.table(id1=1:2, v1=1:2, v2=1:2),
155+
right = data.table(id1=1:3, v1=c(1:2,NA), v2=1:3),
156+
full = data.table(id1=1:3, v1=c(1:2,NA), v2=1:3)
157+
)
158+
for (how in c("inner","left","right","full")) {
159+
num = trunc(num*10)/10 + 0.1
160+
for (mult in c("all","first","last","error")) {
161+
num = trunc(num*100)/100 + 0.01
162+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=TRUE), expected[[how]])
163+
test(num<-num+0.001, copied(ans, l))
164+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=FALSE), expected[[how]])
165+
test(num<-num+0.001, notcopied(ans, l, how=how, unless=unless))
166+
}
167+
}
168+
### LHS includes RHS: no copy in left, right, full
169+
num = 23.000
170+
unless = "inner"
171+
l = list(
172+
lhs = data.table(id1=1:3, v1=1:3),
173+
rhs = data.table(id1=1:2, v2=1:2)
174+
)
175+
expected = list(
176+
inner = data.table(id1=1:2, v1=1:2, v2=1:2),
177+
left = data.table(id1=1:3, v1=1:3, v2=c(1:2,NA)),
178+
right = data.table(id1=1:2, v1=1:2, v2=1:2),
179+
full = data.table(id1=1:3, v1=1:3, v2=c(1:2,NA))
180+
)
181+
for (how in c("inner","left","right","full")) {
182+
num = trunc(num*10)/10 + 0.1
183+
for (mult in c("all","first","last","error")) {
184+
num = trunc(num*100)/100 + 0.01
185+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=TRUE), expected[[how]])
186+
test(num<-num+0.001, copied(ans, l))
187+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=FALSE), expected[[how]])
188+
test(num<-num+0.001, notcopied(ans, l, how=how, unless=unless))
189+
}
190+
}
191+
### LHS single nonmatch RHS on both sides: no copy in left, right
192+
num = 24.000
193+
unless = c("inner","full")
194+
l = list(
195+
lhs = data.table(id1=3:1, v1=1:3),
196+
rhs = data.table(id1=c(4L,2:1), v2=1:3)
197+
)
198+
expected = list(
199+
inner = data.table(id1=2:1, v1=2:3, v2=2:3),
200+
left = data.table(id1=3:1, v1=1:3, v2=c(NA,2:3)),
201+
right = data.table(id1=c(4L,2:1), v1=c(NA,2:3), v2=1:3),
202+
full = data.table(id1=c(3:1,4L), v1=c(1:3,NA), v2=c(NA,2:3,1L))
203+
)
204+
for (how in c("inner","left","right","full")) {
205+
num = trunc(num*10)/10 + 0.1
206+
for (mult in c("all","first","last","error")) {
207+
num = trunc(num*100)/100 + 0.01
208+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=TRUE), expected[[how]])
209+
test(num<-num+0.001, copied(ans, l))
210+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=FALSE), expected[[how]])
211+
test(num<-num+0.001, notcopied(ans, l, how=how, unless=unless))
212+
}
213+
}
214+
### LHS zero match RHS: no copy in left, right
215+
num = 25.000
216+
unless = c("inner","full")
217+
l = list(
218+
lhs = data.table(id1=2:1, v1=1:2),
219+
rhs = data.table(id1=3:4, v2=1:2)
220+
)
221+
expected = list(
222+
inner = data.table(id1=integer(), v1=integer(), v2=integer()),
223+
left = data.table(id1=2:1, v1=1:2, v2=c(NA_integer_,NA)),
224+
right = data.table(id1=3:4, v1=c(NA_integer_,NA), v2=1:2),
225+
full = data.table(id1=c(2:1,3:4), v1=c(1:2,NA,NA), v2=c(NA,NA,1:2))
226+
)
227+
for (how in c("inner","left","right","full")) {
228+
num = trunc(num*10)/10 + 0.1
229+
for (mult in c("all","first","last","error")) {
230+
num = trunc(num*100)/100 + 0.01
231+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=TRUE), expected[[how]])
232+
test(num<-num+0.001, copied(ans, l))
233+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=FALSE), expected[[how]])
234+
test(num<-num+0.001, notcopied(ans, l, how=how, unless=unless))
235+
}
236+
}
237+
### LHS and RHS zero nrow: no copies
238+
num = 26.000
239+
unless = character()
240+
l = list(
241+
lhs = data.table(id1=integer(), v1=integer()),
242+
rhs = data.table(id1=integer(), v2=integer())
243+
)
244+
expected = list(
245+
inner = data.table(id1=integer(), v1=integer(), v2=integer()),
246+
left = data.table(id1=integer(), v1=integer(), v2=integer()),
247+
right = data.table(id1=integer(), v1=integer(), v2=integer()),
248+
full = data.table(id1=integer(), v1=integer(), v2=integer())
249+
)
250+
for (how in c("inner","left","right","full")) {
251+
num = trunc(num*10)/10 + 0.1
252+
for (mult in c("all","first","last","error")) {
253+
num = trunc(num*100)/100 + 0.01
254+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=TRUE), expected[[how]])
255+
test(num<-num+0.001, copied(ans, l))
256+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=FALSE), expected[[how]])
257+
test(num<-num+0.001, notcopied(ans, l, how=how, unless=unless))
258+
}
259+
}
260+
### LHS has zero nrow: no copies
261+
num = 27.000
262+
unless = character()
263+
l = list(
264+
lhs = data.table(id1=integer(), v1=integer()),
265+
rhs = data.table(id1=2:1, v2=1:2)
266+
)
267+
expected = list(
268+
inner = data.table(id1=integer(), v1=integer(), v2=integer()),
269+
left = data.table(id1=integer(), v1=integer(), v2=integer()),
270+
right = data.table(id1=2:1, v1=c(NA_integer_,NA), v2=1:2),
271+
full = data.table(id1=2:1, v1=c(NA_integer_,NA), v2=1:2)
272+
)
273+
for (how in c("inner","left","right","full")) {
274+
num = trunc(num*10)/10 + 0.1
275+
for (mult in c("all","first","last","error")) {
276+
num = trunc(num*100)/100 + 0.01
277+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=TRUE), expected[[how]])
278+
test(num<-num+0.001, copied(ans, l))
279+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=FALSE), expected[[how]])
280+
test(num<-num+0.001, notcopied(ans, l, how=how, unless=unless))
281+
}
282+
}
283+
### RHS has zero nrow
284+
num = 28.000
285+
unless = "inner"
286+
l = list(
287+
lhs = data.table(id1=2:1, v1=1:2),
288+
rhs = data.table(id1=integer(), v2=integer())
289+
)
290+
expected = list(
291+
inner = data.table(id1=integer(), v1=integer(), v2=integer()),
292+
left = data.table(id1=2:1, v1=1:2, v2=c(NA_integer_,NA)),
293+
right = data.table(id1=integer(), v1=integer(), v2=integer()),
294+
full = data.table(id1=2:1, v1=1:2, v2=c(NA_integer_,NA))
295+
)
296+
for (how in c("inner","left","right","full")) {
297+
num = trunc(num*10)/10 + 0.1
298+
for (mult in c("all","first","last","error")) {
299+
num = trunc(num*100)/100 + 0.01
300+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=TRUE), expected[[how]])
301+
test(num<-num+0.001, copied(ans, l))
302+
test(num<-num+0.001, ans <- mergepair(l$lhs, l$rhs, on="id1", how=how, mult=mult, copy=FALSE), expected[[how]])
303+
test(num<-num+0.001, notcopied(ans, l, how=how, unless=unless))
304+
}
305+
}
306+
113307
## fdistinct, another round
114308

115309
dt = data.table(x =

0 commit comments

Comments
 (0)