new join timings with higher memory limit

duckdb-latest/join-duckdb-latest.R

@@ -44,7 +44,7 @@ if (less_cores) {
   ncores = min(ncores, 40)
 }
 invisible(dbExecute(con, sprintf("PRAGMA THREADS=%d", ncores)))
-invisible(dbExecute(con, "SET memory_limit='210GB'"))
+invisible(dbExecute(con, "SET memory_limit='220GB'"))
 git = dbGetQuery(con, "SELECT source_id FROM pragma_version()")[[1L]]
 
 invisible({

duckdb/join-duckdb.R

@@ -41,7 +41,7 @@ if (less_cores) {
 }
 
 invisible(dbExecute(con, sprintf("PRAGMA THREADS=%d", ncores)))
-invisible(dbExecute(con, "SET memory_limit='210GB'"))
+invisible(dbExecute(con, "SET memory_limit='220GB'"))
 git = dbGetQuery(con, "SELECT source_id FROM pragma_version()")[[1L]]
 
 invisible({

logs.csv

@@ -1743,3 +1743,5 @@ ip-172-31-31-147,1720089751,duckdb,1.0.0,1f98600c2c,join,J1_1e8_NA_5_0,172009118
 ip-172-31-31-147,1720089751,duckdb,1.0.0,1f98600c2c,join,J1_1e8_NA_5_0,1720091211.49617,finish,0,0
 ip-172-31-31-147,1720089751,duckdb,1.0.0,1f98600c2c,join,J1_1e8_NA_0_1,1720091226.51143,start,,
 ip-172-31-31-147,1720089751,duckdb,1.0.0,1f98600c2c,join,J1_1e8_NA_0_1,1720091262.47306,finish,0,0
+ip-172-31-31-147,1720529276,duckdb,1.0.0,1f98600c2c,join,J1_1e9_NA_0_0,1720529277.33854,start,,
+ip-172-31-31-147,1720529276,duckdb,1.0.0,1f98600c2c,join,J1_1e9_NA_0_0,1720530147.30009,finish,0,0

time.csv

@@ -12615,3 +12615,13 @@ ip-172-31-31-147,1720089751,1720091255,join,J1_1e8_NA_0_1,100000000,medium inner
 ip-172-31-31-147,1720089751,1720091255,join,J1_1e8_NA_0_1,100000000,medium inner on factor,89995511,11,duckdb,1.0.0,1f98600c2c,inner_join,2,0.516,NA,TRUE,4499224468;4506958891,0.042,NA,FALSE
 ip-172-31-31-147,1720089751,1720091258,join,J1_1e8_NA_0_1,100000000,big inner on int,90000000,13,duckdb,1.0.0,1f98600c2c,inner_join,1,2.285,NA,TRUE,4499618843;4499951833,0.019,NA,FALSE
 ip-172-31-31-147,1720089751,1720091260,join,J1_1e8_NA_0_1,100000000,big inner on int,90000000,13,duckdb,1.0.0,1f98600c2c,inner_join,2,2.307,NA,TRUE,4499618843;4499951833,0.02,NA,FALSE
+ip-172-31-31-147,1720529276,1720529971,join,J1_1e9_NA_0_0,1000000000,small inner on int,899999033,9,duckdb,1.0.0,1f98600c2c,inner_join,1,4.804,NA,TRUE,44998904641;45286789554,0.238,NA,TRUE
+ip-172-31-31-147,1720529276,1720529976,join,J1_1e9_NA_0_0,1000000000,small inner on int,899999033,9,duckdb,1.0.0,1f98600c2c,inner_join,2,4.58,NA,TRUE,44998904641;45286789554,0.122,NA,TRUE
+ip-172-31-31-147,1720529276,1720529988,join,J1_1e9_NA_0_0,1000000000,medium inner on int,899989698,11,duckdb,1.0.0,1f98600c2c,inner_join,1,8.366,NA,TRUE,44998219782;44956313963,0.203,NA,TRUE
+ip-172-31-31-147,1720529276,1720529998,join,J1_1e9_NA_0_0,1000000000,medium inner on int,899989698,11,duckdb,1.0.0,1f98600c2c,inner_join,2,8.027,NA,TRUE,44998219782;44956313963,0.177,NA,TRUE
+ip-172-31-31-147,1720529276,1720530013,join,J1_1e9_NA_0_0,1000000000,medium outer on int,1000000000,11,duckdb,1.0.0,1f98600c2c,left_join,1,8.907,NA,TRUE,49998622440;44956313963,0.219,NA,TRUE
+ip-172-31-31-147,1720529276,1720530024,join,J1_1e9_NA_0_0,1000000000,medium outer on int,1000000000,11,duckdb,1.0.0,1f98600c2c,left_join,2,8.769,NA,TRUE,49998622440;44956313963,0.189,NA,TRUE
+ip-172-31-31-147,1720529276,1720530036,join,J1_1e9_NA_0_0,1000000000,medium inner on factor,899989698,11,duckdb,1.0.0,1f98600c2c,inner_join,1,6.988,NA,TRUE,44998219782;44956313963,0.155,NA,TRUE
+ip-172-31-31-147,1720529276,1720530044,join,J1_1e9_NA_0_0,1000000000,medium inner on factor,899989698,11,duckdb,1.0.0,1f98600c2c,inner_join,2,6.859,NA,TRUE,44998219782;44956313963,0.144,NA,TRUE
+ip-172-31-31-147,1720529276,1720530081,join,J1_1e9_NA_0_0,1000000000,big inner on int,900000000,13,duckdb,1.0.0,1f98600c2c,inner_join,1,34.159,NA,TRUE,44999013966;45000583182,0.583,NA,TRUE
+ip-172-31-31-147,1720529276,1720530123,join,J1_1e9_NA_0_0,1000000000,big inner on int,900000000,13,duckdb,1.0.0,1f98600c2c,inner_join,2,34.749,NA,TRUE,44999013966;45000583182,0.514,NA,TRUE