Skip to content

Commit

Permalink
Merge pull request #2021 from Expensify/flo_sqliteoptims
Browse files Browse the repository at this point in the history
  • Loading branch information
@cead22
cead22 authored Dec 12, 2024
2 parents cf54b48 + d93a7dd commit 7488bd1
Show file tree
Hide file tree
Showing 2 changed files with 140 additions and 42 deletions.
180 changes: 139 additions & 41 deletions libstuff/sqlite3.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
** separate file. This file contains only code for the core SQLite library.
**
** The content in this amalgamation comes from Fossil check-in
** df4183ace93b788b798b258274bf6b651906.
** 3c25c69c93e55738cdbfdd87fa3c879b8786.
*/
#define SQLITE_CORE 1
#define SQLITE_AMALGAMATION 1
Expand Down Expand Up @@ -465,7 +465,7 @@ extern "C" {
*/
#define SQLITE_VERSION "3.47.0"
#define SQLITE_VERSION_NUMBER 3047000
#define SQLITE_SOURCE_ID "2024-12-10 14:56:20 df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523"
#define SQLITE_SOURCE_ID "2024-12-12 20:39:56 3c25c69c93e55738cdbfdd87fa3c879b878674973955490770f5e274da1ca9a4"

/*
** CAPI3REF: Run-Time Library Version Numbers
Expand Down Expand Up @@ -14125,9 +14125,13 @@ struct fts5_api {

/*
** The maximum number of arguments to an SQL function.
**
** This value has a hard upper limit of 32767 due to storage
** constraints (it needs to fit inside a i16). We keep it
** lower than that to prevent abuse.
*/
#ifndef SQLITE_MAX_FUNCTION_ARG
# define SQLITE_MAX_FUNCTION_ARG 127
# define SQLITE_MAX_FUNCTION_ARG 1000
#endif

/*
Expand Down Expand Up @@ -18236,14 +18240,15 @@ struct sqlite3 {

#define SCHEMA_TIME_STAT4_Q1_BODY 14
#define SCHEMA_TIME_AFTER_STAT4_Q1 15
#define SCHEMA_TIME_AFTER_STAT4_Q2 16
#define SCHEMA_TIME_AFTER_STAT4 17
#define SCHEMA_TIME_STAT4_Q2_BODY 16
#define SCHEMA_TIME_AFTER_STAT4_Q2 17
#define SCHEMA_TIME_AFTER_STAT4 18

#define SCHEMA_TIME_END_ANALYZE_LOAD 18
#define SCHEMA_TIME_FINISH 19
#define SCHEMA_TIME_END_ANALYZE_LOAD 19
#define SCHEMA_TIME_FINISH 20

#define SCHEMA_TIME_N 20
#define SCHEMA_TIME_TIMEOUT (0 * 1000 * 1000)
#define SCHEMA_TIME_N 21
#define SCHEMA_TIME_TIMEOUT (500 * 1000)



Expand Down Expand Up @@ -18418,7 +18423,7 @@ SQLITE_PRIVATE void sqlite3SchemaTimeLog(u64 *aSchemaTime, const char *zFile);
** field is used by per-connection app-def functions.
*/
struct FuncDef {
i8 nArg; /* Number of arguments. -1 means unlimited */
i16 nArg; /* Number of arguments. -1 means unlimited */
u32 funcFlags; /* Some combination of SQLITE_FUNC_* */
void *pUserData; /* User data parameter */
FuncDef *pNext; /* Next function with same name */
Expand Down Expand Up @@ -19247,7 +19252,7 @@ struct Index {
** expression, or a reference to a VIRTUAL column */
#ifdef SQLITE_ENABLE_STAT4
int nSample; /* Number of elements in aSample[] */
int mxSample; /* Number of slots allocated to aSample[] */
int nSampleAlloc; /* Number of slots allocated to aSample[] */
int nSampleCol; /* Size of IndexSample.anEq[] and so on */
tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */
IndexSample *aSample; /* Samples of the left-most key */
Expand Down Expand Up @@ -24030,7 +24035,7 @@ struct sqlite3_context {
int isError; /* Error code returned by the function. */
u8 enc; /* Encoding to use for results */
u8 skipFlag; /* Skip accumulator loading if true */
u8 argc; /* Number of arguments */
u16 argc; /* Number of arguments */
sqlite3_value *argv[1]; /* Argument set */
};

Expand Down Expand Up @@ -67624,9 +67629,9 @@ static int walIndexAppend(Wal *pWal, int iWal, u32 iFrame, u32 iPage){
*/
if( pWal->aCommitTime ) t = sqlite3STimeNow();
if( idx==1 && sLoc.aPgno[0]!=0 ){
int nByte = (int)((u8*)&sLoc.aHash[HASHTABLE_NSLOT] - (u8*)sLoc.aPgno);
assert( nByte>=0 && (nByte & 0x07)==0 );
zero64((void*)sLoc.aPgno, nByte);
/* Special for BEDROCK branch: Zero only the aHash[] part. Not the
** aPgno[] part of the page. */
zero64((void*)sLoc.aHash, HASHTABLE_NSLOT * sizeof(sLoc.aHash[0]));
}
if( pWal->aCommitTime ){
pWal->aCommitTime[COMMIT_TIME_WALINDEX_MEMSETUS]+=sqlite3STimeNow()-t;
Expand All @@ -67637,11 +67642,23 @@ static int walIndexAppend(Wal *pWal, int iWal, u32 iFrame, u32 iPage){
** writing one or more dirty pages to the WAL to free up memory).
** Remove the remnants of that writers uncommitted transaction from
** the hash-table before writing any new entries.
*/
if( sLoc.aPgno[idx-1] ){
**
** Special for BEDROCK branch: On this branch we do not assume that
** the aPgno[] part of each hash-table has been zeroed. Therefore, we
** only need to clear out the remnants of an old writer's transaction if
** the hash table matches the aPgno[] entry (as it would if a write
** transaction was interrupted). And, because this makes the test more
** expensive, we only do the check for the first frame written by each
** transaction. */
if( sLoc.aPgno[idx-1] && iFrame-1==walidxGetMxFrame(&pWal->hdr, iWal) ){
if( pWal->aCommitTime ) t = sqlite3STimeNow();
walCleanupHash(pWal);
assert( !sLoc.aPgno[idx-1] );
nCollide = idx;
for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){
if( sLoc.aHash[iKey]==idx ){
walCleanupHash(pWal);
}
if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT;
}
if( pWal->aCommitTime ){
pWal->aCommitTime[COMMIT_TIME_WALINDEX_CLEANUPUS]+=sqlite3STimeNow()-t;
}
Expand Down Expand Up @@ -93474,7 +93491,7 @@ SQLITE_PRIVATE void sqlite3CommitTimeLog(u64 *aCommit){
}
zStr = sqlite3_mprintf("%z%s%s%d%s", zStr, (zStr?", ":""),zHash,iVal,zU);
}
sqlite3_log(SQLITE_WARNING, "slow commit (v=17): (%s)", zStr);
sqlite3_log(SQLITE_WARNING, "slow commit (v=18): (%s)", zStr);
sqlite3_free(zStr);
}
}
Expand Down Expand Up @@ -93502,7 +93519,7 @@ SQLITE_PRIVATE void sqlite3PrepareTimeLog(const char *zSql, int nSql, u64 *aPrep
}
if( nByte<0 ){ nByte = sqlite3Strlen30(zSql); }
sqlite3_log(SQLITE_WARNING,
"slow prepare (v=17): (%s) [%.*s]", zStr, nByte, zSql
"slow prepare (v=18): (%s) [%.*s]", zStr, nByte, zSql
);
sqlite3_free(zStr);
}
Expand All @@ -93515,12 +93532,15 @@ SQLITE_PRIVATE void sqlite3SchemaTimeLog(u64 *aSchema, const char *zFile){
int ii;
for(ii=1; ii<SCHEMA_TIME_N; ii++){
int val = aSchema[ii];
if( val!=0 && ii!=SCHEMA_TIME_STAT4_Q1_BODY ){
if( val!=0
&& ii!=SCHEMA_TIME_STAT4_Q1_BODY
&& ii!=SCHEMA_TIME_STAT4_Q2_BODY
){
val -= i1;
}
zStr = sqlite3_mprintf("%z%s%d", zStr, (zStr?", ":""), val);
}
sqlite3_log(SQLITE_WARNING, "slow schema (%s) (v=17): (%s)", zFile, zStr);
sqlite3_log(SQLITE_WARNING, "slow schema (%s) (v=18): (%s)", zFile, zStr);
sqlite3_free(zStr);
}
}
Expand Down Expand Up @@ -112893,7 +112913,7 @@ static int codeCompare(
p5 = binaryCompareP5(pLeft, pRight, jumpIfNull);
addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1,
(void*)p4, P4_COLLSEQ);
sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5);
sqlite3VdbeChangeP5(pParse->pVdbe, (u16)p5);
return addr;
}

Expand Down Expand Up @@ -123675,6 +123695,7 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){
if( db->pnBytesFreed==0 ){
pIdx->nSample = 0;
pIdx->aSample = 0;
pIdx->nSampleAlloc = 0;
}
#else
UNUSED_PARAMETER(db);
Expand Down Expand Up @@ -123759,6 +123780,70 @@ static Index *findIndexOrPrimaryKey(
return pIdx;
}

/*
** Grow the pIdx->aSample[] array. Return SQLITE_OK if successful, or
** SQLITE_NOMEM otherwise.
*/
static int growSampleArray(sqlite3 *db, Index *pIdx){
int nIdxCol = pIdx->nSampleCol;
int nNew = 0;
IndexSample *aNew = 0;
int nByte = 0;
tRowcnt *pSpace; /* Available allocated memory space */
u8 *pPtr; /* Available memory as a u8 for easier manipulation */
int i;

/* In production set the initial allocation to SQLITE_STAT4_SAMPLES. This
** means that reallocation will almost never be required. But for debug
** builds, set the initial allocation size to 6 entries so that the
** reallocation code gets tested. todo: use real tests for this. */
assert( pIdx->nSample==pIdx->nSampleAlloc );
#ifdef SQLITE_DEBUG
nNew = 6;
#else
nNew = SQLITE_STAT4_SAMPLES;
#endif
if( pIdx->nSample ){
nNew = pIdx->nSample*2;
}

nByte = ROUND8(sizeof(IndexSample) * nNew);
nByte += sizeof(tRowcnt) * nIdxCol * 3 * nNew;
nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */

aNew = (IndexSample*)sqlite3DbMallocZero(db, nByte);
if( aNew==0 ) return SQLITE_NOMEM_BKPT;

pPtr = (u8*)aNew;
pPtr += ROUND8(nNew*sizeof(pIdx->aSample[0]));
pSpace = (tRowcnt*)pPtr;

pIdx->aAvgEq = pSpace; pSpace += nIdxCol;
assert( EIGHT_BYTE_ALIGNMENT( pSpace ) );

if( pIdx->nSample ){
/* Copy the contents of the anEq[], anLt[], anDLt[] arrays for all
** extant samples to the new location. */
int nByte = nIdxCol * 3 * sizeof(tRowcnt) * pIdx->nSample;
memcpy(pSpace, pIdx->aSample[0].anEq, nByte);
}
for(i=0; i<nNew; i++){
aNew[i].anEq = pSpace; pSpace += nIdxCol;
aNew[i].anLt = pSpace; pSpace += nIdxCol;
aNew[i].anDLt = pSpace; pSpace += nIdxCol;
if( i<pIdx->nSample ){
aNew[i].p = pIdx->aSample[i].p;
aNew[i].n = pIdx->aSample[i].n;
}
}
assert( ((u8*)pSpace)-nByte==(u8*)aNew );

sqlite3DbFree(db, pIdx->aSample);
pIdx->aSample = aNew;
pIdx->nSampleAlloc = nNew;
return SQLITE_OK;
}

/*
** Load the content from either the sqlite_stat4
** into the relevant Index.aSample[] arrays.
Expand All @@ -123784,6 +123869,7 @@ static int loadStatTbl(
IndexSample *pSample; /* A slot in pIdx->aSample[] */

assert( db->lookaside.bDisable );
#if 0
zSql = sqlite3MPrintf(db, zSql1, zDb);
if( !zSql ){
return SQLITE_NOMEM_BKPT;
Expand Down Expand Up @@ -123850,6 +123936,9 @@ static int loadStatTbl(
}
rc = sqlite3_finalize(pStmt);
if( rc ) return rc;
#endif

sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1);

zSql = sqlite3MPrintf(db, zSql2, zDb);
if( !zSql ){
Expand All @@ -123859,29 +123948,34 @@ static int loadStatTbl(
sqlite3DbFree(db, zSql);
if( rc ) return rc;

sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1);

while( sqlite3_step(pStmt)==SQLITE_ROW ){
char *zIndex; /* Index name */
Index *pIdx; /* Pointer to the index object */
int nCol = 1; /* Number of columns in index */
u64 t = sqlite3STimeNow();

zIndex = (char *)sqlite3_column_text(pStmt, 0);
if( zIndex==0 ) continue;
pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
if( pIdx==0 ) continue;
if( pIdx->nSample>=pIdx->mxSample ){
/* Too many slots used because the same index appears in
** sqlite_stat4 using multiple names */
continue;

if( pIdx->nSample==pIdx->nSampleAlloc ){
pIdx->pTable->tabFlags |= TF_HasStat4;
assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 );
if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
pIdx->nSampleCol = pIdx->nKeyCol;
}else{
pIdx->nSampleCol = pIdx->nColumn;
}
if( growSampleArray(db, pIdx) ) break;
}
/* This next condition is true if data has already been loaded from
** the sqlite_stat4 table. */
nCol = pIdx->nSampleCol;

if( pIdx!=pPrevIdx ){
initAvgEq(pPrevIdx);
pPrevIdx = pIdx;
}

nCol = pIdx->nSampleCol;
pSample = &pIdx->aSample[pIdx->nSample];
decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0);
decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0);
Expand All @@ -123904,6 +123998,10 @@ static int loadStatTbl(
memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n);
}
pIdx->nSample++;

if( db->aSchemaTime ){
db->aSchemaTime[SCHEMA_TIME_STAT4_Q2_BODY] += (sqlite3STimeNow() - t);
}
}
rc = sqlite3_finalize(pStmt);
sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q2);
Expand Down Expand Up @@ -144195,7 +144293,7 @@ SQLITE_PRIVATE void sqlite3Pragma(

/* Do the b-tree integrity checks */
sqlite3VdbeAddOp4(v, OP_IntegrityCk, 1, cnt, 8, (char*)aRoot,P4_INTARRAY);
sqlite3VdbeChangeP5(v, (u8)i);
sqlite3VdbeChangeP5(v, (u16)i);
addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v);
sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0,
sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName),
Expand Down Expand Up @@ -153462,7 +153560,7 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){
}
sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i));
sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
sqlite3VdbeChangeP5(v, (u8)nArg);
sqlite3VdbeChangeP5(v, (u16)nArg);
sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v);
sqlite3VdbeJumpHere(v, iTop);
sqlite3ReleaseTempRange(pParse, regAgg, nArg);
Expand Down Expand Up @@ -153625,7 +153723,7 @@ static void updateAccumulator(
}
sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i));
sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
sqlite3VdbeChangeP5(v, (u8)nArg);
sqlite3VdbeChangeP5(v, (u16)nArg);
sqlite3ReleaseTempRange(pParse, regAgg, nArg);
}
if( addrNext ){
Expand Down Expand Up @@ -157019,7 +157117,7 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect(
** invocation is disallowed if (a) the sub-program is really a trigger,
** not a foreign key action, and (b) the flag to enable recursive triggers
** is clear. */
sqlite3VdbeChangeP5(v, (u8)bRecursive);
sqlite3VdbeChangeP5(v, (u16)bRecursive);
}
}

Expand Down Expand Up @@ -175428,7 +175526,7 @@ static void windowAggStep(
sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep,
bInverse, regArg, pWin->regAccum);
sqlite3VdbeAppendP4(v, pFunc, P4_FUNCDEF);
sqlite3VdbeChangeP5(v, (u8)nArg);
sqlite3VdbeChangeP5(v, (u16)nArg);
if( pWin->bExprArgs ){
sqlite3ReleaseTempRange(pParse, regArg, nArg);
}
Expand Down Expand Up @@ -187037,8 +187135,8 @@ static const int aHardLimit[] = {
#if SQLITE_MAX_VDBE_OP<40
# error SQLITE_MAX_VDBE_OP must be at least 40
#endif
#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127
# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127
#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>32767
# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 32767
#endif
#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125
# error SQLITE_MAX_ATTACHED must be between 0 and 125
Expand Down Expand Up @@ -257919,7 +258017,7 @@ static void fts5SourceIdFunc(
){
assert( nArg==0 );
UNUSED_PARAM2(nArg, apUnused);
sqlite3_result_text(pCtx, "fts5: 2024-12-10 14:56:20 df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523", -1, SQLITE_TRANSIENT);
sqlite3_result_text(pCtx, "fts5: 2024-12-12 20:39:56 3c25c69c93e55738cdbfdd87fa3c879b878674973955490770f5e274da1ca9a4", -1, SQLITE_TRANSIENT);
}

/*
Expand Down
2 changes: 1 addition & 1 deletion libstuff/sqlite3.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ extern "C" {
*/
#define SQLITE_VERSION "3.47.0"
#define SQLITE_VERSION_NUMBER 3047000
#define SQLITE_SOURCE_ID "2024-12-10 14:56:20 df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523"
#define SQLITE_SOURCE_ID "2024-12-12 20:39:56 3c25c69c93e55738cdbfdd87fa3c879b878674973955490770f5e274da1ca9a4"

/*
** CAPI3REF: Run-Time Library Version Numbers
Expand Down

0 comments on commit 7488bd1

Please sign in to comment.