Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update expensify_prod branch #2023

Merged
merged 2 commits into from
Dec 12, 2024
Merged

Update expensify_prod branch #2023

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d93a7dd
Update SQLite version:
flodnv Dec 12, 2024
7488bd1
Merge pull request #2021 from Expensify/flo_sqliteoptims
cead22 Dec 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 139 additions & 41 deletions libstuff/sqlite3.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
** separate file. This file contains only code for the core SQLite library.
**
** The content in this amalgamation comes from Fossil check-in
** df4183ace93b788b798b258274bf6b651906.
** 3c25c69c93e55738cdbfdd87fa3c879b8786.
*/
#define SQLITE_CORE 1
#define SQLITE_AMALGAMATION 1
Expand Down Expand Up @@ -465,7 +465,7 @@ extern "C" {
*/
#define SQLITE_VERSION "3.47.0"
#define SQLITE_VERSION_NUMBER 3047000
#define SQLITE_SOURCE_ID "2024-12-10 14:56:20 df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523"
#define SQLITE_SOURCE_ID "2024-12-12 20:39:56 3c25c69c93e55738cdbfdd87fa3c879b878674973955490770f5e274da1ca9a4"

/*
** CAPI3REF: Run-Time Library Version Numbers
Expand Down Expand Up @@ -14125,9 +14125,13 @@ struct fts5_api {

/*
** The maximum number of arguments to an SQL function.
**
** This value has a hard upper limit of 32767 due to storage
** constraints (it needs to fit inside a i16). We keep it
** lower than that to prevent abuse.
*/
#ifndef SQLITE_MAX_FUNCTION_ARG
# define SQLITE_MAX_FUNCTION_ARG 127
# define SQLITE_MAX_FUNCTION_ARG 1000
#endif

/*
Expand Down Expand Up @@ -18236,14 +18240,15 @@ struct sqlite3 {

#define SCHEMA_TIME_STAT4_Q1_BODY 14
#define SCHEMA_TIME_AFTER_STAT4_Q1 15
#define SCHEMA_TIME_AFTER_STAT4_Q2 16
#define SCHEMA_TIME_AFTER_STAT4 17
#define SCHEMA_TIME_STAT4_Q2_BODY 16
#define SCHEMA_TIME_AFTER_STAT4_Q2 17
#define SCHEMA_TIME_AFTER_STAT4 18

#define SCHEMA_TIME_END_ANALYZE_LOAD 18
#define SCHEMA_TIME_FINISH 19
#define SCHEMA_TIME_END_ANALYZE_LOAD 19
#define SCHEMA_TIME_FINISH 20

#define SCHEMA_TIME_N 20
#define SCHEMA_TIME_TIMEOUT (0 * 1000 * 1000)
#define SCHEMA_TIME_N 21
#define SCHEMA_TIME_TIMEOUT (500 * 1000)



Expand Down Expand Up @@ -18418,7 +18423,7 @@ SQLITE_PRIVATE void sqlite3SchemaTimeLog(u64 *aSchemaTime, const char *zFile);
** field is used by per-connection app-def functions.
*/
struct FuncDef {
i8 nArg; /* Number of arguments. -1 means unlimited */
i16 nArg; /* Number of arguments. -1 means unlimited */
u32 funcFlags; /* Some combination of SQLITE_FUNC_* */
void *pUserData; /* User data parameter */
FuncDef *pNext; /* Next function with same name */
Expand Down Expand Up @@ -19247,7 +19252,7 @@ struct Index {
** expression, or a reference to a VIRTUAL column */
#ifdef SQLITE_ENABLE_STAT4
int nSample; /* Number of elements in aSample[] */
int mxSample; /* Number of slots allocated to aSample[] */
int nSampleAlloc; /* Number of slots allocated to aSample[] */
int nSampleCol; /* Size of IndexSample.anEq[] and so on */
tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */
IndexSample *aSample; /* Samples of the left-most key */
Expand Down Expand Up @@ -24030,7 +24035,7 @@ struct sqlite3_context {
int isError; /* Error code returned by the function. */
u8 enc; /* Encoding to use for results */
u8 skipFlag; /* Skip accumulator loading if true */
u8 argc; /* Number of arguments */
u16 argc; /* Number of arguments */
sqlite3_value *argv[1]; /* Argument set */
};

Expand Down Expand Up @@ -67624,9 +67629,9 @@ static int walIndexAppend(Wal *pWal, int iWal, u32 iFrame, u32 iPage){
*/
if( pWal->aCommitTime ) t = sqlite3STimeNow();
if( idx==1 && sLoc.aPgno[0]!=0 ){
int nByte = (int)((u8*)&sLoc.aHash[HASHTABLE_NSLOT] - (u8*)sLoc.aPgno);
assert( nByte>=0 && (nByte & 0x07)==0 );
zero64((void*)sLoc.aPgno, nByte);
/* Special for BEDROCK branch: Zero only the aHash[] part. Not the
** aPgno[] part of the page. */
zero64((void*)sLoc.aHash, HASHTABLE_NSLOT * sizeof(sLoc.aHash[0]));
}
if( pWal->aCommitTime ){
pWal->aCommitTime[COMMIT_TIME_WALINDEX_MEMSETUS]+=sqlite3STimeNow()-t;
Expand All @@ -67637,11 +67642,23 @@ static int walIndexAppend(Wal *pWal, int iWal, u32 iFrame, u32 iPage){
** writing one or more dirty pages to the WAL to free up memory).
** Remove the remnants of that writers uncommitted transaction from
** the hash-table before writing any new entries.
*/
if( sLoc.aPgno[idx-1] ){
**
** Special for BEDROCK branch: On this branch we do not assume that
** the aPgno[] part of each hash-table has been zeroed. Therefore, we
** only need to clear out the remnants of an old writer's transaction if
** the hash table matches the aPgno[] entry (as it would if a write
** transaction was interrupted). And, because this makes the test more
** expensive, we only do the check for the first frame written by each
** transaction. */
if( sLoc.aPgno[idx-1] && iFrame-1==walidxGetMxFrame(&pWal->hdr, iWal) ){
if( pWal->aCommitTime ) t = sqlite3STimeNow();
walCleanupHash(pWal);
assert( !sLoc.aPgno[idx-1] );
nCollide = idx;
for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){
if( sLoc.aHash[iKey]==idx ){
walCleanupHash(pWal);
}
if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT;
}
if( pWal->aCommitTime ){
pWal->aCommitTime[COMMIT_TIME_WALINDEX_CLEANUPUS]+=sqlite3STimeNow()-t;
}
Expand Down Expand Up @@ -93474,7 +93491,7 @@ SQLITE_PRIVATE void sqlite3CommitTimeLog(u64 *aCommit){
}
zStr = sqlite3_mprintf("%z%s%s%d%s", zStr, (zStr?", ":""),zHash,iVal,zU);
}
sqlite3_log(SQLITE_WARNING, "slow commit (v=17): (%s)", zStr);
sqlite3_log(SQLITE_WARNING, "slow commit (v=18): (%s)", zStr);
sqlite3_free(zStr);
}
}
Expand Down Expand Up @@ -93502,7 +93519,7 @@ SQLITE_PRIVATE void sqlite3PrepareTimeLog(const char *zSql, int nSql, u64 *aPrep
}
if( nByte<0 ){ nByte = sqlite3Strlen30(zSql); }
sqlite3_log(SQLITE_WARNING,
"slow prepare (v=17): (%s) [%.*s]", zStr, nByte, zSql
"slow prepare (v=18): (%s) [%.*s]", zStr, nByte, zSql
);
sqlite3_free(zStr);
}
Expand All @@ -93515,12 +93532,15 @@ SQLITE_PRIVATE void sqlite3SchemaTimeLog(u64 *aSchema, const char *zFile){
int ii;
for(ii=1; ii<SCHEMA_TIME_N; ii++){
int val = aSchema[ii];
if( val!=0 && ii!=SCHEMA_TIME_STAT4_Q1_BODY ){
if( val!=0
&& ii!=SCHEMA_TIME_STAT4_Q1_BODY
&& ii!=SCHEMA_TIME_STAT4_Q2_BODY
){
val -= i1;
}
zStr = sqlite3_mprintf("%z%s%d", zStr, (zStr?", ":""), val);
}
sqlite3_log(SQLITE_WARNING, "slow schema (%s) (v=17): (%s)", zFile, zStr);
sqlite3_log(SQLITE_WARNING, "slow schema (%s) (v=18): (%s)", zFile, zStr);
sqlite3_free(zStr);
}
}
Expand Down Expand Up @@ -112893,7 +112913,7 @@ static int codeCompare(
p5 = binaryCompareP5(pLeft, pRight, jumpIfNull);
addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1,
(void*)p4, P4_COLLSEQ);
sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5);
sqlite3VdbeChangeP5(pParse->pVdbe, (u16)p5);
return addr;
}

Expand Down Expand Up @@ -123675,6 +123695,7 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){
if( db->pnBytesFreed==0 ){
pIdx->nSample = 0;
pIdx->aSample = 0;
pIdx->nSampleAlloc = 0;
}
#else
UNUSED_PARAMETER(db);
Expand Down Expand Up @@ -123759,6 +123780,70 @@ static Index *findIndexOrPrimaryKey(
return pIdx;
}

/*
** Grow the pIdx->aSample[] array. Return SQLITE_OK if successful, or
** SQLITE_NOMEM otherwise.
*/
static int growSampleArray(sqlite3 *db, Index *pIdx){
int nIdxCol = pIdx->nSampleCol;
int nNew = 0;
IndexSample *aNew = 0;
int nByte = 0;
tRowcnt *pSpace; /* Available allocated memory space */
u8 *pPtr; /* Available memory as a u8 for easier manipulation */
int i;

/* In production set the initial allocation to SQLITE_STAT4_SAMPLES. This
** means that reallocation will almost never be required. But for debug
** builds, set the initial allocation size to 6 entries so that the
** reallocation code gets tested. todo: use real tests for this. */
assert( pIdx->nSample==pIdx->nSampleAlloc );
#ifdef SQLITE_DEBUG
nNew = 6;
#else
nNew = SQLITE_STAT4_SAMPLES;
#endif
if( pIdx->nSample ){
nNew = pIdx->nSample*2;
}

nByte = ROUND8(sizeof(IndexSample) * nNew);
nByte += sizeof(tRowcnt) * nIdxCol * 3 * nNew;
nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */

aNew = (IndexSample*)sqlite3DbMallocZero(db, nByte);
if( aNew==0 ) return SQLITE_NOMEM_BKPT;

pPtr = (u8*)aNew;
pPtr += ROUND8(nNew*sizeof(pIdx->aSample[0]));
pSpace = (tRowcnt*)pPtr;

pIdx->aAvgEq = pSpace; pSpace += nIdxCol;
assert( EIGHT_BYTE_ALIGNMENT( pSpace ) );

if( pIdx->nSample ){
/* Copy the contents of the anEq[], anLt[], anDLt[] arrays for all
** extant samples to the new location. */
int nByte = nIdxCol * 3 * sizeof(tRowcnt) * pIdx->nSample;
memcpy(pSpace, pIdx->aSample[0].anEq, nByte);
}
for(i=0; i<nNew; i++){
aNew[i].anEq = pSpace; pSpace += nIdxCol;
aNew[i].anLt = pSpace; pSpace += nIdxCol;
aNew[i].anDLt = pSpace; pSpace += nIdxCol;
if( i<pIdx->nSample ){
aNew[i].p = pIdx->aSample[i].p;
aNew[i].n = pIdx->aSample[i].n;
}
}
assert( ((u8*)pSpace)-nByte==(u8*)aNew );

sqlite3DbFree(db, pIdx->aSample);
pIdx->aSample = aNew;
pIdx->nSampleAlloc = nNew;
return SQLITE_OK;
}

/*
** Load the content from either the sqlite_stat4
** into the relevant Index.aSample[] arrays.
Expand All @@ -123784,6 +123869,7 @@ static int loadStatTbl(
IndexSample *pSample; /* A slot in pIdx->aSample[] */

assert( db->lookaside.bDisable );
#if 0
zSql = sqlite3MPrintf(db, zSql1, zDb);
if( !zSql ){
return SQLITE_NOMEM_BKPT;
Expand Down Expand Up @@ -123850,6 +123936,9 @@ static int loadStatTbl(
}
rc = sqlite3_finalize(pStmt);
if( rc ) return rc;
#endif

sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1);

zSql = sqlite3MPrintf(db, zSql2, zDb);
if( !zSql ){
Expand All @@ -123859,29 +123948,34 @@ static int loadStatTbl(
sqlite3DbFree(db, zSql);
if( rc ) return rc;

sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1);

while( sqlite3_step(pStmt)==SQLITE_ROW ){
char *zIndex; /* Index name */
Index *pIdx; /* Pointer to the index object */
int nCol = 1; /* Number of columns in index */
u64 t = sqlite3STimeNow();

zIndex = (char *)sqlite3_column_text(pStmt, 0);
if( zIndex==0 ) continue;
pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
if( pIdx==0 ) continue;
if( pIdx->nSample>=pIdx->mxSample ){
/* Too many slots used because the same index appears in
** sqlite_stat4 using multiple names */
continue;

if( pIdx->nSample==pIdx->nSampleAlloc ){
pIdx->pTable->tabFlags |= TF_HasStat4;
assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 );
if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
pIdx->nSampleCol = pIdx->nKeyCol;
}else{
pIdx->nSampleCol = pIdx->nColumn;
}
if( growSampleArray(db, pIdx) ) break;
}
/* This next condition is true if data has already been loaded from
** the sqlite_stat4 table. */
nCol = pIdx->nSampleCol;

if( pIdx!=pPrevIdx ){
initAvgEq(pPrevIdx);
pPrevIdx = pIdx;
}

nCol = pIdx->nSampleCol;
pSample = &pIdx->aSample[pIdx->nSample];
decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0);
decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0);
Expand All @@ -123904,6 +123998,10 @@ static int loadStatTbl(
memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n);
}
pIdx->nSample++;

if( db->aSchemaTime ){
db->aSchemaTime[SCHEMA_TIME_STAT4_Q2_BODY] += (sqlite3STimeNow() - t);
}
}
rc = sqlite3_finalize(pStmt);
sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q2);
Expand Down Expand Up @@ -144195,7 +144293,7 @@ SQLITE_PRIVATE void sqlite3Pragma(

/* Do the b-tree integrity checks */
sqlite3VdbeAddOp4(v, OP_IntegrityCk, 1, cnt, 8, (char*)aRoot,P4_INTARRAY);
sqlite3VdbeChangeP5(v, (u8)i);
sqlite3VdbeChangeP5(v, (u16)i);
addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v);
sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0,
sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName),
Expand Down Expand Up @@ -153462,7 +153560,7 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){
}
sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i));
sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
sqlite3VdbeChangeP5(v, (u8)nArg);
sqlite3VdbeChangeP5(v, (u16)nArg);
sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v);
sqlite3VdbeJumpHere(v, iTop);
sqlite3ReleaseTempRange(pParse, regAgg, nArg);
Expand Down Expand Up @@ -153625,7 +153723,7 @@ static void updateAccumulator(
}
sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i));
sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
sqlite3VdbeChangeP5(v, (u8)nArg);
sqlite3VdbeChangeP5(v, (u16)nArg);
sqlite3ReleaseTempRange(pParse, regAgg, nArg);
}
if( addrNext ){
Expand Down Expand Up @@ -157019,7 +157117,7 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect(
** invocation is disallowed if (a) the sub-program is really a trigger,
** not a foreign key action, and (b) the flag to enable recursive triggers
** is clear. */
sqlite3VdbeChangeP5(v, (u8)bRecursive);
sqlite3VdbeChangeP5(v, (u16)bRecursive);
}
}

Expand Down Expand Up @@ -175428,7 +175526,7 @@ static void windowAggStep(
sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep,
bInverse, regArg, pWin->regAccum);
sqlite3VdbeAppendP4(v, pFunc, P4_FUNCDEF);
sqlite3VdbeChangeP5(v, (u8)nArg);
sqlite3VdbeChangeP5(v, (u16)nArg);
if( pWin->bExprArgs ){
sqlite3ReleaseTempRange(pParse, regArg, nArg);
}
Expand Down Expand Up @@ -187037,8 +187135,8 @@ static const int aHardLimit[] = {
#if SQLITE_MAX_VDBE_OP<40
# error SQLITE_MAX_VDBE_OP must be at least 40
#endif
#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127
# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127
#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>32767
# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 32767
#endif
#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125
# error SQLITE_MAX_ATTACHED must be between 0 and 125
Expand Down Expand Up @@ -257919,7 +258017,7 @@ static void fts5SourceIdFunc(
){
assert( nArg==0 );
UNUSED_PARAM2(nArg, apUnused);
sqlite3_result_text(pCtx, "fts5: 2024-12-10 14:56:20 df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523", -1, SQLITE_TRANSIENT);
sqlite3_result_text(pCtx, "fts5: 2024-12-12 20:39:56 3c25c69c93e55738cdbfdd87fa3c879b878674973955490770f5e274da1ca9a4", -1, SQLITE_TRANSIENT);
}

/*
Expand Down
2 changes: 1 addition & 1 deletion libstuff/sqlite3.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ extern "C" {
*/
#define SQLITE_VERSION "3.47.0"
#define SQLITE_VERSION_NUMBER 3047000
#define SQLITE_SOURCE_ID "2024-12-10 14:56:20 df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523"
#define SQLITE_SOURCE_ID "2024-12-12 20:39:56 3c25c69c93e55738cdbfdd87fa3c879b878674973955490770f5e274da1ca9a4"

/*
** CAPI3REF: Run-Time Library Version Numbers
Expand Down
Loading