From d93a7dd957e2cc9fcc452e632c9b28ac89b7f9ed Mon Sep 17 00:00:00 2001 From: Florent De Neve Date: Thu, 12 Dec 2024 16:48:26 -0400 Subject: [PATCH] Update SQLite version: - raises the limit on the number of arguments to an SQL function to 1000 - in cases where the writer has to do the memset() slowing down COMMIT (i.e. when you use passive checkpoints), the memset() is now only 16KB, not 32KB - we attempt to load stat4 data using only a single pass. And only the simple query (formerly the second query). --- libstuff/sqlite3.c | 180 ++++++++++++++++++++++++++++++++++----------- libstuff/sqlite3.h | 2 +- 2 files changed, 140 insertions(+), 42 deletions(-) diff --git a/libstuff/sqlite3.c b/libstuff/sqlite3.c index a7111ef6c..e02dd0087 100644 --- a/libstuff/sqlite3.c +++ b/libstuff/sqlite3.c @@ -18,7 +18,7 @@ ** separate file. This file contains only code for the core SQLite library. ** ** The content in this amalgamation comes from Fossil check-in -** df4183ace93b788b798b258274bf6b651906. +** 3c25c69c93e55738cdbfdd87fa3c879b8786. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -465,7 +465,7 @@ extern "C" { */ #define SQLITE_VERSION "3.47.0" #define SQLITE_VERSION_NUMBER 3047000 -#define SQLITE_SOURCE_ID "2024-12-10 14:56:20 df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523" +#define SQLITE_SOURCE_ID "2024-12-12 20:39:56 3c25c69c93e55738cdbfdd87fa3c879b878674973955490770f5e274da1ca9a4" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -14125,9 +14125,13 @@ struct fts5_api { /* ** The maximum number of arguments to an SQL function. +** +** This value has a hard upper limit of 32767 due to storage +** constraints (it needs to fit inside a i16). We keep it +** lower than that to prevent abuse. */ #ifndef SQLITE_MAX_FUNCTION_ARG -# define SQLITE_MAX_FUNCTION_ARG 127 +# define SQLITE_MAX_FUNCTION_ARG 1000 #endif /* @@ -18236,14 +18240,15 @@ struct sqlite3 { #define SCHEMA_TIME_STAT4_Q1_BODY 14 #define SCHEMA_TIME_AFTER_STAT4_Q1 15 -#define SCHEMA_TIME_AFTER_STAT4_Q2 16 -#define SCHEMA_TIME_AFTER_STAT4 17 +#define SCHEMA_TIME_STAT4_Q2_BODY 16 +#define SCHEMA_TIME_AFTER_STAT4_Q2 17 +#define SCHEMA_TIME_AFTER_STAT4 18 -#define SCHEMA_TIME_END_ANALYZE_LOAD 18 -#define SCHEMA_TIME_FINISH 19 +#define SCHEMA_TIME_END_ANALYZE_LOAD 19 +#define SCHEMA_TIME_FINISH 20 -#define SCHEMA_TIME_N 20 -#define SCHEMA_TIME_TIMEOUT (0 * 1000 * 1000) +#define SCHEMA_TIME_N 21 +#define SCHEMA_TIME_TIMEOUT (500 * 1000) @@ -18418,7 +18423,7 @@ SQLITE_PRIVATE void sqlite3SchemaTimeLog(u64 *aSchemaTime, const char *zFile); ** field is used by per-connection app-def functions. */ struct FuncDef { - i8 nArg; /* Number of arguments. -1 means unlimited */ + i16 nArg; /* Number of arguments. -1 means unlimited */ u32 funcFlags; /* Some combination of SQLITE_FUNC_* */ void *pUserData; /* User data parameter */ FuncDef *pNext; /* Next function with same name */ @@ -19247,7 +19252,7 @@ struct Index { ** expression, or a reference to a VIRTUAL column */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ - int mxSample; /* Number of slots allocated to aSample[] */ + int nSampleAlloc; /* Number of slots allocated to aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ @@ -24030,7 +24035,7 @@ struct sqlite3_context { int isError; /* Error code returned by the function. */ u8 enc; /* Encoding to use for results */ u8 skipFlag; /* Skip accumulator loading if true */ - u8 argc; /* Number of arguments */ + u16 argc; /* Number of arguments */ sqlite3_value *argv[1]; /* Argument set */ }; @@ -67624,9 +67629,9 @@ static int walIndexAppend(Wal *pWal, int iWal, u32 iFrame, u32 iPage){ */ if( pWal->aCommitTime ) t = sqlite3STimeNow(); if( idx==1 && sLoc.aPgno[0]!=0 ){ - int nByte = (int)((u8*)&sLoc.aHash[HASHTABLE_NSLOT] - (u8*)sLoc.aPgno); - assert( nByte>=0 && (nByte & 0x07)==0 ); - zero64((void*)sLoc.aPgno, nByte); + /* Special for BEDROCK branch: Zero only the aHash[] part. Not the + ** aPgno[] part of the page. */ + zero64((void*)sLoc.aHash, HASHTABLE_NSLOT * sizeof(sLoc.aHash[0])); } if( pWal->aCommitTime ){ pWal->aCommitTime[COMMIT_TIME_WALINDEX_MEMSETUS]+=sqlite3STimeNow()-t; @@ -67637,11 +67642,23 @@ static int walIndexAppend(Wal *pWal, int iWal, u32 iFrame, u32 iPage){ ** writing one or more dirty pages to the WAL to free up memory). ** Remove the remnants of that writers uncommitted transaction from ** the hash-table before writing any new entries. - */ - if( sLoc.aPgno[idx-1] ){ + ** + ** Special for BEDROCK branch: On this branch we do not assume that + ** the aPgno[] part of each hash-table has been zeroed. Therefore, we + ** only need to clear out the remnants of an old writer's transaction if + ** the hash table matches the aPgno[] entry (as it would if a write + ** transaction was interrupted). And, because this makes the test more + ** expensive, we only do the check for the first frame written by each + ** transaction. */ + if( sLoc.aPgno[idx-1] && iFrame-1==walidxGetMxFrame(&pWal->hdr, iWal) ){ if( pWal->aCommitTime ) t = sqlite3STimeNow(); - walCleanupHash(pWal); - assert( !sLoc.aPgno[idx-1] ); + nCollide = idx; + for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ + if( sLoc.aHash[iKey]==idx ){ + walCleanupHash(pWal); + } + if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; + } if( pWal->aCommitTime ){ pWal->aCommitTime[COMMIT_TIME_WALINDEX_CLEANUPUS]+=sqlite3STimeNow()-t; } @@ -93474,7 +93491,7 @@ SQLITE_PRIVATE void sqlite3CommitTimeLog(u64 *aCommit){ } zStr = sqlite3_mprintf("%z%s%s%d%s", zStr, (zStr?", ":""),zHash,iVal,zU); } - sqlite3_log(SQLITE_WARNING, "slow commit (v=17): (%s)", zStr); + sqlite3_log(SQLITE_WARNING, "slow commit (v=18): (%s)", zStr); sqlite3_free(zStr); } } @@ -93502,7 +93519,7 @@ SQLITE_PRIVATE void sqlite3PrepareTimeLog(const char *zSql, int nSql, u64 *aPrep } if( nByte<0 ){ nByte = sqlite3Strlen30(zSql); } sqlite3_log(SQLITE_WARNING, - "slow prepare (v=17): (%s) [%.*s]", zStr, nByte, zSql + "slow prepare (v=18): (%s) [%.*s]", zStr, nByte, zSql ); sqlite3_free(zStr); } @@ -93515,12 +93532,15 @@ SQLITE_PRIVATE void sqlite3SchemaTimeLog(u64 *aSchema, const char *zFile){ int ii; for(ii=1; iipVdbe, opcode, in2, dest, in1, (void*)p4, P4_COLLSEQ); - sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5); + sqlite3VdbeChangeP5(pParse->pVdbe, (u16)p5); return addr; } @@ -123675,6 +123695,7 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ if( db->pnBytesFreed==0 ){ pIdx->nSample = 0; pIdx->aSample = 0; + pIdx->nSampleAlloc = 0; } #else UNUSED_PARAMETER(db); @@ -123759,6 +123780,70 @@ static Index *findIndexOrPrimaryKey( return pIdx; } +/* +** Grow the pIdx->aSample[] array. Return SQLITE_OK if successful, or +** SQLITE_NOMEM otherwise. +*/ +static int growSampleArray(sqlite3 *db, Index *pIdx){ + int nIdxCol = pIdx->nSampleCol; + int nNew = 0; + IndexSample *aNew = 0; + int nByte = 0; + tRowcnt *pSpace; /* Available allocated memory space */ + u8 *pPtr; /* Available memory as a u8 for easier manipulation */ + int i; + + /* In production set the initial allocation to SQLITE_STAT4_SAMPLES. This + ** means that reallocation will almost never be required. But for debug + ** builds, set the initial allocation size to 6 entries so that the + ** reallocation code gets tested. todo: use real tests for this. */ + assert( pIdx->nSample==pIdx->nSampleAlloc ); +#ifdef SQLITE_DEBUG + nNew = 6; +#else + nNew = SQLITE_STAT4_SAMPLES; +#endif + if( pIdx->nSample ){ + nNew = pIdx->nSample*2; + } + + nByte = ROUND8(sizeof(IndexSample) * nNew); + nByte += sizeof(tRowcnt) * nIdxCol * 3 * nNew; + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + + aNew = (IndexSample*)sqlite3DbMallocZero(db, nByte); + if( aNew==0 ) return SQLITE_NOMEM_BKPT; + + pPtr = (u8*)aNew; + pPtr += ROUND8(nNew*sizeof(pIdx->aSample[0])); + pSpace = (tRowcnt*)pPtr; + + pIdx->aAvgEq = pSpace; pSpace += nIdxCol; + assert( EIGHT_BYTE_ALIGNMENT( pSpace ) ); + + if( pIdx->nSample ){ + /* Copy the contents of the anEq[], anLt[], anDLt[] arrays for all + ** extant samples to the new location. */ + int nByte = nIdxCol * 3 * sizeof(tRowcnt) * pIdx->nSample; + memcpy(pSpace, pIdx->aSample[0].anEq, nByte); + } + for(i=0; inSample ){ + aNew[i].p = pIdx->aSample[i].p; + aNew[i].n = pIdx->aSample[i].n; + } + } + assert( ((u8*)pSpace)-nByte==(u8*)aNew ); + + sqlite3DbFree(db, pIdx->aSample); + pIdx->aSample = aNew; + pIdx->nSampleAlloc = nNew; + return SQLITE_OK; +} + /* ** Load the content from either the sqlite_stat4 ** into the relevant Index.aSample[] arrays. @@ -123784,6 +123869,7 @@ static int loadStatTbl( IndexSample *pSample; /* A slot in pIdx->aSample[] */ assert( db->lookaside.bDisable ); +#if 0 zSql = sqlite3MPrintf(db, zSql1, zDb); if( !zSql ){ return SQLITE_NOMEM_BKPT; @@ -123850,6 +123936,9 @@ static int loadStatTbl( } rc = sqlite3_finalize(pStmt); if( rc ) return rc; +#endif + + sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1); zSql = sqlite3MPrintf(db, zSql2, zDb); if( !zSql ){ @@ -123859,29 +123948,34 @@ static int loadStatTbl( sqlite3DbFree(db, zSql); if( rc ) return rc; - sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q1); - while( sqlite3_step(pStmt)==SQLITE_ROW ){ char *zIndex; /* Index name */ Index *pIdx; /* Pointer to the index object */ int nCol = 1; /* Number of columns in index */ + u64 t = sqlite3STimeNow(); zIndex = (char *)sqlite3_column_text(pStmt, 0); if( zIndex==0 ) continue; pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); if( pIdx==0 ) continue; - if( pIdx->nSample>=pIdx->mxSample ){ - /* Too many slots used because the same index appears in - ** sqlite_stat4 using multiple names */ - continue; + + if( pIdx->nSample==pIdx->nSampleAlloc ){ + pIdx->pTable->tabFlags |= TF_HasStat4; + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + pIdx->nSampleCol = pIdx->nKeyCol; + }else{ + pIdx->nSampleCol = pIdx->nColumn; + } + if( growSampleArray(db, pIdx) ) break; } - /* This next condition is true if data has already been loaded from - ** the sqlite_stat4 table. */ - nCol = pIdx->nSampleCol; + if( pIdx!=pPrevIdx ){ initAvgEq(pPrevIdx); pPrevIdx = pIdx; } + + nCol = pIdx->nSampleCol; pSample = &pIdx->aSample[pIdx->nSample]; decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); @@ -123904,6 +123998,10 @@ static int loadStatTbl( memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); } pIdx->nSample++; + + if( db->aSchemaTime ){ + db->aSchemaTime[SCHEMA_TIME_STAT4_Q2_BODY] += (sqlite3STimeNow() - t); + } } rc = sqlite3_finalize(pStmt); sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_AFTER_STAT4_Q2); @@ -144195,7 +144293,7 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Do the b-tree integrity checks */ sqlite3VdbeAddOp4(v, OP_IntegrityCk, 1, cnt, 8, (char*)aRoot,P4_INTARRAY); - sqlite3VdbeChangeP5(v, (u8)i); + sqlite3VdbeChangeP5(v, (u16)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName), @@ -153462,7 +153560,7 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ } sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3VdbeChangeP5(v, (u16)nArg); sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, iTop); sqlite3ReleaseTempRange(pParse, regAgg, nArg); @@ -153625,7 +153723,7 @@ static void updateAccumulator( } sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3VdbeChangeP5(v, (u16)nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); } if( addrNext ){ @@ -157019,7 +157117,7 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( ** invocation is disallowed if (a) the sub-program is really a trigger, ** not a foreign key action, and (b) the flag to enable recursive triggers ** is clear. */ - sqlite3VdbeChangeP5(v, (u8)bRecursive); + sqlite3VdbeChangeP5(v, (u16)bRecursive); } } @@ -175428,7 +175526,7 @@ static void windowAggStep( sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep, bInverse, regArg, pWin->regAccum); sqlite3VdbeAppendP4(v, pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3VdbeChangeP5(v, (u16)nArg); if( pWin->bExprArgs ){ sqlite3ReleaseTempRange(pParse, regArg, nArg); } @@ -187037,8 +187135,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_VDBE_OP<40 # error SQLITE_MAX_VDBE_OP must be at least 40 #endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>32767 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 32767 #endif #if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 # error SQLITE_MAX_ATTACHED must be between 0 and 125 @@ -257919,7 +258017,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2024-12-10 14:56:20 df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2024-12-12 20:39:56 3c25c69c93e55738cdbfdd87fa3c879b878674973955490770f5e274da1ca9a4", -1, SQLITE_TRANSIENT); } /* diff --git a/libstuff/sqlite3.h b/libstuff/sqlite3.h index 8d452cc85..2983455f0 100644 --- a/libstuff/sqlite3.h +++ b/libstuff/sqlite3.h @@ -148,7 +148,7 @@ extern "C" { */ #define SQLITE_VERSION "3.47.0" #define SQLITE_VERSION_NUMBER 3047000 -#define SQLITE_SOURCE_ID "2024-12-10 14:56:20 df4183ace93b788b798b258274bf6b651906c9f1cf2af4983e447cdf52904523" +#define SQLITE_SOURCE_ID "2024-12-12 20:39:56 3c25c69c93e55738cdbfdd87fa3c879b878674973955490770f5e274da1ca9a4" /* ** CAPI3REF: Run-Time Library Version Numbers