diff --git a/libstuff/sqlite3.c b/libstuff/sqlite3.c
index 5a10f1179..1af6bcf0e 100644
--- a/libstuff/sqlite3.c
+++ b/libstuff/sqlite3.c
@@ -1,6 +1,6 @@
/******************************************************************************
** This file is an amalgamation of many separate C source files from SQLite
-** version 3.45.0. By combining all the individual C code files into this
+** version 3.45.2. By combining all the individual C code files into this
** single large file, the entire code can be compiled as a single translation
** unit. This allows many compilers to do optimizations that would not be
** possible if the files were compiled separately. Performance improvements
@@ -18,7 +18,7 @@
** separate file. This file contains only code for the core SQLite library.
**
** The content in this amalgamation comes from Fossil check-in
-** 11a4178f0e6cd760303a5874b5a3529ccd78.
+** 66c885814e1c533f0f45ee01b6b25cd5ec08.
*/
#define SQLITE_CORE 1
#define SQLITE_AMALGAMATION 1
@@ -460,9 +460,9 @@ extern "C" {
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
-#define SQLITE_VERSION "3.45.0"
-#define SQLITE_VERSION_NUMBER 3045000
-#define SQLITE_SOURCE_ID "2024-02-16 11:05:35 11a4178f0e6cd760303a5874b5a3529ccd78a2398b59700e878c7e6b1251de91"
+#define SQLITE_VERSION "3.45.2"
+#define SQLITE_VERSION_NUMBER 3045002
+#define SQLITE_SOURCE_ID "2024-06-04 16:24:12 66c885814e1c533f0f45ee01b6b25cd5ec08770d699e441b1af4f79b23e9f0a7"
/*
** CAPI3REF: Run-Time Library Version Numbers
@@ -734,6 +734,8 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running.
** The application must not modify the SQL statement text passed into
** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running.
+** The application must not dereference the arrays or string pointers
+** passed as the 3rd and 4th callback parameters after it returns.
**
*/
SQLITE_API int sqlite3_exec(
@@ -15246,6 +15248,7 @@ SQLITE_PRIVATE u32 sqlite3TreeTrace;
** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing
** 0x00020000 Transform DISTINCT into GROUP BY
** 0x00040000 SELECT tree dump after all code has been generated
+** 0x00080000 NOT NULL strength reduction
*/
/*
@@ -16087,6 +16090,7 @@ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno);
SQLITE_PRIVATE void sqlite3PagerRekey(DbPage*, Pgno, u16);
#ifndef SQLITE_OMIT_CONCURRENT
+SQLITE_PRIVATE int sqlite3PagerUsePage(Pager*, Pgno);
SQLITE_PRIVATE void sqlite3PagerEndConcurrent(Pager*);
SQLITE_PRIVATE int sqlite3PagerBeginConcurrent(Pager*);
SQLITE_PRIVATE void sqlite3PagerDropExclusiveLock(Pager*);
@@ -16095,6 +16099,7 @@ SQLITE_PRIVATE void sqlite3PagerSetDbsize(Pager *pPager, Pgno);
SQLITE_PRIVATE int sqlite3PagerIsWal(Pager*);
#else
# define sqlite3PagerEndConcurrent(x)
+# define sqlite3PagerUsePage(x, y) SQLITE_OK
#endif
#if defined(SQLITE_DEBUG) || !defined(SQLITE_OMIT_CONCURRENT)
@@ -19555,6 +19560,7 @@ struct NameContext {
#define NC_InAggFunc 0x020000 /* True if analyzing arguments to an agg func */
#define NC_FromDDL 0x040000 /* SQL text comes from sqlite_schema */
#define NC_NoSelect 0x080000 /* Do not descend into sub-selects */
+#define NC_Where 0x100000 /* Processing WHERE clause of a SELECT */
#define NC_OrderAgg 0x8000000 /* Has an aggregate other than count/min/max */
/*
@@ -19578,6 +19584,7 @@ struct Upsert {
Expr *pUpsertWhere; /* WHERE clause for the ON CONFLICT UPDATE */
Upsert *pNextUpsert; /* Next ON CONFLICT clause in the list */
u8 isDoUpdate; /* True for DO UPDATE. False for DO NOTHING */
+ u8 isDup; /* True if 2nd or later with same pUpsertIdx */
/* Above this point is the parse tree for the ON CONFLICT clauses.
** The next group of fields stores intermediate data. */
void *pToFree; /* Free memory when deleting the Upsert object */
@@ -21655,7 +21662,7 @@ SQLITE_PRIVATE With *sqlite3WithPush(Parse*, With*, u8);
SQLITE_PRIVATE Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*);
SQLITE_PRIVATE void sqlite3UpsertDelete(sqlite3*,Upsert*);
SQLITE_PRIVATE Upsert *sqlite3UpsertDup(sqlite3*,Upsert*);
-SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*);
+SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*,Upsert*);
SQLITE_PRIVATE void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int);
SQLITE_PRIVATE Upsert *sqlite3UpsertOfIndex(Upsert*,Index*);
SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert*);
@@ -31521,6 +31528,7 @@ SQLITE_API void sqlite3_str_vappendf(
if( xtype==etFLOAT ){
iRound = -precision;
}else if( xtype==etGENERIC ){
+ if( precision==0 ) precision = 1;
iRound = precision;
}else{
iRound = precision+1;
@@ -35433,6 +35441,9 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en
u64 s2;
rr[0] = (double)s;
s2 = (u64)rr[0];
+#if defined(_MSC_VER) && _MSC_VER<1700
+ if( s2==0x8000000000000000LL ){ s2 = 2*(u64)(0.5*rr[0]); }
+#endif
rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s);
if( e>0 ){
while( e>=100 ){
@@ -35875,7 +35886,7 @@ SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRou
assert( p->n>0 );
assert( p->nzBuf) );
p->iDP = p->n + exp;
- if( iRound<0 ){
+ if( iRound<=0 ){
iRound = p->iDP - iRound;
if( iRound==0 && p->zBuf[i+1]>='5' ){
iRound = 1;
@@ -43825,11 +43836,16 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){
#if SQLITE_MAX_MMAP_SIZE>0
if( pFd->mmapSizeMax>0 ){
+ /* Ensure that there is always at least a 256 byte buffer of addressable
+ ** memory following the returned page. If the database is corrupt,
+ ** SQLite may overread the page slightly (in practice only a few bytes,
+ ** but 256 is safe, round, number). */
+ const int nEofBuffer = 256;
if( pFd->pMapRegion==0 ){
int rc = unixMapfile(pFd, -1);
if( rc!=SQLITE_OK ) return rc;
}
- if( pFd->mmapSize >= iOff+nAmt ){
+ if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){
*pp = &((u8 *)pFd->pMapRegion)[iOff];
pFd->nFetchOut++;
}
@@ -51185,6 +51201,11 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){
#if SQLITE_MAX_MMAP_SIZE>0
if( pFd->mmapSizeMax>0 ){
+ /* Ensure that there is always at least a 256 byte buffer of addressable
+ ** memory following the returned page. If the database is corrupt,
+ ** SQLite may overread the page slightly (in practice only a few bytes,
+ ** but 256 is safe, round, number). */
+ const int nEofBuffer = 256;
if( pFd->pMapRegion==0 ){
int rc = winMapfile(pFd, -1);
if( rc!=SQLITE_OK ){
@@ -51193,7 +51214,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){
return rc;
}
}
- if( pFd->mmapSize >= iOff+nAmt ){
+ if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){
assert( pFd->pMapRegion!=0 );
*pp = &((u8 *)pFd->pMapRegion)[iOff];
pFd->nFetchOut++;
@@ -53672,6 +53693,14 @@ SQLITE_API unsigned char *sqlite3_serialize(
pOut = 0;
}else{
sz = sqlite3_column_int64(pStmt, 0)*szPage;
+ if( sz==0 ){
+ sqlite3_reset(pStmt);
+ sqlite3_exec(db, "BEGIN IMMEDIATE; COMMIT;", 0, 0, 0);
+ rc = sqlite3_step(pStmt);
+ if( rc==SQLITE_ROW ){
+ sz = sqlite3_column_int64(pStmt, 0)*szPage;
+ }
+ }
if( piSize ) *piSize = sz;
if( mFlags & SQLITE_SERIALIZE_NOCOPY ){
pOut = 0;
@@ -62674,6 +62703,23 @@ static void pagerUnlockIfUnused(Pager *pPager){
}
}
+#ifndef SQLITE_OMIT_CONCURRENT
+/*
+** If this pager is currently in a concurrent transaction (pAllRead!=0),
+** then set the bit in the pAllRead vector to indicate that the transaction
+** read from page pgno. Return SQLITE_OK if successful, or an SQLite error
+** code (i.e. SQLITE_NOMEM) if an error occurs.
+*/
+SQLITE_PRIVATE int sqlite3PagerUsePage(Pager *pPager, Pgno pgno){
+ int rc = SQLITE_OK;
+ if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){
+ PAGERTRACE(("USING page %d\n", pgno));
+ rc = sqlite3BitvecSet(pPager->pAllRead, pgno);
+ }
+ return rc;
+}
+#endif
+
/*
** The page getter methods each try to acquire a reference to a
** page with page number pgno. If the requested reference is
@@ -62747,17 +62793,13 @@ static int getPageNormal(
assert( assert_pager_state(pPager) );
assert( pPager->hasHeldSharedLock==1 );
-#ifndef SQLITE_OMIT_CONCURRENT
/* If this is an CONCURRENT transaction and the page being read was
** present in the database file when the transaction was opened,
** mark it as read in the pAllRead vector. */
- pPg = 0;
- if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){
- PAGERTRACE(("USING page %d\n", pgno));
- rc = sqlite3BitvecSet(pPager->pAllRead, pgno);
- if( rc!=SQLITE_OK ) goto pager_acquire_err;
+ if( sqlite3PagerUsePage(pPager, pgno)!=SQLITE_OK ){
+ pPg = 0;
+ goto pager_acquire_err;
}
-#endif
if( pgno==0 ) return SQLITE_CORRUPT_BKPT;
pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3);
@@ -77663,6 +77705,8 @@ static int accessPayload(
u8 *aWrite = &pBuf[-4];
assert( aWrite>=pBufStart ); /* due to (6) */
memcpy(aSave, aWrite, 4);
+ rc = sqlite3PagerUsePage(pBt->pPager, nextPage);
+ if( rc!=SQLITE_OK ) break;
rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1));
nextPage = get4byte(aWrite);
memcpy(aWrite, aSave, 4);
@@ -77675,6 +77719,9 @@ static int accessPayload(
(eOp==0 ? PAGER_GET_READONLY : 0)
);
if( rc==SQLITE_OK ){
+ setMempageRoot(
+ (MemPage*)sqlite3PagerGetExtra(pDbPage), pCur->pgnoRoot
+ );
aPayload = sqlite3PagerGetData(pDbPage);
nextPage = get4byte(aPayload);
rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage);
@@ -78787,7 +78834,10 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){
}
pPage = pCur->pPage;
- assert( pPage->isInit );
+ if( sqlite3FaultSim(412) ) pPage->isInit = 0;
+ if( !pPage->isInit ){
+ return SQLITE_CORRUPT_BKPT;
+ }
if( !pPage->leaf ){
int idx = pCur->ix;
rc = moveToChild(pCur, get4byte(findCell(pPage, idx)));
@@ -79470,7 +79520,10 @@ static int fillInCell(
n = nHeader + nPayload;
testcase( n==3 );
testcase( n==4 );
- if( n<4 ) n = 4;
+ if( n<4 ){
+ n = 4;
+ pPayload[nPayload] = 0;
+ }
*pnSize = n;
assert( nSrc<=nPayload );
testcase( nSrcpgnoRoot);
if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){
rc = SQLITE_CORRUPT_BKPT;
}else{
@@ -81917,7 +81971,10 @@ SQLITE_PRIVATE int sqlite3BtreeInsert(
if( flags & BTREE_PREFORMAT ){
rc = SQLITE_OK;
szNew = p->pBt->nPreformatSize;
- if( szNew<4 ) szNew = 4;
+ if( szNew<4 ){
+ szNew = 4;
+ newCell[3] = 0;
+ }
if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){
CellInfo info;
pPage->xParseCell(pPage, newCell, &info);
@@ -82119,6 +82176,9 @@ SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64
pPageIn = 0;
rc = sqlite3PagerGet(pSrcPager, ovflIn, &pPageIn, PAGER_GET_READONLY);
if( rc==SQLITE_OK ){
+ setMempageRoot(
+ (MemPage*)sqlite3PagerGetExtra(pPageIn), pSrc->pgnoRoot
+ );
aIn = (const u8*)sqlite3PagerGetData(pPageIn);
ovflIn = get4byte(aIn);
aIn += 4;
@@ -90927,6 +90987,23 @@ static void serialGet(
pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real;
}
}
+static int serialGet7(
+ const unsigned char *buf, /* Buffer to deserialize from */
+ Mem *pMem /* Memory cell to write value into */
+){
+ u64 x = FOUR_BYTE_UINT(buf);
+ u32 y = FOUR_BYTE_UINT(buf+4);
+ x = (x<<32) + y;
+ assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 );
+ swapMixedEndianFloat(x);
+ memcpy(&pMem->u.r, &x, sizeof(x));
+ if( IsNaN(x) ){
+ pMem->flags = MEM_Null;
+ return 1;
+ }
+ pMem->flags = MEM_Real;
+ return 0;
+}
SQLITE_PRIVATE void sqlite3VdbeSerialGet(
const unsigned char *buf, /* Buffer to deserialize from */
u32 serial_type, /* Serial type to deserialize */
@@ -91606,7 +91683,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(
}else if( serial_type==0 ){
rc = -1;
}else if( serial_type==7 ){
- sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
+ serialGet7(&aKey1[d1], &mem1);
rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r);
}else{
i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]);
@@ -91631,14 +91708,18 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(
}else if( serial_type==0 ){
rc = -1;
}else{
- sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
if( serial_type==7 ){
- if( mem1.u.ru.r ){
+ if( serialGet7(&aKey1[d1], &mem1) ){
+ rc = -1; /* mem1 is a NaN */
+ }else if( mem1.u.ru.r ){
rc = -1;
}else if( mem1.u.r>pRhs->u.r ){
rc = +1;
+ }else{
+ assert( rc==0 );
}
}else{
+ sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1);
rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r);
}
}
@@ -91708,7 +91789,14 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(
/* RHS is null */
else{
serial_type = aKey1[idx1];
- rc = (serial_type!=0 && serial_type!=10);
+ if( serial_type==0
+ || serial_type==10
+ || (serial_type==7 && serialGet7(&aKey1[d1], &mem1)!=0)
+ ){
+ assert( rc==0 );
+ }else{
+ rc = 1;
+ }
}
if( rc!=0 ){
@@ -97406,7 +97494,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */
}
}
}else if( affinity==SQLITE_AFF_TEXT && ((flags1 | flags3) & MEM_Str)!=0 ){
- if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){
+ if( (flags1 & MEM_Str)!=0 ){
+ pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal);
+ }else if( (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){
testcase( pIn1->flags & MEM_Int );
testcase( pIn1->flags & MEM_Real );
testcase( pIn1->flags & MEM_IntReal );
@@ -97415,7 +97505,9 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */
flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask);
if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str;
}
- if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){
+ if( (flags3 & MEM_Str)!=0 ){
+ pIn3->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal);
+ }else if( (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){
testcase( pIn3->flags & MEM_Int );
testcase( pIn3->flags & MEM_Real );
testcase( pIn3->flags & MEM_IntReal );
@@ -108814,6 +108906,8 @@ static void resolveAlias(
assert( iCol>=0 && iColnExpr );
pOrig = pEList->a[iCol].pExpr;
assert( pOrig!=0 );
+ assert( !ExprHasProperty(pExpr, EP_Reduced|EP_TokenOnly) );
+ if( pExpr->pAggInfo ) return;
db = pParse->db;
pDup = sqlite3ExprDup(db, pOrig, 0);
if( db->mallocFailed ){
@@ -109699,6 +109793,19 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
** resolved. This prevents "column" from being counted as having been
** referenced, which might prevent a SELECT from being erroneously
** marked as correlated.
+ **
+ ** 2024-03-28: Beware of aggregates. A bare column of aggregated table
+ ** can still evaluate to NULL even though it is marked as NOT NULL.
+ ** Example:
+ **
+ ** CREATE TABLE t1(a INT NOT NULL);
+ ** SELECT a, a IS NULL, a IS NOT NULL, count(*) FROM t1;
+ **
+ ** The "a IS NULL" and "a IS NOT NULL" expressions cannot be optimized
+ ** here because at the time this case is hit, we do not yet know whether
+ ** or not t1 is being aggregated. We have to assume the worst and omit
+ ** the optimization. The only time it is safe to apply this optimization
+ ** is within the WHERE clause.
*/
case TK_NOTNULL:
case TK_ISNULL: {
@@ -109709,19 +109816,36 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
anRef[i] = p->nRef;
}
sqlite3WalkExpr(pWalker, pExpr->pLeft);
- if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){
- testcase( ExprHasProperty(pExpr, EP_OuterON) );
- assert( !ExprHasProperty(pExpr, EP_IntValue) );
- pExpr->u.iValue = (pExpr->op==TK_NOTNULL);
- pExpr->flags |= EP_IntValue;
- pExpr->op = TK_INTEGER;
+ if( IN_RENAME_OBJECT ) return WRC_Prune;
+ if( sqlite3ExprCanBeNull(pExpr->pLeft) ){
+ /* The expression can be NULL. So the optimization does not apply */
+ return WRC_Prune;
+ }
- for(i=0, p=pNC; p && ipNext, i++){
- p->nRef = anRef[i];
+ for(i=0, p=pNC; p; p=p->pNext, i++){
+ if( (p->ncFlags & NC_Where)==0 ){
+ return WRC_Prune; /* Not in a WHERE clause. Unsafe to optimize. */
}
- sqlite3ExprDelete(pParse->db, pExpr->pLeft);
- pExpr->pLeft = 0;
}
+ testcase( ExprHasProperty(pExpr, EP_OuterON) );
+ assert( !ExprHasProperty(pExpr, EP_IntValue) );
+#if TREETRACE_ENABLED
+ if( sqlite3TreeTrace & 0x80000 ){
+ sqlite3DebugPrintf(
+ "NOT NULL strength reduction converts the following to %d:\n",
+ pExpr->op==TK_NOTNULL
+ );
+ sqlite3ShowExpr(pExpr);
+ }
+#endif /* TREETRACE_ENABLED */
+ pExpr->u.iValue = (pExpr->op==TK_NOTNULL);
+ pExpr->flags |= EP_IntValue;
+ pExpr->op = TK_INTEGER;
+ for(i=0, p=pNC; p && ipNext, i++){
+ p->nRef = anRef[i];
+ }
+ sqlite3ExprDelete(pParse->db, pExpr->pLeft);
+ pExpr->pLeft = 0;
return WRC_Prune;
}
@@ -110621,7 +110745,9 @@ static int resolveSelectStep(Walker *pWalker, Select *p){
}
if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort;
}
+ sNC.ncFlags |= NC_Where;
if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort;
+ sNC.ncFlags &= ~NC_Where;
/* Resolve names in table-valued-function arguments */
for(i=0; ipSrc->nSrc; i++){
@@ -131551,13 +131677,13 @@ SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){
double r1, r2;
const char *zVal;
r1 = sqlite3_value_double(pValue);
- sqlite3_str_appendf(pStr, "%!.15g", r1);
+ sqlite3_str_appendf(pStr, "%!0.15g", r1);
zVal = sqlite3_str_value(pStr);
if( zVal ){
sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8);
if( r1!=r2 ){
sqlite3_str_reset(pStr);
- sqlite3_str_appendf(pStr, "%!.20e", r1);
+ sqlite3_str_appendf(pStr, "%!0.20e", r1);
}
}
break;
@@ -131859,7 +131985,7 @@ static void replaceFunc(
}
if( zPattern[0]==0 ){
assert( sqlite3_value_type(argv[1])!=SQLITE_NULL );
- sqlite3_result_value(context, argv[0]);
+ sqlite3_result_text(context, (const char*)zStr, nStr, SQLITE_TRANSIENT);
return;
}
nPattern = sqlite3_value_bytes(argv[1]);
@@ -135779,7 +135905,7 @@ SQLITE_PRIVATE void sqlite3Insert(
pNx->iDataCur = iDataCur;
pNx->iIdxCur = iIdxCur;
if( pNx->pUpsertTarget ){
- if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx) ){
+ if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx, pUpsert) ){
goto insert_cleanup;
}
}
@@ -142088,31 +142214,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
int mxCol; /* Maximum non-virtual column number */
if( pObjTab && pObjTab!=pTab ) continue;
- if( !IsOrdinaryTable(pTab) ){
-#ifndef SQLITE_OMIT_VIRTUALTABLE
- sqlite3_vtab *pVTab;
- int a1;
- if( !IsVirtual(pTab) ) continue;
- if( pTab->nCol<=0 ){
- const char *zMod = pTab->u.vtab.azArg[0];
- if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue;
- }
- sqlite3ViewGetColumnNames(pParse, pTab);
- if( pTab->u.vtab.p==0 ) continue;
- pVTab = pTab->u.vtab.p->pVtab;
- if( NEVER(pVTab==0) ) continue;
- if( NEVER(pVTab->pModule==0) ) continue;
- if( pVTab->pModule->iVersion<4 ) continue;
- if( pVTab->pModule->xIntegrity==0 ) continue;
- sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick);
- pTab->nTabRef++;
- sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF);
- a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v);
- integrityCheckResultRow(v);
- sqlite3VdbeJumpHere(v, a1);
-#endif
- continue;
- }
+ if( !IsOrdinaryTable(pTab) ) continue;
if( isQuick || HasRowid(pTab) ){
pPk = 0;
r2 = 0;
@@ -142247,6 +142349,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
** is REAL, we have to load the actual data using OP_Column
** to reliably determine if the value is a NULL. */
sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3);
+ sqlite3ColumnDefault(v, pTab, j, 3);
jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk);
VdbeCoverage(v);
}
@@ -142437,6 +142540,38 @@ SQLITE_PRIVATE void sqlite3Pragma(
}
}
}
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ /* Second pass to invoke the xIntegrity method on all virtual
+ ** tables.
+ */
+ for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){
+ Table *pTab = sqliteHashData(x);
+ sqlite3_vtab *pVTab;
+ int a1;
+ if( pObjTab && pObjTab!=pTab ) continue;
+ if( IsOrdinaryTable(pTab) ) continue;
+ if( !IsVirtual(pTab) ) continue;
+ if( pTab->nCol<=0 ){
+ const char *zMod = pTab->u.vtab.azArg[0];
+ if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue;
+ }
+ sqlite3ViewGetColumnNames(pParse, pTab);
+ if( pTab->u.vtab.p==0 ) continue;
+ pVTab = pTab->u.vtab.p->pVtab;
+ if( NEVER(pVTab==0) ) continue;
+ if( NEVER(pVTab->pModule==0) ) continue;
+ if( pVTab->pModule->iVersion<4 ) continue;
+ if( pVTab->pModule->xIntegrity==0 ) continue;
+ sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick);
+ pTab->nTabRef++;
+ sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF);
+ a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v);
+ integrityCheckResultRow(v);
+ sqlite3VdbeJumpHere(v, a1);
+ continue;
+ }
+#endif
}
{
static const int iLn = VDBE_OFFSET_LINENO(2);
@@ -156085,7 +156220,8 @@ SQLITE_PRIVATE Upsert *sqlite3UpsertNew(
SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(
Parse *pParse, /* The parsing context */
SrcList *pTabList, /* Table into which we are inserting */
- Upsert *pUpsert /* The ON CONFLICT clauses */
+ Upsert *pUpsert, /* The ON CONFLICT clauses */
+ Upsert *pAll /* Complete list of all ON CONFLICT clauses */
){
Table *pTab; /* That table into which we are inserting */
int rc; /* Result code */
@@ -156188,6 +156324,14 @@ SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(
continue;
}
pUpsert->pUpsertIdx = pIdx;
+ if( sqlite3UpsertOfIndex(pAll,pIdx)!=pUpsert ){
+ /* Really this should be an error. The isDup ON CONFLICT clause will
+ ** never fire. But this problem was not discovered until three years
+ ** after multi-CONFLICT upsert was added, and so we silently ignore
+ ** the problem to prevent breaking applications that might actually
+ ** have redundant ON CONFLICT clauses. */
+ pUpsert->isDup = 1;
+ }
break;
}
if( pUpsert->pUpsertIdx==0 ){
@@ -156214,9 +156358,13 @@ SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert *pUpsert){
Upsert *pNext;
if( NEVER(pUpsert==0) ) return 0;
pNext = pUpsert->pNextUpsert;
- if( pNext==0 ) return 1;
- if( pNext->pUpsertTarget==0 ) return 1;
- if( pNext->pUpsertIdx==0 ) return 1;
+ while( 1 /*exit-by-return*/ ){
+ if( pNext==0 ) return 1;
+ if( pNext->pUpsertTarget==0 ) return 1;
+ if( pNext->pUpsertIdx==0 ) return 1;
+ if( !pNext->isDup ) return 0;
+ pNext = pNext->pNextUpsert;
+ }
return 0;
}
@@ -169451,7 +169599,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
/* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */
testcase( pOrderBy && pOrderBy->nExpr==BMS-1 );
- if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0;
+ if( pOrderBy && pOrderBy->nExpr>=BMS ){
+ pOrderBy = 0;
+ wctrlFlags &= ~WHERE_WANT_DISTINCT;
+ }
/* The number of tables in the FROM clause is limited by the number of
** bits in a Bitmask
@@ -187459,6 +187610,8 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int);
SQLITE_PRIVATE int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);
+SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk);
+
#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
#endif /* _FTSINT_H */
@@ -191181,7 +191334,7 @@ static int fts3ShadowName(const char *zName){
** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual
** table.
*/
-static int fts3Integrity(
+static int fts3IntegrityMethod(
sqlite3_vtab *pVtab, /* The virtual table to be checked */
const char *zSchema, /* Name of schema in which pVtab lives */
const char *zTabname, /* Name of the pVTab table */
@@ -191189,30 +191342,21 @@ static int fts3Integrity(
char **pzErr /* Write error message here */
){
Fts3Table *p = (Fts3Table*)pVtab;
- char *zSql;
int rc;
- char *zErr = 0;
+ int bOk = 0;
- assert( pzErr!=0 );
- assert( *pzErr==0 );
UNUSED_PARAMETER(isQuick);
- zSql = sqlite3_mprintf(
- "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');",
- zSchema, zTabname, zTabname);
- if( zSql==0 ){
- return SQLITE_NOMEM;
- }
- rc = sqlite3_exec(p->db, zSql, 0, 0, &zErr);
- sqlite3_free(zSql);
- if( (rc&0xff)==SQLITE_CORRUPT ){
- *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
- p->bFts4 ? 4 : 3, zSchema, zTabname);
- }else if( rc!=SQLITE_OK ){
+ rc = sqlite3Fts3IntegrityCheck(p, &bOk);
+ assert( rc!=SQLITE_CORRUPT_VTAB || bOk==0 );
+ if( rc!=SQLITE_OK && rc!=SQLITE_CORRUPT_VTAB ){
*pzErr = sqlite3_mprintf("unable to validate the inverted index for"
" FTS%d table %s.%s: %s",
- p->bFts4 ? 4 : 3, zSchema, zTabname, zErr);
+ p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc));
+ }else if( bOk==0 ){
+ *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
+ p->bFts4 ? 4 : 3, zSchema, zTabname);
}
- sqlite3_free(zErr);
+ sqlite3Fts3SegmentsClose(p);
return SQLITE_OK;
}
@@ -191243,7 +191387,7 @@ static const sqlite3_module fts3Module = {
/* xRelease */ fts3ReleaseMethod,
/* xRollbackTo */ fts3RollbackToMethod,
/* xShadowName */ fts3ShadowName,
- /* xIntegrity */ fts3Integrity,
+ /* xIntegrity */ fts3IntegrityMethod,
};
/*
@@ -202797,7 +202941,7 @@ static u64 fts3ChecksumIndex(
** If an error occurs (e.g. an OOM or IO error), return an SQLite error
** code. The final value of *pbOk is undefined in this case.
*/
-static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
+SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){
int rc = SQLITE_OK; /* Return code */
u64 cksum1 = 0; /* Checksum based on FTS index contents */
u64 cksum2 = 0; /* Checksum based on %_content contents */
@@ -202875,7 +203019,7 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
sqlite3_finalize(pStmt);
}
- *pbOk = (cksum1==cksum2);
+ *pbOk = (rc==SQLITE_OK && cksum1==cksum2);
return rc;
}
@@ -202915,7 +203059,7 @@ static int fts3DoIntegrityCheck(
){
int rc;
int bOk = 0;
- rc = fts3IntegrityCheck(p, &bOk);
+ rc = sqlite3Fts3IntegrityCheck(p, &bOk);
if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
return rc;
}
@@ -206468,6 +206612,16 @@ static void jsonAppendChar(JsonString *p, char c){
}
}
+/* Remove a single character from the end of the string
+*/
+static void jsonStringTrimOneChar(JsonString *p){
+ if( p->eErr==0 ){
+ assert( p->nUsed>0 );
+ p->nUsed--;
+ }
+}
+
+
/* Make sure there is a zero terminator on p->zBuf[]
**
** Return true on success. Return false if an OOM prevents this
@@ -206475,7 +206629,7 @@ static void jsonAppendChar(JsonString *p, char c){
*/
static int jsonStringTerminate(JsonString *p){
jsonAppendChar(p, 0);
- p->nUsed--;
+ jsonStringTrimOneChar(p);
return p->eErr==0;
}
@@ -207476,6 +207630,7 @@ static int jsonTranslateTextToBlob(JsonParse *pParse, u32 i){
case '[': {
/* Parse array */
iThis = pParse->nBlob;
+ assert( i<=(u32)pParse->nJson );
jsonBlobAppendNode(pParse, JSONB_ARRAY, pParse->nJson - i, 0);
iStart = pParse->nBlob;
if( pParse->oom ) return -1;
@@ -207874,6 +208029,10 @@ static void jsonReturnStringAsBlob(JsonString *pStr){
JsonParse px;
memset(&px, 0, sizeof(px));
jsonStringTerminate(pStr);
+ if( pStr->eErr ){
+ sqlite3_result_error_nomem(pStr->pCtx);
+ return;
+ }
px.zJson = pStr->zBuf;
px.nJson = pStr->nUsed;
px.db = sqlite3_context_db_handle(pStr->pCtx);
@@ -207941,8 +208100,8 @@ static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){
(pParse->aBlob[i+7]<<8) + pParse->aBlob[i+8];
n = 9;
}
- if( i+sz+n > pParse->nBlob
- && i+sz+n > pParse->nBlob-pParse->delta
+ if( (i64)i+sz+n > pParse->nBlob
+ && (i64)i+sz+n > pParse->nBlob-pParse->delta
){
sz = 0;
n = 0;
@@ -207992,6 +208151,7 @@ static u32 jsonTranslateBlobToText(
}
case JSONB_INT:
case JSONB_FLOAT: {
+ if( sz==0 ) goto malformed_jsonb;
jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz);
break;
}
@@ -208000,6 +208160,7 @@ static u32 jsonTranslateBlobToText(
sqlite3_uint64 u = 0;
const char *zIn = (const char*)&pParse->aBlob[i+n];
int bOverflow = 0;
+ if( sz==0 ) goto malformed_jsonb;
if( zIn[0]=='-' ){
jsonAppendChar(pOut, '-');
k++;
@@ -208022,6 +208183,7 @@ static u32 jsonTranslateBlobToText(
case JSONB_FLOAT5: { /* Float literal missing digits beside "." */
u32 k = 0;
const char *zIn = (const char*)&pParse->aBlob[i+n];
+ if( sz==0 ) goto malformed_jsonb;
if( zIn[0]=='-' ){
jsonAppendChar(pOut, '-');
k++;
@@ -208135,11 +208297,12 @@ static u32 jsonTranslateBlobToText(
jsonAppendChar(pOut, '[');
j = i+n;
iEnd = j+sz;
- while( jeErr==0 ){
j = jsonTranslateBlobToText(pParse, j, pOut);
jsonAppendChar(pOut, ',');
}
- if( sz>0 ) pOut->nUsed--;
+ if( j>iEnd ) pOut->eErr |= JSTRING_MALFORMED;
+ if( sz>0 ) jsonStringTrimOneChar(pOut);
jsonAppendChar(pOut, ']');
break;
}
@@ -208148,17 +208311,18 @@ static u32 jsonTranslateBlobToText(
jsonAppendChar(pOut, '{');
j = i+n;
iEnd = j+sz;
- while( jeErr==0 ){
j = jsonTranslateBlobToText(pParse, j, pOut);
jsonAppendChar(pOut, (x++ & 1) ? ',' : ':');
}
- if( x & 1 ) pOut->eErr |= JSTRING_MALFORMED;
- if( sz>0 ) pOut->nUsed--;
+ if( (x & 1)!=0 || j>iEnd ) pOut->eErr |= JSTRING_MALFORMED;
+ if( sz>0 ) jsonStringTrimOneChar(pOut);
jsonAppendChar(pOut, '}');
break;
}
default: {
+ malformed_jsonb:
pOut->eErr |= JSTRING_MALFORMED;
break;
}
@@ -209085,6 +209249,38 @@ static void jsonInsertIntoBlob(
return;
}
+/*
+** If pArg is a blob that seems like a JSONB blob, then initialize
+** p to point to that JSONB and return TRUE. If pArg does not seem like
+** a JSONB blob, then return FALSE;
+**
+** This routine is only called if it is already known that pArg is a
+** blob. The only open question is whether or not the blob appears
+** to be a JSONB blob.
+*/
+static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){
+ u32 n, sz = 0;
+ p->aBlob = (u8*)sqlite3_value_blob(pArg);
+ p->nBlob = (u32)sqlite3_value_bytes(pArg);
+ if( p->nBlob==0 ){
+ p->aBlob = 0;
+ return 0;
+ }
+ if( NEVER(p->aBlob==0) ){
+ return 0;
+ }
+ if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT
+ && (n = jsonbPayloadSize(p, 0, &sz))>0
+ && sz+n==p->nBlob
+ && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0)
+ ){
+ return 1;
+ }
+ p->aBlob = 0;
+ p->nBlob = 0;
+ return 0;
+}
+
/*
** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob,
** from the SQL function argument pArg. Return a pointer to the new
@@ -209141,34 +209337,30 @@ static JsonParse *jsonParseFuncArg(
return p;
}
if( eType==SQLITE_BLOB ){
- u32 n, sz = 0;
- p->aBlob = (u8*)sqlite3_value_blob(pArg);
- p->nBlob = (u32)sqlite3_value_bytes(pArg);
- if( p->nBlob==0 ){
- goto json_pfa_malformed;
- }
- if( NEVER(p->aBlob==0) ){
- goto json_pfa_oom;
- }
- if( (p->aBlob[0] & 0x0f)>JSONB_OBJECT ){
- goto json_pfa_malformed;
- }
- n = jsonbPayloadSize(p, 0, &sz);
- if( n==0
- || sz+n!=p->nBlob
- || ((p->aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0)
- ){
- goto json_pfa_malformed;
- }
- if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){
- goto json_pfa_oom;
+ if( jsonArgIsJsonb(pArg,p) ){
+ if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){
+ goto json_pfa_oom;
+ }
+ return p;
}
- return p;
+ /* If the blob is not valid JSONB, fall through into trying to cast
+ ** the blob into text which is then interpreted as JSON. (tag-20240123-a)
+ **
+ ** This goes against all historical documentation about how the SQLite
+ ** JSON functions were suppose to work. From the beginning, blob was
+ ** reserved for expansion and a blob value should have raised an error.
+ ** But it did not, due to a bug. And many applications came to depend
+ ** upon this buggy behavior, espeically when using the CLI and reading
+ ** JSON text using readfile(), which returns a blob. For this reason
+ ** we will continue to support the bug moving forward.
+ ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d
+ */
}
p->zJson = (char*)sqlite3_value_text(pArg);
p->nJson = sqlite3_value_bytes(pArg);
+ if( db->mallocFailed ) goto json_pfa_oom;
if( p->nJson==0 ) goto json_pfa_malformed;
- if( NEVER(p->zJson==0) ) goto json_pfa_oom;
+ assert( p->zJson!=0 );
if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){
if( flgs & JSON_KEEPERROR ){
p->nErr = 1;
@@ -209334,10 +209526,10 @@ static void jsonDebugPrintBlob(
if( sz==0 && x<=JSONB_FALSE ){
sqlite3_str_append(pOut, "\n", 1);
}else{
- u32 i;
+ u32 j;
sqlite3_str_appendall(pOut, ": \"");
- for(i=iStart+n; iaBlob[i];
+ for(j=iStart+n; jaBlob[j];
if( c<0x20 || c>=0x7f ) c = '.';
sqlite3_str_append(pOut, (char*)&c, 1);
}
@@ -210139,12 +210331,12 @@ static void jsonValidFunc(
return;
}
case SQLITE_BLOB: {
- if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){
+ if( jsonFuncArgMightBeBinary(argv[0]) ){
if( flags & 0x04 ){
/* Superficial checking only - accomplished by the
** jsonFuncArgMightBeBinary() call above. */
res = 1;
- }else{
+ }else if( flags & 0x08 ){
/* Strict checking. Check by translating BLOB->TEXT->BLOB. If
** no errors occur, call that a "strict check". */
JsonParse px;
@@ -210155,8 +210347,11 @@ static void jsonValidFunc(
iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1);
res = iErr==0;
}
+ break;
}
- break;
+ /* Fall through into interpreting the input as text. See note
+ ** above at tag-20240123-a. */
+ /* no break */ deliberate_fall_through
}
default: {
JsonParse px;
@@ -210281,7 +210476,7 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){
if( isFinal ){
if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf);
}else{
- pStr->nUsed--;
+ jsonStringTrimOneChar(pStr);
}
return;
}else if( isFinal ){
@@ -210291,7 +210486,7 @@ static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){
pStr->bStatic = 1;
}else{
sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
- pStr->nUsed--;
+ jsonStringTrimOneChar(pStr);
}
}else{
sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC);
@@ -210401,7 +210596,7 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){
if( isFinal ){
if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf);
}else{
- pStr->nUsed--;
+ jsonStringTrimOneChar(pStr);
}
return;
}else if( isFinal ){
@@ -210411,7 +210606,7 @@ static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){
pStr->bStatic = 1;
}else{
sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
- pStr->nUsed--;
+ jsonStringTrimOneChar(pStr);
}
}else{
sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC);
@@ -210742,6 +210937,9 @@ static int jsonEachColumn(
case JEACH_VALUE: {
u32 i = jsonSkipLabel(p);
jsonReturnFromBlob(&p->sParse, i, ctx, 1);
+ if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY ){
+ sqlite3_result_subtype(ctx, JSON_SUBTYPE);
+ }
break;
}
case JEACH_TYPE: {
@@ -210788,9 +210986,9 @@ static int jsonEachColumn(
case JEACH_JSON: {
if( p->sParse.zJson==0 ){
sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob,
- SQLITE_STATIC);
+ SQLITE_TRANSIENT);
}else{
- sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC);
+ sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_TRANSIENT);
}
break;
}
@@ -210892,13 +211090,9 @@ static int jsonEachFilter(
memset(&p->sParse, 0, sizeof(p->sParse));
p->sParse.nJPRef = 1;
p->sParse.db = p->db;
- if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
- if( jsonFuncArgMightBeBinary(argv[0]) ){
- p->sParse.nBlob = sqlite3_value_bytes(argv[0]);
- p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]);
- }else{
- goto json_each_malformed_input;
- }
+ if( jsonFuncArgMightBeBinary(argv[0]) ){
+ p->sParse.nBlob = sqlite3_value_bytes(argv[0]);
+ p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]);
}else{
p->sParse.zJson = (char*)sqlite3_value_text(argv[0]);
p->sParse.nJson = sqlite3_value_bytes(argv[0]);
@@ -211820,11 +212014,9 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){
** Clear the Rtree.pNodeBlob object
*/
static void nodeBlobReset(Rtree *pRtree){
- if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){
- sqlite3_blob *pBlob = pRtree->pNodeBlob;
- pRtree->pNodeBlob = 0;
- sqlite3_blob_close(pBlob);
- }
+ sqlite3_blob *pBlob = pRtree->pNodeBlob;
+ pRtree->pNodeBlob = 0;
+ sqlite3_blob_close(pBlob);
}
/*
@@ -211868,7 +212060,6 @@ static int nodeAcquire(
&pRtree->pNodeBlob);
}
if( rc ){
- nodeBlobReset(pRtree);
*ppNode = 0;
/* If unable to open an sqlite3_blob on the desired row, that can only
** be because the shadow tables hold erroneous data. */
@@ -211928,6 +212119,7 @@ static int nodeAcquire(
}
*ppNode = pNode;
}else{
+ nodeBlobReset(pRtree);
if( pNode ){
pRtree->nNodeRef--;
sqlite3_free(pNode);
@@ -212072,6 +212264,7 @@ static void nodeGetCoord(
int iCoord, /* Which coordinate to extract */
RtreeCoord *pCoord /* OUT: Space to write result to */
){
+ assert( iCellzData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord);
}
@@ -212261,7 +212454,9 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){
sqlite3_finalize(pCsr->pReadAux);
sqlite3_free(pCsr);
pRtree->nCursor--;
- nodeBlobReset(pRtree);
+ if( pRtree->nCursor==0 && pRtree->inWrTrans==0 ){
+ nodeBlobReset(pRtree);
+ }
return SQLITE_OK;
}
@@ -212846,7 +213041,11 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){
int rc = SQLITE_OK;
RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc);
if( rc==SQLITE_OK && ALWAYS(p) ){
- *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell);
+ if( p->iCell>=NCELL(pNode) ){
+ rc = SQLITE_ABORT;
+ }else{
+ *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell);
+ }
}
return rc;
}
@@ -212864,6 +213063,7 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
if( rc ) return rc;
if( NEVER(p==0) ) return SQLITE_OK;
+ if( p->iCell>=NCELL(pNode) ) return SQLITE_ABORT;
if( i==0 ){
sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell));
}else if( i<=pRtree->nDim2 ){
@@ -214345,8 +214545,7 @@ static int rtreeUpdate(
*/
static int rtreeBeginTransaction(sqlite3_vtab *pVtab){
Rtree *pRtree = (Rtree *)pVtab;
- assert( pRtree->inWrTrans==0 );
- pRtree->inWrTrans++;
+ pRtree->inWrTrans = 1;
return SQLITE_OK;
}
@@ -214360,6 +214559,9 @@ static int rtreeEndTransaction(sqlite3_vtab *pVtab){
nodeBlobReset(pRtree);
return SQLITE_OK;
}
+static int rtreeRollback(sqlite3_vtab *pVtab){
+ return rtreeEndTransaction(pVtab);
+}
/*
** The xRename method for rtree module virtual tables.
@@ -214478,7 +214680,7 @@ static sqlite3_module rtreeModule = {
rtreeBeginTransaction, /* xBegin - begin transaction */
rtreeEndTransaction, /* xSync - sync transaction */
rtreeEndTransaction, /* xCommit - commit transaction */
- rtreeEndTransaction, /* xRollback - rollback transaction */
+ rtreeRollback, /* xRollback - rollback transaction */
0, /* xFindFunction - function overloading */
rtreeRename, /* xRename - rename the table */
rtreeSavepoint, /* xSavepoint */
@@ -248069,23 +248271,26 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){
static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){
int ii;
Fts5TokenDataIter *pT = pIter->pTokenDataIter;
+ Fts5Index *pIndex = pIter->pIndex;
for(ii=0; iinIter; ii++){
Fts5Iter *p = pT->apIter[ii];
if( p->base.bEof==0
&& (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowidpIndex, p, bFrom, iFrom);
+ fts5MultiIterNext(pIndex, p, bFrom, iFrom);
while( bFrom && p->base.bEof==0
&& p->base.iRowidpIndex->rc==SQLITE_OK
+ && pIndex->rc==SQLITE_OK
){
- fts5MultiIterNext(p->pIndex, p, 0, 0);
+ fts5MultiIterNext(pIndex, p, 0, 0);
}
}
}
- fts5IterSetOutputsTokendata(pIter);
+ if( pIndex->rc==SQLITE_OK ){
+ fts5IterSetOutputsTokendata(pIter);
+ }
}
/*
@@ -253240,7 +253445,7 @@ static void fts5SourceIdFunc(
){
assert( nArg==0 );
UNUSED_PARAM2(nArg, apUnused);
- sqlite3_result_text(pCtx, "fts5: 2024-02-16 11:05:35 11a4178f0e6cd760303a5874b5a3529ccd78a2398b59700e878c7e6b1251de91", -1, SQLITE_TRANSIENT);
+ sqlite3_result_text(pCtx, "fts5: 2024-06-04 16:24:12 66c885814e1c533f0f45ee01b6b25cd5ec08770d699e441b1af4f79b23e9f0a7", -1, SQLITE_TRANSIENT);
}
/*
@@ -253271,27 +253476,21 @@ static int fts5IntegrityMethod(
char **pzErr /* Write error message here */
){
Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
- Fts5Config *pConfig = pTab->p.pConfig;
- char *zSql;
- char *zErr = 0;
int rc;
+
assert( pzErr!=0 && *pzErr==0 );
UNUSED_PARAM(isQuick);
- zSql = sqlite3_mprintf(
- "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');",
- zSchema, zTabname, pConfig->zName);
- if( zSql==0 ) return SQLITE_NOMEM;
- rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr);
- sqlite3_free(zSql);
+ rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0);
if( (rc&0xff)==SQLITE_CORRUPT ){
*pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
zSchema, zTabname);
}else if( rc!=SQLITE_OK ){
*pzErr = sqlite3_mprintf("unable to validate the inverted index for"
" FTS5 table %s.%s: %s",
- zSchema, zTabname, zErr);
+ zSchema, zTabname, sqlite3_errstr(rc));
}
- sqlite3_free(zErr);
+ sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
+
return SQLITE_OK;
}
diff --git a/libstuff/sqlite3.h b/libstuff/sqlite3.h
index 232c6afb6..31159190e 100644
--- a/libstuff/sqlite3.h
+++ b/libstuff/sqlite3.h
@@ -146,9 +146,9 @@ extern "C" {
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
*/
-#define SQLITE_VERSION "3.45.0"
-#define SQLITE_VERSION_NUMBER 3045000
-#define SQLITE_SOURCE_ID "2024-02-16 11:05:35 11a4178f0e6cd760303a5874b5a3529ccd78a2398b59700e878c7e6b1251de91"
+#define SQLITE_VERSION "3.45.2"
+#define SQLITE_VERSION_NUMBER 3045002
+#define SQLITE_SOURCE_ID "2024-06-04 16:24:12 66c885814e1c533f0f45ee01b6b25cd5ec08770d699e441b1af4f79b23e9f0a7"
/*
** CAPI3REF: Run-Time Library Version Numbers
@@ -420,6 +420,8 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**);
** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running.
** The application must not modify the SQL statement text passed into
** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running.
+** The application must not dereference the arrays or string pointers
+** passed as the 3rd and 4th callback parameters after it returns.
**
*/
SQLITE_API int sqlite3_exec(
diff --git a/sqlitecluster/SQLiteNode.cpp b/sqlitecluster/SQLiteNode.cpp
index 01cb75cd2..1d20e46c8 100644
--- a/sqlitecluster/SQLiteNode.cpp
+++ b/sqlitecluster/SQLiteNode.cpp
@@ -1716,6 +1716,12 @@ void SQLiteNode::_onMESSAGE(SQLitePeer* peer, const SData& message) {
uint64_t threadAttemptStartTimestamp = STimeNow();
thread(&SQLiteNode::_replicate, this, peer, message, _dbPool->getIndex(false), threadAttemptStartTimestamp).detach();
} catch (const system_error& e) {
+ // If the server is strugling and falling behind on replication, we might have too many threads
+ // causing a resource exhaustion. If that happens, all the transactions that are already threaded
+ // and waiting for the transaction that failed will be stuck in an infinite loop. To prevent that
+ // we're changing the state to SEARCHING and sending the cancelAfter property to drop all threads
+ // that depend on the transaction that failed to be threaded.
+ _changeState(SQLiteNodeState::SEARCHING, message.calcU64("NewCount") - 1);
SWARN("Caught system_error starting _replicate thread with " << _replicationThreadCount.load() << " threads. e.what()=" << e.what());
STHROW("Error starting replicate thread so giving up and reconnecting.");
}
@@ -1921,7 +1927,7 @@ void SQLiteNode::_sendToAllPeers(const SData& message, bool subscribedOnly) {
}
}
-void SQLiteNode::_changeState(SQLiteNodeState newState) {
+void SQLiteNode::_changeState(SQLiteNodeState newState, uint64_t commitIDToCancelAfter) {
SINFO("[NOTIFY] setting commit count to: " << _db.getCommitCount());
_localCommitNotifier.notifyThrough(_db.getCommitCount());
@@ -1932,7 +1938,7 @@ void SQLiteNode::_changeState(SQLiteNodeState newState) {
// If we were following, and now we're not, we give up an any replications.
if (_state == SQLiteNodeState::FOLLOWING) {
_replicationThreadsShouldExit = true;
- uint64_t cancelAfter = _leaderCommitNotifier.getValue();
+ uint64_t cancelAfter = commitIDToCancelAfter ? commitIDToCancelAfter : _leaderCommitNotifier.getValue();
SINFO("Replication threads should exit, canceling commits after current leader commit " << cancelAfter);
_localCommitNotifier.cancel(cancelAfter);
_leaderCommitNotifier.cancel(cancelAfter);
diff --git a/sqlitecluster/SQLiteNode.h b/sqlitecluster/SQLiteNode.h
index 44aa6b8c6..8212ae18a 100644
--- a/sqlitecluster/SQLiteNode.h
+++ b/sqlitecluster/SQLiteNode.h
@@ -226,7 +226,7 @@ class SQLiteNode : public STCPManager {
// Add required headers for messages being sent to peers.
SData _addPeerHeaders(SData message);
- void _changeState(SQLiteNodeState newState);
+ void _changeState(SQLiteNodeState newState, uint64_t commitIDToCancelAfter = 0);
// Handlers for transaction messages.
void _handleBeginTransaction(SQLite& db, SQLitePeer* peer, const SData& message, bool wasConflict);