Complete fix for [53fdb87e49]: "Glyph indexing is still unperfect". T…

…k 8.7 now uses UTF-32 indices everywhere, just as Tk 9.0.
tcltk · Mar 15, 2024 · f48cf63 · f48cf63
2 parents cbda117 + 1498eb2
commit f48cf63
Show file tree

Hide file tree

Showing 17 changed files with 191 additions and 128 deletions.
diff --git a/generic/tkCanvText.c b/generic/tkCanvText.c
@@ -594,7 +594,7 @@ ConfigureText(
      */
 
     textPtr->numBytes = strlen(textPtr->text);
-    textPtr->numChars = Tcl_NumUtfChars(textPtr->text, textPtr->numBytes);
+    textPtr->numChars = TkNumUtfChars(textPtr->text, textPtr->numBytes);
     if (textInfoPtr->selItemPtr == itemPtr) {
 
 	if (textInfoPtr->selectFirst >= textPtr->numChars) {
@@ -1119,7 +1119,7 @@ TextInsert(
 
     ckfree(text);
     textPtr->text = newStr;
-    charsAdded = Tcl_NumUtfChars(string, byteCount);
+    charsAdded = TkNumUtfChars(string, byteCount);
     textPtr->numChars += charsAdded;
     textPtr->numBytes += byteCount;
 
@@ -1189,8 +1189,8 @@ TextDeleteChars(
     }
     charsRemoved = last + 1 - first;
 
-    byteIndex = Tcl_UtfAtIndex(text, first) - text;
-    byteCount = Tcl_UtfAtIndex(text + byteIndex, charsRemoved)
+    byteIndex = TkUtfAtIndex(text, first) - text;
+    byteCount = TkUtfAtIndex(text + byteIndex, charsRemoved)
 	- (text + byteIndex);
 
     newStr = (char *)ckalloc(textPtr->numBytes + 1 - byteCount);
@@ -1612,8 +1612,8 @@ GetSelText(
 	return 0;
     }
     text = textPtr->text;
-    selStart = Tcl_UtfAtIndex(text, textInfoPtr->selectFirst);
-    selEnd = Tcl_UtfAtIndex(selStart,
+    selStart = TkUtfAtIndex(text, textInfoPtr->selectFirst);
+    selEnd = TkUtfAtIndex(selStart,
 	    textInfoPtr->selectLast + 1 - textInfoPtr->selectFirst);
     if (selEnd  <= selStart + offset) {
 	return 0;

diff --git a/generic/tkEntry.c b/generic/tkEntry.c
@@ -2164,7 +2164,7 @@ InsertChars(
     char *newStr;
 
     string = entryPtr->string;
-    byteIndex = Tcl_UtfAtIndex(string, index) - string;
+    byteIndex = TkUtfAtIndex(string, index) - string;
     byteCount = strlen(value);
     if (byteCount == 0) {
 	return TCL_OK;
@@ -2197,7 +2197,7 @@ InsertChars(
      */
 
     oldChars = entryPtr->numChars;
-    entryPtr->numChars = Tcl_NumUtfChars(newStr, TCL_INDEX_NONE);
+    entryPtr->numChars = TkNumUtfChars(newStr, TCL_INDEX_NONE);
     charsAdded = entryPtr->numChars - oldChars;
     entryPtr->numBytes += byteCount;
 
@@ -2268,8 +2268,8 @@ DeleteChars(
     }
 
     string = entryPtr->string;
-    byteIndex = Tcl_UtfAtIndex(string, index) - string;
-    byteCount = Tcl_UtfAtIndex(string + byteIndex, count) - (string+byteIndex);
+    byteIndex = TkUtfAtIndex(string, index) - string;
+    byteCount = TkUtfAtIndex(string + byteIndex, count) - (string+byteIndex);
 
     newByteCount = entryPtr->numBytes + 1 - byteCount;
     newStr = (char *)ckalloc(newByteCount);
@@ -2497,7 +2497,7 @@ EntrySetValue(
 	entryPtr->string = tmp;
     }
     entryPtr->numBytes = valueLen;
-    entryPtr->numChars = Tcl_NumUtfChars(value, valueLen);
+    entryPtr->numChars = TkNumUtfChars(value, valueLen);
 
     if (entryPtr->displayString == oldSource) {
 	entryPtr->displayString = entryPtr->string;
@@ -2927,8 +2927,8 @@ EntryFetchSelection(
 	return -1;
     }
     string = entryPtr->displayString;
-    selStart = Tcl_UtfAtIndex(string, entryPtr->selectFirst);
-    selEnd = Tcl_UtfAtIndex(selStart,
+    selStart = TkUtfAtIndex(string, entryPtr->selectFirst);
+    selEnd = TkUtfAtIndex(selStart,
 	    entryPtr->selectLast - entryPtr->selectFirst);
     if (selEnd <= selStart + offset) {
 	return 0;

diff --git a/generic/tkFont.c b/generic/tkFont.c
@@ -2013,7 +2013,7 @@ Tk_ComputeTextLayout(
     height = fmPtr->ascent + fmPtr->descent;
 
     if (numChars < 0) {
-	numChars = Tcl_NumUtfChars(string, TCL_INDEX_NONE);
+	numChars = TkNumUtfChars(string, TCL_INDEX_NONE);
     }
     if (wrapLength == 0) {
 	wrapLength = -1;
@@ -2036,7 +2036,7 @@ Tk_ComputeTextLayout(
 
     curX = 0;
 
-    endp = Tcl_UtfAtIndex(string, numChars);
+    endp = TkUtfAtIndex(string, numChars);
     special = string;
 
     flags &= TK_IGNORE_TABS | TK_IGNORE_NEWLINES;
@@ -2153,7 +2153,7 @@ Tk_ComputeTextLayout(
 		bytesThisChunk = Tk_MeasureChars(tkfont, end, bytesThisChunk,
 			-1, 0, &chunkPtr->totalWidth);
 		chunkPtr->numBytes += bytesThisChunk;
-		chunkPtr->numChars += Tcl_NumUtfChars(end, bytesThisChunk);
+		chunkPtr->numChars += TkNumUtfChars(end, bytesThisChunk);
 		chunkPtr->totalWidth += curX;
 	    }
 	}
@@ -2345,14 +2345,14 @@ Tk_DrawTextLayout(
 		firstChar = 0;
 		firstByte = chunkPtr->start;
 	    } else {
-		firstByte = Tcl_UtfAtIndex(chunkPtr->start, firstChar);
+		firstByte = TkUtfAtIndex(chunkPtr->start, firstChar);
 		Tk_MeasureChars(layoutPtr->tkfont, chunkPtr->start,
 			firstByte - chunkPtr->start, -1, 0, &drawX);
 	    }
 	    if (lastChar < numDisplayChars) {
 		numDisplayChars = lastChar;
 	    }
-	    lastByte = Tcl_UtfAtIndex(chunkPtr->start, numDisplayChars);
+	    lastByte = TkUtfAtIndex(chunkPtr->start, numDisplayChars);
 #ifdef TK_DRAW_IN_CONTEXT
 	    TkpDrawCharsInContext(display, drawable, gc, layoutPtr->tkfont,
 		    chunkPtr->start, chunkPtr->numBytes,
@@ -2415,14 +2415,14 @@ TkDrawAngledTextLayout(
 		firstChar = 0;
 		firstByte = chunkPtr->start;
 	    } else {
-		firstByte = Tcl_UtfAtIndex(chunkPtr->start, firstChar);
+		firstByte = TkUtfAtIndex(chunkPtr->start, firstChar);
 		Tk_MeasureChars(layoutPtr->tkfont, chunkPtr->start,
 			firstByte - chunkPtr->start, -1, 0, &drawX);
 	    }
 	    if (lastChar < numDisplayChars) {
 		numDisplayChars = lastChar;
 	    }
-	    lastByte = Tcl_UtfAtIndex(chunkPtr->start, numDisplayChars);
+	    lastByte = TkUtfAtIndex(chunkPtr->start, numDisplayChars);
 #ifdef TK_DRAW_IN_CONTEXT
 	    dx = cosA * (chunkPtr->x) + sinA * (chunkPtr->y);
 	    dy = -sinA * (chunkPtr->x) + cosA * (chunkPtr->y);
@@ -2675,7 +2675,7 @@ Tk_PointToChar(
 		    }
 		    n = Tk_MeasureChars((Tk_Font) fontPtr, chunkPtr->start,
 			    chunkPtr->numBytes, x - chunkPtr->x, 0, &dummy);
-		    return numChars + Tcl_NumUtfChars(chunkPtr->start, n);
+		    return numChars + TkNumUtfChars(chunkPtr->start, n);
 		}
 		numChars += chunkPtr->numChars;
 		lastPtr = chunkPtr;
@@ -2784,7 +2784,7 @@ Tk_CharBbox(
 		goto check;
 	    }
 	} else if (index < chunkPtr->numChars) {
-	    end = Tcl_UtfAtIndex(chunkPtr->start, index);
+	    end = TkUtfAtIndex(chunkPtr->start, index);
 	    if (xPtr != NULL) {
 		Tk_MeasureChars(tkfont, chunkPtr->start,
 			end - chunkPtr->start, -1, 0, &x);
@@ -3859,7 +3859,7 @@ NewChunk(
 	*layoutPtrPtr = layoutPtr;
 	*maxPtr = maxChunks;
     }
-    numChars = Tcl_NumUtfChars(start, numBytes);
+    numChars = TkNumUtfChars(start, numBytes);
     chunkPtr = &layoutPtr->chunks[layoutPtr->numChunks];
     chunkPtr->start		= start;
     chunkPtr->numBytes		= numBytes;

diff --git a/generic/tkIcu.c b/generic/tkIcu.c
@@ -92,7 +92,8 @@ startEndOfCmd(
     str = Tcl_GetStringFromObj(objv[1], &len);
     Tcl_UtfToChar16DString(str, len, &ds);
     len = Tcl_DStringLength(&ds)/2;
-    if (TkGetIntForIndex(objv[2], len-1, 0, &idx) != TCL_OK) {
+    Tcl_Size ulen = TkGetCharLength(objv[1]);
+    if (TkGetIntForIndex(objv[2], ulen-1, 0, &idx) != TCL_OK) {
 	Tcl_DStringFree(&ds);
 	Tcl_SetObjResult(interp, Tcl_ObjPrintf("bad index \"%s\": must be integer?[+-]integer?, end?[+-]integer?, or \"\"", Tcl_GetString(objv[2])));
 	Tcl_SetErrorCode(interp, "TK", "ICU", "INDEX", NULL);
@@ -111,6 +112,14 @@ startEndOfCmd(
     	Tcl_SetErrorCode(interp, "TK", "ICU", "CANNOTOPEN", NULL);
     	return TCL_ERROR;
     }
+    if (idx > 0 && len != ulen) {
+	/* The string contains codepoints > \uFFFF. Determine UTF-16 index */
+	Tcl_Size newIdx = 0;
+	for (Tcl_Size i = 0; i < idx; i++) {
+	    newIdx += 1 + (((newIdx < (Tcl_Size)len-1) && (ustr[newIdx]&0xFC00) == 0xD800) && ((ustr[newIdx+1]&0xFC00) == 0xDC00));
+	}
+	idx = newIdx;
+    }
     if (flags & FLAG_FOLLOWING) {
 	if ((idx < 0) && (flags & FLAG_WORD)) {
 	    idx = 0;
@@ -145,6 +154,14 @@ startEndOfCmd(
     icu_close(it);
     Tcl_DStringFree(&ds);
     if (idx != TCL_INDEX_NONE) {
+	if (idx > 0 && len != ulen) {
+	    /* The string contains codepoints > \uFFFF. Determine UTF-32 index */
+	    Tcl_Size newIdx = 1;
+	    for (Tcl_Size i = 1; i < idx; i++) {
+    	if (((ustr[i-1]&0xFC00) != 0xD800) || ((ustr[i]&0xFC00) != 0xDC00)) newIdx++;
+	    }
+	    idx = newIdx;
+	}
 	Tcl_SetObjResult(interp, TkNewIndexObj(idx));
     }
     return TCL_OK;

diff --git a/generic/tkInt.h b/generic/tkInt.h
@@ -76,13 +76,40 @@
 #endif
 
 #if (TCL_MAJOR_VERSION == 8) && (TCL_MINOR_VERSION < 7)
-# define Tcl_WCharToUtfDString ((char * (*)(const WCHAR *, int len, Tcl_DString *))Tcl_UniCharToUtfDString)
-# define Tcl_UtfToWCharDString ((WCHAR * (*)(const char *, int len, Tcl_DString *))Tcl_UtfToUniCharDString)
+# define Tcl_WCharToUtfDString ((char * (*)(const WCHAR *, int, Tcl_DString *))Tcl_UniCharToUtfDString)
+# define Tcl_UtfToWCharDString ((WCHAR * (*)(const char *, int, Tcl_DString *))Tcl_UtfToUniCharDString)
 # define Tcl_Char16ToUtfDString Tcl_UniCharToUtfDString
 # define Tcl_UtfToChar16DString Tcl_UtfToUniCharDString
 # define TCL_COMBINE		0
 #endif
 
+/* Make available UTF-32 versions of the API, even though we compile with TCL_UTF_MAX=3 */
+#if TCL_MAJOR_VERSION > 8
+#   define TkUtfToUniChar (tclStubsPtr->tcl_UtfToUniChar) /* 646 */
+#   define TkUniCharToUtf (tclStubsPtr->tcl_UniCharToUtf) /* 324 (without TCL_COMBINE) */
+#   define TkNumUtfChars (tclStubsPtr->tcl_NumUtfChars) /* 669 */
+#   define TkGetCharLength (tclStubsPtr->tcl_GetCharLength) /* 670 */
+#   define TkUtfAtIndex (tclStubsPtr->tcl_UtfAtIndex) /* 671 */
+#else
+    MODULE_SCOPE Tcl_Size TkUtfToUniChar(const char *, int *);
+    MODULE_SCOPE Tcl_Size TkUniCharToUtf(int, char *);
+#   ifdef USE_TCL_STUBS
+#	define TkNumUtfChars (((&tclStubsPtr->tcl_PkgProvideEx)[631]) ? \
+		((Tcl_Size (*)(const char *, Tcl_Size))(void *)((&tclStubsPtr->tcl_PkgProvideEx)[669])) \
+		: (tclStubsPtr->tcl_NumUtfChars) /* 312 */)
+#	define TkGetCharLength (((&tclStubsPtr->tcl_PkgProvideEx)[631]) ? \
+		((Tcl_Size (*)(Tcl_Obj *))(void *)((&tclStubsPtr->tcl_PkgProvideEx)[670])) \
+		: (tclStubsPtr->tcl_GetCharLength) /* 380 */)
+#	define TkUtfAtIndex (((&tclStubsPtr->tcl_PkgProvideEx)[631]) ? \
+		((const char *(*)(const char *, Tcl_Size))(void *)((&tclStubsPtr->tcl_PkgProvideEx)[671])) \
+		: (tclStubsPtr->tcl_UtfAtIndex) /* 325 */)
+#   else
+#	define TkNumUtfChars TclNumUtfChars
+#	define TkGetCharLength TclGetCharLength
+#	define TkUtfAtIndex TclUtfAtIndex
+#   endif
+#endif
+
 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
 #   define TKFLEXARRAY
 #elif defined(__GNUC__) && (__GNUC__ > 2)
@@ -1377,15 +1404,6 @@ MODULE_SCOPE void	TkpCopyRegion(TkRegion dst, TkRegion src);
 # define c_class class
 #endif
 
-/* Tcl 8.6 has a different definition of Tcl_UniChar than other Tcl versions for TCL_UTF_MAX > 3 */
-#if TCL_UTF_MAX > (3 + (TCL_MAJOR_VERSION == 8 && TCL_MINOR_VERSION == 6))
-#   define TkUtfToUniChar(src, ch) (size_t)(((int (*)(const char *, int *))Tcl_UtfToUniChar)(src, ch))
-#   define TkUniCharToUtf(ch, src) (size_t)(((int (*)(int, char *))Tcl_UniCharToUtf)(ch, src))
-#else
-    MODULE_SCOPE size_t TkUtfToUniChar(const char *, int *);
-    MODULE_SCOPE size_t TkUniCharToUtf(int, char *);
-#endif
-
 #if defined(_WIN32) && !defined(STATIC_BUILD) && TCL_MAJOR_VERSION < 9
 #   define tcl_CreateFileHandler reserved9
 #endif

diff --git a/generic/tkMessage.c b/generic/tkMessage.c
@@ -501,7 +501,7 @@ ConfigureMessage(
      * be specified to Tk_ConfigureWidget.
      */
 
-    msgPtr->numChars = Tcl_NumUtfChars(msgPtr->string, TCL_INDEX_NONE);
+    msgPtr->numChars = TkNumUtfChars(msgPtr->string, TCL_INDEX_NONE);
 
     if (msgPtr->highlightWidth < 0) {
 	msgPtr->highlightWidth = 0;
@@ -902,7 +902,7 @@ MessageTextVarProc(
     if (msgPtr->string != NULL) {
 	ckfree(msgPtr->string);
     }
-    msgPtr->numChars = Tcl_NumUtfChars(value, TCL_INDEX_NONE);
+    msgPtr->numChars = TkNumUtfChars(value, TCL_INDEX_NONE);
     msgPtr->string = (char *)ckalloc(strlen(value) + 1);
     strcpy(msgPtr->string, value);
     ComputeMessageGeometry(msgPtr);

diff --git a/generic/tkSelect.c b/generic/tkSelect.c
@@ -1401,15 +1401,15 @@ HandleTclCommand(
 
 	if (cmdInfoPtr->interp != NULL) {
 	    if (length <= maxBytes) {
-		cmdInfoPtr->charOffset += Tcl_NumUtfChars(string, TCL_INDEX_NONE);
+		cmdInfoPtr->charOffset += TkNumUtfChars(string, TCL_INDEX_NONE);
 		cmdInfoPtr->buffer[0] = '\0';
 	    } else {
-		Tcl_UniChar ch = 0;
+		int ch = 0;
 		p = string;
 		string += count;
 		numChars = 0;
 		while (p < string) {
-		    p += Tcl_UtfToUniChar(p, &ch);
+		    p += TkUtfToUniChar(p, &ch);
 		    numChars++;
 		}
 		cmdInfoPtr->charOffset += numChars;

diff --git a/generic/tkText.c b/generic/tkText.c
@@ -1461,7 +1461,7 @@ TextWidgetObjCmd(
 
 		insertLength = 0;
 		for (j = 4; j < objc; j += 2) {
-		    insertLength += Tcl_GetCharLength(objv[j]);
+		    insertLength += TkGetCharLength(objv[j]);
 		}
 
 		/*
@@ -4092,12 +4092,12 @@ TextSearchIndexInLine(
 		if (searchSpecPtr->exact) {
 		    index += leftToScan;
 		} else {
-		    index += Tcl_NumUtfChars(segPtr->body.chars, leftToScan);
+		    index += TkNumUtfChars(segPtr->body.chars, leftToScan);
 		}
 	    } else if (searchSpecPtr->exact) {
 		index += segPtr->size;
 	    } else {
-		index += Tcl_NumUtfChars(segPtr->body.chars, -1);
+		index += TkNumUtfChars(segPtr->body.chars, -1);
 	    }
 	}
 	leftToScan -= segPtr->size;
@@ -4222,7 +4222,7 @@ TextSearchAddNextLine(
 	    Tcl_GetString(theLine);
 	    *lenPtr = theLine->length;
 	} else {
-	    *lenPtr = Tcl_GetCharLength(theLine);
+	    *lenPtr = TkGetCharLength(theLine);
 	}
     }
     return linePtr;
@@ -4292,7 +4292,7 @@ TextSearchFoundMatch(
     if (searchSpecPtr->exact) {
 	const char *startOfLine = Tcl_GetString(theLine);
 
-	numChars = Tcl_NumUtfChars(startOfLine + matchOffset, matchLength);
+	numChars = TkNumUtfChars(startOfLine + matchOffset, matchLength);
     } else {
 	numChars = matchLength;
     }
@@ -4351,13 +4351,13 @@ TextSearchFoundMatch(
 		if (searchSpecPtr->exact) {
 		    matchOffset += segPtr->size;
 		} else {
-		    matchOffset += Tcl_NumUtfChars(segPtr->body.chars, -1);
+		    matchOffset += TkNumUtfChars(segPtr->body.chars, -1);
 		}
 	    } else {
 		if (searchSpecPtr->exact) {
 		    leftToScan -= (int)segPtr->size;
 		} else {
-		    leftToScan -= Tcl_NumUtfChars(segPtr->body.chars, -1);
+		    leftToScan -= TkNumUtfChars(segPtr->body.chars, -1);
 		}
 	    }
 	    curIndex.byteIndex += segPtr->size;
@@ -4442,13 +4442,13 @@ TextSearchFoundMatch(
 	    continue;
 	} else if (!searchSpecPtr->searchElide
 		&& TkTextIsElided(textPtr, &curIndex, NULL)) {
-	    numChars += Tcl_NumUtfChars(segPtr->body.chars, -1);
+	    numChars += TkNumUtfChars(segPtr->body.chars, -1);
 	    continue;
 	}
 	if (searchSpecPtr->exact) {
 	    leftToScan -= segPtr->size;
 	} else {
-	    leftToScan -= Tcl_NumUtfChars(segPtr->body.chars, -1);
+	    leftToScan -= TkNumUtfChars(segPtr->body.chars, -1);
 	}
     }