Emojis and some weird chars are broken

[rdbende]

@Moosems:

It works in normal Text pretty well (although with its own issues)but doesn't in chlorophyll

@rdbende:

Can you provide an example?
In general, Tkinter (Tk) doesn't support most emojis (any character that's beyond U+FFFF, to be precise), so I'm wondering what is what works with the normal text widget but not with Chlorophyll

@Moosems:

Tkinter recently made them not break widgets so example chars are:

😀 😃 😄 😁 😆 😅 😂 🤣 😊 😇 🙂 🙃 😉 😌

Instructions: Paste and then press backspace

@rdbende:

simplescreenrecorder-2022-08-01_21.49.24.mp4

@Moosems:

Very odd... honestly I wish tkinter could support it cause some people use emojis in code and if it doesn't work I'm sure I'm gonna hear about it a lot later on

@rdbende It happens in mantaray too though so it seems less like a pygments thing

Happens in Porcupine too, probably due to some Pygments coloring tag that confuses Tk

[Moosems]

When this is done on a normal Text widget it works fine so there's definitely something going on, I'll have to look into this more and make an MRE.

[Moosems]

Here's what we know: The issue appears in both Chlorophyll and Mantaray however the issues are different. In Mantaray, deleting the emojis crashes the window, in Chlorophyll, it simply makes the chars act really weird. That means that we can not cross-compare them. However, when I started working on making an MRE I first deleted all the highlighting code believing it was not the cause of the issue but found that it did, in fact, alleviate the issue, That means that this is either a misuse of pygments or an issue in pygments. (Using pygments version 2.15.0 and fresh install of Chlorophyll (not from any PR's))

Code without pygments that works fine:

from __future__ import annotations

from chlorophyll import CodeView
from tkinter import Tk, Text, Misc, TclError, BaseWidget, Frame, Event, Scrollbar
from contextlib import suppress
from tkinter.font import Font
from typing import Any
from pyperclip import copy

root = Tk()
cv = CodeView(root)
cv.pack()


class SpecialText(Text):
    _w: str

    def __init__(
        self,
        master: Misc | None = None,
        tab_width: int = 4,
        **kwargs,
    ) -> None:
        self._frame = Frame(master)
        self._frame.grid_rowconfigure(0, weight=1)
        self._frame.grid_columnconfigure(0, weight=1)

        kwargs.setdefault("wrap", "none")
        kwargs.setdefault("font", ("monospace", 11))

        super().__init__(self._frame, **kwargs)
        super().grid(row=0, column=1, sticky="nswe")

        self._hs = Scrollbar(self._frame, orient="horizontal", command=self.xview)
        self._vs = Scrollbar(self._frame, orient="vertical", command=self.yview)

        self._hs.grid(row=1, column=1, sticky="we")
        self._vs.grid(row=0, column=2, sticky="ns")

        super().configure(
            xscrollcommand=self._hs.set,
            yscrollcommand=self._vs.set,
            tabs=Font(font=kwargs["font"]).measure(" " * tab_width)
        )

        contmand = "Command" if self._windowingsystem == "aqua" else "Control"

        super().bind(f"<{contmand}-c>", self._copy, add=True)
        super().bind(f"<{contmand}-v>", self._paste, add=True)
        super().bind(f"<{contmand}-a>", self._select_all, add=True)
        super().bind(f"<{contmand}-Shift-Z>", self.redo, add=True)

        self._orig = f"{self._w}_widget"
        self.tk.call("rename", self._w, self._orig)
        self.tk.createcommand(self._w, self._cmd_proxy)


    def _select_all(self, *_) -> str:
        self.tag_add("sel", "1.0", "end")
        self.mark_set("insert", "end")
        return "break"

    def redo(self, event: Event | None = None) -> None:
        try:
            self.edit_redo()
        except TclError:
            pass

    def _paste(self, *_):
        insert = self.index(f"@0,0 + {self.cget('height') // 2} lines")

        with suppress(TclError):
            self.delete("sel.first", "sel.last")
            self.tag_remove("sel", "1.0", "end")
            self.insert("insert", self.clipboard_get())

        self.see(insert)

        return "break"

    def _copy(self, *_):
        text = self.get("sel.first", "sel.last")
        if not text:
            text = self.get("insert linestart", "insert lineend")

        copy(text)

        return "break"

    def _cmd_proxy(self, command: str, *args) -> Any:
        cmd = (self._orig, command) + args
        try:
            result = self.tk.call(cmd)
        except TclError as e:
            error = str(e)
            if 'tagged with "sel"' in error or "nothing to" in error:
                return ""
            raise e from None

        if command in {"insert", "replace", "delete"}:
            self.event_generate("<<ContentChanged>>")

        return result

    def __setitem__(self, key: str, value) -> None:
        self.configure(**{key: value})

    def __getitem__(self, key: str) -> Any:
        return self.cget(key)

    def configure(self, **kwargs) -> None:
        lexer = kwargs.pop("lexer", None)
        color_scheme = kwargs.pop("color_scheme", None)

        if lexer is not None:
            self._set_lexer(lexer)

        if color_scheme is not None:
            self._set_color_scheme(color_scheme)

        super().configure(**kwargs)

    config = configure

    def pack(self, *args, **kwargs) -> None:
        self._frame.pack(*args, **kwargs)

    def grid(self, *args, **kwargs) -> None:
        self._frame.grid(*args, **kwargs)

    def place(self, *args, **kwargs) -> None:
        self._frame.place(*args, **kwargs)

    def pack_forget(self) -> None:
        self._frame.pack_forget()

    def grid_forget(self) -> None:
        self._frame.grid_forget()

    def place_forget(self) -> None:
        self._frame.place_forget()

    def destroy(self) -> None:
        for widget in self._frame.winfo_children():
            BaseWidget.destroy(widget)
        BaseWidget.destroy(self._frame)

tt = SpecialText(root)
tt.pack()
root.mainloop()

[Moosems]

@rdbende we could check that chard getting tags don't include emojis and if they do include them we can remove the areas with them.

[Moosems]

If the char is an emoji don't highlight.