From bfae211c1cfde4e621dcc239721d6b639ae690bb Mon Sep 17 00:00:00 2001 From: Darius Jahandarie Date: Mon, 19 Feb 2024 21:25:01 +0900 Subject: [PATCH] [WIP] attempt to use OffscreenCanvas --- .eslintrc.json | 4 +- .vscode/settings.json | 2 + ext/css/display.css | 2 + ext/css/structured-content.css | 3 - ext/js/background/backend.js | 19 ++- ext/js/background/offscreen-proxy.js | 30 +++- ext/js/background/offscreen.js | 55 ++++--- ext/js/comm/api.js | 14 +- ext/js/comm/frame-client.js | 14 +- ext/js/comm/frame-endpoint.js | 36 ++++- ext/js/dictionary/dictionary-database.js | 95 ++++++++++-- ext/js/display/display-content-manager.js | 55 +------ ext/js/display/display-generator.js | 6 + ext/js/display/display.js | 139 +++++++++-------- .../display/structured-content-generator.js | 144 ++++++++---------- jsconfig.json | 1 + package-lock.json | 13 ++ package.json | 1 + test/jsconfig.json | 1 + types/ext/api.d.ts | 6 +- types/ext/dictionary-database.d.ts | 15 +- types/ext/display-content-manager.d.ts | 6 +- types/ext/offscreen.d.ts | 6 +- 23 files changed, 397 insertions(+), 270 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index faee1c51d0..9968b096bb 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -86,7 +86,7 @@ "no-shadow": ["error", {"builtinGlobals": false}], "no-shadow-restricted-names": "error", "no-template-curly-in-string": "error", - "no-undef": "error", + "no-undef": "off", "no-undefined": "error", "no-underscore-dangle": ["error", {"allowAfterThis": true, "allowAfterSuper": false, "allowAfterThisConstructor": false}], "no-unexpected-multiline": "error", @@ -581,7 +581,6 @@ "ext/js/core/to-error.js", "ext/js/core/utilities.js", "ext/js/data/database.js", - "ext/js/dictionary/dictionary-database.js", "ext/js/dictionary/dictionary-importer.js", "ext/js/dictionary/dictionary-worker-handler.js", "ext/js/dictionary/dictionary-worker-main.js", @@ -623,7 +622,6 @@ "ext/js/data/json-schema.js", "ext/js/data/options-util.js", "ext/js/data/permissions-util.js", - "ext/js/dictionary/dictionary-database.js", "ext/js/dom/native-simple-dom-parser.js", "ext/js/dom/simple-dom-parser.js", "ext/js/extension/environment.js", diff --git a/.vscode/settings.json b/.vscode/settings.json index e1bf61a8c3..b76078bf78 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,6 +17,8 @@ "eslint.format.enable": true, "javascript.format.insertSpaceAfterOpeningAndBeforeClosingNonemptyBraces": false, "typescript.format.insertSpaceAfterOpeningAndBeforeClosingNonemptyBraces": false, + "javascript.format.enable": false, + "typescript.format.enable": false, "javascript.preferences.importModuleSpecifierEnding": "js", "editor.tabSize": 4, "editor.insertSpaces": true, diff --git a/ext/css/display.css b/ext/css/display.css index 631c4811dd..c16c21f167 100644 --- a/ext/css/display.css +++ b/ext/css/display.css @@ -827,6 +827,8 @@ button.action-button:active { .entry { padding: var(--entry-vertical-padding) var(--entry-horizontal-padding); position: relative; + content-visibility: auto; + contain-intrinsic-height: auto 500px; } .entry+.entry { border-top: var(--thin-border-size) solid var(--light-border-color); diff --git a/ext/css/structured-content.css b/ext/css/structured-content.css index f3ef41984e..db15d4980d 100644 --- a/ext/css/structured-content.css +++ b/ext/css/structured-content.css @@ -75,9 +75,6 @@ outline: none; width: 100%; } -.gloss-image:not([src]) { - display: none; -} .gloss-image-link[data-image-rendering=pixelated] .gloss-image { image-rendering: auto; image-rendering: -moz-crisp-edges; diff --git a/ext/js/background/backend.js b/ext/js/background/backend.js index 2fb00708aa..c8cb55908c 100644 --- a/ext/js/background/backend.js +++ b/ext/js/background/backend.js @@ -72,7 +72,6 @@ export class Backend { this._translator = new Translator(this._dictionaryDatabase); /** @type {ClipboardReader|ClipboardReaderProxy} */ this._clipboardReader = new ClipboardReader( - // eslint-disable-next-line no-undef (typeof document === 'object' && document !== null ? document : null), '#clipboard-paste-target', '#clipboard-rich-content-paste-target', @@ -172,7 +171,7 @@ export class Backend { ['getDictionaryInfo', this._onApiGetDictionaryInfo.bind(this)], ['purgeDatabase', this._onApiPurgeDatabase.bind(this)], ['getMedia', this._onApiGetMedia.bind(this)], - ['getMediaObjects', this._onApiGetMediaObjects.bind(this)], + ['drawMedia', this._onApiDrawMedia.bind(this)], ['logGenericErrorBackend', this._onApiLogGenericErrorBackend.bind(this)], ['logIndicatorClear', this._onApiLogIndicatorClear.bind(this)], ['modifySettings', this._onApiModifySettings.bind(this)], @@ -242,6 +241,15 @@ export class Backend { const onMessage = this._onMessageWrapper.bind(this); chrome.runtime.onMessage.addListener(onMessage); + // This is for receiving messages sent with navigator.serviceWorker, which has the benefit of being able to transfer objects, but doesn't accept callbacks + addEventListener('message', (event) => { + if (event.data.action === 'drawMedia') { + this._dictionaryDatabase.drawMedia(event.data.params); + } else if (event.data.action === 'registerOffscreenPort' && this._offscreen !== null) { + this._offscreen.registerOffscreenPort(event.ports[0]); + } + }); + if (this._canObservePermissionsChanges()) { const onPermissionsChanged = this._onWebExtensionEventWrapper(this._onPermissionsChanged.bind(this)); chrome.permissions.onAdded.addListener(onPermissionsChanged); @@ -796,9 +804,10 @@ export class Backend { return await this._getNormalizedDictionaryDatabaseMedia(targets); } - /** @type {import('api').ApiHandler<'getMediaObjects'>} */ - async _onApiGetMediaObjects({targets}) { - return await this._dictionaryDatabase.getMediaObjects(targets); + /** @type {import('api').ApiHandler<'drawMedia'>} */ + async _onApiDrawMedia({targets}) { + console.log('_onApiDrawMedia', targets); + await this._dictionaryDatabase.drawMedia(targets); } /** @type {import('api').ApiHandler<'logGenericErrorBackend'>} */ diff --git a/ext/js/background/offscreen-proxy.js b/ext/js/background/offscreen-proxy.js index 2f888fb598..082389296e 100644 --- a/ext/js/background/offscreen-proxy.js +++ b/ext/js/background/offscreen-proxy.js @@ -58,6 +58,9 @@ export class OffscreenProxy { this._webExtension = webExtension; /** @type {?Promise} */ this._creatingOffscreen = null; + + /** @type {?MessagePort} */ + this._currentOffscreenPort = null; } /** @@ -136,6 +139,24 @@ export class OffscreenProxy { } return response.result; } + + /** + * @param {MessagePort} port + */ + async registerOffscreenPort(port) { + if (this._currentOffscreenPort) { + this._currentOffscreenPort.close(); + } + this._currentOffscreenPort = port; + } + + /** + * @param {any} message + * @param {Transferable[]} transfers + */ + sendMessageViaPort(message, transfers) { + this._currentOffscreenPort?.postMessage(message, transfers); + } } export class DictionaryDatabaseProxy { @@ -178,12 +199,11 @@ export class DictionaryDatabaseProxy { } /** - * @param {import('dictionary-database').MediaRequest[]} targets - * @returns {Promise} + * @param {import('dictionary-database').DrawMediaRequest[]} targets + * @returns {Promise} */ - async getMediaObjects(targets) { - console.log('offscreen getMediaObjects', targets); - return await this._offscreen.sendMessagePromise({action: 'databaseGetMediaObjectsOffscreen', params: {targets}}); + async drawMedia(targets) { + this._offscreen.sendMessageViaPort({action: 'drawMedia', params: targets}, targets.map((t) => t.canvas)); } } diff --git a/ext/js/background/offscreen.js b/ext/js/background/offscreen.js index a732d5a89d..05b75e26cf 100644 --- a/ext/js/background/offscreen.js +++ b/ext/js/background/offscreen.js @@ -46,19 +46,19 @@ export class Offscreen { /* eslint-disable @stylistic/no-multi-spaces */ /** @type {import('offscreen').ApiMap} */ this._apiMap = createApiMap([ - ['clipboardGetTextOffscreen', this._getTextHandler.bind(this)], - ['clipboardGetImageOffscreen', this._getImageHandler.bind(this)], + ['clipboardGetTextOffscreen', this._getTextHandler.bind(this)], + ['clipboardGetImageOffscreen', this._getImageHandler.bind(this)], ['clipboardSetBrowserOffscreen', this._setClipboardBrowser.bind(this)], - ['databasePrepareOffscreen', this._prepareDatabaseHandler.bind(this)], - ['getDictionaryInfoOffscreen', this._getDictionaryInfoHandler.bind(this)], - ['databasePurgeOffscreen', this._purgeDatabaseHandler.bind(this)], - ['databaseGetMediaOffscreen', this._getMediaHandler.bind(this)], - ['databaseGetMediaObjectsOffscreen', this._getMediaObjectsHandler.bind(this)], - ['translatorPrepareOffscreen', this._prepareTranslatorHandler.bind(this)], - ['findKanjiOffscreen', this._findKanjiHandler.bind(this)], - ['findTermsOffscreen', this._findTermsHandler.bind(this)], - ['getTermFrequenciesOffscreen', this._getTermFrequenciesHandler.bind(this)], - ['clearDatabaseCachesOffscreen', this._clearDatabaseCachesHandler.bind(this)], + ['databasePrepareOffscreen', this._prepareDatabaseHandler.bind(this)], + ['getDictionaryInfoOffscreen', this._getDictionaryInfoHandler.bind(this)], + ['databasePurgeOffscreen', this._purgeDatabaseHandler.bind(this)], + ['databaseGetMediaOffscreen', this._getMediaHandler.bind(this)], + ['databaseDrawMediaOffscreen', this._drawMediaHandler.bind(this)], + ['translatorPrepareOffscreen', this._prepareTranslatorHandler.bind(this)], + ['findKanjiOffscreen', this._findKanjiHandler.bind(this)], + ['findTermsOffscreen', this._findTermsHandler.bind(this)], + ['getTermFrequenciesOffscreen', this._getTermFrequenciesHandler.bind(this)], + ['clearDatabaseCachesOffscreen', this._clearDatabaseCachesHandler.bind(this)] ]); /* eslint-enable @stylistic/no-multi-spaces */ @@ -69,6 +69,18 @@ export class Offscreen { /** */ prepare() { chrome.runtime.onMessage.addListener(this._onMessage.bind(this)); + + const registerPort = () => { + const mc = new MessageChannel(); + mc.port1.onmessage = (e) => { + this._onSWMessage(e.data); + }; + void navigator.serviceWorker.ready.then((swr) => { + swr.active?.postMessage({action: 'registerOffscreenPort'}, [mc.port2]); + }); + }; + navigator.serviceWorker.addEventListener("controllerchange", registerPort); + registerPort(); } /** @type {import('offscreen').ApiHandler<'clipboardGetTextOffscreen'>} */ @@ -111,9 +123,9 @@ export class Offscreen { return media.map((m) => ({...m, content: arrayBufferToBase64(m.content)})); } - /** @type {import('offscreen').ApiHandler<'databaseGetMediaObjectsOffscreen'>} */ - async _getMediaObjectsHandler({targets}) { - return await this._dictionaryDatabase.getMediaObjects(targets); + /** @type {import('offscreen').ApiHandler<'databaseDrawMediaOffscreen'>} */ + async _drawMediaHandler({targets}) { + await this._dictionaryDatabase.drawMedia(targets); } /** @type {import('offscreen').ApiHandler<'translatorPrepareOffscreen'>} */ @@ -136,11 +148,11 @@ export class Offscreen { const enabledDictionaryMap = new Map(options.enabledDictionaryMap); const excludeDictionaryDefinitions = ( options.excludeDictionaryDefinitions !== null ? - new Set(options.excludeDictionaryDefinitions) : - null + new Set(options.excludeDictionaryDefinitions) : + null ); const textReplacements = options.textReplacements.map((group) => { - if (group === null) { return null; } + if (group === null) {return null;} return group.map((opt) => { // https://stackoverflow.com/a/33642463 const match = opt.pattern.match(/\/(.*?)\/([a-z]*)?$/i); @@ -172,4 +184,11 @@ export class Offscreen { _onMessage({action, params}, _sender, callback) { return invokeApiMapHandler(this._apiMap, action, params, [], callback); } + + /** @param {{action: string, params: any}} obj */ + _onSWMessage({action, params}) { + if (action === 'drawMedia') { + this._dictionaryDatabase.drawMedia(params); + } + } } diff --git a/ext/js/comm/api.js b/ext/js/comm/api.js index c9e84903e6..566174a708 100644 --- a/ext/js/comm/api.js +++ b/ext/js/comm/api.js @@ -255,12 +255,12 @@ export class API { } /** - * @param {import('api').ApiParam<'getMediaObjects', 'targets'>} targets - * @returns {Promise>} + * @param {import('api').ApiParam<'drawMedia', 'targets'>} targets + * @returns {Promise>} */ - getMediaObjects(targets) { - console.log('getMediaObjects', targets); - return this._invoke('getMediaObjects', {targets}); + drawMedia(targets) { + console.log('drawMedia', targets); + return this._invoke('drawMedia', {targets}); } /** @@ -399,10 +399,10 @@ export class API { if (response !== null && typeof response === 'object') { const {error} = /** @type {import('core').UnknownObject} */ (response); if (typeof error !== 'undefined') { - reject(ExtensionError.deserialize(/** @type {import('core').SerializedError} */ (error))); + reject(ExtensionError.deserialize(/** @type {import('core').SerializedError} */(error))); } else { const {result} = /** @type {import('core').UnknownObject} */ (response); - resolve(/** @type {import('api').ApiReturn} */ (result)); + resolve(/** @type {import('api').ApiReturn} */(result)); } } else { const message = response === null ? 'Unexpected null response. You may need to refresh the page.' : `Unexpected response of type ${typeof response}. You may need to refresh the page.`; diff --git a/ext/js/comm/frame-client.js b/ext/js/comm/frame-client.js index 6a49fe209e..6eb1e348e5 100644 --- a/ext/js/comm/frame-client.js +++ b/ext/js/comm/frame-client.js @@ -85,6 +85,8 @@ export class FrameClient { return new Promise((resolve, reject) => { /** @type {Map} */ const tokenMap = new Map(); + /** @type {MessagePort | null} */ + let messagePort = null; /** @type {?import('core').Timeout} */ let timer = null; const deferPromiseDetails = /** @type {import('core').DeferredPromiseDetails} */ (deferPromise()); @@ -95,9 +97,10 @@ export class FrameClient { /** * @param {string} action * @param {import('core').SerializableObject} params + * @param {Transferable[]} transfers * @throws {Error} */ - const postMessage = (action, params) => { + const postMessage = (action, params, transfers) => { const contentWindow = frame.contentWindow; if (contentWindow === null) { throw new Error('Frame missing content window'); } @@ -109,7 +112,7 @@ export class FrameClient { } if (!validOrigin) { throw new Error('Unexpected frame origin'); } - contentWindow.postMessage({action, params}, targetOrigin); + contentWindow.postMessage({action, params}, targetOrigin, transfers); }; /** @type {import('extension').ChromeRuntimeOnMessageCallback} */ @@ -132,10 +135,12 @@ export class FrameClient { switch (action) { case 'frameEndpointReady': { + const mc = new MessageChannel(); const {secret} = params; const token = generateId(16); tokenMap.set(secret, token); - postMessage('frameEndpointConnect', {secret, token, hostFrameId}); + messagePort = mc.port1; + postMessage('frameEndpointConnect', {secret, token, hostFrameId}, [mc.port2]); } break; case 'frameEndpointConnected': @@ -176,6 +181,9 @@ export class FrameClient { if (timer === null) { return; } // Done clearTimeout(timer); timer = null; + if (messagePort !== null) { + messagePort.close(); + } frameLoadedResolve = null; if (frameLoadedReject !== null) { diff --git a/ext/js/comm/frame-endpoint.js b/ext/js/comm/frame-endpoint.js index 69d28a1e7a..373ea5ea4c 100644 --- a/ext/js/comm/frame-endpoint.js +++ b/ext/js/comm/frame-endpoint.js @@ -17,6 +17,7 @@ */ import {EventListenerCollection} from '../core/event-listener-collection.js'; +import {log} from '../core/log.js'; import {generateId} from '../core/utilities.js'; export class FrameEndpoint { @@ -68,21 +69,44 @@ export class FrameEndpoint { _onMessage(event) { if (this._token !== null) { return; } // Already initialized - const {data} = event; - if (typeof data !== 'object' || data === null) { return; } // Invalid message + const {data, ports} = event; + if (typeof data !== 'object' || data === null) { + log.error('Invalid message'); + return; + } const {action} = /** @type {import('core').SerializableObject} */ (data); - if (action !== 'frameEndpointConnect') { return; } // Invalid message + if (action !== 'frameEndpointConnect') { + log.error('Invalid action'); + return; + } const {params} = /** @type {import('core').SerializableObject} */ (data); - if (typeof params !== 'object' || params === null) { return; } // Invalid data + if (typeof params !== 'object' || params === null) { + log.error('Invalid data'); + return; + } const {secret} = /** @type {import('core').SerializableObject} */ (params); - if (secret !== this._secret) { return; } // Invalid authentication + if (secret !== this._secret) { + log.error('Invalid authentication'); + return; + } const {token, hostFrameId} = /** @type {import('core').SerializableObject} */ (params); - if (typeof token !== 'string' || typeof hostFrameId !== 'number') { return; } // Invalid target + if (typeof token !== 'string' || typeof hostFrameId !== 'number') { + log.error('Invalid target'); + return; + } + + if (typeof ports !== 'object' || ports.length !== 1) { + log.error('Invalid transfer'); + return; + } + void navigator.serviceWorker.ready.then((swr) => { + swr.active?.postMessage('mcp', [ports[0]]); + }); this._token = token; this._eventListeners.removeAllEventListeners(); diff --git a/ext/js/dictionary/dictionary-database.js b/ext/js/dictionary/dictionary-database.js index 039e72ec74..5854c6e3be 100644 --- a/ext/js/dictionary/dictionary-database.js +++ b/ext/js/dictionary/dictionary-database.js @@ -34,6 +34,8 @@ export class DictionaryDatabase { this._createOnlyQuery3 = (item) => IDBKeyRange.only(item.term); /** @type {import('dictionary-database').CreateQuery} */ this._createOnlyQuery4 = (item) => IDBKeyRange.only(item.path); + /** @type {import('dictionary-database').CreateQuery} */ + this._createOnlyQuery5 = (item) => IDBKeyRange.only(item.path); /** @type {import('dictionary-database').CreateQuery} */ this._createBoundQuery1 = (item) => IDBKeyRange.bound(item, `${item}\uffff`, false, false); /** @type {import('dictionary-database').CreateQuery} */ @@ -53,6 +55,8 @@ export class DictionaryDatabase { this._createKanjiMetaBind = this._createKanjiMeta.bind(this); /** @type {import('dictionary-database').CreateResult} */ this._createMediaBind = this._createMedia.bind(this); + /** @type {import('dictionary-database').CreateResult} */ + this._createDrawMediaBind = this._createDrawMedia.bind(this); } /** */ @@ -347,18 +351,65 @@ export class DictionaryDatabase { } /** - * This requires the ability to call URL.createObjectURL, which is not available in a service worker. Therefore Chrome must go through the Offscreen API as opposed to directly calling this. - * @param {import('dictionary-database').MediaRequest[]} items - * @returns {Promise} + * @param {import('dictionary-database').DrawMediaRequest[]} items */ - async getMediaObjects(items) { + async drawMedia(items) { + performance.mark('drawMedia:start'); + + // merge items with the same path to reduce the number of database queries. collects the canvases into a single array for each path. + /** @type {Map} */ + const groupedItems = new Map(); + for (const item of items) { + const {path, dictionary, canvas} = item; + const key = `${path}:::${dictionary}`; + if (!groupedItems.has(key)) { + groupedItems.set(key, {path, dictionary, canvases: []}); + } + groupedItems.get(key)?.canvases.push(canvas); + } + const groupedItemsArray = [...groupedItems.values()]; + console.log(groupedItemsArray); + /** @type {import('dictionary-database').FindPredicate} */ const predicate = (row, item) => (row.dictionary === item.dictionary); - return (await this._findMultiBulk('media', ['path'], items, this._createOnlyQuery4, predicate, this._createMediaBind)).map((m) => { - const blob = new Blob([m.content], {type: m.mediaType}); - const url = URL.createObjectURL(blob); - return {...m, content: null, url}; - }); + const drawPromises = []; + // performance.mark('drawMedia:findMultiBulk:start'); + const results = await this._findMultiBulk('media', ['path'], groupedItemsArray, this._createOnlyQuery5, predicate, this._createDrawMediaBind); + // performance.mark('drawMedia:findMultiBulk:end'); + // performance.measure('drawMedia:findMultiBulk', 'drawMedia:findMultiBulk:start', 'drawMedia:findMultiBulk:end'); + + performance.mark('drawMedia:decode:start'); + for (const m of results) { + if (m.mediaType === 'image/svg+xml') { + const blob = new Blob([m.content], {type: m.mediaType}); + const url = URL.createObjectURL(blob); + const image = new Image(m.width, m.height); + image.src = url; + drawPromises.push(image.decode().then(() => { + URL.revokeObjectURL(url); + return {canvases: m.canvases, image}; + })); + } else { + const imageDecoder = new ImageDecoder({type: m.mediaType, data: m.content}); + drawPromises.push(imageDecoder.decode().then((decodedImage) => { + return {canvases: m.canvases, image: decodedImage.image}; + })); + } + } + performance.mark('drawMedia:decode:end'); + performance.measure('drawMedia:decode', 'drawMedia:decode:start', 'drawMedia:decode:end'); + + performance.mark('drawMedia:draw:start'); + for (const {canvases, image} of await Promise.all(drawPromises)) { + for (const c of canvases) { + c.getContext('2d')?.drawImage(image, 0, 0, c.width, c.height); + } + } + performance.mark('drawMedia:draw:end'); + performance.measure('drawMedia:draw', 'drawMedia:draw:start', 'drawMedia:draw:end'); + + performance.mark('drawMedia:end'); + // performance.measure('drawMedia', 'drawMedia:start', 'drawMedia:end'); } /** @@ -491,8 +542,13 @@ export class DictionaryDatabase { /** * @param {TRow[]} rows * @param {import('dictionary-database').FindMultiBulkData} data + * @param item */ - const onGetAll = (rows, data) => { + const onGetAll = (item) => (rows, data) => { + if (typeof item === 'object' && 'path' in item) { + performance.mark(`findMultiBulk:onGetAll:${item.path}:end`); + performance.measure(`findMultiBulk:onGetAll:${item.path}`, `findMultiBulk:onGetAll:${item.path}:start`, `findMultiBulk:onGetAll:${item.path}:end`); + } for (const row of rows) { if (predicate(row, data.item)) { results.push(createResult(row, data)); @@ -502,15 +558,22 @@ export class DictionaryDatabase { resolve(results); } }; + performance.mark('findMultiBulk:getAll:start'); + // console.log('?'); for (let i = 0; i < itemCount; ++i) { const item = items[i]; const query = createQuery(item); for (let j = 0; j < indexCount; ++j) { /** @type {import('dictionary-database').FindMultiBulkData} */ const data = {item, itemIndex: i, indexIndex: j}; - this._db.getAll(indexList[j], query, onGetAll, reject, data); + if (typeof item === 'object' && 'path' in item) { + performance.mark(`findMultiBulk:onGetAll:${item.path}:start`); + } + this._db.getAll(indexList[j], query, onGetAll(item), reject, data); } } + performance.mark('findMultiBulk:getAll:end'); + performance.measure('findMultiBulk:getAll', 'findMultiBulk:getAll:start', 'findMultiBulk:getAll:end'); }); } @@ -672,6 +735,16 @@ export class DictionaryDatabase { return {index, dictionary, path, mediaType, width, height, content}; } + /** + * @param {import('dictionary-database').MediaDataArrayBufferContent} row + * @param {import('dictionary-database').FindMultiBulkData} data + * @returns {import('dictionary-database').DrawMedia} + */ + _createDrawMedia(row, {itemIndex: index, item: {canvases}}) { + const {dictionary, path, mediaType, width, height, content} = row; + return {index, dictionary, path, mediaType, width, height, content, canvases: canvases}; + } + /** * @param {unknown} field * @returns {string[]} diff --git a/ext/js/display/display-content-manager.js b/ext/js/display/display-content-manager.js index e491b8e05d..11cadfa200 100644 --- a/ext/js/display/display-content-manager.js +++ b/ext/js/display/display-content-manager.js @@ -31,8 +31,6 @@ export class DisplayContentManager { this._display = display; /** @type {import('core').TokenObject} */ this._token = {}; - /** @type {Map>} */ - this._mediaCache = new Map(); /** @type {EventListenerCollection} */ this._eventListeners = new EventListenerCollection(); /** @type {import('display-content-manager').LoadMediaRequest[]} */ @@ -48,22 +46,16 @@ export class DisplayContentManager { * Queues loading media file from a given dictionary. * @param {string} path * @param {string} dictionary - * @param {import('display-content-manager').OnLoadCallback} onLoad - * @param {import('display-content-manager').OnUnloadCallback} onUnload + * @param {OffscreenCanvas} canvas */ - loadMedia(path, dictionary, onLoad, onUnload) { - this._loadMediaRequests.push({path, dictionary, onLoad, onUnload}); + loadMedia(path, dictionary, canvas) { + this._loadMediaRequests.push({path, dictionary, canvas}); } /** * Unloads all media that has been loaded. */ unloadAll() { - for (const mediaObject of this._mediaCache.values()) { - URL.revokeObjectURL(mediaObject.url); - } - this._mediaCache.clear(); - this._token = {}; this._eventListeners.removeAllEventListeners(); @@ -90,42 +82,11 @@ export class DisplayContentManager { * Execute media requests */ async executeMediaRequests() { - /** @type {Map} */ - const uncachedRequests = new Map(); - for (const request of this._loadMediaRequests) { - const cacheKey = this._cacheKey(request.path, request.dictionary); - const mediaObject = this._mediaCache.get(cacheKey); - if (typeof mediaObject !== 'undefined' && mediaObject !== null) { - await request.onLoad(mediaObject.url); - } else { - const cache = uncachedRequests.get(cacheKey); - if (typeof cache === 'undefined') { - uncachedRequests.set(cacheKey, [request]); - } else { - cache.push(request); - } - } - } - - performance.mark('display-content-manager:executeMediaRequests:getMediaObjects:start'); - const mediaObjects = await this._display.application.api.getMediaObjects([...uncachedRequests.values()].map((r) => ({path: r[0].path, dictionary: r[0].dictionary}))); - performance.mark('display-content-manager:executeMediaRequests:getMediaObjects:end'); - performance.measure('display-content-manager:executeMediaRequests:getMediaObjects', 'display-content-manager:executeMediaRequests:getMediaObjects:start', 'display-content-manager:executeMediaRequests:getMediaObjects:end'); - const promises = []; - for (const mediaObject of mediaObjects) { - const cacheKey = this._cacheKey(mediaObject.path, mediaObject.dictionary); - this._mediaCache.set(cacheKey, mediaObject); - const requests = uncachedRequests.get(cacheKey); - if (typeof requests !== 'undefined') { - for (const request of requests) { - promises.push(request.onLoad(mediaObject.url)); - } - } - } - performance.mark('display-content-manager:executeMediaRequests:runCallbacks:start'); - await Promise.allSettled(promises); - performance.mark('display-content-manager:executeMediaRequests:runCallbacks:end'); - performance.measure('display-content-manager:executeMediaRequests:runCallbacks', 'display-content-manager:executeMediaRequests:runCallbacks:start', 'display-content-manager:executeMediaRequests:runCallbacks:end'); + await navigator.serviceWorker.ready.then((swr) => { + const canvases = this._loadMediaRequests.map(({canvas}) => /** @type {Transferable} */ (canvas)); + swr.active?.postMessage({action: 'drawMedia', params: this._loadMediaRequests}, canvases); + }); + // await this._display.application.api.drawMedia(this._loadMediaRequests); this._loadMediaRequests = []; } diff --git a/ext/js/display/display-generator.js b/ext/js/display/display-generator.js index fb28410cfb..5e30bf3516 100644 --- a/ext/js/display/display-generator.js +++ b/ext/js/display/display-generator.js @@ -117,19 +117,25 @@ export class DisplayGenerator { node.dataset.groupedFrequencyCount = `${groupedFrequencies.length}`; node.dataset.primaryMatchTypes = [...primaryMatchTypes].join(' '); + performance.mark('displayGenerator:createTermEntry:createTermHeadword:start'); for (let i = 0, ii = headwords.length; i < ii; ++i) { const node2 = this._createTermHeadword(headwords[i], i, pronunciations); node2.dataset.index = `${i}`; headwordsContainer.appendChild(node2); } headwordsContainer.dataset.count = `${headwords.length}`; + performance.mark('displayGenerator:createTermEntry:createTermHeadword:end'); + performance.measure('displayGenerator:createTermEntry:createTermHeadword', 'displayGenerator:createTermEntry:createTermHeadword:start', 'displayGenerator:createTermEntry:createTermHeadword:end'); + performance.mark('displayGenerator:createTermEntry:promises:start'); await Promise.all([ this._appendMultiple(inflectionRuleChainsContainer, this._createInflectionRuleChain.bind(this), inflectionRuleChainCandidates), this._appendMultiple(frequencyGroupListContainer, this._createFrequencyGroup.bind(this), groupedFrequencies, false), this._appendMultiple(groupedPronunciationsContainer, this._createGroupedPronunciation.bind(this), groupedPronunciations), this._appendMultiple(headwordTagsContainer, this._createTermTag.bind(this), termTags, headwords.length) ]); + performance.mark('displayGenerator:createTermEntry:promises:end'); + performance.measure('displayGenerator:createTermEntry:promises', 'displayGenerator:createTermEntry:promises:start', 'displayGenerator:createTermEntry:promises:end'); for (const term of uniqueTerms) { headwordTagsContainer.appendChild(await this._createSearchTag(term)); diff --git a/ext/js/display/display.js b/ext/js/display/display.js index 81eccabe5f..ba6fe2f5c2 100644 --- a/ext/js/display/display.js +++ b/ext/js/display/display.js @@ -197,20 +197,20 @@ export class Display extends EventDispatcher { /** @type {import('language').LanguageSummary[]} */ this._languageSummaries = []; - /* eslint-disable @stylistic/no-multi-spaces */ + this._hotkeyHandler.registerActions([ - ['close', () => {this._onHotkeyClose();}], + ['close', () => { this._onHotkeyClose(); }], ['nextEntry', this._onHotkeyActionMoveRelative.bind(this, 1)], ['previousEntry', this._onHotkeyActionMoveRelative.bind(this, -1)], - ['lastEntry', () => {this._focusEntry(this._dictionaryEntries.length - 1, 0, true);}], - ['firstEntry', () => {this._focusEntry(0, 0, true);}], - ['historyBackward', () => {this._sourceTermView();}], - ['historyForward', () => {this._nextTermView();}], - ['profilePrevious', async () => {await setProfile(-1, this._application);}], - ['profileNext', async () => {await setProfile(1, this._application);}], + ['lastEntry', () => { this._focusEntry(this._dictionaryEntries.length - 1, 0, true); }], + ['firstEntry', () => { this._focusEntry(0, 0, true); }], + ['historyBackward', () => { this._sourceTermView(); }], + ['historyForward', () => { this._nextTermView(); }], + ['profilePrevious', async () => { await setProfile(-1, this._application); }], + ['profileNext', async () => { await setProfile(1, this._application); }], ['copyHostSelection', () => this._copyHostSelection()], - ['nextEntryDifferentDictionary', () => {this._focusEntryWithDifferentDictionary(1, true);}], - ['previousEntryDifferentDictionary', () => {this._focusEntryWithDifferentDictionary(-1, true);}], + ['nextEntryDifferentDictionary', () => { this._focusEntryWithDifferentDictionary(1, true); }], + ['previousEntryDifferentDictionary', () => { this._focusEntryWithDifferentDictionary(-1, true); }], ]); this.registerDirectMessageHandlers([ ['displaySetOptionsContext', this._onMessageSetOptionsContext.bind(this)], @@ -223,7 +223,6 @@ export class Display extends EventDispatcher { this.registerWindowMessageHandlers([ ['displayExtensionUnloaded', this._onMessageExtensionUnloaded.bind(this)], ]); - /* eslint-enable @stylistic/no-multi-spaces */ } /** @type {import('../application.js').Application} */ @@ -392,7 +391,7 @@ export class Display extends EventDispatcher { * @param {Error} error */ onError(error) { - if (this._application.webExtension.unloaded) {return;} + if (this._application.webExtension.unloaded) { return; } log.error(error); } @@ -408,7 +407,7 @@ export class Display extends EventDispatcher { * @throws {Error} */ getLanguageSummary() { - if (this._options === null) {throw new Error('Options is null');} + if (this._options === null) { throw new Error('Options is null'); } const language = this._options.general.language; return /** @type {import('language').LanguageSummary} */ (this._languageSummaries.find(({iso}) => iso === language)); } @@ -489,7 +488,7 @@ export class Display extends EventDispatcher { const urlSearchParams = new URLSearchParams(); for (const [key, value] of Object.entries(params)) { - if (typeof value !== 'string') {continue;} + if (typeof value !== 'string') { continue; } urlSearchParams.append(key, value); } const url = `${location.protocol}//${location.host}${location.pathname}?${urlSearchParams.toString()}`; @@ -518,7 +517,7 @@ export class Display extends EventDispatcher { */ setCustomCss(css) { if (this._styleNode === null) { - if (css.length === 0) {return;} + if (css.length === 0) { return; } this._styleNode = document.createElement('style'); } @@ -581,7 +580,7 @@ export class Display extends EventDispatcher { */ searchLast(updateOptionsContext) { const type = this._contentType; - if (type === 'clear') {return;} + if (type === 'clear') { return; } const query = this._query; const {state} = this._history; const hasState = typeof state === 'object' && state !== null; @@ -649,9 +648,9 @@ export class Display extends EventDispatcher { */ getElementDictionaryEntryIndex(element) { const node = /** @type {?HTMLElement} */ (element.closest('.entry')); - if (node === null) {return -1;} + if (node === null) { return -1; } const {index} = node.dataset; - if (typeof index !== 'string') {return -1;} + if (typeof index !== 'string') { return -1; } const indexNumber = Number.parseInt(index, 10); return Number.isFinite(indexNumber) ? indexNumber : -1; } @@ -716,7 +715,7 @@ export class Display extends EventDispatcher { try { const {action, params} = data2; - const callback = () => { }; // NOP + const callback = () => {}; // NOP invokeApiMapHandler(this._windowApiMap, action, params, [], callback); } catch (e) { // NOP @@ -782,7 +781,7 @@ export class Display extends EventDispatcher { /** */ async _onStateChanged() { - if (this._historyChangeIgnore) {return;} + if (this._historyChangeIgnore) { return; } performance.mark('display:onStateChanged:start'); @@ -808,7 +807,7 @@ export class Display extends EventDispatcher { performance.mark('display:prepare:start'); const urlSearchParams = new URLSearchParams(location.search); let type = urlSearchParams.get('type'); - if (type === null && urlSearchParams.get('query') !== null) {type = 'terms';} + if (type === null && urlSearchParams.get('query') !== null) { type = 'terms'; } const fullVisible = urlSearchParams.get('full-visible'); this._queryParserVisibleOverride = (fullVisible === null ? null : (fullVisible !== 'false')); @@ -851,8 +850,8 @@ export class Display extends EventDispatcher { const historyState = this._history.state; const historyMode = ( eventType === 'click' || - !(typeof historyState === 'object' && historyState !== null) || - historyState.cause !== 'queryParser' ? + !(typeof historyState === 'object' && historyState !== null) || + historyState.cause !== 'queryParser' ? 'new' : 'overwrite' ); @@ -877,7 +876,7 @@ export class Display extends EventDispatcher { /** */ _onExtensionUnloaded() { const type = 'unloaded'; - if (this._contentType === type) {return;} + if (this._contentType === type) { return; } const {tabId, frameId} = this._application; /** @type {import('display').ContentDetails} */ const details = { @@ -945,15 +944,15 @@ export class Display extends EventDispatcher { try { e.preventDefault(); const {state} = this._history; - if (!(typeof state === 'object' && state !== null)) {return;} + if (!(typeof state === 'object' && state !== null)) { return; } let {sentence, url, documentTitle} = state; - if (typeof url !== 'string') {url = window.location.href;} - if (typeof documentTitle !== 'string') {documentTitle = document.title;} + if (typeof url !== 'string') { url = window.location.href; } + if (typeof documentTitle !== 'string') { documentTitle = document.title; } const optionsContext = this.getOptionsContext(); const element = /** @type {Element} */ (e.currentTarget); let query = element.textContent; - if (query === null) {query = '';} + if (query === null) { query = ''; } const dictionaryEntries = await this._application.api.kanjiFind(query, optionsContext); /** @type {import('display').ContentDetails} */ const details = { @@ -996,7 +995,7 @@ export class Display extends EventDispatcher { * @param {WheelEvent} e */ _onHistoryWheel(e) { - if (e.altKey) {return;} + if (e.altKey) { return; } const delta = -e.deltaX || e.deltaY; if (delta > 0) { this._sourceTermView(); @@ -1060,12 +1059,12 @@ export class Display extends EventDispatcher { * @param {MouseEvent} e */ _onEntryClick(e) { - if (e.button !== 0) {return;} + if (e.button !== 0) { return; } const node = /** @type {HTMLElement} */ (e.currentTarget); const {index} = node.dataset; - if (typeof index !== 'string') {return;} + if (typeof index !== 'string') { return; } const indexNumber = Number.parseInt(index, 10); - if (!Number.isFinite(indexNumber)) {return;} + if (!Number.isFinite(indexNumber)) { return; } this._entrySetCurrent(indexNumber); } @@ -1133,7 +1132,7 @@ export class Display extends EventDispatcher { */ _showTagNotification(tagNode) { const parent = tagNode.parentNode; - if (parent === null || !(parent instanceof HTMLElement)) {return;} + if (parent === null || !(parent instanceof HTMLElement)) { return; } if (this._tagNotification === null) { this._tagNotification = this.createNotification(true); @@ -1152,7 +1151,7 @@ export class Display extends EventDispatcher { */ _showInflectionNotification(inflectionNode) { const description = inflectionNode.title; - if (!description || !(inflectionNode instanceof HTMLSpanElement)) {return;} + if (!description || !(inflectionNode instanceof HTMLSpanElement)) { return; } if (this._inflectionNotification === null) { this._inflectionNotification = this.createNotification(true); @@ -1166,7 +1165,7 @@ export class Display extends EventDispatcher { * @param {boolean} animate */ _hideTagNotification(animate) { - if (this._tagNotification === null) {return;} + if (this._tagNotification === null) { return; } this._tagNotification.close(animate); } @@ -1174,7 +1173,7 @@ export class Display extends EventDispatcher { * @param {boolean} animate */ _hideInflectionNotification(animate) { - if (this._inflectionNotification === null) {return;} + if (this._inflectionNotification === null) { return; } this._inflectionNotification.close(animate); } @@ -1274,12 +1273,12 @@ export class Display extends EventDispatcher { const {findDetails, source: source2} = this._getFindDetails(source, wildcardsEnabled); if (isKanji) { dictionaryEntries = await this._application.api.kanjiFind(source, optionsContext); - if (dictionaryEntries.length > 0) {return dictionaryEntries;} + if (dictionaryEntries.length > 0) { return dictionaryEntries; } dictionaryEntries = (await this._application.api.termsFind(source2, findDetails, optionsContext)).dictionaryEntries; } else { dictionaryEntries = (await this._application.api.termsFind(source2, findDetails, optionsContext)).dictionaryEntries; - if (dictionaryEntries.length > 0) {return dictionaryEntries;} + if (dictionaryEntries.length > 0) { return dictionaryEntries; } dictionaryEntries = await this._application.api.kanjiFind(source, optionsContext); } @@ -1323,7 +1322,7 @@ export class Display extends EventDispatcher { // Set query performance.mark('display:setQuery:start'); let query = urlSearchParams.get('query'); - if (query === null) {query = '';} + if (query === null) { query = ''; } let queryFull = urlSearchParams.get('full'); queryFull = (queryFull !== null ? queryFull : query); const queryOffsetString = urlSearchParams.get('offset'); @@ -1348,7 +1347,7 @@ export class Display extends EventDispatcher { } let {focusEntry, scrollX, scrollY, optionsContext} = state; - if (typeof focusEntry !== 'number') {focusEntry = 0;} + if (typeof focusEntry !== 'number') { focusEntry = 0; } if (!(typeof optionsContext === 'object' && optionsContext !== null)) { optionsContext = this.getOptionsContext(); state.optionsContext = optionsContext; @@ -1361,7 +1360,7 @@ export class Display extends EventDispatcher { dictionaryEntries = hasEnabledDictionaries && lookup && query.length > 0 ? await this._findDictionaryEntries(type === 'kanji', query, wildcardsEnabled, optionsContext) : []; performance.mark('display:findDictionaryEntries:end'); performance.measure('display:findDictionaryEntries', 'display:findDictionaryEntries:start', 'display:findDictionaryEntries:end'); - if (this._setContentToken !== token) {return;} + if (this._setContentToken !== token) { return; } content.dictionaryEntries = dictionaryEntries; changeHistory = true; } @@ -1382,11 +1381,11 @@ export class Display extends EventDispatcher { } await this._setOptionsContextIfDifferent(optionsContext); - if (this._setContentToken !== token) {return;} + if (this._setContentToken !== token) { return; } if (this._options === null) { await this.updateOptions(); - if (this._setContentToken !== token) {return;} + if (this._setContentToken !== token) { return; } } if (changeHistory) { @@ -1415,9 +1414,11 @@ export class Display extends EventDispatcher { if (i > 0) { await promiseTimeout(1); - if (this._setContentToken !== token) {return;} + if (this._setContentToken !== token) { return; } } + performance.mark('display:createEntryReal:start'); + const entry = ( dictionaryEntry.type === 'term' ? await this._displayGenerator.createTermEntry(dictionaryEntry) : @@ -1429,9 +1430,10 @@ export class Display extends EventDispatcher { this._triggerContentUpdateEntry(dictionaryEntry, entry, i); performance.mark('display:waitMedia:start'); await this._contentManager.executeMediaRequests(); + console.log('done drawing'); performance.mark('display:waitMedia:end'); performance.measure('display:waitMedia', 'display:waitMedia:start', 'display:waitMedia:end'); - if (this._setContentToken !== token) {return;} + if (this._setContentToken !== token) { return; } container.appendChild(entry); if (focusEntry === i) { @@ -1440,6 +1442,9 @@ export class Display extends EventDispatcher { this._elementOverflowController.addElements(entry); + performance.mark('display:createEntryReal:end'); + performance.measure('display:createEntryReal', 'display:createEntryReal:start', 'display:createEntryReal:end'); + performance.mark('display:createEntry:end'); performance.measure('display:createEntry', 'display:createEntry:start', 'display:createEntry:end'); @@ -1448,8 +1453,8 @@ export class Display extends EventDispatcher { if (typeof scrollX === 'number' || typeof scrollY === 'number') { let {x, y} = this._windowScroll; - if (typeof scrollX === 'number') {x = scrollX;} - if (typeof scrollY === 'number') {y = scrollY;} + if (typeof scrollX === 'number') { x = scrollX; } + if (typeof scrollY === 'number') { y = scrollY; } this._windowScroll.stop(); this._windowScroll.to(x, y); } @@ -1652,22 +1657,22 @@ export class Display extends EventDispatcher { */ _focusEntryWithDifferentDictionary(offset, smooth) { const sign = Math.sign(offset); - if (sign === 0) {return false;} + if (sign === 0) { return false; } let index = this._index; const count = Math.min(this._dictionaryEntries.length, this._dictionaryEntryNodes.length); - if (index < 0 || index >= count) {return false;} + if (index < 0 || index >= count) { return false; } const dictionaryEntry = this._dictionaryEntries[index]; const visibleDefinitionIndex = this._getDictionaryEntryVisibleDefinitionIndex(index, sign); - if (visibleDefinitionIndex === null) {return false;} + if (visibleDefinitionIndex === null) { return false; } let focusDefinitionIndex = null; if (dictionaryEntry.type === 'term') { const {dictionary} = dictionaryEntry.definitions[visibleDefinitionIndex]; for (let i = index; i >= 0 && i < count; i += sign) { const otherDictionaryEntry = this._dictionaryEntries[i]; - if (otherDictionaryEntry.type !== 'term') {continue;} + if (otherDictionaryEntry.type !== 'term') { continue; } const {definitions} = otherDictionaryEntry; const jj = definitions.length; let j = (i === index ? visibleDefinitionIndex + sign : (sign > 0 ? 0 : jj - 1)); @@ -1682,7 +1687,7 @@ export class Display extends EventDispatcher { } } - if (focusDefinitionIndex === null) {return false;} + if (focusDefinitionIndex === null) { return false; } this._focusEntry(index, focusDefinitionIndex, smooth); return true; @@ -1699,13 +1704,13 @@ export class Display extends EventDispatcher { const {definitions} = this._dictionaryEntries[index]; const nodes = this._getDictionaryEntryDefinitionNodes(index); const definitionCount = Math.min(definitions.length, nodes.length); - if (definitionCount <= 0) {return null;} + if (definitionCount <= 0) { return null; } let visibleIndex = null; let visibleCoverage = 0; for (let i = (sign > 0 ? 0 : definitionCount - 1); i >= 0 && i < definitionCount; i += sign) { const {top, bottom} = nodes[i].getBoundingClientRect(); - if (bottom <= scrollTop || top >= scrollBottom) {continue;} + if (bottom <= scrollTop || top >= scrollBottom) { continue; } const top2 = Math.max(scrollTop, Math.min(scrollBottom, top)); const bottom2 = Math.max(scrollTop, Math.min(scrollBottom, bottom)); const coverage = (bottom2 - top2) / (bottom - top); @@ -1770,7 +1775,7 @@ export class Display extends EventDispatcher { /** */ _updateHistoryState() { const {state, content} = this._history; - if (!(typeof state === 'object' && state !== null)) {return;} + if (!(typeof state === 'object' && state !== null)) { return; } state.focusEntry = this._index; state.scrollX = this._windowScroll.x; @@ -1843,7 +1848,7 @@ export class Display extends EventDispatcher { * @param {import('settings').OptionsContext} optionsContext */ async _setOptionsContextIfDifferent(optionsContext) { - if (deepEqual(this._optionsContext, optionsContext)) {return;} + if (deepEqual(this._optionsContext, optionsContext)) { return; } await this.setOptionsContext(optionsContext); } @@ -1852,7 +1857,7 @@ export class Display extends EventDispatcher { */ _setContentScale(scale) { const body = document.body; - if (body === null) {return;} + if (body === null) { return; } body.style.fontSize = `${scale}em`; } @@ -1861,7 +1866,7 @@ export class Display extends EventDispatcher { */ async _updateNestedFrontend(options) { const {tabId, frameId} = this._application; - if (tabId === null || frameId === null) {return;} + if (tabId === null || frameId === null) { return; } const isSearchPage = (this._pageType === 'search'); const isEnabled = ( @@ -1874,7 +1879,7 @@ export class Display extends EventDispatcher { ); if (this._frontend === null) { - if (!isEnabled) {return;} + if (!isEnabled) { return; } try { if (this._frontendSetupPromise === null) { @@ -1927,9 +1932,9 @@ export class Display extends EventDispatcher { * @returns {boolean} */ _copyHostSelection() { - if (typeof this._contentOriginFrameId !== 'number') {return false;} + if (typeof this._contentOriginFrameId !== 'number') { return false; } const selection = window.getSelection(); - if (selection !== null && selection.toString().length > 0) {return false;} + if (selection !== null && selection.toString().length > 0) { return false; } void this._copyHostSelectionSafe(); return true; } @@ -1970,7 +1975,7 @@ export class Display extends EventDispatcher { */ _copyText(text) { const parent = document.body; - if (parent === null) {return;} + if (parent === null) { return; } let textarea = this._copyTextarea; if (textarea === null) { @@ -2178,15 +2183,15 @@ export class Display extends EventDispatcher { /** */ async _closeTab() { const tab = await this._getCurrentTab(); - if (tab === null) {return;} + if (tab === null) { return; } const tabId = tab.id; - if (typeof tabId === 'undefined') {return;} + if (typeof tabId === 'undefined') { return; } await this._removeTab(tabId); } /** */ _onHotkeyClose() { - if (this._closeSinglePopupMenu()) {return;} + if (this._closeSinglePopupMenu()) { return; } this.close(); } @@ -2196,7 +2201,7 @@ export class Display extends EventDispatcher { */ _onHotkeyActionMoveRelative(sign, argument) { let count = typeof argument === 'number' ? argument : (typeof argument === 'string' ? Number.parseInt(argument, 10) : 0); - if (!Number.isFinite(count)) {count = 1;} + if (!Number.isFinite(count)) { count = 1; } count = Math.max(0, Math.floor(count)); this._focusEntry(this._index + count * sign, 0, true); } @@ -2223,7 +2228,7 @@ export class Display extends EventDispatcher { * @param {number} index */ async _logDictionaryEntryData(index) { - if (index < 0 || index >= this._dictionaryEntries.length) {return;} + if (index < 0 || index >= this._dictionaryEntries.length) { return; } const dictionaryEntry = this._dictionaryEntries[index]; const result = {dictionaryEntry}; diff --git a/ext/js/display/structured-content-generator.js b/ext/js/display/structured-content-generator.js index f51aeb0862..6ec5d963b9 100644 --- a/ext/js/display/structured-content-generator.js +++ b/ext/js/display/structured-content-generator.js @@ -16,6 +16,7 @@ * along with this program. If not, see . */ +import {DisplayContentManager} from '../display/display-content-manager.js'; import {getLanguageFromText} from '../language/text-utilities.js'; export class StructuredContentGenerator { @@ -113,20 +114,34 @@ export class StructuredContentGenerator { node.dataset.sizeUnits = sizeUnits; } - imageContainer.style.width = `${usedWidth}em`; - if (typeof border === 'string') { imageContainer.style.border = border; } - if (typeof borderRadius === 'string') { imageContainer.style.borderRadius = borderRadius; } + console.log('dimension', usedWidth, invAspectRatio, width, height, preferredWidth, preferredHeight); + if (typeof border === 'string') {imageContainer.style.border = border;} + if (typeof borderRadius === 'string') {imageContainer.style.borderRadius = borderRadius;} if (typeof title === 'string') { imageContainer.title = title; } + const canvas = /** @type {HTMLCanvasElement} */ (this._createElement('canvas', 'gloss-image')); + if (sizeUnits === 'em' && (hasPreferredWidth || hasPreferredHeight)) { + canvas.style.width = `${usedWidth}em`; + canvas.style.height = `${usedWidth * invAspectRatio}em`; + canvas.width = usedWidth * 16; + canvas.height = usedWidth * invAspectRatio * 16; + } else { + canvas.width = usedWidth; + canvas.height = usedWidth * invAspectRatio; + } + imageContainer.appendChild(canvas); if (this._contentManager !== null) { - this._contentManager.loadMedia( - path, - dictionary, - async (url) => await this._setImageData(node, imageContainer, alt, invAspectRatio, url, false), - async () => await this._setImageData(node, imageContainer, alt, invAspectRatio, null, true), - ); + if (this._contentManager instanceof DisplayContentManager) { + this._contentManager.loadMedia( + path, + dictionary, + canvas.transferControlToOffscreen() + ); + } else { + // TODO: figure out anki + } } return node; @@ -205,43 +220,6 @@ export class StructuredContentGenerator { } } - /** - * @param {HTMLAnchorElement} node - * @param {HTMLElement} imageContainer - * @param {string|undefined} alt - * @param {number} invAspectRatio - * @param {?string} url - * @param {boolean} unloaded - */ - async _setImageData(node, imageContainer, alt, invAspectRatio, url, unloaded) { - const img = /** @type {HTMLImageElement} */ (this._createElement('img', 'gloss-image')); - if (url !== null) { - if (typeof alt === 'string') { - img.alt = alt; - } - img.src = url; - performance.mark('structured-content-generator:_setImageData:decode:[' + url + ']:start'); - await img.decode().then(() => { - performance.mark('structured-content-generator:_setImageData:decode:[' + url + ']:end'); - performance.measure('structured-content-generator:_setImageData:decode:[' + url + ']', 'structured-content-generator:_setImageData:decode:[' + url + ']:start', 'structured-content-generator:_setImageData:decode:[' + url + ']:end'); - node.dataset.imageLoadState = 'loaded'; - node.href = url; - performance.mark('structured-content-generator:_setImageData:appendChild:start'); - imageContainer.appendChild(img); - performance.mark('structured-content-generator:_setImageData:appendChild:end'); - performance.measure('structured-content-generator:_setImageData:appendChild', 'structured-content-generator:_setImageData:appendChild:start', 'structured-content-generator:_setImageData:appendChild:end'); - }).catch(() => { - node.dataset.imageLoadState = 'load-error'; - }); - } else { - if (node.children.length > 0) { - node.children[0].remove(); - } - node.removeAttribute('href'); - node.dataset.imageLoadState = unloaded ? 'unloaded' : 'load-error'; - } - } - /** * @param {import('structured-content').Element} content * @param {string} dictionary @@ -310,7 +288,7 @@ export class StructuredContentGenerator { async _createStructuredContentElement(tag, content, dictionary, language, type, hasChildren, hasStyle) { const node = this._createElement(tag, `gloss-sc-${tag}`); const {data, lang} = content; - if (typeof data === 'object' && data !== null) { this._setElementDataset(node, data); } + if (typeof data === 'object' && data !== null) {this._setElementDataset(node, data);} if (typeof lang === 'string') { node.lang = lang; language = lang; @@ -320,8 +298,8 @@ export class StructuredContentGenerator { { const cell = /** @type {HTMLTableCellElement} */ (node); const {colSpan, rowSpan} = /** @type {import('structured-content').TableElement} */ (content); - if (typeof colSpan === 'number') { cell.colSpan = colSpan; } - if (typeof rowSpan === 'number') { cell.rowSpan = rowSpan; } + if (typeof colSpan === 'number') {cell.colSpan = colSpan;} + if (typeof rowSpan === 'number') {cell.rowSpan = rowSpan;} } break; } @@ -330,7 +308,7 @@ export class StructuredContentGenerator { if (typeof style === 'object' && style !== null) { this._setStructuredContentElementStyle(node, style); } - if (typeof title === 'string') { node.title = title; } + if (typeof title === 'string') {node.title = title;} } if (hasChildren) { await this._appendStructuredContent(node, content.content, dictionary, language); @@ -378,16 +356,16 @@ export class StructuredContentGenerator { cursor, listStyleType, } = contentStyle; - if (typeof fontStyle === 'string') { style.fontStyle = fontStyle; } - if (typeof fontWeight === 'string') { style.fontWeight = fontWeight; } - if (typeof fontSize === 'string') { style.fontSize = fontSize; } - if (typeof color === 'string') { style.color = color; } - if (typeof background === 'string') { style.background = background; } - if (typeof backgroundColor === 'string') { style.backgroundColor = backgroundColor; } - if (typeof verticalAlign === 'string') { style.verticalAlign = verticalAlign; } - if (typeof textAlign === 'string') { style.textAlign = textAlign; } - if (typeof textEmphasis === 'string') { style.textEmphasis = textEmphasis; } - if (typeof textShadow === 'string') { style.textShadow = textShadow; } + if (typeof fontStyle === 'string') {style.fontStyle = fontStyle;} + if (typeof fontWeight === 'string') {style.fontWeight = fontWeight;} + if (typeof fontSize === 'string') {style.fontSize = fontSize;} + if (typeof color === 'string') {style.color = color;} + if (typeof background === 'string') {style.background = background;} + if (typeof backgroundColor === 'string') {style.backgroundColor = backgroundColor;} + if (typeof verticalAlign === 'string') {style.verticalAlign = verticalAlign;} + if (typeof textAlign === 'string') {style.textAlign = textAlign;} + if (typeof textEmphasis === 'string') {style.textEmphasis = textEmphasis;} + if (typeof textShadow === 'string') {style.textShadow = textShadow;} if (typeof textDecorationLine === 'string') { style.textDecoration = textDecorationLine; } else if (Array.isArray(textDecorationLine)) { @@ -399,29 +377,29 @@ export class StructuredContentGenerator { if (typeof textDecorationColor === 'string') { style.textDecorationColor = textDecorationColor; } - if (typeof borderColor === 'string') { style.borderColor = borderColor; } - if (typeof borderStyle === 'string') { style.borderStyle = borderStyle; } - if (typeof borderRadius === 'string') { style.borderRadius = borderRadius; } - if (typeof borderWidth === 'string') { style.borderWidth = borderWidth; } - if (typeof clipPath === 'string') { style.clipPath = clipPath; } - if (typeof margin === 'string') { style.margin = margin; } - if (typeof marginTop === 'number') { style.marginTop = `${marginTop}em`; } - if (typeof marginTop === 'string') { style.marginTop = marginTop; } - if (typeof marginLeft === 'number') { style.marginLeft = `${marginLeft}em`; } - if (typeof marginLeft === 'string') { style.marginLeft = marginLeft; } - if (typeof marginRight === 'number') { style.marginRight = `${marginRight}em`; } - if (typeof marginRight === 'string') { style.marginRight = marginRight; } - if (typeof marginBottom === 'number') { style.marginBottom = `${marginBottom}em`; } - if (typeof marginBottom === 'string') { style.marginBottom = marginBottom; } - if (typeof padding === 'string') { style.padding = padding; } - if (typeof paddingTop === 'string') { style.paddingTop = paddingTop; } - if (typeof paddingLeft === 'string') { style.paddingLeft = paddingLeft; } - if (typeof paddingRight === 'string') { style.paddingRight = paddingRight; } - if (typeof paddingBottom === 'string') { style.paddingBottom = paddingBottom; } - if (typeof wordBreak === 'string') { style.wordBreak = wordBreak; } - if (typeof whiteSpace === 'string') { style.whiteSpace = whiteSpace; } - if (typeof cursor === 'string') { style.cursor = cursor; } - if (typeof listStyleType === 'string') { style.listStyleType = listStyleType; } + if (typeof borderColor === 'string') {style.borderColor = borderColor;} + if (typeof borderStyle === 'string') {style.borderStyle = borderStyle;} + if (typeof borderRadius === 'string') {style.borderRadius = borderRadius;} + if (typeof borderWidth === 'string') {style.borderWidth = borderWidth;} + if (typeof clipPath === 'string') {style.clipPath = clipPath;} + if (typeof margin === 'string') {style.margin = margin;} + if (typeof marginTop === 'number') {style.marginTop = `${marginTop}em`;} + if (typeof marginTop === 'string') {style.marginTop = marginTop;} + if (typeof marginLeft === 'number') {style.marginLeft = `${marginLeft}em`;} + if (typeof marginLeft === 'string') {style.marginLeft = marginLeft;} + if (typeof marginRight === 'number') {style.marginRight = `${marginRight}em`;} + if (typeof marginRight === 'string') {style.marginRight = marginRight;} + if (typeof marginBottom === 'number') {style.marginBottom = `${marginBottom}em`;} + if (typeof marginBottom === 'string') {style.marginBottom = marginBottom;} + if (typeof padding === 'string') {style.padding = padding;} + if (typeof paddingTop === 'string') {style.paddingTop = paddingTop;} + if (typeof paddingLeft === 'string') {style.paddingLeft = paddingLeft;} + if (typeof paddingRight === 'string') {style.paddingRight = paddingRight;} + if (typeof paddingBottom === 'string') {style.paddingBottom = paddingBottom;} + if (typeof wordBreak === 'string') {style.wordBreak = wordBreak;} + if (typeof whiteSpace === 'string') {style.whiteSpace = whiteSpace;} + if (typeof cursor === 'string') {style.cursor = cursor;} + if (typeof listStyleType === 'string') {style.listStyleType = listStyleType;} } /** diff --git a/jsconfig.json b/jsconfig.json index 4ab1310628..9c389c0dd9 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -17,6 +17,7 @@ }, "types": [ "chrome", + "dom-webcodecs", "firefox-webext-browser", "handlebars", "jszip", diff --git a/package-lock.json b/package-lock.json index 2aa0a412bf..468e406fee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "@types/browserify": "^12.0.40", "@types/chrome": "^0.0.268", "@types/css": "^0.0.37", + "@types/dom-webcodecs": "^0.1.13", "@types/events": "^3.0.3", "@types/firefox-webext-browser": "^120.0.3", "@types/jsdom": "^21.1.6", @@ -1649,6 +1650,12 @@ "integrity": "sha512-IVhWCNH1mw3VRjkOMHsxVAcnANhee9w//TX1fqmALP628Dzf6VMG1LRnOngpptnrilcWCkmcY1tj6QkKGUy0CA==", "dev": true }, + "node_modules/@types/dom-webcodecs": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz", + "integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==", + "dev": true + }, "node_modules/@types/eslint": { "version": "8.56.5", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.5.tgz", @@ -11155,6 +11162,12 @@ "integrity": "sha512-IVhWCNH1mw3VRjkOMHsxVAcnANhee9w//TX1fqmALP628Dzf6VMG1LRnOngpptnrilcWCkmcY1tj6QkKGUy0CA==", "dev": true }, + "@types/dom-webcodecs": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz", + "integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==", + "dev": true + }, "@types/eslint": { "version": "8.56.5", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.5.tgz", diff --git a/package.json b/package.json index ec074190bb..84cb7af53f 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "@types/browserify": "^12.0.40", "@types/chrome": "^0.0.268", "@types/css": "^0.0.37", + "@types/dom-webcodecs": "^0.1.13", "@types/events": "^3.0.3", "@types/firefox-webext-browser": "^120.0.3", "@types/jsdom": "^21.1.6", diff --git a/test/jsconfig.json b/test/jsconfig.json index 4f7f175b7d..9395d140c7 100644 --- a/test/jsconfig.json +++ b/test/jsconfig.json @@ -22,6 +22,7 @@ }, "types": [ "chrome", + "dom-webcodecs", "firefox-webext-browser", "handlebars", "jszip", diff --git a/types/ext/api.d.ts b/types/ext/api.d.ts index 6ea84bf52b..0d022effec 100644 --- a/types/ext/api.d.ts +++ b/types/ext/api.d.ts @@ -292,11 +292,11 @@ type ApiSurface = { }; return: DictionaryDatabase.MediaDataStringContent[]; }; - getMediaObjects: { + drawMedia: { params: { - targets: GetMediaDetailsTarget[]; + targets: DictionaryDatabase.DrawMediaRequest[]; }; - return: DictionaryDatabase.MediaObject[]; + return: void; }; logGenericErrorBackend: { params: { diff --git a/types/ext/dictionary-database.d.ts b/types/ext/dictionary-database.d.ts index 447c8e56c5..d8b2c416dd 100644 --- a/types/ext/dictionary-database.d.ts +++ b/types/ext/dictionary-database.d.ts @@ -40,8 +40,7 @@ type MediaType = ArrayBuffer | string | null; export type Media = {index: number} & MediaDataBase; -// eslint-disable-next-line @typescript-eslint/no-unused-vars -export type MediaObject<_T extends MediaType = ArrayBuffer> = {index: number} & MediaDataBase & {url: string}; +export type DrawMedia = {index: number} & MediaDataBase & {canvases: [OffscreenCanvas]}; export type DatabaseTermEntry = { expression: string; @@ -242,6 +241,18 @@ export type MediaRequest = { dictionary: string; }; +export type DrawMediaRequest = { + path: string; + dictionary: string; + canvas: OffscreenCanvas; +}; + +export type DrawMediaGroupedRequest = { + path: string; + dictionary: string; + canvases: OffscreenCanvas[]; +}; + export type FindMultiBulkData = { item: TItem; itemIndex: number; diff --git a/types/ext/display-content-manager.d.ts b/types/ext/display-content-manager.d.ts index 7c216f8830..b1e88c93fc 100644 --- a/types/ext/display-content-manager.d.ts +++ b/types/ext/display-content-manager.d.ts @@ -39,8 +39,6 @@ export type LoadMediaRequest = { path: string; /** The name of the dictionary. */ dictionary: string; - /** The callback that is executed if the media was loaded successfully. */ - onLoad: OnLoadCallback; - /** The callback that is executed when the media should be unloaded. */ - onUnload: OnUnloadCallback; + /** The canvas to draw the image onto. */ + canvas: OffscreenCanvas; }; diff --git a/types/ext/offscreen.d.ts b/types/ext/offscreen.d.ts index 4d978614f7..98c9edebb0 100644 --- a/types/ext/offscreen.d.ts +++ b/types/ext/offscreen.d.ts @@ -49,11 +49,11 @@ type ApiSurface = { }; return: DictionaryDatabase.Media[]; }; - databaseGetMediaObjectsOffscreen: { + databaseDrawMediaOffscreen: { params: { - targets: DictionaryDatabase.MediaRequest[]; + targets: DictionaryDatabase.DrawMediaRequest[]; }; - return: DictionaryDatabase.MediaObject[]; + return: void; }; translatorPrepareOffscreen: { params: void;