Merge pull request #1 from StefanVukovic99/martholow-latin

fix tests, merge master
yomidevs · Mar 27, 2024 · 9d1712a · 9d1712a
2 parents d96d0fb + 368c67e
commit 9d1712a
Show file tree

Hide file tree

Showing 44 changed files with 3,284 additions and 2,728 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -642,14 +642,18 @@
                 "ext/js/general/text-source-map.js",
                 "ext/js/language/ar/arabic-text-preprocessors.js",
                 "ext/js/language/de/german-text-preprocessors.js",
+                "ext/js/language/en/english-transforms.js",
                 "ext/js/language/ja/japanese-text-preprocessors.js",
+                "ext/js/language/ja/japanese-transforms.js",
                 "ext/js/language/ja/japanese-wanakana.js",
                 "ext/js/language/ja/japanese.js",
                 "ext/js/language/language-descriptors.js",
                 "ext/js/language/language-transformer.js",
+                "ext/js/language/language-transforms.js",
                 "ext/js/language/languages.js",
                 "ext/js/language/multi-language-transformer.js",
                 "ext/js/language/ru/russian-text-preprocessors.js",
+                "ext/js/language/sq/albanian-transforms.js",
                 "ext/js/language/text-preprocessors.js",
                 "ext/js/language/translator.js",
                 "ext/js/media/audio-downloader.js",

diff --git a/benches/language-transformer.bench.js b/benches/language-transformer.bench.js
@@ -15,19 +15,12 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
-import fs from 'fs';
-import {fileURLToPath} from 'node:url';
-import path from 'path';
 import {bench, describe} from 'vitest';
-import {parseJson} from '../dev/json.js';
+import {japaneseTransforms} from '../ext/js/language/ja/japanese-transforms.js';
 import {LanguageTransformer} from '../ext/js/language/language-transformer.js';
 
-const dirname = path.dirname(fileURLToPath(import.meta.url));
-
-/** @type {import('language-transformer').LanguageTransformDescriptor} */
-const descriptor = parseJson(fs.readFileSync(path.join(dirname, '..', 'ext', 'js/language/ja/japanese-transforms.json'), {encoding: 'utf8'}));
 const languageTransformer = new LanguageTransformer();
-languageTransformer.addDescriptor(descriptor);
+languageTransformer.addDescriptor(japaneseTransforms);
 
 describe('language transformer', () => {
     describe('basic tests', () => {

diff --git a/ext/css/display.css b/ext/css/display.css
@@ -669,6 +669,8 @@ button.action-button:active {
 .icon[data-icon=view-note]      { background-image: url('/images/view-note.svg'); }
 .icon[data-icon=add-term-kanji] { background-image: url('/images/add-term-kanji.svg'); }
 .icon[data-icon=add-term-kana]  { background-image: url('/images/add-term-kana.svg'); }
+.icon[data-icon=add-duplicate-term-kanji] { background-image: url('/images/add-duplicate-term-kanji.svg'); }
+.icon[data-icon=add-duplicate-term-kana]  { background-image: url('/images/add-duplicate-term-kana.svg'); }
 .icon[data-icon=play-audio]     { background-image: url('/images/play-audio.svg'); }
 .icon[data-icon=source-term]    { background-image: url('/images/source-term.svg'); }
 .icon[data-icon=entry-current]  { background-image: url('/images/entry-current.svg'); }

diff --git a/ext/images/add-duplicate-term-kana.svg b/ext/images/add-duplicate-term-kana.svg
diff --git a/ext/images/add-duplicate-term-kanji.svg b/ext/images/add-duplicate-term-kanji.svg
diff --git a/ext/js/background/backend.js b/ext/js/background/backend.js
@@ -23,7 +23,7 @@ import {ClipboardReader} from '../comm/clipboard-reader.js';
 import {Mecab} from '../comm/mecab.js';
 import {createApiMap, invokeApiMapHandler} from '../core/api-map.js';
 import {ExtensionError} from '../core/extension-error.js';
-import {fetchJson, fetchText} from '../core/fetch-utilities.js';
+import {fetchText} from '../core/fetch-utilities.js';
 import {logErrorLevelToNumber} from '../core/log-utilities.js';
 import {log} from '../core/log.js';
 import {isObjectNotArray} from '../core/object-utilities.js';
@@ -275,16 +275,7 @@ export class Backend {
                 log.error(e);
             }
 
-            /** @type {import('language-transformer').LanguageTransformDescriptor[]} */
-            const descriptors = [];
-            const languageSummaries = getLanguageSummaries();
-            for (const {languageTransformsFile} of languageSummaries) {
-                if (!languageTransformsFile) { continue; }
-                /** @type {import('language-transformer').LanguageTransformDescriptor} */
-                const descriptor = await fetchJson(languageTransformsFile);
-                descriptors.push(descriptor);
-            }
-            void this._translator.prepare(descriptors);
+            void this._translator.prepare();
 
             await this._optionsUtil.prepare();
             this._defaultAnkiFieldTemplates = (await fetchText('/data/templates/default-anki-field-templates.handlebars')).trim();
@@ -544,22 +535,89 @@ export class Backend {
         return await this._anki.addNote(note);
     }
 
+    /**
+     * @param {import('anki').Note[]} notes
+     * @returns {Promise<({ canAdd: true; } | { canAdd: false; error: string; })[]>}
+     */
+    async detectDuplicateNotes(notes) {
+        // `allowDuplicate` is on for all notes by default, so we temporarily set it to false
+        // to check which notes are duplicates.
+        const notesNoDuplicatesAllowed = notes.map((note) => ({...note, options: {...note.options, allowDuplicate: false}}));
+
+        return await this._anki.canAddNotesWithErrorDetail(notesNoDuplicatesAllowed);
+    }
+
+    /**
+     * Partitions notes between those that can / cannot be added.
+     * It further sets the `isDuplicate` strings for notes that have a duplicate.
+     * @param {import('anki').Note[]} notes
+     * @returns {Promise<import('backend').CanAddResults>}
+     */
+    async partitionAddibleNotes(notes) {
+        const canAddResults = await this.detectDuplicateNotes(notes);
+
+        /** @type {{ note: import('anki').Note, isDuplicate: boolean }[]} */
+        const canAddArray = [];
+
+        /** @type {import('anki').Note[]} */
+        const cannotAddArray = [];
+
+        for (let i = 0; i < canAddResults.length; i++) {
+            const result = canAddResults[i];
+
+            // If the note is a duplicate, the error is "cannot create note because it is a duplicate".
+            if (result.canAdd) {
+                canAddArray.push({note: notes[i], isDuplicate: false});
+            } else if (result.error.endsWith('duplicate')) {
+                canAddArray.push({note: notes[i], isDuplicate: true});
+            } else {
+                cannotAddArray.push(notes[i]);
+            }
+        }
+
+        return {canAddArray, cannotAddArray};
+    }
+
     /** @type {import('api').ApiHandler<'getAnkiNoteInfo'>} */
     async _onApiGetAnkiNoteInfo({notes, fetchAdditionalInfo}) {
-        /** @type {import('anki').NoteInfoWrapper[]} */
-        const results = [];
+        const {canAddArray, cannotAddArray} = await this.partitionAddibleNotes(notes);
+
         /** @type {{note: import('anki').Note, info: import('anki').NoteInfoWrapper}[]} */
-        const cannotAdd = [];
-        const canAddArray = await this._anki.canAddNotes(notes);
+        const cannotAdd = cannotAddArray.filter((note) => isNoteDataValid(note)).map((note) => ({note, info: {canAdd: false, valid: false, noteIds: null}}));
+
+        /** @type {import('anki').NoteInfoWrapper[]} */
+        const results = cannotAdd.map(({info}) => info);
+
+        /** @type {import('anki').Note[]} */
+        const duplicateNotes = [];
+
+        /** @type {number[]} */
+        const originalIndices = [];
+
+        for (let i = 0; i < canAddArray.length; i++) {
+            if (canAddArray[i].isDuplicate) {
+                duplicateNotes.push(canAddArray[i].note);
+                // Keep original indices to locate duplicate inside `duplicateNoteIds`
+                originalIndices.push(i);
+            }
+        }
+
+        const duplicateNoteIds = await this._anki.findNoteIds(duplicateNotes);
+
+        for (let i = 0; i < canAddArray.length; ++i) {
+            const {note, isDuplicate} = canAddArray[i];
 
-        for (let i = 0; i < notes.length; ++i) {
-            const note = notes[i];
-            let canAdd = canAddArray[i];
             const valid = isNoteDataValid(note);
-            if (!valid) { canAdd = false; }
-            const info = {canAdd, valid, noteIds: null};
+
+            const info = {
+                canAdd: valid,
+                valid,
+                noteIds: isDuplicate ? duplicateNoteIds[originalIndices.indexOf(i)] : null
+            };
+
             results.push(info);
-            if (!canAdd && valid) {
+
+            if (!valid) {
                 cannotAdd.push({note, info});
             }
         }

diff --git a/ext/js/background/offscreen-proxy.js b/ext/js/background/offscreen-proxy.js
@@ -183,11 +183,9 @@ export class TranslatorProxy {
         this._offscreen = offscreen;
     }
 
-    /**
-     * @param {import('language-transformer').LanguageTransformDescriptor[]} descriptors
-     */
-    async prepare(descriptors) {
-        await this._offscreen.sendMessagePromise({action: 'translatorPrepareOffscreen', params: {descriptors}});
+    /** */
+    async prepare() {
+        await this._offscreen.sendMessagePromise({action: 'translatorPrepareOffscreen'});
     }
 
     /**

diff --git a/ext/js/background/offscreen.js b/ext/js/background/offscreen.js
@@ -111,8 +111,8 @@ export class Offscreen {
     }
 
     /** @type {import('offscreen').ApiHandler<'translatorPrepareOffscreen'>} */
-    _prepareTranslatorHandler({descriptors}) {
-        this._translator.prepare(descriptors);
+    _prepareTranslatorHandler() {
+        this._translator.prepare();
     }
 
     /** @type {import('offscreen').ApiHandler<'findKanjiOffscreen'>} */