Skip to content

Commit

Permalink
Merge branch 'master' into debug-translator
Browse files Browse the repository at this point in the history
  • Loading branch information
djahandarie authored Feb 23, 2024
2 parents 3e1fb1c + 8078e0e commit b782e6e
Show file tree
Hide file tree
Showing 18 changed files with 557 additions and 112 deletions.
4 changes: 4 additions & 0 deletions .eslintrc.json
Original file line number Diff line number Diff line change
Expand Up @@ -638,12 +638,16 @@
"ext/js/general/object-property-accessor.js",
"ext/js/general/regex-util.js",
"ext/js/general/text-source-map.js",
"ext/js/language/ar/arabic-text-preprocessors.js",
"ext/js/language/de/german-text-preprocessors.js",
"ext/js/language/ja/japanese-text-preprocessors.js",
"ext/js/language/ja/japanese-wanakana.js",
"ext/js/language/ja/japanese.js",
"ext/js/language/la/latin-text-preprocessors.js",
"ext/js/language/language-descriptors.js",
"ext/js/language/language-transformer.js",
"ext/js/language/languages.js",
"ext/js/language/ru/russian-text-preprocessors.js",
"ext/js/language/text-preprocessors.js",
"ext/js/language/translator.js",
"ext/js/media/audio-downloader.js",
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/bench.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
run: npm ci

- name: Build Libs
run: npm run build-libs
run: npm run build:libs

- name: Run Benchmarks
uses: CodSpeedHQ/action@v2
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/broken-links.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
- name: Install dependencies
run: npm ci
- name: Build Legal
run: npm run license-report
run: npm run license-report:html
- uses: lycheeverse/lychee-action@c053181aa0c3d17606addfe97a9075a32723548a
with:
fail: true
Expand Down
99 changes: 66 additions & 33 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,72 @@ on:
merge_group:

jobs:
test:
static-analysis:
name: Static Analysis
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Setup node
uses: actions/setup-node@v4
with:
node-version-file: "package.json"

- name: Install dependencies
run: npm ci

- name: Build third-party libraries
run: npm run build:libs

- name: Run static analysis tests
run: npm run test:static-analysis

static-analysis-json:
name: Static Analysis (JSON)
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Setup node
uses: actions/setup-node@v4
with:
node-version-file: "package.json"

- name: Install dependencies
run: npm ci

- name: Build third-party libraries
run: npm run build:libs

- name: Run static analysis tests
run: npm run test:json

unit-tests:
name: Unit Tests
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Setup node
uses: actions/setup-node@v4
with:
node-version-file: "package.json"

- name: Install dependencies
run: npm ci

- name: Build third-party libraries
run: npm run build:libs

- name: Run unit tests
run: npm run test:unit

test-build:
name: Test Build
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
Expand All @@ -26,40 +89,10 @@ jobs:
run: npm ci

- name: Build Libs
run: npm run build-libs

- name: Lint JS
run: npm run test-lint-js
env:
CI: true

- name: Validate JS Types
run: npm run test-ts
env:
CI: true

- name: Lint CSS
run: npm run test-lint-css
env:
CI: true

- name: Lint HTML
run: npm run test-lint-html
env:
CI: true

- name: Tests
run: npm run test-code
env:
CI: true

- name: Test JSON
run: npm run test-json
env:
CI: true
run: npm run build:libs

- name: Build Legal
run: npm run license-report
run: npm run license-report:html

- name: Build
run: npm run build
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/create-prerelease-on-tag.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
run: npm ci

- name: Build Legal
run: npm run license-report
run: npm run license-report:html

- name: Build
run: npm run-script build -- --all --version ${{ github.ref_name }}
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ Required licensing notices for this project follow below:

Yomitan uses several third-party libraries to function.

<!-- The following table is generated using the command `npm run license-report-markdown`. -->
<!-- The following table is generated using the command `npm run license-report:markdown`. -->

| Name | Installed version | License type | Link |
| :------------------ | :---------------- | :----------- | :----------------------------------------------- |
Expand Down
3 changes: 2 additions & 1 deletion benches/jsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
"test/*": ["../types/test/*"],
"rollup/parseAst": ["../types/other/rollup-parse-ast"],
"ext/json-schema": ["../types/ext/json-schema"],
"json-schema": ["json-schema"]
"json-schema": ["json-schema"],
"chai": ["../node_modules/@vitest/expect/dist/chai.d.cts"]
},
"types": [
"chrome",
Expand Down
86 changes: 86 additions & 0 deletions docs/development/npm-scripts.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# npm Scripts

This file documents the scripts available in the [package.json](../../package.json) file.
Scripts can be executed by running `npm run <name>`.

- `bench`
Runs performance benchmarks.

- `build`
Builds packages for all of the primary build targets and outputs them to the builds folder in the root project directory.

- `build:libs`
Rebuilds all of the third-party dependencies that the extension uses.

- `test`
Runs all of the tests.

- `test:fast`
Runs most of the tests that are used more frequently in the typical development process.

- `test:static-analysis`
Runs all of the static analysis tests.

- `test:js`
Runs [eslint](https://eslint.org/) on all of the JavaScript and TypeScript files in the project.

- `test:json`
Runs all JSON tests.

- `test:json:format`
Runs eslint on all of the JSON files in the project.

- `test:json:types`
Performs type checking on all of the JSON files in the project.
Running this script often takes a long time since it has to validate a lot of files with complex types.

- `test:css`
Runs [stylelint](https://stylelint.io/) on all of the CSS files in the project.

- `test:html`
Runs [html-validate](https://html-validate.org/) on all of the HTML files in the project.

- `test:md`
Runs [prettier](https://prettier.io/) on all of the Markdown files in the project.

- `test:md:write`
Uses prettier to fix all issues it encounters with files.

- `test:ts`
Runs [TypeScript](https://www.typescriptlang.org/) validation on all of the JavaScript and TypeScript files in the project.

- `test:ts:main`
Runs [TypeScript](https://www.typescriptlang.org/) validation on the files in the [ext](../../ext/) folder.

- `test:ts:dev`
Runs [TypeScript](https://www.typescriptlang.org/) validation on the files in the [dev](../../dev/) folder.

- `test:ts:test`
Runs [TypeScript](https://www.typescriptlang.org/) validation on the files in the [test](../../test/) folder.

- `test:ts:bench`
Runs [TypeScript](https://www.typescriptlang.org/) validation on the files in the [benches](../../benches/) folder.

- `test:unit`
Runs all of the unit tests in the project using [vitest](https://vitest.dev/).

- `test:unit:write`
Overwrites the expected test output data for some of the larger tests.
This usually only needs to be run when something modifies the format of dictionary entries or Anki data.

- `test:unit:options`
Runs unit tests related to the extension's options and their upgrade process.

- `test:build`
Performs a dry run of the build process without generating any files.

- `license-report:html`
Generates a file containing license information about the third-party dependencies the extension uses.
The resulting file is located at ext/legal-npm.html.

- `license-report:markdown`
Generates a Markdown table containing license information about the third-party dependencies the extension uses.
This table is located in the [README.md](../../README.md#third-party-libraries) file

- `prepare`
Sets up [husky](https://typicode.github.io/husky/) for some git pre-commit tasks.
28 changes: 28 additions & 0 deletions ext/js/language/ar/arabic-text-preprocessors.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
* Copyright (C) 2024 Yomitan Authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/

import {basicTextPreprocessorOptions} from '../text-preprocessors.js';

/** @type {import('language').TextPreprocessor<boolean>} */
export const removeArabicScriptDiacritics = {
name: 'Remove diacritics',
description: 'وَلَدَ ⬅️ ولد',
options: basicTextPreprocessorOptions,
process: (text, setting) => {
return setting ? text.replace(/[\u064E-\u0650]/g, '') : text;
}
};
34 changes: 34 additions & 0 deletions ext/js/language/de/german-text-preprocessors.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright (C) 2024 Yomitan Authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/


/** @type {import('language').BidirectionalConversionPreprocessor} */
export const eszettPreprocessor = {
name: 'Convert "ß" to "ss"',
description: 'ß → ss, ẞ → SS and vice versa',
options: ['off', 'direct', 'inverse'],
process: (str, setting) => {
switch (setting) {
case 'off':
return str;
case 'direct':
return str.replace(//g, 'SS').replace(/ß/g, 'ss');
case 'inverse':
return str.replace(/SS/g, 'ẞ').replace(/ss/g, 'ß');
}
}
};
31 changes: 13 additions & 18 deletions ext/js/language/ja/japanese-text-preprocessors.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ export const convertHalfWidthCharacters = {
name: 'Convert half width characters to full width',
description: 'ヨミチャン → ヨミチャン',
options: basicTextPreprocessorOptions,
/** @type {import('language').TextPreprocessorFunction<boolean>} */
process: (str, setting, sourceMap) => (setting ? convertHalfWidthKanaToFullWidth(str, sourceMap) : str)
};

Expand All @@ -39,7 +38,6 @@ export const convertNumericCharacters = {
name: 'Convert numeric characters to full width',
description: '1234 → 1234',
options: basicTextPreprocessorOptions,
/** @type {import('language').TextPreprocessorFunction<boolean>} */
process: (str, setting) => (setting ? convertNumericToFullWidth(str) : str)
};

Expand All @@ -48,34 +46,31 @@ export const convertAlphabeticCharacters = {
name: 'Convert alphabetic characters to hiragana',
description: 'yomichan → よみちゃん',
options: basicTextPreprocessorOptions,
/** @type {import('language').TextPreprocessorFunction<boolean>} */
process: (str, setting, sourceMap) => (setting ? convertAlphabeticToKana(str, sourceMap) : str)
};

/** @type {import('language').TextPreprocessor<boolean>} */
/** @type {import('language').BidirectionalConversionPreprocessor} */
export const convertHiraganaToKatakana = {
name: 'Convert hiragana to katakana',
description: 'よみちゃん → ヨミチャン',
options: basicTextPreprocessorOptions,
/** @type {import('language').TextPreprocessorFunction<boolean>} */
process: (str, setting) => (setting ? convertHiraganaToKatakanaFunction(str) : str)
};

/** @type {import('language').TextPreprocessor<boolean>} */
export const convertKatakanaToHiragana = {
name: 'Convert katakana to hiragana',
description: 'ヨミチャン → よみちゃん',
options: basicTextPreprocessorOptions,
/** @type {import('language').TextPreprocessorFunction<boolean>} */
process: (str, setting) => (setting ? convertKatakanaToHiraganaFunction(str) : str)
description: 'よみちゃん → ヨミチャン and vice versa',
options: ['off', 'direct', 'inverse'],
process: (str, setting) => {
switch (setting) {
case 'off':
return str;
case 'direct':
return convertHiraganaToKatakanaFunction(str);
case 'inverse':
return convertKatakanaToHiraganaFunction(str);
}
}
};

/** @type {import('language').TextPreprocessor<[collapseEmphatic: boolean, collapseEmphaticFull: boolean]>} */
export const collapseEmphaticSequences = {
name: 'Collapse emphatic character sequences',
description: 'すっっごーーい → すっごーい / すごい',
options: [[false, false], [true, false], [true, true]],
/** @type {import('language').TextPreprocessorFunction<[collapseEmphatic: boolean, collapseEmphaticFull: boolean]>} */
process: (str, setting, sourceMap) => {
const [collapseEmphatic, collapseEmphaticFull] = setting;
if (collapseEmphatic) {
Expand Down
Loading

0 comments on commit b782e6e

Please sign in to comment.