diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 894e2c1..d645cbc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -23,6 +23,14 @@ jobs: java-version: 11 distribution: 'temurin' cache: 'maven' + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + cache-dependency-path: 'rascal-textmate-core/package-lock.json' + - name: install-tokenizer + working-directory: rascal-textmate-core + run: npm install vscode-tmgrammar-test - name: run-tests working-directory: rascal-textmate-core run: mvn test -B -Drascal.compile.skip -Drascal.tutor.skip diff --git a/README.md b/README.md index 35a7c5d..5d66b6d 100644 --- a/README.md +++ b/README.md @@ -16,5 +16,18 @@ TextMate grammars, this project applies partial conversion. Alternatively, a previous [project](https://github.com/TarVK/syntax-highlighter) by [@TarVK](https://github.com/TarVK) applies total conversion. -The [walkthrough](src/main/rascal/lang/textmate/conversiontests/Walkthrough.rsc) +## Documentation + +The [walkthrough](rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Walkthrough.rsc) explains the main ideas behind the conversion algorithm in this project. + +## Tests + +To test tokenization (as part of the conversion +[tests](rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests)), +the [`vscode-tmgrammar-test`](https://github.com/PanAeon/vscode-tmgrammar-test) +tool is used. Install it locally in directory `rascal-textmate-core` as follows: + +``` +npm install vscode-tmgrammar-test +``` \ No newline at end of file diff --git a/rascal-textmate-core/.gitignore b/rascal-textmate-core/.gitignore index 1de5659..5feb907 100644 --- a/rascal-textmate-core/.gitignore +++ b/rascal-textmate-core/.gitignore @@ -1 +1,2 @@ -target \ No newline at end of file +target +node_modules \ No newline at end of file diff --git a/rascal-textmate-core/package-lock.json b/rascal-textmate-core/package-lock.json new file mode 100644 index 0000000..cae3904 --- /dev/null +++ b/rascal-textmate-core/package-lock.json @@ -0,0 +1,383 @@ +{ + "name": "rascal-textmate-core", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "dependencies": { + "vscode-tmgrammar-test": "^0.1.3" + } + }, + "node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==", + "engines": { + "node": "^12.20.0 || >=14" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/vscode-oniguruma": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz", + "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==" + }, + "node_modules/vscode-textmate": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-7.0.4.tgz", + "integrity": "sha512-9hJp0xL7HW1Q5OgGe03NACo7yiCTMEk3WU/rtKXUbncLtdg6rVVNJnHwD88UhbIYU2KoxY0Dih0x+kIsmUKn2A==" + }, + "node_modules/vscode-tmgrammar-test": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/vscode-tmgrammar-test/-/vscode-tmgrammar-test-0.1.3.tgz", + "integrity": "sha512-Wg6Pz+ePAT1O+F/A1Fc4wS5vY2X+HNtgN4qMdL+65NLQYd1/zdDWH4fhwsLjX8wTzeXkMy49Cr4ZqWTJ7VnVxg==", + "dependencies": { + "bottleneck": "^2.19.5", + "chalk": "^2.4.2", + "commander": "^9.2.0", + "diff": "^4.0.2", + "glob": "^7.1.6", + "vscode-oniguruma": "^1.5.1", + "vscode-textmate": "^7.0.1" + }, + "bin": { + "vscode-tmgrammar-snap": "dist/snapshot.js", + "vscode-tmgrammar-test": "dist/unit.js" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + } + }, + "dependencies": { + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "bottleneck": { + "version": "2.19.5", + "resolved": "https://registry.npmjs.org/bottleneck/-/bottleneck-2.19.5.tgz", + "integrity": "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "commander": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-9.5.0.tgz", + "integrity": "sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==" + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==" + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==" + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "requires": { + "wrappy": "1" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } + }, + "vscode-oniguruma": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz", + "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==" + }, + "vscode-textmate": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-7.0.4.tgz", + "integrity": "sha512-9hJp0xL7HW1Q5OgGe03NACo7yiCTMEk3WU/rtKXUbncLtdg6rVVNJnHwD88UhbIYU2KoxY0Dih0x+kIsmUKn2A==" + }, + "vscode-tmgrammar-test": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/vscode-tmgrammar-test/-/vscode-tmgrammar-test-0.1.3.tgz", + "integrity": "sha512-Wg6Pz+ePAT1O+F/A1Fc4wS5vY2X+HNtgN4qMdL+65NLQYd1/zdDWH4fhwsLjX8wTzeXkMy49Cr4ZqWTJ7VnVxg==", + "requires": { + "bottleneck": "^2.19.5", + "chalk": "^2.4.2", + "commander": "^9.2.0", + "diff": "^4.0.2", + "glob": "^7.1.6", + "vscode-oniguruma": "^1.5.1", + "vscode-textmate": "^7.0.1" + } + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + } + } +} diff --git a/rascal-textmate-core/package.json b/rascal-textmate-core/package.json new file mode 100644 index 0000000..85010cc --- /dev/null +++ b/rascal-textmate-core/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "vscode-tmgrammar-test": "^0.1.3" + } +} diff --git a/rascal-textmate-core/src/main/rascal/lang/oniguruma/Conversion.rsc b/rascal-textmate-core/src/main/rascal/lang/oniguruma/Conversion.rsc index 9203ae3..d05303a 100644 --- a/rascal-textmate-core/src/main/rascal/lang/oniguruma/Conversion.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/oniguruma/Conversion.rsc @@ -158,7 +158,9 @@ default RegExp toRegExp(Grammar _, Condition c) { Converts a character range to a regular expression. } -RegExp toRegExp(Grammar _, range(begin, end)) +RegExp toRegExp(Grammar _, range(char, char)) + = regExp("", []); +default RegExp toRegExp(Grammar _, range(begin, end)) = regExp("[-]", []); @synopsis{ diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/ConversionTests.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/ConversionTests.rsc index c7bf8f2..7507b0f 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/ConversionTests.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/ConversionTests.rsc @@ -5,16 +5,24 @@ module lang::textmate::ConversionTests import Grammar; +import IO; import List; import Map; import ParseTree; +import String; +import util::ShellExec; +import util::SystemAPI; import lang::textmate::Conversion; import lang::textmate::Grammar; -bool doAnalyzeTest(RscGrammar rsc, list[ConversionUnit] expect) { +bool doAnalyzeTest(RscGrammar rsc, list[ConversionUnit] expect, bool printActual = false) { list[ConversionUnit] actual = analyze(rsc); + for (printActual, u <- actual) { + println("unit(rsc, ),"); + } + for (u <- actual) { assert u in expect : "Actual but not expected: "; } @@ -32,11 +40,13 @@ bool doAnalyzeTest(RscGrammar rsc, list[ConversionUnit] expect) { return true; } -bool doTransformTest(list[ConversionUnit] units, RepositoryStats expect) { - TmGrammar tm = transform(units); +bool doTransformTest(list[ConversionUnit] units, RepositoryStats expect, str name = "") { + TmGrammar tm = transform(units)[scopeName = ""]; Repository repo = tm.repository; list[TmRule] pats = tm.patterns; + // Test structural properties of the TextMate grammar + RepositoryStats actual = < (0 | it + 1 | s <- repo, repo[s] is match), (0 | it + 1 | s <- repo, repo[s] is beginEnd), @@ -59,10 +69,44 @@ bool doTransformTest(list[ConversionUnit] units, RepositoryStats expect) { assert size(pats) == size(repo) : "Actual patterns list size: . Expected: ."; assert (true | it && r is include | r <- pats) : "Patterns list contains pattern(s) of unexpected kind"; assert (true | it && s in repo | r <- pats, include(/#$/) := r) : "Patterns list contains pattern(s) outside repository"; + + // Test behavioral properties of the TextMate grammar + + loc lProject = |project://rascal-textmate-core|; + loc lGrammar = lProject + "/target/generated-test-grammars/.tmLanguage.json"; + toJSON(tm, l = resolveLocation(lGrammar)); + + loc lTest = lProject + "/src/main/rascal/lang/textmate/conversiontests/.test"; + loc lTester = lProject + "/node_modules/vscode-tmgrammar-test"; + if (!exists(lTest)) { + println("[LOG] No tokenization tests available for `` (`` does not exist)"); + } elseif (!exists(lTester)) { + println("[LOG] No tokenizer available (`` does not exist)"); + } else { + bool windows = startsWith(getSystemProperty("os.name"), "Windows"); + loc lExec = |PATH:///npx|; + list[str] args = [ + "vscode-tmgrammar-test", + "--grammar", + resolveLocation(lGrammar).path[(windows ? 1 : 0)..], + resolveLocation(lTest).path[(windows ? 1 : 0)..] + ]; + + if ( := execWithCode(lExec, args = args) && exitCode != 0) { + println(output); + assert false : "Actual tokenization does not match expected tokenization (see output above for details)"; + } + } + return true; } -alias RepositoryStats = tuple[int match, int beginEnd, int include]; +alias RepositoryStats = tuple[ + int match, + int beginEnd, + int include]; int sum(RepositoryStats stats) - = stats.match + stats.beginEnd + stats.include; \ No newline at end of file + = stats.match + + stats.beginEnd + + stats.include; \ No newline at end of file diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Emoji.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Emoji.rsc index 71d1ff2..200a0c3 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Emoji.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Emoji.rsc @@ -21,4 +21,4 @@ list[ConversionUnit] units = [ ]; test bool analyzeTest() = doAnalyzeTest(rsc, units); -test bool transformTest() = doTransformTest(units, <3, 0, 0>); +test bool transformTest() = doTransformTest(units, <3, 0, 0>, name = "Emoji"); diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Emoji.test b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Emoji.test new file mode 100644 index 0000000..9399ce9 --- /dev/null +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Emoji.test @@ -0,0 +1,17 @@ +# SYNTAX TEST "Emoji" + +🌊 +# <-- constant.language + +foo 🌊 bar +# ^ -constant.language +# ^^ constant.language +# ^ -constant.language + +🙂🙁 +# <---- constant.language + +foo 🙂 bar 🙁 +# ^^ constant.language +# ^^^^^ -constant.language +# ^^ constant.language \ No newline at end of file diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc index 5766d34..b053b5f 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.rsc @@ -15,4 +15,4 @@ list[ConversionUnit] units = [ ]; test bool analyzeTest() = doAnalyzeTest(rsc, units); -test bool transformTest() = doTransformTest(units, <3, 0, 0>); \ No newline at end of file +test bool transformTest() = doTransformTest(units, <3, 0, 0>, name = "Pico"); \ No newline at end of file diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.test b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.test new file mode 100644 index 0000000..3557861 --- /dev/null +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/Pico.test @@ -0,0 +1,11 @@ +# SYNTAX TEST "Pico" + +%% foo bar +# <-- Comment +# ^^^^^^^ Comment + +%% do +# ^^ Comment + +do +# <-- keyword.control \ No newline at end of file diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/PicoWithCategories.rsc b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/PicoWithCategories.rsc index 509551d..9f6744b 100644 --- a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/PicoWithCategories.rsc +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/PicoWithCategories.rsc @@ -60,4 +60,4 @@ list[ConversionUnit] units = [ ]; test bool analyzeTest() = doAnalyzeTest(rsc, units); -test bool transformTest() = doTransformTest(units, <8, 0, 0>); \ No newline at end of file +test bool transformTest() = doTransformTest(units, <8, 0, 0>, name = "PicoWithCategories"); \ No newline at end of file diff --git a/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/PicoWithCategories.test b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/PicoWithCategories.test new file mode 100644 index 0000000..8e8c421 --- /dev/null +++ b/rascal-textmate-core/src/main/rascal/lang/textmate/conversiontests/PicoWithCategories.test @@ -0,0 +1,41 @@ +# SYNTAX TEST "PicoWithCategories" + +%% foo bar +# <-- comment.line +# ^^^^^^^ comment.line + +%% do +# ^^ comment.line + +do +# <-- variable.other +# Note: Keywords can be identifiers in Pico + +123 +# <--- constant.numeric + +foo123 +# <------- variable.other + +foo_123 +# <--- variable.other +# ^ -variable.other +# ^ -constant.numeric +# ^^^ constant.numeric +# Note: Underscores cannot occur in identifiers in Pico + +#begin +## <----- keyword.control +## TODO: `begin` outside `begin`-`end` can never be an identifier in Pico + +#"foo" "bar" +## <----- string.quoted.double +## ^ -string.quoted.double +## ^^^^^ string.quoted.double +## TODO: Support multiline productions + +#"foo\"bar" +## <---- string.quoted.double +## ^^ string.quoted.double +## ^^^^ string.quoted.double +## TODO: Support multiline productions with escaping \ No newline at end of file diff --git a/vscode-extension/syntaxes/rascal.tmLanguage.json b/vscode-extension/syntaxes/rascal.tmLanguage.json index 05935b3..c92421c 100644 --- a/vscode-extension/syntaxes/rascal.tmLanguage.json +++ b/vscode-extension/syntaxes/rascal.tmLanguage.json @@ -1,7 +1,7 @@ { "repository": { "prod(lex(\"Comment\"),[lit(\"//\"),conditional(\\iter-star(\\char-class([range(1,9),range(11,1114111)])),{\\not-follow(\\char-class([range(9,9),range(13,13),range(32,32),range(160,160),range(5760,5760),range(8192,8202),range(8239,8239),range(8287,8287),range(12288,12288)])),\\end-of-line()})],{tag(\"category\"(\"comment\"))})": { - "match": "((?:\\/\\/)(?:(?:[\\x01-\\t]|[\\x0B-\\x{10FFFF}])*?(?!(?:[\\t-\\t]|[\\r-\\r]|[\\x20-\\x20]|[\\xA0-\\xA0]|[\\x{1680}-\\x{1680}]|[\\x{2000}-\\x{200A}]|[\\x{202F}-\\x{202F}]|[\\x{205F}-\\x{205F}]|[\\x{3000}-\\x{3000}]))(?:$)))", + "match": "((?:\\x{2F}\\x{2F})(?:(?:[\\x{1}-\\x{9}]|[\\x{B}-\\x{10FFFF}])*?(?!(?:\\x{9}|\\x{D}|\\x{20}|\\x{A0}|\\x{1680}|[\\x{2000}-\\x{200A}]|\\x{202F}|\\x{205F}|\\x{3000}))(?:$)))", "name": "prod(lex(\"Comment\"),[lit(\"//\"),conditional(\\iter-star(\\char-class([range(1,9),range(11,1114111)])),{\\not-follow(\\char-class([range(9,9),range(13,13),range(32,32),range(160,160),range(5760,5760),range(8192,8202),range(8239,8239),range(8287,8287),range(12288,12288)])),\\end-of-line()})],{tag(\"category\"(\"comment\"))})", "captures": { "1": { @@ -19,16 +19,16 @@ } }, "prod(lex(\"delimiters\"),[alt({lit(\"bottom-up-break\"),lit(\")\"),lit(\"≫\"),lit(\"\\n\"),lit(\"!:=\"),lit(\"\\'\"),lit(\"!=\"),lit(\"\\>=\"),lit(\"://\"),lit(\"non-assoc\"),lit(\"&=\"),lit(\"\\<-\"),lit(\"*=\"),lit(\"+=\"),lit(\"top-down-break\"),lit(\",\"),lit(\"...\"),lit(\"/=\"),lit(\"⇨\"),lit(\"(\"),lit(\"*/\"),lit(\"%\"),lit(\"!\\<\\<\"),lit(\"=\\>\"),lit(\"!\\>\\>\"),lit(\"||\"),lit(\"\\>\\>\"),lit(\"::\"),lit(\"⚠\"),lit(\"&&\"),lit(\"@\"),lit(\":=\"),lit(\"#\"),lit(\"\\<==\\>\"),lit(\"\\\"\"),lit(\"\\<\\<=\"),lit(\"}\"),lit(\"?=\"),lit(\"\\<:\"),lit(\"==\\>\"),lit(\"^\"),lit(\"/*\"),lit(\";\"),lit(\"{\"),lit(\"-=\"),lit(\"$T\")})],{})": { - "match": "(?:(?:bottom\\-up\\-break)|(?:\\))|(?:\\x{226B})|(?:\\n)|(?:\\!\\:\\=)|(?:\\')|(?:\\!\\=)|(?:\\>\\=)|(?:\\:\\/\\/)|(?:non\\-assoc)|(?:\\&\\=)|(?:\\<\\-)|(?:\\*\\=)|(?:\\+\\=)|(?:top\\-down\\-break)|(?:\\,)|(?:\\.\\.\\.)|(?:\\/\\=)|(?:\\x{21E8})|(?:\\()|(?:\\*\\/)|(?:\\%)|(?:\\!\\<\\<)|(?:\\=\\>)|(?:\\!\\>\\>)|(?:\\|\\|)|(?:\\>\\>)|(?:\\:\\:)|(?:\\x{26A0})|(?:\\&\\&)|(?:\\@)|(?:\\:\\=)|(?:\\#)|(?:\\<\\=\\=\\>)|(?:\\\")|(?:\\<\\<\\=)|(?:\\})|(?:\\?\\=)|(?:\\<\\:)|(?:\\=\\=\\>)|(?:\\^)|(?:\\/\\*)|(?:\\;)|(?:\\{)|(?:\\-\\=)|(?:\\$T))", + "match": "(?:(?:bottom\\x{2D}up\\x{2D}break)|(?:\\x{29})|(?:\\x{226B})|(?:\\x{A})|(?:\\x{21}\\x{3A}\\x{3D})|(?:\\x{27})|(?:\\x{21}\\x{3D})|(?:\\x{3E}\\x{3D})|(?:\\x{3A}\\x{2F}\\x{2F})|(?:non\\x{2D}assoc)|(?:\\x{26}\\x{3D})|(?:\\x{3C}\\x{2D})|(?:\\x{2A}\\x{3D})|(?:\\x{2B}\\x{3D})|(?:top\\x{2D}down\\x{2D}break)|(?:\\x{2C})|(?:\\x{2E}\\x{2E}\\x{2E})|(?:\\x{2F}\\x{3D})|(?:\\x{21E8})|(?:\\x{28})|(?:\\x{2A}\\x{2F})|(?:\\x{25})|(?:\\x{21}\\x{3C}\\x{3C})|(?:\\x{3D}\\x{3E})|(?:\\x{21}\\x{3E}\\x{3E})|(?:\\x{7C}\\x{7C})|(?:\\x{3E}\\x{3E})|(?:\\x{3A}\\x{3A})|(?:\\x{26A0})|(?:\\x{26}\\x{26})|(?:\\x{40})|(?:\\x{3A}\\x{3D})|(?:\\x{23})|(?:\\x{3C}\\x{3D}\\x{3D}\\x{3E})|(?:\\x{22})|(?:\\x{3C}\\x{3C}\\x{3D})|(?:\\x{7D})|(?:\\x{3F}\\x{3D})|(?:\\x{3C}\\x{3A})|(?:\\x{3D}\\x{3D}\\x{3E})|(?:\\x{5E})|(?:\\x{2F}\\x{2A})|(?:\\x{3B})|(?:\\x{7B})|(?:\\x{2D}\\x{3D})|(?:\\x{24}T))", "name": "prod(lex(\"delimiters\"),[alt({lit(\"bottom-up-break\"),lit(\")\"),lit(\"≫\"),lit(\"\\n\"),lit(\"!:=\"),lit(\"\\'\"),lit(\"!=\"),lit(\"\\>=\"),lit(\"://\"),lit(\"non-assoc\"),lit(\"&=\"),lit(\"\\<-\"),lit(\"*=\"),lit(\"+=\"),lit(\"top-down-break\"),lit(\",\"),lit(\"...\"),lit(\"/=\"),lit(\"⇨\"),lit(\"(\"),lit(\"*/\"),lit(\"%\"),lit(\"!\\<\\<\"),lit(\"=\\>\"),lit(\"!\\>\\>\"),lit(\"||\"),lit(\"\\>\\>\"),lit(\"::\"),lit(\"⚠\"),lit(\"&&\"),lit(\"@\"),lit(\":=\"),lit(\"#\"),lit(\"\\<==\\>\"),lit(\"\\\"\"),lit(\"\\<\\<=\"),lit(\"}\"),lit(\"?=\"),lit(\"\\<:\"),lit(\"==\\>\"),lit(\"^\"),lit(\"/*\"),lit(\";\"),lit(\"{\"),lit(\"-=\"),lit(\"$T\")})],{})", - "captures": {} + "captures": [] }, "lit(\"`\"):lit(\"`\")": { - "begin": "(?:\\`)", - "end": "(?:\\`)", + "begin": "(?:\\x{60})", + "end": "(?:\\x{60})", "patterns": [ { - "match": "(\\\\\\\\)", + "match": "(\\x{5C}\\x{5C})", "name": "prod(label(\"bs\",lex(\"ConcretePart\")),[lit(\"\\\\\\\\\")],{tag(\"category\"(\"string\"))})", "captures": { "1": { @@ -37,7 +37,7 @@ } }, { - "match": "(\\\\\\>)", + "match": "(\\x{5C}\\x{3E})", "name": "prod(label(\"gt\",lex(\"ConcretePart\")),[lit(\"\\\\\\>\")],{tag(\"category\"(\"string\"))})", "captures": { "1": { @@ -46,7 +46,7 @@ } }, { - "match": "((?:[\\x01-\\t]|[\\x0B-\\;]|[\\=-\\=]|[\\?-\\[]|[\\]-\\_]|[a-\\x{10FFFF}])+?(?!(?:[\\x01-\\t]|[\\x0B-\\;]|[\\=-\\=]|[\\?-\\[]|[\\]-\\_]|[a-\\x{10FFFF}])))", + "match": "((?:[\\x{1}-\\x{9}]|[\\x{B}-\\x{3B}]|\\x{3D}|[\\x{3F}-\\x{5B}]|[\\x{5D}-\\x{5F}]|[a-\\x{10FFFF}])+?(?!(?:[\\x{1}-\\x{9}]|[\\x{B}-\\x{3B}]|\\x{3D}|[\\x{3F}-\\x{5B}]|[\\x{5D}-\\x{5F}]|[a-\\x{10FFFF}])))", "name": "prod(label(\"text\",lex(\"ConcretePart\")),[conditional(iter(\\char-class([range(1,9),range(11,59),range(61,61),range(63,91),range(93,95),range(97,1114111)])),{\\not-follow(\\char-class([range(1,9),range(11,59),range(61,61),range(63,91),range(93,95),range(97,1114111)]))})],{tag(\"category\"(\"string\"))})", "captures": { "1": { @@ -55,7 +55,7 @@ } }, { - "match": "(\\\\\\<)", + "match": "(\\x{5C}\\x{3C})", "name": "prod(label(\"lt\",lex(\"ConcretePart\")),[lit(\"\\\\\\<\")],{tag(\"category\"(\"string\"))})", "captures": { "1": { @@ -64,7 +64,7 @@ } }, { - "match": "(\\\\\\`)", + "match": "(\\x{5C}\\x{60})", "name": "prod(label(\"bq\",lex(\"ConcretePart\")),[lit(\"\\\\`\")],{tag(\"category\"(\"string\"))})", "captures": { "1": { @@ -76,11 +76,11 @@ "name": "lit(\"`\"):lit(\"`\")" }, "lit(\"[\"):lit(\"]\")": { - "begin": "(?:\\[)", - "end": "(?:\\])", + "begin": "(?:\\x{5B})", + "end": "(?:\\x{5D})", "patterns": [ { - "match": "((?:\\\\)(?:[\\x20-\\x20]|[\\\"-\\\"]|[\\'-\\']|[\\--\\-]|[\\<-\\<]|[\\>-\\>]|[\\[-\\]]|[b-b]|[f-f]|[n-n]|[r-r]|[t-t]))", + "match": "((?:\\x{5C})(?:\\x{20}|\\x{22}|\\x{27}|\\x{2D}|\\x{3C}|\\x{3E}|[\\x{5B}-\\x{5D}]|b|f|n|r|t))", "name": "prod(lex(\"Char\"),[lit(\"\\\\\"),\\char-class([range(32,32),range(34,34),range(39,39),range(45,45),range(60,60),range(62,62),range(91,93),range(98,98),range(102,102),range(110,110),range(114,114),range(116,116)])],{tag(\"category\"(\"constant\"))})", "captures": { "1": { @@ -89,7 +89,7 @@ } }, { - "match": "((?:(?:\\\\)[U-U](?:(?:\\b10\\b)|(?:(?:\\b0\\b)(?:[0-9]|[A-F]|[a-f])))(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f]))|(?:(?:\\\\)[u-u](?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f]))|(?:(?:\\\\)[a-a][0-7](?:[0-9]|[A-F]|[a-f])))", + "match": "((?:(?:\\x{5C})U(?:(?:\\b10\\b)|(?:(?:\\b0\\b)(?:[0-9]|[A-F]|[a-f])))(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f]))|(?:(?:\\x{5C})u(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f])(?:[0-9]|[A-F]|[a-f]))|(?:(?:\\x{5C})a[0-7](?:[0-9]|[A-F]|[a-f])))", "name": "prod(lex(\"Char\"),[lex(\"UnicodeEscape\")],{tag(\"category\"(\"constant\"))})", "captures": { "1": {