From 0b60259b002a61964c0f0a85589d625946f508e9 Mon Sep 17 00:00:00 2001 From: Kelly Joseph Price Date: Thu, 6 Jun 2024 16:18:52 -0700 Subject: [PATCH] feat: plain (#898) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit | [![PR App][icn]][demo] | RM-9817 | | :--------------------: | :-----: | ## 🧰 Changes Updates `plain` and `hast`. These are used in the main app for indexing. ## 🧬 QA & Testing - [Broken on production][prod]. - [Working in this PR app][demo]. [demo]: https://markdown-pr-PR_NUMBER.herokuapp.com [prod]: https://SUBDOMAIN.readme.io [icn]: https://user-images.githubusercontent.com/886627/160426047-1bee9488-305a-4145-bb2b-09d8b757d38a.svg --- __tests__/astToPlainText.test.js | 118 ------------------ __tests__/lib/hast.test.ts | 24 ++++ __tests__/lib/plain.test.ts | 56 +++++++++ index.tsx | 28 +---- lib/hast.ts | 22 ++++ lib/index.ts | 7 +- lib/mdast.ts | 11 ++ .../plugin/plain-text.js => lib/plain.ts | 26 ++-- package-lock.json | 1 + package.json | 1 + processor/transform/index.ts | 5 +- processor/transform/inject-components.ts | 27 ++++ types.d.ts | 2 + 13 files changed, 164 insertions(+), 164 deletions(-) delete mode 100644 __tests__/astToPlainText.test.js create mode 100644 __tests__/lib/hast.test.ts create mode 100644 __tests__/lib/plain.test.ts create mode 100644 lib/hast.ts create mode 100644 lib/mdast.ts rename processor/plugin/plain-text.js => lib/plain.ts (66%) create mode 100644 processor/transform/inject-components.ts diff --git a/__tests__/astToPlainText.test.js b/__tests__/astToPlainText.test.js deleted file mode 100644 index 3eecdbc14..000000000 --- a/__tests__/astToPlainText.test.js +++ /dev/null @@ -1,118 +0,0 @@ -import { hast, astToPlainText } from '../index'; - -const find = (node, matcher) => { - if (matcher(node)) return node; - if (node.children) { - return node.children.find(child => find(child, matcher)); - } - - return null; -}; - -describe.skip('astToPlainText()', () => { - it("converts br's to ''", () => { - const txt = '
'; - - expect(astToPlainText(hast(txt))).toBe(''); - }); - - it("converts hr's to ''", () => { - const txt = '
'; - - expect(astToPlainText(hast(txt))).toBe(''); - }); - - it('converts flavored callouts', () => { - const txt = ` -> 📘 Title -> -> Some body - `; - - expect(astToPlainText(hast(txt))).toBe('Title Some body'); - }); - - it('converts markdown tables', () => { - const txt = ` -| Header 1 | Header 2 | -| :------- | :------- | -| Cell 1 | Cell 2 | - `; - - expect(astToPlainText(hast(txt))).toBe('Header 1 Header 2 Cell 1 Cell 2'); - }); - - it('converts magic block tables', () => { - const txt = ` -[block:parameters] -${JSON.stringify( - { - data: { - 'h-0': 'Header 1', - 'h-1': 'Header 2', - '0-0': 'Cell 1', - '0-1': 'Cell 2 \nCell 2.1', - }, - cols: 2, - rows: 1, - align: ['left', 'left', 'left'], - }, - null, - 2, -)} -[/block] - `; - - expect(astToPlainText(hast(txt))).toBe('Header 1 Header 2 Cell 1 Cell 2 \nCell 2.1'); - }); - - it('converts images', () => { - const txt = ` -![image **label**](http://placekitten.com/600/600 "entitled kittens") - `; - - expect(astToPlainText(hast(txt))).toBe('entitled kittens'); - }); - - it('converts a single image', () => { - const txt = ` -![image **label**](http://placekitten.com/600/600 "entitled kittens") - `; - const ast = hast(txt); - - expect(astToPlainText(find(ast, n => n.tagName === 'img'))).toBe('entitled kittens'); - }); - - it('converts magic block images', () => { - const txt = ` - [block:image] - { - "images": [ - { - "image": ["https://files.readme.io/test.png", "Test Image Title", 100, 100, "#fff"] - } - ] - } - [/block] - `; - - expect(astToPlainText(hast(txt))).toBe('Test Image Title'); - }); - - it('converts a lone magic block image', () => { - const txt = ` - [block:image] - { - "images": [ - { - "image": ["https://files.readme.io/test.png", "Test Image Title", 100, 100, "#fff"] - } - ] - } - [/block] - `; - const img = find(hast(txt), n => n.tagName === 'img'); - - expect(astToPlainText(img)).toBe('Test Image Title'); - }); -}); diff --git a/__tests__/lib/hast.test.ts b/__tests__/lib/hast.test.ts new file mode 100644 index 000000000..35fb91198 --- /dev/null +++ b/__tests__/lib/hast.test.ts @@ -0,0 +1,24 @@ +import { hast } from '../../lib'; +import { h } from 'hastscript'; + +describe('hast transformer', () => { + it('parses components into the tree', () => { + const md = ` +## Test + + + `; + const components = { + Example: "## It's coming from within the component!", + }; + + const expected = h( + undefined, + h('h2', undefined, 'Test'), + '\n', + h('h2', undefined, "It's coming from within the component!"), + ); + + expect(hast(md, { components })).toStrictEqualExceptPosition(expected); + }); +}); diff --git a/__tests__/lib/plain.test.ts b/__tests__/lib/plain.test.ts new file mode 100644 index 000000000..6821fc775 --- /dev/null +++ b/__tests__/lib/plain.test.ts @@ -0,0 +1,56 @@ +import { hast, plain } from '../../index'; + +describe('plain compiler', () => { + it('returns plain text of markdown components', () => { + const md = ` +## Hello! + +Is it _me_ you're looking for? +`; + + const tree = hast(md); + expect(plain(tree)).toEqual("Hello! Is it me you're looking for?"); + }); + + it("compiles br's to ''", () => { + const txt = '
'; + + expect(plain(hast(txt))).toBe(''); + }); + + it("compiles hr's to ''", () => { + const txt = '
'; + + expect(plain(hast(txt))).toBe(''); + }); + + it('compiles callouts', () => { + const txt = ` +> 📘 Title +> +> Some body + `; + const tree = hast(txt); + + expect(plain(tree)).toBe('Title Some body'); + }); + + it('compiles markdown tables', () => { + const txt = ` +| Header 1 | Header 2 | +| :------- | :------- | +| Cell 1 | Cell 2 | + `; + + expect(plain(hast(txt))).toBe('Header 1 Header 2 Cell 1 Cell 2'); + }); + + it('compiles images to their title', () => { + const txt = ` +![image **label**](http://placekitten.com/600/600 "entitled kittens") + `; + const tree = hast(txt); + + expect(plain(tree)).toBe('entitled kittens'); + }); +}); diff --git a/index.tsx b/index.tsx index 64c56bd39..e0b42152a 100644 --- a/index.tsx +++ b/index.tsx @@ -1,5 +1,4 @@ import debug from 'debug'; -import remarkRehype from 'remark-rehype'; import { createProcessor } from '@mdx-js/mdx'; @@ -7,17 +6,12 @@ import * as Components from './components'; import { getHref } from './components/Anchor'; import { options } from './options'; -import { readmeComponentsTransformer } from './processor/transform'; -import { compile, run, mdx, astProcessor, remarkPlugins } from './lib'; +import { compile, hast, run, mdast, mdx, plain, remarkPlugins } from './lib'; import './styles/main.scss'; const unimplemented = debug('mdx:unimplemented'); -type MdastOpts = { - components?: Record; -}; - const utils = { get options() { return { ...options }; @@ -27,8 +21,6 @@ const utils = { calloutIcons: {}, }; -export { compile, run, mdx, Components, utils }; - export const reactProcessor = (opts = {}) => { return createProcessor({ remarkPlugins, ...opts }); }; @@ -37,24 +29,8 @@ export const html = (text: string, opts = {}) => { unimplemented('html export'); }; -export const mdast: any = (text: string, opts: MdastOpts = {}) => { - const processor = astProcessor(opts).use(readmeComponentsTransformer({ components: opts.components })); - - const tree = processor.parse(text); - return processor.runSync(tree); -}; - -export const hast = (text: string, opts = {}) => { - const processor = astProcessor(opts).use(remarkRehype); - - const tree = processor.parse(text); - return processor.runSync(tree); -}; - export const esast = (text: string, opts = {}) => { unimplemented('esast export'); }; -export const plain = (text: string, opts = {}) => { - unimplemented('plain export'); -}; +export { compile, hast, run, mdast, mdx, plain, Components, utils }; diff --git a/lib/hast.ts b/lib/hast.ts new file mode 100644 index 000000000..bc13fd7bb --- /dev/null +++ b/lib/hast.ts @@ -0,0 +1,22 @@ +import astProcessor from './ast-processor'; +import remarkRehype from 'remark-rehype'; +import { injectComponents } from '../processor/transform'; +import { MdastComponents } from '../types'; +import mdast from './mdast'; + +interface Options { + components?: Record; +} + +const hast = (text: string, opts: Options = {}) => { + const components: MdastComponents = Object.entries(opts.components || {}).reduce((memo, [name, doc]) => { + memo[name] = mdast(doc); + return memo; + }, {}); + + const processor = astProcessor(opts).use(injectComponents({ components })).use(remarkRehype); + + return processor.runSync(processor.parse(text)); +}; + +export default hast; diff --git a/lib/index.ts b/lib/index.ts index 3109aa683..4f9876c69 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -1,7 +1,10 @@ import astProcessor, { MdastOpts, remarkPlugins } from './ast-processor'; -import compile from './compile' +import compile from './compile'; +import hast from './hast'; +import mdast from './mdast'; import mdx from './mdx'; +import plain from './plain'; import run from './run'; export type { MdastOpts }; -export { astProcessor, compile, mdx, run, remarkPlugins } +export { astProcessor, compile, hast, mdast, mdx, plain, run, remarkPlugins }; diff --git a/lib/mdast.ts b/lib/mdast.ts new file mode 100644 index 000000000..8e0eb26d3 --- /dev/null +++ b/lib/mdast.ts @@ -0,0 +1,11 @@ +import { readmeComponentsTransformer } from '../processor/transform'; +import astProcessor, { MdastOpts } from './ast-processor'; + +const mdast: any = (text: string, opts: MdastOpts = {}) => { + const processor = astProcessor(opts).use(readmeComponentsTransformer({ components: opts.components })); + + const tree = processor.parse(text); + return processor.runSync(tree); +}; + +export default mdast; diff --git a/processor/plugin/plain-text.js b/lib/plain.ts similarity index 66% rename from processor/plugin/plain-text.js rename to lib/plain.ts index b45be5b52..b7bbd796f 100644 --- a/processor/plugin/plain-text.js +++ b/lib/plain.ts @@ -1,11 +1,13 @@ +import { Node } from 'hast-util-to-text'; + /* @note: copied from https://github.com/rehypejs/rehype-minify/blob/main/packages/hast-util-to-string/index.js */ -function toString(node) { - // eslint-disable-next-line no-use-before-define + +const plain = (node: Node, opts = {}) => { return 'children' in node ? all(node) || one(node) : one(node); -} +}; -function one(node) { +const one = (node: Node) => { if (node.tagName === 'img') { return node.properties?.title || ''; } @@ -20,9 +22,9 @@ function one(node) { // eslint-disable-next-line no-use-before-define return 'children' in node ? all(node) : ' '; -} +}; -function all(node) { +const all = (node: Node) => { let index = -1; const result = []; @@ -31,15 +33,7 @@ function all(node) { result[index] = one(node.children[index]); } - return result.join(' ').trim().replace(/ +/, ' '); -} - -const Compiler = node => { - return toString(node); -}; - -const toPlainText = function () { - Object.assign(this, { Compiler }); + return result.join(' ').replaceAll(/\s+/g, ' ').trim(); }; -module.exports = toPlainText; +export default plain; diff --git a/package-lock.json b/package-lock.json index 7220e7a4b..b763d0479 100644 --- a/package-lock.json +++ b/package-lock.json @@ -77,6 +77,7 @@ "codemirror": "^5.54.0", "css-loader": "^6.7.3", "eslint": "^8.37.0", + "hast-util-to-text": "^4.0.2", "identity-obj-proxy": "^3.0.0", "jest": "^29.5.0", "jest-environment-jsdom": "^29.5.0", diff --git a/package.json b/package.json index 4bf5e61a8..2e15f18bb 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "codemirror": "^5.54.0", "css-loader": "^6.7.3", "eslint": "^8.37.0", + "hast-util-to-text": "^4.0.2", "identity-obj-proxy": "^3.0.0", "jest": "^29.5.0", "jest-environment-jsdom": "^29.5.0", diff --git a/processor/transform/index.ts b/processor/transform/index.ts index ece1548ec..3c8687815 100644 --- a/processor/transform/index.ts +++ b/processor/transform/index.ts @@ -2,10 +2,11 @@ import calloutTransformer from './callouts'; import codeTabsTransfromer from './code-tabs'; import embedTransformer from './embeds'; import gemojiTransformer from './gemoji+'; +import injectComponents from './inject-components'; import readmeComponentsTransformer from './readme-components'; -import rehypeToc from './rehype-toc'; import readmeToMdx from './readme-to-mdx'; +import rehypeToc from './rehype-toc'; -export { readmeComponentsTransformer, rehypeToc, readmeToMdx }; +export { readmeComponentsTransformer, rehypeToc, readmeToMdx, injectComponents }; export default [calloutTransformer, codeTabsTransfromer, embedTransformer, gemojiTransformer]; diff --git a/processor/transform/inject-components.ts b/processor/transform/inject-components.ts new file mode 100644 index 000000000..c90ce23c7 --- /dev/null +++ b/processor/transform/inject-components.ts @@ -0,0 +1,27 @@ +import { MdastComponents } from '../../types'; +import { visit } from 'unist-util-visit'; +import { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx'; +import { Transform } from 'mdast-util-from-markdown'; +import { Parents } from 'mdast'; + +interface Options { + components?: MdastComponents; +} + +const inject = + ({ components }: Options = {}) => + (node: MdxJsxFlowElement | MdxJsxTextElement, index: number, parent: Parents) => { + if (!(node.name in components)) return; + + const { children } = components[node.name]; + parent.children.splice(index, children.length, ...(children as any)); + }; + +const injectComponents = (opts: Options) => (): Transform => tree => { + visit(tree, 'mdxJsxFlowElement', inject(opts)); + visit(tree, 'mdxJsxTextElement', inject(opts)); + + return tree; +}; + +export default injectComponents; diff --git a/types.d.ts b/types.d.ts index 0f1768962..8e227f717 100644 --- a/types.d.ts +++ b/types.d.ts @@ -105,3 +105,5 @@ type VFileWithToc = VFile & { }; interface CompiledComponents extends Record {} + +interface MdastComponents extends Record {}