add: --includes-map and --ext-links-check

README.md

@@ -27,7 +27,7 @@ npm i foliant-md-linter
 
 You can invoke foliant-md-linter in a Foliant building process.
 
-#### Prerequisites 
+#### Prerequisites
 
 - Install _foliant-md-linter_ into your Docker-image if you're using a building within Docker:
 
@@ -68,56 +68,60 @@ if you haven't installed it yet:
   $ foliant make site --with mkdocs
   Parsing config... Done
   Applying preprocessor runcommands... markdownlint-cli2 v0.4.0 (markdownlint v0.25.1)
-  
+
   Finding: src/**/*.md
   ...
   Found 5 styleguide and formatting errors
   Full markdownlint log see in /usr/src/app/.markdownlint_full.log
-  
+
   removing /usr/src/app/.markdownlint-cli2.jsonc ...
-  
+
   Done
   Applying preprocessor mkdocs... Done
   ...
   ```
 
 ## Usage
 
-Run _foliant-md-linter_ from the project root with following commands and options
+Run _foliant-md-linter_ from the project root with following commands and options:
+
+- `full-check` - check md files with markdownlint and markdown-link-check
+    - `-v`, `--verbose` - print full linting results (default: false)
+    - `-s`, `--source <path-to-sources>` - specify source directory (default: _src_)
+    - `-c`, `--config` - do not create a new markdownlint config file and use default or one in root directory instead (default: false)
+    - `-p`, `--project <project-name>` - specify project name
+    - `-d`, `--debug` - print executing command (default: false)
+    - `-f`, `--allowfailure` - allow exit with failure if errors (default: false)
 
-- `full-check` Check md files with markdownlint and markdown-link-check
-    - `-v`, `--verbose` Print full linting results (default: false)
-    - `-s`, `--source <path-to-sources>` specify source directory (default: _src_)
-    - `-c`, `--config` Do not create a new markdownlint config file and use default or one in root directory instead (default: false)
-    - `-p`, `--project <project-name>` specify project name
-    - `-d`, `--debug` print executing command (default: false)
-    - `-f`, `--allowfailure` allow exit with failure if errors (default: false)
-
       _helpful in CI/CD, as you can cause pipelines to fail in case of linting errors_
-  
-    - `-l`, `--clearconfig` remove markdownlint config after execution (default: false)
+
+    - `-l`, `--clearconfig` - remove markdownlint config after execution (default: false)
 
       _helpful within docker, otherwise annoying bugs are occurred with the markdownlint extension for VSCode_
 
-- `essential` Check md files for critical formatting errors with markdownlint and validate external links ith markdown-link-check
-  - `-v`, `-s`, `-c`, `-p`, `-d`, `-f`, `-l`
-- `urls` Validate external links with markdown-link-check
-    - `-v`, `-s`, `-d`, `-f`, `-l`
-- `styleguide` Check for styleguide adherence with markdownlint
+    - `--includes-map` - set the path to the includes map (default: `./includes_map.json`)
+
+    - `--ext-links-check <command-for-get-list>` - specify command for [get list of files](#ext-links-check).
+
+- `essential` check md files for critical formatting errors with markdownlint and validate external links ith markdown-link-check
+  - `-v`, `-s`, `-c`, `-p`, `-d`, `-f`, `-l`, `--includes-map`,`--ext-links-check`
+- `urls` validate external links with markdown-link-check
+    - `-v`, `-s`, `-d`, `-f`, `-l`, `--includes-map`,`--ext-links-check`
+- `styleguide` check for styleguide adherence with markdownlint
+    - `-v`, `-s`, `-c`, `-p`, `-d`, `-f`, `-l`, `--includes-map`,`--ext-links-check`
+- `slim` check for critical errors with markdownlint
+    - `-v`, `-s`, `-c`, `-p`, `-d`, `-f`, `-l`, `--includes-map`,`--ext-links-check`
+- `fix` fix formatting errors with markdownlint
+    - `-v`, `-s`, `-c`, `-p`, `-d`, `-f`, `-l`, `--includes-map`,`--ext-links-check`
+- `typograph` fix typograph errors with markdownlint
     - `-v`, `-s`, `-c`, `-p`, `-d`, `-f`, `-l`
-- `slim` Check for critical errors with markdownlint
-    - `-v`, `-s`, `-c`, `-p`, `-d`, `-f`, `-l`
-- `fix` Fix formatting errors with markdownlint
-    - `-v`, `-s`, `-c`, `-p`, `-d`, `-f`, `-l`
-- `typograph` Fix typograph errors with markdownlint
-    - `-v`, `-s`, `-c`, `-p`, `-d`, `-f`, `-l`
-- `print` Print linting results
+- `print` print linting results
     - `-v`
-- `create-full-config` Create markdownlint config for styleguide adherence
+- `create-full-config` create markdownlint config for styleguide adherence
     - `-s`, `-p`, `-d`
-- `create-slim-config` Create markdownlint config for critical errors check
+- `create-slim-config` create markdownlint config for critical errors check
     - `-s`, `-p`, `-d`
-- `create-typograph-config` Create typograph config for typograph errors check
+- `create-typograph-config` create typograph config for typograph errors check
     - `-s`, `-p`, `-d`
 
 ### Examples
@@ -206,3 +210,23 @@ and use it for the next _markdownlint_ runs by `-c` option
 ```bash
 $ npx foliant-md-linter full-check -c
 ```
+
+### Checking external links only in the file list {#ext-links-check}
+
+Checking external links takes a long time, so you can run the external link check only in a given list of files.
+In this case, internal links will be checked in all files.
+
+Example, commands to check external links only in modified files.
+The list of modified files was obtained by the `git` command comparing the current branch with `origin/master`:
+
+**Unix**
+
+```bash
+$ npx foliant-md-linter full-check -v -p project-name -l --ext-links-check 'git diff --name-only HEAD origin/master | grep .md$'
+```
+
+**Windows**
+
+```bash
+$ npx foliant-md-linter full-check -v -p project-name -l --ext-links-check 'git diff --name-only HEAD origin/master | findstr .md'
+```

generate.js

@@ -6,7 +6,7 @@ const fs = require('fs')
 const program = new Command()
 const cwd = process.cwd().toString()
 
-function createConfig (mode = 'full', source = '', project = '') {
+function createConfig (mode = 'full', source = '', project = '', includesMap = '') {
   let customRules
   if (fs.existsSync(path.join(__dirname, '/node_modules/markdownlint-rules-foliant/package.json'))) {
     customRules = [
@@ -74,7 +74,8 @@ function createConfig (mode = 'full', source = '', project = '') {
     typograph: true,
     'validate-internal-links': {
       src: source.length === 0 ? undefined : source,
-      project: project.length === 0 ? undefined : project
+      project: project.length === 0 ? undefined : project,
+      includesMap: project.length === 0 ? undefined : includesMap
     },
     'frontmatter-tags-exist': false
   }
@@ -123,7 +124,8 @@ function createConfig (mode = 'full', source = '', project = '') {
     typograph: false,
     'validate-internal-links': {
       src: source.length === 0 ? undefined : source,
-      project: project.length === 0 ? undefined : project
+      project: project.length === 0 ? undefined : project,
+      includesMap: project.length === 0 ? undefined : includesMap
     },
     'frontmatter-tags-exist': false
   }
@@ -209,8 +211,9 @@ program
   .option('-m, --mode <mode>', 'full, slim, typograph or default config', 'full')
   .option('-s, --source <source>', 'relative path to source directory', '')
   .option('-p, --project <project>', 'project name', '')
+  .option('--includes-map <includes-map>', 'includes map path', '')
 
 program.parse()
 
 const options = program.opts()
-createConfig(options.mode, options.source, options.project)
+createConfig(options.mode, options.source, options.project, options.includesMap)

linter.js

@@ -132,18 +132,24 @@ function writeLog (logFile) {
   return (isWin === true) ? `>> ${logFile} 2>&1` : `2>&1 | tee ${logFile}`
 }
 
-const commandsGen = function (src = defaultSrc, customConfig = false, project = '') {
+const commandsGen = function (src = defaultSrc, customConfig = false, project = '', includesMap = '', listOfFiles = '') {
+  let findCommandUnix = `${src}/ -type f -name '*.md'`
+  let findCommandWin = `del ${path.join(cwd, markdownLinkCheckLog)} & forfiles /P ${src} /S /M *.md`
+  if (listOfFiles !== '*.md' && listOfFiles !== '') {
+    findCommandUnix = `$(${listOfFiles})`
+    findCommandWin = `${listOfFiles}`
+  }
   const commands = {}
   const and = (isWin === true) ? '&' : ';'
-  commands.createFullMarkdownlintConfig = (customConfig === false) ? `node ${path.join(__dirname, '/generate.js')} -m full -s ${src} -p "${project}"` : 'echo "using custom config"'
-  commands.createSlimMarkdownlintConfig = (customConfig === false) ? `node ${path.join(__dirname, '/generate.js')} -m slim -s ${src} -p "${project}"` : 'echo "using custom config"'
+  commands.createFullMarkdownlintConfig = (customConfig === false) ? `node ${path.join(__dirname, '/generate.js')} -m full -s ${src} --includes-map ${includesMap} -p "${project}"` : 'echo "using custom config"'
+  commands.createSlimMarkdownlintConfig = (customConfig === false) ? `node ${path.join(__dirname, '/generate.js')} -m slim -s ${src} --includes-map ${includesMap} -p "${project}"` : 'echo "using custom config"'
   commands.createTypographMarkdownlintConfig = (customConfig === false) ? `node ${path.join(__dirname, '/generate.js')} -m typograph -s ${src} -p "${project}"` : 'echo "using custom config"'
   commands.markdownlintSrcSlim = `${commands.createSlimMarkdownlintConfig} && ${execPath}/markdownlint-cli2 "${src}/**/*.md" ${writeLog(markdownLintSlimLog)}`
   commands.markdownlintSrcFull = `${commands.markdownlintSrcSlim} ${and} ${commands.createFullMarkdownlintConfig} && ${execPath}/markdownlint-cli2 "${src}/**/*.md" ${writeLog(markdownLintFullLog)}`
   commands.markdownlintSrcFix = `${commands.markdownlintSrcSlim} ${and} ${commands.createFullMarkdownlintConfig} && ${execPath}/markdownlint-cli2-fix "${src}/**/*.md" ${writeLog(markdownLintFullLog)}`
   commands.markdownlintSrcTypograph = `${commands.createTypographMarkdownlintConfig} && ${execPath}/markdownlint-cli2-fix "${src}/**/*.md" ${writeLog(markdownLintFullLog)}`
-  commands.markdownlinkcheckSrcUnix = `find ${src}/ -type f -name '*.md' -print0 | xargs -0 -n1 ${execPath}/markdown-link-check -p -c ${path.join(__dirname, '/configs/mdLinkCheckConfig.json')} ${writeLog(path.join(cwd, markdownLinkCheckLog))}`
-  commands.markdownlinkcheckSrcWin = `del ${path.join(cwd, markdownLinkCheckLog)} & forfiles /P ${src} /S /M *.md /C "cmd /c npx markdown-link-check @file -p -c ${path.join(__dirname, '/configs/mdLinkCheckConfig.json')} ${writeLog(path.join(cwd, markdownLinkCheckLog))}"`
+  commands.markdownlinkcheckSrcUnix = `find ${findCommandUnix} -print0 | xargs -0 -n1 ${execPath}/markdown-link-check -p -c ${path.join(__dirname, '/configs/mdLinkCheckConfig.json')} ${writeLog(path.join(cwd, markdownLinkCheckLog))}`
+  commands.markdownlinkcheckSrcWin = `${findCommandWin} /C "cmd /c npx markdown-link-check @file -p -c ${path.join(__dirname, '/configs/mdLinkCheckConfig.json')} ${writeLog(path.join(cwd, markdownLinkCheckLog))}"`
   commands.markdownlinkcheckSrc = (isWin === true) ? commands.markdownlinkcheckSrcWin : commands.markdownlinkcheckSrcUnix
   commands.lintSrcEssential = `${commands.markdownlintSrcSlim} & ${commands.markdownlinkcheckSrc}`
   commands.lintSrcFull = `${commands.markdownlintSrcFull} & ${commands.markdownlinkcheckSrc}`
@@ -214,6 +220,8 @@ const projectOption = new Option('-p, --project <project-name>', 'project name')
 const debugOption = new Option('-d, --debug', 'print executing command').default(false)
 const allowfailureOption = new Option('-f, --allowfailure', 'allow exit with failure if errors').default(false)
 const clearconfigOption = new Option('-l, --clearconfig', 'remove markdownlint config after execution').default(false)
+const includesMap = new Option('--includes-map <includes-map>', 'includes map path').default('./includes_map.json')
+const listOfFiles = new Option('--ext-links-check <list-of-files>', 'the list of files to be checked').default('*.md')
 
 program
   .name('foliant-md-linter')
@@ -229,8 +237,10 @@ program.command('full-check')
   .addOption(debugOption)
   .addOption(allowfailureOption)
   .addOption(clearconfigOption)
+  .addOption(includesMap)
+  .addOption(listOfFiles)
   .action((options) => {
-    execute(commandsGen(options.source, options.config, options.project).commands.lintSrcFull, options.verbose, options.debug, options.allowfailure, options.clearconfig)
+    execute(commandsGen(options.source, options.config, options.project, options.includesMap, options.extLinksCheck).commands.lintSrcFull, options.verbose, options.debug, options.allowfailure, options.clearconfig)
   })
 
 program.command('essential')
@@ -242,8 +252,10 @@ program.command('essential')
   .addOption(debugOption)
   .addOption(allowfailureOption)
   .addOption(clearconfigOption)
+  .addOption(includesMap)
+  .addOption(listOfFiles)
   .action((options) => {
-    execute(commandsGen(options.source, options.config, options.project).commands.lintSrcEssential, options.verbose, options.debug, options.allowfailure, options.clearconfig)
+    execute(commandsGen(options.source, options.config, options.project, options.includesMap, options.extLinksCheck).commands.lintSrcEssential, options.verbose, options.debug, options.allowfailure, options.clearconfig)
   })
 
 program.command('urls')
@@ -266,8 +278,10 @@ program.command('styleguide')
   .addOption(debugOption)
   .addOption(allowfailureOption)
   .addOption(clearconfigOption)
+  .addOption(includesMap)
+  .addOption(listOfFiles)
   .action((options) => {
-    execute(commandsGen(options.source, options.config, options.project).commands.markdownlintSrcFull, options.verbose, options.debug, options.allowfailure, options.clearconfig)
+    execute(commandsGen(options.source, options.config, options.project, options.includesMap, options.extLinksCheck).commands.markdownlintSrcFull, options.verbose, options.debug, options.allowfailure, options.clearconfig)
   })
 
 program.command('slim')
@@ -279,8 +293,10 @@ program.command('slim')
   .addOption(debugOption)
   .addOption(allowfailureOption)
   .addOption(clearconfigOption)
+  .addOption(includesMap)
+  .addOption(listOfFiles)
   .action((options) => {
-    execute(commandsGen(options.source, options.config, options.project).commands.markdownlintSrcSlim, options.verbose, options.debug, options.allowfailure, options.clearconfig)
+    execute(commandsGen(options.source, options.config, options.project, options.includesMap, options.extLinksCheck).commands.markdownlintSrcSlim, options.verbose, options.debug, options.allowfailure, options.clearconfig)
   })
 
 program.command('fix')
@@ -292,8 +308,10 @@ program.command('fix')
   .addOption(debugOption)
   .addOption(allowfailureOption)
   .addOption(clearconfigOption)
+  .addOption(includesMap)
+  .addOption(listOfFiles)
   .action((options) => {
-    execute(commandsGen(options.source, options.config, options.project).commands.markdownlintSrcFix, options.verbose, options.debug, options.allowfailure, options.clearconfig)
+    execute(commandsGen(options.source, options.config, options.project, options.includesMap, options.extLinksCheck).commands.markdownlintSrcFix, options.verbose, options.debug, options.allowfailure, options.clearconfig)
   })
 
 program.command('typograph')
@@ -322,7 +340,7 @@ program.command('create-full-config')
   .addOption(projectOption)
   .addOption(debugOption)
   .action((options) => {
-    execute(commandsGen(options.source, options.config, options.project).commands.createFullMarkdownlintConfig, options.verbose, options.debug)
+    execute(commandsGen(options.source, options.config, options.project, options.includesMap, options.extLinksCheck).commands.createFullMarkdownlintConfig, options.verbose, options.debug)
   })
 
 program.command('create-slim-config')
@@ -331,7 +349,7 @@ program.command('create-slim-config')
   .addOption(projectOption)
   .addOption(debugOption)
   .action((options) => {
-    execute(commandsGen(options.source, options.config, options.project).commands.createSlimMarkdownlintConfig, options.verbose, options.debug)
+    execute(commandsGen(options.source, options.config, options.project, options.includesMap, options.extLinksCheck).commands.createSlimMarkdownlintConfig, options.verbose, options.debug)
   })
 
 program.command('create-typograph-config')
@@ -340,7 +358,7 @@ program.command('create-typograph-config')
   .addOption(projectOption)
   .addOption(debugOption)
   .action((options) => {
-    execute(commandsGen(options.source, options.config, options.project).commands.createTypographMarkdownlintConfig, options.verbose, options.debug)
+    execute(commandsGen(options.source, options.config, options.project, options.includesMap, options.extLinksCheck).commands.createTypographMarkdownlintConfig, options.verbose, options.debug)
   })
 
 program.parse()

test/linter.spec.js

@@ -1385,6 +1385,7 @@ function cli (args, cwd, clearLogs = true) {
   if (clearLogs) {
     clearLogsCmd = (isWin === true) ? 'DEL /Q /F ".markdownlin*" &&' : 'rm -f .markdownlin* &&'
   }
+  console.log(`isWin: ${isWin}`, `args: ${args}`, `cwd: ${cwd}`, 'command', `${clearLogsCmd} node ${path.resolve('./linter')} ${args.join(' ')}`)
   return new Promise(resolve => {
     exec(`${clearLogsCmd} node ${path.resolve('./linter')} ${args.join(' ')}`,
       { cwd },