From 6dee72c50879e41a956b60368b49e5e7cf945754 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Thu, 19 Oct 2023 10:13:55 +1100 Subject: [PATCH 01/15] Show correct counts --- api/functions/queries.js | 8 ++++---- docker/utils.js | 4 ++-- .../buildlistcardcomponents/HistoryChart.svelte | 2 +- ui/src/components/misccomponents/Tabs.svelte | 2 +- .../scancomparecomponents/ScanCompareListItem.svelte | 8 ++++---- .../components/summaryitemcomponents/LinkSummary.svelte | 8 ++++---- ui/src/containers/ScanCompare.svelte | 2 +- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/api/functions/queries.js b/api/functions/queries.js index 18302097..320fe4bf 100644 --- a/api/functions/queries.js +++ b/api/functions/queries.js @@ -279,7 +279,7 @@ exports.getAllScanSummaryFromUrl = (url, api) => for await (const item of iterator) { if (item[0]) { const existing = await getExistingBrokenLinkCount(item[0].runId); - item[0].totalUnique404Existing = existing; + item[0].totalUniqueBrokenLinksExisting = existing; } resolve(item); @@ -321,9 +321,9 @@ exports.compareScans = (api, url) => isHtmlErrorsUp: result[0].htmlErrors > result[1].htmlErrors, prevHtmlErrors: result[1].htmlErrors, currHtmlErrors: result[0].htmlErrors, - isBrokenLinksUp: result[0].totalUnique404 > result[1].totalUnique404, - prevBrokenLinks: result[1].totalUnique404, - currBrokenLinks: result[0].totalUnique404, + isBrokenLinksUp: result[0].uniqueBrokenLinks > result[1].uniqueBrokenLinks, + prevBrokenLinks: result[1].uniqueBrokenLinks, + currBrokenLinks: result[0].uniqueBrokenLinks, latestRunId: result[0].runId } resolve(isErrorUp) diff --git a/docker/utils.js b/docker/utils.js index be30c39e..24bcdb0f 100644 --- a/docker/utils.js +++ b/docker/utils.js @@ -29,7 +29,7 @@ exports.sendAlertEmail = async (email, emailConfig, scanSummary) => { from: 'foo@example.com', // sender address to: email, // list of receivers subject: `SSW CodeAuditor Scan Result - ${scanSummary.url}`, // Subject line - html: `

Hi there,

This is the result from SSW CodeAuditor scan on ${scanSummary.url} on ${fns.format(new Date(scanSummary.buildDate), 'dd MMM yyyy, hh:mm aaaa')}

⏳ Duration: ${scanSummary.scanDuration} seconds

🚨 Broken Links: ${scanSummary.totalUnique404} / ${scanSummary.totalScanned} Bad links

⚠️ HTML Warnings: ${scanSummary.htmlWarnings}

❌ HTML Errors: ${scanSummary.htmlErrors}

See https://codeauditor.com/build/${scanSummary.runId} for full scan result

<This is the automated alert email from SSW CodeAuditor>

`, + html: `

Hi there,

This is the result from SSW CodeAuditor scan on ${scanSummary.url} on ${fns.format(new Date(scanSummary.buildDate), 'dd MMM yyyy, hh:mm aaaa')}

⏳ Duration: ${scanSummary.scanDuration} seconds

🚨 Broken Links: ${scanSummary.uniqueBrokenLinks} / ${scanSummary.totalScanned} Bad links

⚠️ HTML Warnings: ${scanSummary.htmlWarnings}

❌ HTML Errors: ${scanSummary.htmlErrors}

See https://codeauditor.com/build/${scanSummary.runId} for full scan result

<This is the automated alert email from SSW CodeAuditor>

`, }); } @@ -418,7 +418,7 @@ exports.processBrokenLinks = ( const __getBadResults = (allUrls) => allUrls // Allow successful 2xx status code range (200-299) - .filter((url) => !(url["StatusCode"]?.startsWith('2') && url["StatusCode"]?.length === 3)) + .filter((url) => !((url["StatusCode"]?.startsWith('2') || url["StatusCode"]?.startsWith('3')) && url["StatusCode"]?.length === 3)) .map((x) => ({ src: x.Source || "", dst: x.Destination || "", diff --git a/ui/src/components/buildlistcardcomponents/HistoryChart.svelte b/ui/src/components/buildlistcardcomponents/HistoryChart.svelte index 3cba0590..0b1928fd 100644 --- a/ui/src/components/buildlistcardcomponents/HistoryChart.svelte +++ b/ui/src/components/buildlistcardcomponents/HistoryChart.svelte @@ -12,7 +12,7 @@ // Categorize and populate charts with data, title and color if (dataType === historyChartType.BadLinks) { chartTitle = historyChartType.BadLinks; - allDataToDisplay = value.map((i) => i.totalUnique404); + allDataToDisplay = value.map((i) => i.uniqueBrokenLinks); barColor = 'red' } else if (dataType === historyChartType.WarningCode) { chartTitle = historyChartType.WarningCode; diff --git a/ui/src/components/misccomponents/Tabs.svelte b/ui/src/components/misccomponents/Tabs.svelte index a9baad88..8f3adaba 100644 --- a/ui/src/components/misccomponents/Tabs.svelte +++ b/ui/src/components/misccomponents/Tabs.svelte @@ -48,7 +48,7 @@
  • - Links{build.summary ? ` (${build.summary.totalUnique404})` : ''} + Links{build.summary ? ` (${build.summary.uniqueBrokenLinks})` : ''}
    • diff --git a/ui/src/components/scancomparecomponents/ScanCompareListItem.svelte b/ui/src/components/scancomparecomponents/ScanCompareListItem.svelte index 11fdd120..0ba316ad 100644 --- a/ui/src/components/scancomparecomponents/ScanCompareListItem.svelte +++ b/ui/src/components/scancomparecomponents/ScanCompareListItem.svelte @@ -12,11 +12,11 @@
    0} - class:textgrey={value.totalUnique404 === 0} + class:textred={value.uniqueBrokenLinks > 0} + class:textgrey={value.uniqueBrokenLinks === 0} > - {value.totalUnique404} + {value.uniqueBrokenLinks}
    {#if Object.keys(comparisonDifferences).length > 0} @@ -24,7 +24,7 @@ {numberWithCommas(Math.abs(comparisonDifferences.brokenLinksDifference))} {/if}
    -

    Broken Links {value.totalUnique404Existing !== undefined ? `(${value.totalUnique404Existing || 0} Existing)` : ''}

    +

    Broken Links {value.totalUniqueBrokenLinksExisting !== undefined ? `(${value.totalUniqueBrokenLinksExisting || 0} Existing)` : ''}

    diff --git a/ui/src/components/summaryitemcomponents/LinkSummary.svelte b/ui/src/components/summaryitemcomponents/LinkSummary.svelte index 5c60448e..b03ff3f1 100644 --- a/ui/src/components/summaryitemcomponents/LinkSummary.svelte +++ b/ui/src/components/summaryitemcomponents/LinkSummary.svelte @@ -19,10 +19,10 @@ 0} - class:textgrey={value.totalUnique404 === 0}> - {value.totalUnique404} - {#if value.totalUnique404 === 0} + class:text-red-600={value.uniqueBrokenLinks > 0} + class:textgrey={value.uniqueBrokenLinks === 0}> + {value.uniqueBrokenLinks} + {#if value.uniqueBrokenLinks === 0} {/if} diff --git a/ui/src/containers/ScanCompare.svelte b/ui/src/containers/ScanCompare.svelte index d06bd4de..8c33df9b 100644 --- a/ui/src/containers/ScanCompare.svelte +++ b/ui/src/containers/ScanCompare.svelte @@ -36,7 +36,7 @@ if (allScans.length > 0 && selectedScan) { comparisonDifferences = { brokenLinksDifference: - allScans[0].totalUnique404 - selectedScan.totalUnique404, + allScans[0].uniqueBrokenLinks - selectedScan.uniqueBrokenLinks, codeWarningDifference: allScans[0].htmlWarnings - selectedScan.htmlWarnings, codeErrorDifference: allScans[0].htmlErrors - selectedScan.htmlErrors, From 7d5ed9f4873384161431c365352c78756e79d434 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Thu, 19 Oct 2023 10:17:51 +1100 Subject: [PATCH 02/15] Update text --- ui/src/components/summaryitemcomponents/LinkSummary.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/summaryitemcomponents/LinkSummary.svelte b/ui/src/components/summaryitemcomponents/LinkSummary.svelte index b03ff3f1..c3471151 100644 --- a/ui/src/components/summaryitemcomponents/LinkSummary.svelte +++ b/ui/src/components/summaryitemcomponents/LinkSummary.svelte @@ -15,7 +15,7 @@
    - 404 Errors + Broken Links Date: Thu, 19 Oct 2023 10:19:25 +1100 Subject: [PATCH 03/15] Show commas --- ui/src/components/summaryitemcomponents/LinkSummary.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/summaryitemcomponents/LinkSummary.svelte b/ui/src/components/summaryitemcomponents/LinkSummary.svelte index c3471151..ea628477 100644 --- a/ui/src/components/summaryitemcomponents/LinkSummary.svelte +++ b/ui/src/components/summaryitemcomponents/LinkSummary.svelte @@ -21,7 +21,7 @@ class="font-sans font-bold block lg:inline-block" class:text-red-600={value.uniqueBrokenLinks > 0} class:textgrey={value.uniqueBrokenLinks === 0}> - {value.uniqueBrokenLinks} + {numberWithCommas(value.uniqueBrokenLinks)} {#if value.uniqueBrokenLinks === 0} {/if} From 072bfe72ff6d215f8d8da13d499351d9b81e9c12 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Thu, 19 Oct 2023 12:36:22 +1100 Subject: [PATCH 04/15] Sanitize strings before writing tab-delimited CSV --- docker/sswlinkauditor.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker/sswlinkauditor.go b/docker/sswlinkauditor.go index 66844341..c56d59ba 100644 --- a/docker/sswlinkauditor.go +++ b/docker/sswlinkauditor.go @@ -208,12 +208,23 @@ func writeResultFile(allUrls map[string]LinkStatus) { f.WriteString("Source" + "\t" + "Destination" + "\t" + "Status" + "\t" + "StatusCode" + "\t" + "Anchor" + "\n") for _, v := range allUrls { - f.WriteString(v.srcUrl + "\t" + v.url + "\t" + v.status + "\t" + strconv.Itoa(v.statusCode) + "\t" + strings.ReplaceAll(v.anchor,"\"","") + "\n") + f.WriteString(sanitizeString(v.srcUrl) + "\t" + sanitizeString(v.url) + "\t" + sanitizeString(v.status) + "\t" + strconv.Itoa(v.statusCode) + "\t" + sanitizeString(v.anchor) + "\n") } f.Close() } +func sanitizeString(s string) string { + replacer := strings.NewReplacer( + "\"", "", + "\t", " ", + "\r\n", "", + "\n", "", + ); + + return replacer.Replace(s); +} + func isLinkUnscannable(a string, unscannableLinks []string) bool { for _, b := range unscannableLinks { if strings.HasPrefix(strings.ToLower(a), strings.ToLower(b)) { From fc0e772c7792834047b8f3b6ddc957dbc4ce2f16 Mon Sep 17 00:00:00 2001 From: tombui99 Date: Thu, 19 Oct 2023 13:26:40 +1100 Subject: [PATCH 05/15] Display Load Test errors --- ui/src/containers/ArtilleryReport.svelte | 28 +++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/ui/src/containers/ArtilleryReport.svelte b/ui/src/containers/ArtilleryReport.svelte index cf33d992..455c7d63 100644 --- a/ui/src/containers/ArtilleryReport.svelte +++ b/ui/src/containers/ArtilleryReport.svelte @@ -23,10 +23,13 @@ }; let atrFull = []; + let atrError; const getAtrFull = async (path) => { await fetch(`${CONSTS.BlobURL}/atr/${path}.json`) .then((x) => x.json()) .then((res) => { + atrError = res.aggregate.errors; + res.intermediate.forEach((i) => { atrFull.push({ fullTimestamp: i.timestamp, @@ -36,7 +39,7 @@ }); }); }); - return atrFull; + return {atrFull, atrError}; }; let getAtrData = getAtrFull(currentRoute.namedParams.id); @@ -76,11 +79,26 @@ {#await getAtrData} - {:then atrFull} - - {/await} + {:then atrData} + + + - +
    + Errors +
    + {#if Object.keys(atrData.atrError).length > 0} +
    + + {`${Object.keys(atrData.atrError)} : ${Object.values(atrData.atrError)}`} +
    + {:else} +
    + + Test completed without network or OS errors +
    + {/if} + {/await} {:else}
    From 52dd890cf5662f34ed2d384b7e5c9cedb1eddd1f Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Thu, 19 Oct 2023 16:26:10 +1100 Subject: [PATCH 06/15] Update packages --- docker/package-lock.json | 38 ++++++++++++++------------------------ docker/package.json | 6 +++--- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/docker/package-lock.json b/docker/package-lock.json index 57537c47..ba7e8069 100644 --- a/docker/package-lock.json +++ b/docker/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "boxen": "^4.2.0", "chalk": "^3.0.0", - "csv-parser": "^2.3.5", + "csv-parser": "^3.0.0", "csv-writer": "^1.6.0", "date-fns": "^2.30.0", "eslint-plugin-promise": "^4.3.1", @@ -20,10 +20,10 @@ "html5parser": "^1.2.1", "htmlhint": "^0.11.0", "js-beautify": "^1.14.9", - "libphonenumber-js": "^1.10.47", + "libphonenumber-js": "^1.10.48", "minimatch": "^3.1.2", "mocha": "^9.2.2", - "node-fetch": "^2.6.12", + "node-fetch": "^2.7.0", "nodemailer": "^6.9.3", "puppeteer": "^3.3.0", "ramda": "^0.27.2", @@ -1158,18 +1158,17 @@ "dev": true }, "node_modules/csv-parser": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-2.3.5.tgz", - "integrity": "sha512-LCHolC4AlNwL+5EuD5LH2VVNKpD8QixZW2zzK1XmrVYUaslFY4c5BooERHOCIubG9iv/DAyFjs4x0HvWNZuyWg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-3.0.0.tgz", + "integrity": "sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ==", "dependencies": { - "minimist": "^1.2.0", - "through2": "^3.0.1" + "minimist": "^1.2.0" }, "bin": { "csv-parser": "bin/csv-parser" }, "engines": { - "node": ">= 8.16.0" + "node": ">= 10" } }, "node_modules/csv-writer": { @@ -2948,9 +2947,9 @@ } }, "node_modules/libphonenumber-js": { - "version": "1.10.47", - "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.10.47.tgz", - "integrity": "sha512-b4t7VQDV29xx/ni+58yl9KWPGjnDLDXCeCTLrD4V8vDpObXZRZBrg7uX/HWZ7YXiJKqdBDGgc+barUUTNB6Slw==" + "version": "1.10.48", + "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.10.48.tgz", + "integrity": "sha512-Vvcgt4+o8+puIBJZLdMshPYx9nRN3/kTT7HPtOyfYrSQuN9PGBF1KUv0g07fjNzt4E4GuA7FnsLb+WeAMzyRQg==" }, "node_modules/locate-path": { "version": "6.0.0", @@ -3434,9 +3433,9 @@ "dev": true }, "node_modules/node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "dependencies": { "whatwg-url": "^5.0.0" }, @@ -4662,15 +4661,6 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, - "node_modules/through2": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz", - "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==", - "dependencies": { - "inherits": "^2.0.4", - "readable-stream": "2 || 3" - } - }, "node_modules/timed-out": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", diff --git a/docker/package.json b/docker/package.json index 22269fc7..ab4e91fc 100644 --- a/docker/package.json +++ b/docker/package.json @@ -11,7 +11,7 @@ "dependencies": { "boxen": "^4.2.0", "chalk": "^3.0.0", - "csv-parser": "^2.3.5", + "csv-parser": "^3.0.0", "csv-writer": "^1.6.0", "date-fns": "^2.30.0", "eslint-plugin-promise": "^4.3.1", @@ -20,10 +20,10 @@ "html5parser": "^1.2.1", "htmlhint": "^0.11.0", "js-beautify": "^1.14.9", - "libphonenumber-js": "^1.10.47", + "libphonenumber-js": "^1.10.48", "minimatch": "^3.1.2", "mocha": "^9.2.2", - "node-fetch": "^2.6.12", + "node-fetch": "^2.7.0", "nodemailer": "^6.9.3", "puppeteer": "^3.3.0", "ramda": "^0.27.2", From f50fc246e41638ee18f282ac4456c7141fe99839 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Fri, 20 Oct 2023 09:39:53 +1100 Subject: [PATCH 07/15] Upload all rules by default --- docker/index.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/index.js b/docker/index.js index 35788fac..9dcfe063 100644 --- a/docker/index.js +++ b/docker/index.js @@ -28,6 +28,7 @@ const { runLighthouseReport, runArtilleryLoadTest } = require("./utils"); +const { htmlHintConfig } = require("./api"); const LIGHTHOUSEFOLDER = "./lhr.json"; const ARTILLERYFOLDER = "./artilleryOut.json"; @@ -315,7 +316,7 @@ const processAndUpload = async ( }); } catch (error) { console.error( - `Error: Unabled to push data to dashboard service => ${error.message}` + `Error: Unable to push data to dashboard service => ${error.message}` ); } } @@ -334,13 +335,13 @@ const processAndUpload = async ( // Upload selected HTMLHint Rules to the scan if (args.htmlhint && args.token && runId) { const result = await getHTMLHintRules(args.token, args.url); + const selectedRules = result?.selectedRules ?? Object.keys(htmlHintConfig).join(","); - if (result && result.selectedRules?.split(",").length > 0) { - const selectedRules = result.selectedRules; + if (selectedRules?.length > 0) { const res = await addHTMLHintRulesForScan(args.token, args.url, runId, selectedRules) if (res) { - console.log('Upload selected HTMLHint Rules successfully') + console.log('Uploaded selected HTMLHint Rules successfully'); } else { throw new Error("Failed to add custom html rules for each scan"); } From 6fb65df96837a9a6777d2ab612ee9c7ab02a580b Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Fri, 20 Oct 2023 10:20:28 +1100 Subject: [PATCH 08/15] Add error handling --- api/functions/queries.js | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/api/functions/queries.js b/api/functions/queries.js index 320fe4bf..e5ab17f4 100644 --- a/api/functions/queries.js +++ b/api/functions/queries.js @@ -314,17 +314,21 @@ exports.compareScans = (api, url) => for await (const item of entity) { result.push(item); } + + const latestResult = result[0] || {}; + const prevResult = result[1] || {}; + let isErrorUp = { - isHtmlWarningsUp: result[0].htmlWarnings > result[1].htmlWarnings, - prevHtmlWarnings: result[1].htmlWarnings, - currHtmlWarnings: result[0].htmlWarnings, - isHtmlErrorsUp: result[0].htmlErrors > result[1].htmlErrors, - prevHtmlErrors: result[1].htmlErrors, - currHtmlErrors: result[0].htmlErrors, - isBrokenLinksUp: result[0].uniqueBrokenLinks > result[1].uniqueBrokenLinks, - prevBrokenLinks: result[1].uniqueBrokenLinks, - currBrokenLinks: result[0].uniqueBrokenLinks, - latestRunId: result[0].runId + isHtmlWarningsUp: latestResult.htmlWarnings > prevResult.htmlWarnings, + prevHtmlWarnings: prevResult.htmlWarnings || 0, + currHtmlWarnings: latestResult.htmlWarnings || 0, + isHtmlErrorsUp: latestResult.htmlErrors > prevResult.htmlErrors, + prevHtmlErrors: prevResult.htmlErrors || 0, + currHtmlErrors: latestResult.htmlErrors || 0, + isBrokenLinksUp: latestResult.uniqueBrokenLinks > prevResult.uniqueBrokenLinks, + prevBrokenLinks: prevResult.uniqueBrokenLinks || 0, + currBrokenLinks: latestResult.uniqueBrokenLinks || 0, + latestRunId: latestResult.runId } resolve(isErrorUp) }); From 367081205190a72153996aed81ab206637208ed3 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Fri, 20 Oct 2023 14:17:32 +1100 Subject: [PATCH 09/15] Always show unscannable links --- .../linkauditorcomponents/DetailsTable.svelte | 99 ++++++++++--------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/ui/src/components/linkauditorcomponents/DetailsTable.svelte b/ui/src/components/linkauditorcomponents/DetailsTable.svelte index 618afff0..aad7f8f0 100644 --- a/ui/src/components/linkauditorcomponents/DetailsTable.svelte +++ b/ui/src/components/linkauditorcomponents/DetailsTable.svelte @@ -61,21 +61,7 @@ } -{#if builds.length === 0} -
    - - - - No broken links in this build! - - - -
    -{:else} +{#if builds.length}
    hideShow()} @@ -128,39 +116,56 @@ Unscannable Links ({foundUnscannableLinks.length}): - {#if !hiddenRows} - - Some working links are reported as broken by CodeAuditor. They're marked as "unscannable". Learn more on our KB. - - {#each foundUnscannableLinks as url} - - - - - - - - - - - - - - - - -
    Source - {url.src} -
    Anchor Text{url.link || ''}
    Link - {url.dst} -
    - {/each} - {/if} +
    + {#if !hiddenRows} + + Some working links are reported as broken by CodeAuditor. They're marked as "unscannable". Learn more on our KB. + + {#each foundUnscannableLinks as url} + + + + + + + + + + + + + + + + +
    Source + {url.src} +
    Anchor Text{url.link || ''}
    Link + {url.dst} +
    + {/each} {/if} +{/if} +{#if !builds.length} +
    + + + + No broken links in this build! + + + +
    +{/if} +{#if builds.length} {#if displayMode === 0} {:else if displayMode === 1} From a87758fa50ecba187203cee7dbc2297fe9882152 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Fri, 20 Oct 2023 14:33:00 +1100 Subject: [PATCH 10/15] Use api in function --- api/functions/index.js | 8 ++++---- api/functions/queries.js | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/functions/index.js b/api/functions/index.js index 50a7949a..dee5f6e3 100644 --- a/api/functions/index.js +++ b/api/functions/index.js @@ -4,7 +4,6 @@ const admin = require('firebase-admin'); const R = require('ramda'); const fetch = require('node-fetch'); const Queue = require('better-queue'); -const { unscannableLinks } = require('./consts'); require('dotenv').config(); const { @@ -231,6 +230,7 @@ app.post('/scanresult/:api/:buildId', async (req, res) => { const buildId = req.params.buildId; const runId = newGuid(); const buildDate = new Date(); + const unscannableLinks = await getUnscannableLinks(); const uid = await getUserIdFromApiKey(apikey); if (!uid) { @@ -262,11 +262,11 @@ app.post('/scanresult/:api/:buildId', async (req, res) => { url, cloc, totalBrokenLinks: badUrls.length, - uniqueBrokenLinks: R.uniqBy(R.prop('dst'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link.url)))).length, - pagesWithBrokenLink: R.uniqBy(R.prop('src'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link.url)))).length, + uniqueBrokenLinks: R.uniqBy(R.prop('dst'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link)))).length, + pagesWithBrokenLink: R.uniqBy(R.prop('src'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link)))).length, totalUnique404: R.uniqBy( R.prop('dst'), - badUrls.filter((x) => x.statuscode === '404' && !unscannableLinks.some(link => x.dst.includes(link.url))) + badUrls.filter((x) => x.statuscode === '404' && !unscannableLinks.some(link => x.dst.includes(link))) ).length, htmlWarnings: htmlWarnings ? htmlWarnings : 0, htmlErrors: htmlErrors ? htmlErrors : 0, diff --git a/api/functions/queries.js b/api/functions/queries.js index e5ab17f4..89fe8601 100644 --- a/api/functions/queries.js +++ b/api/functions/queries.js @@ -42,8 +42,8 @@ const getExistingBrokenLinkCount = async (runId) => { const existingCount = result.reduce((count, item) => { if (item.runId === runId) { - if (!previousFailures.has(item.dst) && !unscannableLinks.find((i) => item.dst.startsWith(i))) { - const hasPrevious = result.find((i) => i.dst === item.dst && i.buildDate < item.buildDate); + if (!previousFailures.has(item.dst) && !unscannableLinks.some((i) => item.dst.includes(i))) { + const hasPrevious = result.some((i) => i.dst === item.dst && i.buildDate < item.buildDate); previousFailures.set(item.dst, hasPrevious); if (hasPrevious) { From 64d89182e1507684cb2910b4f594f3d71d569567 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Fri, 20 Oct 2023 15:01:42 +1100 Subject: [PATCH 11/15] Get unscannable links from API --- .../linkauditorcomponents/DetailsTable.svelte | 4 +-- ui/src/containers/LinkReport.svelte | 28 +++++++++++++------ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/ui/src/components/linkauditorcomponents/DetailsTable.svelte b/ui/src/components/linkauditorcomponents/DetailsTable.svelte index aad7f8f0..81b1eb28 100644 --- a/ui/src/components/linkauditorcomponents/DetailsTable.svelte +++ b/ui/src/components/linkauditorcomponents/DetailsTable.svelte @@ -14,10 +14,10 @@ export let unscannableLinks; let foundUnscannableLinks = []; - foundUnscannableLinks = builds.filter(build => unscannableLinks.some(link => build.dst.includes(link.url))); + foundUnscannableLinks = builds.filter(build => unscannableLinks.some(link => build.dst.includes(link))); // Filter out unscannable links - builds = builds.filter(build => !unscannableLinks.some(link => build.dst.includes(link.url))); + builds = builds.filter(build => !unscannableLinks.some(link => build.dst.includes(link))); let displayMode = 0; diff --git a/ui/src/containers/LinkReport.svelte b/ui/src/containers/LinkReport.svelte index f7a3c855..8032e661 100644 --- a/ui/src/containers/LinkReport.svelte +++ b/ui/src/containers/LinkReport.svelte @@ -11,21 +11,31 @@ import { Navigate } from "svelte-router-spa"; import LoadingFlat from "../components/misccomponents/LoadingFlat.svelte"; import UpdateIgnoreUrl from "../components/misccomponents/UpdateIgnoreUrl.svelte"; - import { unscannableLinks } from "../../../api/functions/consts"; import BuildDetailsSlot from "../components/detailslotcomponents/BuildDetailsSlot.svelte"; + import { CONSTS } from "../utils/utils"; export let currentRoute; - let promise + let promise; let userNotLoginToast; let ignoreUrlShown; let urlToIgnore; let scanUrl; if (currentRoute.namedParams.id) { - promise = getBuildDetails(currentRoute.namedParams.id); + promise = new Promise(async (resolve) => { + const buildDetails = await getBuildDetails(currentRoute.namedParams.id); + const resp = await fetch(`${CONSTS.API}/api/unscannablelinks`); + const unscannableLinks = await resp.json(); + resolve({ buildDetails, unscannableLinks }); + }); } else { - promise = getLatestBuildDetails(currentRoute.namedParams.api, currentRoute.namedParams.url) + promise = new Promise(async (resolve) => { + const buildDetails = await getLatestBuildDetails(currentRoute.namedParams.api, currentRoute.namedParams.url); + const resp = await fetch(`${CONSTS.API}/api/unscannablelinks`); + const unscannableLinks = await resp.json(); + resolve({ buildDetails, unscannableLinks }); + }); } const onDownload = data => { @@ -72,15 +82,15 @@ {:then data} onDownload(data)} - on:ignore={url => showIgnore(data.summary.url, url, $userSession$)} - builds={data ? data.brokenLinks : []} + on:download={() => onDownload(data.buildDetails)} + on:ignore={url => showIgnore(data.buildDetails.summary.url, url, $userSession$)} + builds={data.buildDetails ? data.buildDetails.brokenLinks : []} {currentRoute} - {unscannableLinks} + unscannableLinks={data.unscannableLinks} /> {:catch error} From 6753da747c28f0dea999e7811a77138006b7ab29 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Fri, 20 Oct 2023 15:34:44 +1100 Subject: [PATCH 12/15] Remove const --- api/functions/consts.js | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/api/functions/consts.js b/api/functions/consts.js index df07db25..bc62242f 100644 --- a/api/functions/consts.js +++ b/api/functions/consts.js @@ -24,29 +24,3 @@ exports.BLOB = { htmlhint: 'htmlhint', codeAuditor: 'codeauditor', }; - -exports.unscannableLinks = [ - {url: "https://learn.microsoft.com/en-us/"}, - {url: "https://support.google.com/"}, - {url: "https://twitter.com/"}, - {url: "https://marketplace.visualstudio.com/"}, - {url: "https://www.nuget.org/"}, - {url: "https://make.powerautomate.com"}, - {url: "https://www.microsoft.com/"}, - {url: "http://www.microsoft.com/"}, - {url: "https://answers.microsoft.com/"}, - {url: "https://admin.microsoft.com/"}, - {url: "https://ngrx.io"}, - {url: "https://twitter.com"}, - {url: "https://marketplace"}, - {url: "https://www.nuget.org/"}, - {url: "http://nuget.org"}, - {url: "https://t.co"}, - {url: "https://support.google.com"}, - {url: "https://playwright.dev"}, - {url: "https://www.theurlist.com/xamarinstreamers"}, - {url: "https://dev.botframework.com"}, - {url: "https://www.ssw.com.au/rules/rules-to-better-research-and-development/"}, - {url: "https://www.ato.gov.au/Business/Research-and-development-tax-incentive/"}, - {url: "https://learn.microsoft.com/en-us/assessments/?mode=home/"} - ] \ No newline at end of file From 05368d7f7a9f61be960b31333c9412669deae182 Mon Sep 17 00:00:00 2001 From: tombui99 Date: Fri, 20 Oct 2023 16:50:28 +1100 Subject: [PATCH 13/15] Hide title if there is no alert email on the list --- .../misccomponents/SendAlertModal.svelte | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/ui/src/components/misccomponents/SendAlertModal.svelte b/ui/src/components/misccomponents/SendAlertModal.svelte index 1b185447..21ad1f09 100644 --- a/ui/src/components/misccomponents/SendAlertModal.svelte +++ b/ui/src/components/misccomponents/SendAlertModal.svelte @@ -123,22 +123,24 @@
    Invalid email input
    {/if}
    -
    - Currently receiving alerts: -
    - {#each sharedEmailAddresses as item} -
  • {item = { ...item, showDeleteIcon: true } }} - on:focus={() => {item = { ...item, showDeleteIcon: true } }} - on:mouseleave={()=> {item = { ...item, showDeleteIcon: false } }} - > - {item.emailAddress} - {#if item.showDeleteIcon} - removeAlertEmail(item)} on:keydown={undefined}/> - {/if} -
    • - {/each} + {#if sharedEmailAddresses.length > 0} +
    + Currently receiving alerts: +
    + {#each sharedEmailAddresses as item} +
  • {item = { ...item, showDeleteIcon: true } }} + on:focus={() => {item = { ...item, showDeleteIcon: true } }} + on:mouseleave={()=> {item = { ...item, showDeleteIcon: false } }} + > + {item.emailAddress} + {#if item.showDeleteIcon} + removeAlertEmail(item)} on:keydown={undefined}/> + {/if} +
    • + {/each} + {/if}
    {/if} From 899ae3e485200f3e545fe3a8057a76df31d2d6f1 Mon Sep 17 00:00:00 2001 From: tombui99 Date: Mon, 23 Oct 2023 14:29:43 +1100 Subject: [PATCH 14/15] Fixed false positive in spelling mistake custom rule --- docker/customHtmlRules.js | 43 +++++++++++--------- docker/test/common-spelling-mistakes.spec.js | 6 +++ docker/test/grammar-scrum-terms.spec.js | 6 +++ 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/docker/customHtmlRules.js b/docker/customHtmlRules.js index bb7b05f9..6122e7a3 100644 --- a/docker/customHtmlRules.js +++ b/docker/customHtmlRules.js @@ -61,15 +61,18 @@ exports.addCustomHtmlRule = () => { scrumTerms.forEach((i) => { var contentIndex = pageContent.indexOf(i); var col = event.lastEvent.col; - - if (contentIndex >= 0) { - reporter.warn( - "Incorrect Scrum term: '" + i + "'.", - event.line, - col, - self, - event.raw - ); + + // Make sure the character has space and is not part of a long single string + if (pageContent.indexOf(' ') >= 0) { + if (contentIndex >= 0) { + reporter.warn( + "Incorrect Scrum term: '" + i + "'.", + event.line, + col, + self, + event.raw + ); + } } }); } @@ -445,17 +448,19 @@ exports.addCustomHtmlRule = () => { let pageContent = event.lastEvent.raw; if (pageContent) { spellings.forEach((i) => { - var contentIndex = pageContent.indexOf(i); + var contentIndex = pageContent.indexOf(i) >= 0; var col = event.lastEvent.col; - - if (contentIndex >= 0) { - reporter.warn( - "Incorrect spellings: '" + i + "'.", - event.line, - col, - self, - event.raw - ); + // Make sure the character has space and is not part of a long single string + if (pageContent.indexOf(' ') >= 0) { + if (contentIndex) { + reporter.warn( + "Incorrect spellings: '" + i + "'.", + event.line, + col, + self, + event.raw + ); + } } }); } diff --git a/docker/test/common-spelling-mistakes.spec.js b/docker/test/common-spelling-mistakes.spec.js index 3ad4b601..5549b574 100644 --- a/docker/test/common-spelling-mistakes.spec.js +++ b/docker/test/common-spelling-mistakes.spec.js @@ -23,6 +23,12 @@ describe(`Rules: ${ruleId}`, () => { expect(messages.length).to.be(9); }); + it("long string contains the character should not result in an error", () => { + const code = `
    wgARCAAKABQDASIAAhEBAxEB/8QAGAAAAwEBAAAAAAAAAAAAAAAAAAIDBAX/xAAUAQE
    `; + const messages = HTMLHint.verify(code, ruleOptions); + expect(messages.length).to.be(0); + }); + ["meta", "link", "script", "svg"].forEach((tag) => { it(`incorrect terms in a <${tag}> tag should not result in an error`, () => { const code = `<${tag}>a.k.a A.K.A AKA e-mail EMail can not web site user name task bar`; diff --git a/docker/test/grammar-scrum-terms.spec.js b/docker/test/grammar-scrum-terms.spec.js index b0e215ac..cfd68293 100644 --- a/docker/test/grammar-scrum-terms.spec.js +++ b/docker/test/grammar-scrum-terms.spec.js @@ -25,6 +25,12 @@ describe(`Rules: ${ruldId}`, () => { expect(messages.length).to.be(11); }); + it("long string contains the character should not result in an error", () => { + const code = `
    blahblahsprintblahblah
    `; + const messages = HTMLHint.verify(code, ruleOptions); + expect(messages.length).to.be(0); + }); + it("Incorrect Scrum terms in tag should not result in an error", () => { const code = "scrum, sprint, product owner"; From f0d74eb830d5e58fb95d6c3ead7748a7c929d64d Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Mon, 23 Oct 2023 15:43:01 +1100 Subject: [PATCH 15/15] Fix false positives --- docker/customHtmlRules.js | 2 +- docker/test/phone-numbers-without-links.js | 30 ++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/docker/customHtmlRules.js b/docker/customHtmlRules.js index bb7b05f9..524f8026 100644 --- a/docker/customHtmlRules.js +++ b/docker/customHtmlRules.js @@ -473,7 +473,7 @@ exports.addCustomHtmlRule = () => { init: function (parser, reporter) { const self = this; parser.addListener("all", (event) => { - if (event.raw && event.lastEvent && findPhoneNumbersInText(event.raw, "US").length) { + if (event.raw && event.lastEvent && findPhoneNumbersInText(event.raw, "AU").length) { const pageContent = event.lastEvent.raw; if (pageContent && event.lastEvent.tagName) { const tagName = event.lastEvent.tagName.toLowerCase(); diff --git a/docker/test/phone-numbers-without-links.js b/docker/test/phone-numbers-without-links.js index 3267c2ed..97b5cd4e 100644 --- a/docker/test/phone-numbers-without-links.js +++ b/docker/test/phone-numbers-without-links.js @@ -10,16 +10,19 @@ const { addCustomHtmlRule } = require("../customHtmlRules"); ruleOptions[ruleId] = true; const phoneNumbers = [ - "(213) 373-4253", - "(213)373-4253", - "213-373-4253", - "213.373.4253", - "2133734253", - "+12133734253", - "121-33734253", "+61 2 9953 3000", "+61299533000", - "+61 402 123 456" + "+61 402 123 456", + "13 00 00", + "1800 160 401", + "02 5550 4321", + "+33 3 67 39 05 39" +]; + +const nonPhoneNumbers = [ + "2023.05.31.02", + "123.456.7890", + "1234.567.890" ]; describe(`Rules: ${ruleId}`, () => { @@ -37,6 +40,17 @@ describe(`Rules: ${ruleId}`, () => { }); }); + nonPhoneNumbers.forEach((phone) => { + it(`text containing non-phone number "${phone}" without hyperlink should not error`, () => { + const code = + `
    +

    ${phone}

    +
    `; + const messages = HTMLHint.verify(code, ruleOptions); + expect(messages.length).to.be(0); + }); + }); + phoneNumbers.forEach((phone) => { it(`link containing "${phone}" without hyperlink should error`, () => { const code = `${phone}`;