diff --git a/api/functions/consts.js b/api/functions/consts.js index df07db25..bc62242f 100644 --- a/api/functions/consts.js +++ b/api/functions/consts.js @@ -24,29 +24,3 @@ exports.BLOB = { htmlhint: 'htmlhint', codeAuditor: 'codeauditor', }; - -exports.unscannableLinks = [ - {url: "https://learn.microsoft.com/en-us/"}, - {url: "https://support.google.com/"}, - {url: "https://twitter.com/"}, - {url: "https://marketplace.visualstudio.com/"}, - {url: "https://www.nuget.org/"}, - {url: "https://make.powerautomate.com"}, - {url: "https://www.microsoft.com/"}, - {url: "http://www.microsoft.com/"}, - {url: "https://answers.microsoft.com/"}, - {url: "https://admin.microsoft.com/"}, - {url: "https://ngrx.io"}, - {url: "https://twitter.com"}, - {url: "https://marketplace"}, - {url: "https://www.nuget.org/"}, - {url: "http://nuget.org"}, - {url: "https://t.co"}, - {url: "https://support.google.com"}, - {url: "https://playwright.dev"}, - {url: "https://www.theurlist.com/xamarinstreamers"}, - {url: "https://dev.botframework.com"}, - {url: "https://www.ssw.com.au/rules/rules-to-better-research-and-development/"}, - {url: "https://www.ato.gov.au/Business/Research-and-development-tax-incentive/"}, - {url: "https://learn.microsoft.com/en-us/assessments/?mode=home/"} - ] \ No newline at end of file diff --git a/api/functions/index.js b/api/functions/index.js index 50a7949a..dee5f6e3 100644 --- a/api/functions/index.js +++ b/api/functions/index.js @@ -4,7 +4,6 @@ const admin = require('firebase-admin'); const R = require('ramda'); const fetch = require('node-fetch'); const Queue = require('better-queue'); -const { unscannableLinks } = require('./consts'); require('dotenv').config(); const { @@ -231,6 +230,7 @@ app.post('/scanresult/:api/:buildId', async (req, res) => { const buildId = req.params.buildId; const runId = newGuid(); const buildDate = new Date(); + const unscannableLinks = await getUnscannableLinks(); const uid = await getUserIdFromApiKey(apikey); if (!uid) { @@ -262,11 +262,11 @@ app.post('/scanresult/:api/:buildId', async (req, res) => { url, cloc, totalBrokenLinks: badUrls.length, - uniqueBrokenLinks: R.uniqBy(R.prop('dst'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link.url)))).length, - pagesWithBrokenLink: R.uniqBy(R.prop('src'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link.url)))).length, + uniqueBrokenLinks: R.uniqBy(R.prop('dst'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link)))).length, + pagesWithBrokenLink: R.uniqBy(R.prop('src'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link)))).length, totalUnique404: R.uniqBy( R.prop('dst'), - badUrls.filter((x) => x.statuscode === '404' && !unscannableLinks.some(link => x.dst.includes(link.url))) + badUrls.filter((x) => x.statuscode === '404' && !unscannableLinks.some(link => x.dst.includes(link))) ).length, htmlWarnings: htmlWarnings ? htmlWarnings : 0, htmlErrors: htmlErrors ? htmlErrors : 0, diff --git a/api/functions/queries.js b/api/functions/queries.js index 18302097..89fe8601 100644 --- a/api/functions/queries.js +++ b/api/functions/queries.js @@ -42,8 +42,8 @@ const getExistingBrokenLinkCount = async (runId) => { const existingCount = result.reduce((count, item) => { if (item.runId === runId) { - if (!previousFailures.has(item.dst) && !unscannableLinks.find((i) => item.dst.startsWith(i))) { - const hasPrevious = result.find((i) => i.dst === item.dst && i.buildDate < item.buildDate); + if (!previousFailures.has(item.dst) && !unscannableLinks.some((i) => item.dst.includes(i))) { + const hasPrevious = result.some((i) => i.dst === item.dst && i.buildDate < item.buildDate); previousFailures.set(item.dst, hasPrevious); if (hasPrevious) { @@ -279,7 +279,7 @@ exports.getAllScanSummaryFromUrl = (url, api) => for await (const item of iterator) { if (item[0]) { const existing = await getExistingBrokenLinkCount(item[0].runId); - item[0].totalUnique404Existing = existing; + item[0].totalUniqueBrokenLinksExisting = existing; } resolve(item); @@ -314,17 +314,21 @@ exports.compareScans = (api, url) => for await (const item of entity) { result.push(item); } + + const latestResult = result[0] || {}; + const prevResult = result[1] || {}; + let isErrorUp = { - isHtmlWarningsUp: result[0].htmlWarnings > result[1].htmlWarnings, - prevHtmlWarnings: result[1].htmlWarnings, - currHtmlWarnings: result[0].htmlWarnings, - isHtmlErrorsUp: result[0].htmlErrors > result[1].htmlErrors, - prevHtmlErrors: result[1].htmlErrors, - currHtmlErrors: result[0].htmlErrors, - isBrokenLinksUp: result[0].totalUnique404 > result[1].totalUnique404, - prevBrokenLinks: result[1].totalUnique404, - currBrokenLinks: result[0].totalUnique404, - latestRunId: result[0].runId + isHtmlWarningsUp: latestResult.htmlWarnings > prevResult.htmlWarnings, + prevHtmlWarnings: prevResult.htmlWarnings || 0, + currHtmlWarnings: latestResult.htmlWarnings || 0, + isHtmlErrorsUp: latestResult.htmlErrors > prevResult.htmlErrors, + prevHtmlErrors: prevResult.htmlErrors || 0, + currHtmlErrors: latestResult.htmlErrors || 0, + isBrokenLinksUp: latestResult.uniqueBrokenLinks > prevResult.uniqueBrokenLinks, + prevBrokenLinks: prevResult.uniqueBrokenLinks || 0, + currBrokenLinks: latestResult.uniqueBrokenLinks || 0, + latestRunId: latestResult.runId } resolve(isErrorUp) }); diff --git a/docker/customHtmlRules.js b/docker/customHtmlRules.js index bb7b05f9..d5868acd 100644 --- a/docker/customHtmlRules.js +++ b/docker/customHtmlRules.js @@ -61,15 +61,18 @@ exports.addCustomHtmlRule = () => { scrumTerms.forEach((i) => { var contentIndex = pageContent.indexOf(i); var col = event.lastEvent.col; - - if (contentIndex >= 0) { - reporter.warn( - "Incorrect Scrum term: '" + i + "'.", - event.line, - col, - self, - event.raw - ); + + // Make sure the character has space and is not part of a long single string + if (pageContent.indexOf(' ') >= 0) { + if (contentIndex >= 0) { + reporter.warn( + "Incorrect Scrum term: '" + i + "'.", + event.line, + col, + self, + event.raw + ); + } } }); } @@ -445,17 +448,19 @@ exports.addCustomHtmlRule = () => { let pageContent = event.lastEvent.raw; if (pageContent) { spellings.forEach((i) => { - var contentIndex = pageContent.indexOf(i); + var contentIndex = pageContent.indexOf(i) >= 0; var col = event.lastEvent.col; - - if (contentIndex >= 0) { - reporter.warn( - "Incorrect spellings: '" + i + "'.", - event.line, - col, - self, - event.raw - ); + // Make sure the character has space and is not part of a long single string + if (pageContent.indexOf(' ') >= 0) { + if (contentIndex) { + reporter.warn( + "Incorrect spellings: '" + i + "'.", + event.line, + col, + self, + event.raw + ); + } } }); } @@ -473,7 +478,7 @@ exports.addCustomHtmlRule = () => { init: function (parser, reporter) { const self = this; parser.addListener("all", (event) => { - if (event.raw && event.lastEvent && findPhoneNumbersInText(event.raw, "US").length) { + if (event.raw && event.lastEvent && findPhoneNumbersInText(event.raw, "AU").length) { const pageContent = event.lastEvent.raw; if (pageContent && event.lastEvent.tagName) { const tagName = event.lastEvent.tagName.toLowerCase(); diff --git a/docker/index.js b/docker/index.js index 35788fac..9dcfe063 100644 --- a/docker/index.js +++ b/docker/index.js @@ -28,6 +28,7 @@ const { runLighthouseReport, runArtilleryLoadTest } = require("./utils"); +const { htmlHintConfig } = require("./api"); const LIGHTHOUSEFOLDER = "./lhr.json"; const ARTILLERYFOLDER = "./artilleryOut.json"; @@ -315,7 +316,7 @@ const processAndUpload = async ( }); } catch (error) { console.error( - `Error: Unabled to push data to dashboard service => ${error.message}` + `Error: Unable to push data to dashboard service => ${error.message}` ); } } @@ -334,13 +335,13 @@ const processAndUpload = async ( // Upload selected HTMLHint Rules to the scan if (args.htmlhint && args.token && runId) { const result = await getHTMLHintRules(args.token, args.url); + const selectedRules = result?.selectedRules ?? Object.keys(htmlHintConfig).join(","); - if (result && result.selectedRules?.split(",").length > 0) { - const selectedRules = result.selectedRules; + if (selectedRules?.length > 0) { const res = await addHTMLHintRulesForScan(args.token, args.url, runId, selectedRules) if (res) { - console.log('Upload selected HTMLHint Rules successfully') + console.log('Uploaded selected HTMLHint Rules successfully'); } else { throw new Error("Failed to add custom html rules for each scan"); } diff --git a/docker/package-lock.json b/docker/package-lock.json index 57537c47..ba7e8069 100644 --- a/docker/package-lock.json +++ b/docker/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "boxen": "^4.2.0", "chalk": "^3.0.0", - "csv-parser": "^2.3.5", + "csv-parser": "^3.0.0", "csv-writer": "^1.6.0", "date-fns": "^2.30.0", "eslint-plugin-promise": "^4.3.1", @@ -20,10 +20,10 @@ "html5parser": "^1.2.1", "htmlhint": "^0.11.0", "js-beautify": "^1.14.9", - "libphonenumber-js": "^1.10.47", + "libphonenumber-js": "^1.10.48", "minimatch": "^3.1.2", "mocha": "^9.2.2", - "node-fetch": "^2.6.12", + "node-fetch": "^2.7.0", "nodemailer": "^6.9.3", "puppeteer": "^3.3.0", "ramda": "^0.27.2", @@ -1158,18 +1158,17 @@ "dev": true }, "node_modules/csv-parser": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-2.3.5.tgz", - "integrity": "sha512-LCHolC4AlNwL+5EuD5LH2VVNKpD8QixZW2zzK1XmrVYUaslFY4c5BooERHOCIubG9iv/DAyFjs4x0HvWNZuyWg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/csv-parser/-/csv-parser-3.0.0.tgz", + "integrity": "sha512-s6OYSXAK3IdKqYO33y09jhypG/bSDHPuyCme/IdEHfWpLf/jKcpitVFyOC6UemgGk8v7Q5u2XE0vvwmanxhGlQ==", "dependencies": { - "minimist": "^1.2.0", - "through2": "^3.0.1" + "minimist": "^1.2.0" }, "bin": { "csv-parser": "bin/csv-parser" }, "engines": { - "node": ">= 8.16.0" + "node": ">= 10" } }, "node_modules/csv-writer": { @@ -2948,9 +2947,9 @@ } }, "node_modules/libphonenumber-js": { - "version": "1.10.47", - "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.10.47.tgz", - "integrity": "sha512-b4t7VQDV29xx/ni+58yl9KWPGjnDLDXCeCTLrD4V8vDpObXZRZBrg7uX/HWZ7YXiJKqdBDGgc+barUUTNB6Slw==" + "version": "1.10.48", + "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.10.48.tgz", + "integrity": "sha512-Vvcgt4+o8+puIBJZLdMshPYx9nRN3/kTT7HPtOyfYrSQuN9PGBF1KUv0g07fjNzt4E4GuA7FnsLb+WeAMzyRQg==" }, "node_modules/locate-path": { "version": "6.0.0", @@ -3434,9 +3433,9 @@ "dev": true }, "node_modules/node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "dependencies": { "whatwg-url": "^5.0.0" }, @@ -4662,15 +4661,6 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, - "node_modules/through2": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz", - "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==", - "dependencies": { - "inherits": "^2.0.4", - "readable-stream": "2 || 3" - } - }, "node_modules/timed-out": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", diff --git a/docker/package.json b/docker/package.json index 22269fc7..ab4e91fc 100644 --- a/docker/package.json +++ b/docker/package.json @@ -11,7 +11,7 @@ "dependencies": { "boxen": "^4.2.0", "chalk": "^3.0.0", - "csv-parser": "^2.3.5", + "csv-parser": "^3.0.0", "csv-writer": "^1.6.0", "date-fns": "^2.30.0", "eslint-plugin-promise": "^4.3.1", @@ -20,10 +20,10 @@ "html5parser": "^1.2.1", "htmlhint": "^0.11.0", "js-beautify": "^1.14.9", - "libphonenumber-js": "^1.10.47", + "libphonenumber-js": "^1.10.48", "minimatch": "^3.1.2", "mocha": "^9.2.2", - "node-fetch": "^2.6.12", + "node-fetch": "^2.7.0", "nodemailer": "^6.9.3", "puppeteer": "^3.3.0", "ramda": "^0.27.2", diff --git a/docker/sswlinkauditor.go b/docker/sswlinkauditor.go index 66844341..c56d59ba 100644 --- a/docker/sswlinkauditor.go +++ b/docker/sswlinkauditor.go @@ -208,12 +208,23 @@ func writeResultFile(allUrls map[string]LinkStatus) { f.WriteString("Source" + "\t" + "Destination" + "\t" + "Status" + "\t" + "StatusCode" + "\t" + "Anchor" + "\n") for _, v := range allUrls { - f.WriteString(v.srcUrl + "\t" + v.url + "\t" + v.status + "\t" + strconv.Itoa(v.statusCode) + "\t" + strings.ReplaceAll(v.anchor,"\"","") + "\n") + f.WriteString(sanitizeString(v.srcUrl) + "\t" + sanitizeString(v.url) + "\t" + sanitizeString(v.status) + "\t" + strconv.Itoa(v.statusCode) + "\t" + sanitizeString(v.anchor) + "\n") } f.Close() } +func sanitizeString(s string) string { + replacer := strings.NewReplacer( + "\"", "", + "\t", " ", + "\r\n", "", + "\n", "", + ); + + return replacer.Replace(s); +} + func isLinkUnscannable(a string, unscannableLinks []string) bool { for _, b := range unscannableLinks { if strings.HasPrefix(strings.ToLower(a), strings.ToLower(b)) { diff --git a/docker/test/common-spelling-mistakes.spec.js b/docker/test/common-spelling-mistakes.spec.js index 3ad4b601..5549b574 100644 --- a/docker/test/common-spelling-mistakes.spec.js +++ b/docker/test/common-spelling-mistakes.spec.js @@ -23,6 +23,12 @@ describe(`Rules: ${ruleId}`, () => { expect(messages.length).to.be(9); }); + it("long string contains the character should not result in an error", () => { + const code = `
${phone}
+This is the result from SSW CodeAuditor scan on ${scanSummary.url} on ${fns.format(new Date(scanSummary.buildDate), 'dd MMM yyyy, hh:mm aaaa')}
⏳ Duration: ${scanSummary.scanDuration} seconds
🚨 Broken Links: ${scanSummary.totalUnique404} / ${scanSummary.totalScanned} Bad links
⚠️ HTML Warnings: ${scanSummary.htmlWarnings}
❌ HTML Errors: ${scanSummary.htmlErrors}
See https://codeauditor.com/build/${scanSummary.runId} for full scan result
<This is the automated alert email from SSW CodeAuditor>
`, + html: `This is the result from SSW CodeAuditor scan on ${scanSummary.url} on ${fns.format(new Date(scanSummary.buildDate), 'dd MMM yyyy, hh:mm aaaa')}
⏳ Duration: ${scanSummary.scanDuration} seconds
🚨 Broken Links: ${scanSummary.uniqueBrokenLinks} / ${scanSummary.totalScanned} Bad links
⚠️ HTML Warnings: ${scanSummary.htmlWarnings}
❌ HTML Errors: ${scanSummary.htmlErrors}
See https://codeauditor.com/build/${scanSummary.runId} for full scan result
<This is the automated alert email from SSW CodeAuditor>
`, }); } @@ -418,7 +418,7 @@ exports.processBrokenLinks = ( const __getBadResults = (allUrls) => allUrls // Allow successful 2xx status code range (200-299) - .filter((url) => !(url["StatusCode"]?.startsWith('2') && url["StatusCode"]?.length === 3)) + .filter((url) => !((url["StatusCode"]?.startsWith('2') || url["StatusCode"]?.startsWith('3')) && url["StatusCode"]?.length === 3)) .map((x) => ({ src: x.Source || "", dst: x.Destination || "", diff --git a/ui/src/components/buildlistcardcomponents/HistoryChart.svelte b/ui/src/components/buildlistcardcomponents/HistoryChart.svelte index 3cba0590..0b1928fd 100644 --- a/ui/src/components/buildlistcardcomponents/HistoryChart.svelte +++ b/ui/src/components/buildlistcardcomponents/HistoryChart.svelte @@ -12,7 +12,7 @@ // Categorize and populate charts with data, title and color if (dataType === historyChartType.BadLinks) { chartTitle = historyChartType.BadLinks; - allDataToDisplay = value.map((i) => i.totalUnique404); + allDataToDisplay = value.map((i) => i.uniqueBrokenLinks); barColor = 'red' } else if (dataType === historyChartType.WarningCode) { chartTitle = historyChartType.WarningCode; diff --git a/ui/src/components/linkauditorcomponents/DetailsTable.svelte b/ui/src/components/linkauditorcomponents/DetailsTable.svelte index 618afff0..81b1eb28 100644 --- a/ui/src/components/linkauditorcomponents/DetailsTable.svelte +++ b/ui/src/components/linkauditorcomponents/DetailsTable.svelte @@ -14,10 +14,10 @@ export let unscannableLinks; let foundUnscannableLinks = []; - foundUnscannableLinks = builds.filter(build => unscannableLinks.some(link => build.dst.includes(link.url))); + foundUnscannableLinks = builds.filter(build => unscannableLinks.some(link => build.dst.includes(link))); // Filter out unscannable links - builds = builds.filter(build => !unscannableLinks.some(link => build.dst.includes(link.url))); + builds = builds.filter(build => !unscannableLinks.some(link => build.dst.includes(link))); let displayMode = 0; @@ -61,21 +61,7 @@ } -{#if builds.length === 0} -Source | ++ {url.src} + | +
---|---|
Anchor Text | +{url.link || ''} | +
Link | ++ {url.dst} + | +