Merge Staging to Main

api/functions/consts.js

@@ -24,29 +24,3 @@ exports.BLOB = {
 	htmlhint: 'htmlhint',
 	codeAuditor: 'codeauditor',
 };
-
-exports.unscannableLinks = [
-	{url: "https://learn.microsoft.com/en-us/"},
-	{url: "https://support.google.com/"},
-	{url: "https://twitter.com/"},
-	{url: "https://marketplace.visualstudio.com/"},
-	{url: "https://www.nuget.org/"},
-	{url: "https://make.powerautomate.com"},
-	{url: "https://www.microsoft.com/"},
-	{url: "http://www.microsoft.com/"},
-	{url: "https://answers.microsoft.com/"},
-	{url: "https://admin.microsoft.com/"},
-	{url: "https://ngrx.io"},
-	{url: "https://twitter.com"},
-	{url: "https://marketplace"},
-	{url: "https://www.nuget.org/"},
-	{url: "http://nuget.org"},
-	{url: "https://t.co"},
-	{url: "https://support.google.com"},
-	{url: "https://playwright.dev"},
-	{url: "https://www.theurlist.com/xamarinstreamers"},
-	{url: "https://dev.botframework.com"},
-	{url: "https://www.ssw.com.au/rules/rules-to-better-research-and-development/"},
-	{url: "https://www.ato.gov.au/Business/Research-and-development-tax-incentive/"},
-	{url: "https://learn.microsoft.com/en-us/assessments/?mode=home/"}
-  ]

api/functions/index.js

@@ -4,7 +4,6 @@ const admin = require('firebase-admin');
 const R = require('ramda');
 const fetch = require('node-fetch');
 const Queue = require('better-queue');
-const { unscannableLinks } = require('./consts');
 require('dotenv').config();
 
 const {
@@ -231,6 +230,7 @@ app.post('/scanresult/:api/:buildId', async (req, res) => {
 	const buildId = req.params.buildId;
 	const runId = newGuid();
 	const buildDate = new Date();
+	const unscannableLinks = await getUnscannableLinks();
 
 	const uid = await getUserIdFromApiKey(apikey);
 	if (!uid) {
@@ -262,11 +262,11 @@ app.post('/scanresult/:api/:buildId', async (req, res) => {
 		url,
 		cloc,
 		totalBrokenLinks: badUrls.length,
-		uniqueBrokenLinks: R.uniqBy(R.prop('dst'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link.url)))).length,
-		pagesWithBrokenLink: R.uniqBy(R.prop('src'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link.url)))).length,
+		uniqueBrokenLinks: R.uniqBy(R.prop('dst'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link)))).length,
+		pagesWithBrokenLink: R.uniqBy(R.prop('src'), badUrls.filter((x) => !unscannableLinks.some(link => x.dst.includes(link)))).length,
 		totalUnique404: R.uniqBy(
 			R.prop('dst'),
-			badUrls.filter((x) => x.statuscode === '404' && !unscannableLinks.some(link => x.dst.includes(link.url)))
+			badUrls.filter((x) => x.statuscode === '404' && !unscannableLinks.some(link => x.dst.includes(link)))
 		).length,
 		htmlWarnings: htmlWarnings ? htmlWarnings : 0,
 		htmlErrors: htmlErrors ? htmlErrors : 0,

api/functions/queries.js

@@ -42,8 +42,8 @@ const getExistingBrokenLinkCount = async (runId) => {
 
     const existingCount = result.reduce((count, item) => {
         if (item.runId === runId) {
-            if (!previousFailures.has(item.dst) && !unscannableLinks.find((i) => item.dst.startsWith(i))) {
-				const hasPrevious = result.find((i) => i.dst === item.dst && i.buildDate < item.buildDate);
+            if (!previousFailures.has(item.dst) && !unscannableLinks.some((i) => item.dst.includes(i))) {
+				const hasPrevious = result.some((i) => i.dst === item.dst && i.buildDate < item.buildDate);
 				previousFailures.set(item.dst, hasPrevious);
 
 				if (hasPrevious) {
@@ -279,7 +279,7 @@ exports.getAllScanSummaryFromUrl = (url, api) =>
 		for await (const item of iterator) {
 			if (item[0]) {
 				const existing = await getExistingBrokenLinkCount(item[0].runId);
-				item[0].totalUnique404Existing = existing;
+				item[0].totalUniqueBrokenLinksExisting = existing;
 			}
 
 			resolve(item);
@@ -314,17 +314,21 @@ exports.compareScans = (api, url) =>
 		for await (const item of entity) {
 			result.push(item);
 		}
+
+		const latestResult = result[0] || {};
+		const prevResult = result[1] || {};
+
 		let isErrorUp = {
-			isHtmlWarningsUp: result[0].htmlWarnings > result[1].htmlWarnings,
-			prevHtmlWarnings: result[1].htmlWarnings,
-			currHtmlWarnings: result[0].htmlWarnings,
-			isHtmlErrorsUp: result[0].htmlErrors > result[1].htmlErrors,
-			prevHtmlErrors: result[1].htmlErrors,
-			currHtmlErrors: result[0].htmlErrors,
-			isBrokenLinksUp: result[0].totalUnique404 > result[1].totalUnique404,
-			prevBrokenLinks: result[1].totalUnique404,
-			currBrokenLinks: result[0].totalUnique404,
-			latestRunId: result[0].runId
+			isHtmlWarningsUp: latestResult.htmlWarnings > prevResult.htmlWarnings,
+			prevHtmlWarnings: prevResult.htmlWarnings || 0,
+			currHtmlWarnings: latestResult.htmlWarnings || 0,
+			isHtmlErrorsUp: latestResult.htmlErrors > prevResult.htmlErrors,
+			prevHtmlErrors: prevResult.htmlErrors || 0,
+			currHtmlErrors: latestResult.htmlErrors || 0,
+			isBrokenLinksUp: latestResult.uniqueBrokenLinks > prevResult.uniqueBrokenLinks,
+			prevBrokenLinks: prevResult.uniqueBrokenLinks || 0,
+			currBrokenLinks: latestResult.uniqueBrokenLinks || 0,
+			latestRunId: latestResult.runId
 		} 
 		resolve(isErrorUp)
 	});

docker/customHtmlRules.js

@@ -61,15 +61,18 @@ exports.addCustomHtmlRule = () => {
                 scrumTerms.forEach((i) => {
                   var contentIndex = pageContent.indexOf(i);
                   var col = event.lastEvent.col;
-
-                  if (contentIndex >= 0) {
-                    reporter.warn(
-                      "Incorrect Scrum term: '" + i + "'.",
-                      event.line,
-                      col,
-                      self,
-                      event.raw
-                    );
+
+                  // Make sure the character has space and is not part of a long single string 
+                  if (pageContent.indexOf(' ') >= 0) {
+                    if (contentIndex >= 0) {
+                      reporter.warn(
+                        "Incorrect Scrum term: '" + i + "'.",
+                        event.line,
+                        col,
+                        self,
+                        event.raw
+                      );
+                    }
                   }
                 });
               }
@@ -445,17 +448,19 @@ exports.addCustomHtmlRule = () => {
               let pageContent = event.lastEvent.raw;
               if (pageContent) {
                 spellings.forEach((i) => {
-                  var contentIndex = pageContent.indexOf(i);
+                  var contentIndex = pageContent.indexOf(i) >= 0;
                   var col = event.lastEvent.col;
-
-                  if (contentIndex >= 0) {
-                    reporter.warn(
-                      "Incorrect spellings: '" + i + "'.",
-                      event.line,
-                      col,
-                      self,
-                      event.raw
-                    );
+                  // Make sure the character has space and is not part of a long single string 
+                  if (pageContent.indexOf(' ') >= 0) {
+                    if (contentIndex) {
+                      reporter.warn(
+                        "Incorrect spellings: '" + i + "'.",
+                        event.line,
+                        col,
+                        self,
+                        event.raw
+                      );
+                    }
                   }
                 });
               }
@@ -473,7 +478,7 @@ exports.addCustomHtmlRule = () => {
       init: function (parser, reporter) {
         const self = this;
         parser.addListener("all", (event) => {
-          if (event.raw && event.lastEvent && findPhoneNumbersInText(event.raw, "US").length) {
+          if (event.raw && event.lastEvent && findPhoneNumbersInText(event.raw, "AU").length) {
             const pageContent = event.lastEvent.raw;
             if (pageContent && event.lastEvent.tagName) {
               const tagName = event.lastEvent.tagName.toLowerCase();

docker/index.js

@@ -28,6 +28,7 @@ const {
   runLighthouseReport,
   runArtilleryLoadTest
 } = require("./utils");
+const { htmlHintConfig } = require("./api");
 
 const LIGHTHOUSEFOLDER = "./lhr.json";
 const ARTILLERYFOLDER = "./artilleryOut.json";
@@ -315,7 +316,7 @@ const processAndUpload = async (
       });
     } catch (error) {
       console.error(
-        `Error: Unabled to push data to dashboard service => ${error.message}`
+        `Error: Unable to push data to dashboard service => ${error.message}`
       );
     }
   }
@@ -334,13 +335,13 @@ const processAndUpload = async (
   // Upload selected HTMLHint Rules to the scan
   if (args.htmlhint && args.token && runId) {
     const result = await getHTMLHintRules(args.token, args.url);
+    const selectedRules = result?.selectedRules ?? Object.keys(htmlHintConfig).join(",");
 
-    if (result && result.selectedRules?.split(",").length > 0) {
-      const selectedRules = result.selectedRules;
+    if (selectedRules?.length > 0) {
       const res = await addHTMLHintRulesForScan(args.token, args.url, runId, selectedRules)
 
       if (res) {
-        console.log('Upload selected HTMLHint Rules successfully')
+        console.log('Uploaded selected HTMLHint Rules successfully');
       } else {
         throw new Error("Failed to add custom html rules for each scan");
       }

docker/package.json

@@ -11,7 +11,7 @@
 	"dependencies": {
 		"boxen": "^4.2.0",
 		"chalk": "^3.0.0",
-		"csv-parser": "^2.3.5",
+		"csv-parser": "^3.0.0",
 		"csv-writer": "^1.6.0",
 		"date-fns": "^2.30.0",
 		"eslint-plugin-promise": "^4.3.1",
@@ -20,10 +20,10 @@
 		"html5parser": "^1.2.1",
 		"htmlhint": "^0.11.0",
 		"js-beautify": "^1.14.9",
-		"libphonenumber-js": "^1.10.47",
+		"libphonenumber-js": "^1.10.48",
 		"minimatch": "^3.1.2",
 		"mocha": "^9.2.2",
-		"node-fetch": "^2.6.12",
+		"node-fetch": "^2.7.0",
 		"nodemailer": "^6.9.3",
 		"puppeteer": "^3.3.0",
 		"ramda": "^0.27.2",

docker/sswlinkauditor.go

@@ -208,12 +208,23 @@ func writeResultFile(allUrls map[string]LinkStatus) {
 
 	f.WriteString("Source" + "\t" + "Destination" + "\t" + "Status" + "\t" + "StatusCode" + "\t" + "Anchor" + "\n")
 	for _, v := range allUrls {
-		f.WriteString(v.srcUrl + "\t" + v.url + "\t" + v.status + "\t" + strconv.Itoa(v.statusCode) + "\t" + strings.ReplaceAll(v.anchor,"\"","") + "\n")
+		f.WriteString(sanitizeString(v.srcUrl) + "\t" + sanitizeString(v.url) + "\t" + sanitizeString(v.status) + "\t" + strconv.Itoa(v.statusCode) + "\t" + sanitizeString(v.anchor) + "\n")
 	}
 
 	f.Close()
 }
 
+func sanitizeString(s string) string {
+	replacer := strings.NewReplacer(
+		"\"", "",
+		"\t", " ",
+		"\r\n", "",
+		"\n", "",
+	);
+
+	return replacer.Replace(s);
+}
+
 func isLinkUnscannable(a string, unscannableLinks []string) bool {
 	for _, b := range unscannableLinks {
 		if strings.HasPrefix(strings.ToLower(a), strings.ToLower(b)) {

docker/test/common-spelling-mistakes.spec.js

@@ -23,6 +23,12 @@ describe(`Rules: ${ruleId}`, () => {
     expect(messages.length).to.be(9);
   });
 
+  it("long string contains the character should not result in an error", () => {
+    const code = `<div>wgARCAAKABQDASIAAhEBAxEB/8QAGAAAAwEBAAAAAAAAAAAAAAAAAAIDBAX/xAAUAQE</div>`;
+    const messages = HTMLHint.verify(code, ruleOptions);
+    expect(messages.length).to.be(0);
+  });
+
   ["meta", "link", "script", "svg"].forEach((tag) => {
     it(`incorrect terms in a <${tag}> tag should not result in an error`, () => {
       const code = `<${tag}>a.k.a A.K.A AKA e-mail EMail can not web site user name task bar</${tag}>`;

docker/test/grammar-scrum-terms.spec.js

@@ -25,6 +25,12 @@ describe(`Rules: ${ruldId}`, () => {
     expect(messages.length).to.be(11);
   });
 
+  it("long string contains the character should not result in an error", () => {
+    const code = `<div>blahblahsprintblahblah</div>`;
+    const messages = HTMLHint.verify(code, ruleOptions);
+    expect(messages.length).to.be(0);
+  });
+
   it("Incorrect Scrum terms in <a> tag should not result in an error", () => {
     const code =
       "<a href='sprint'>scrum, sprint, product owner</a>";