Merge Staging into Main

api/functions/commands.js

@@ -16,15 +16,26 @@ const {
 const azure = require('azure-storage');
 const slug = require('slug');
 
-exports.insertScanResult = (api, buildId, runId, data, buildDate) => {
+exports.insertScanResult = async (api, buildId, runId, data, buildDate, url) => {
 	const entGen = azure.TableUtilities.entityGenerator;
 	let entity = {
 		PartitionKey: entGen.String(api),
 		RowKey: entGen.String(`${api}-${newGuid()}`),
 		buildId: entGen.String(buildId),
 		runId: entGen.String(runId),
 		buildDate: entGen.DateTime(buildDate),
+		apiKey: entGen.String(api)
 	};
+	let entityRunIndexed = {
+		...entity,
+		PartitionKey: entGen.String(`${api}-${runId}`),
+	};
+	let entityUrlIndexed = {
+		...entity,
+		PartitionKey: entGen.String(`${api}-${slug(url)}`),
+	};
+	await insertEntity(TABLE.ScanResults, replaceProp(data, entityRunIndexed));
+	await insertEntity(TABLE.ScanResults, replaceProp(data, entityUrlIndexed));
 	return insertEntity(TABLE.ScanResults, replaceProp(data, entity));
 };
 
@@ -105,16 +116,28 @@ exports.addHTMLHintRulesForEachRun = (api, data) => {
 	);
 };
 
-exports.insertScanSummary = (api, buildId, runId, buildDate, data) => {
+exports.insertScanSummary = async (api, buildId, runId, buildDate, data) => {
 	var entGen = azure.TableUtilities.entityGenerator;
 	// use Log tail pattern to get native sort from Table Storage
-	var entity = {
+	const entity = {
 		PartitionKey: entGen.String(api),
 		RowKey: entGen.String(getReversedTick()),
 		buildId: entGen.String(buildId),
 		runId: entGen.String(runId),
 		buildDate: entGen.DateTime(buildDate),
+		scanResultVersion: entGen.Int32(2),
+		apiKey: entGen.String(api)
+	};
+	const entityRunIndexed = {
+		...entity,
+		PartitionKey: entGen.String(`${api}-${runId}`),
 	};
+	let entityUrlIndexed = {
+		...entity,
+		PartitionKey: entGen.String(`${api}-${slug(data.url)}`),
+	};
+	await insertEntity(TABLE.Scans, replaceProp(data, entityRunIndexed));
+	await insertEntity(TABLE.Scans, replaceProp(data, entityUrlIndexed));
 	return insertEntity(TABLE.Scans, replaceProp(data, entity));
 };
 
@@ -139,6 +162,17 @@ exports.removeAlertEmailAddress = (api, rowkey) => {
 	return deleteEntity(TABLE.alertEmailAddresses, entity);
 };
 
+exports.addCustomHtmlRuleOptions = (api, data) => {
+	const entGen = azure.TableUtilities.entityGenerator;
+	return updateEntity(
+		TABLE.HtmlRulesCustomOptions,
+		replaceProp(data, {
+			PartitionKey: entGen.String(api),
+			RowKey: entGen.String(data.ruleId),
+		})
+	);
+};
+
 exports.uploadLighthouseReport = (runId, lhr) =>
 	uploadBlob(BLOB.lhr, `${runId}.json`, JSON.stringify(lhr));
 

api/functions/consts.js

@@ -14,7 +14,8 @@ exports.TABLE = {
 	ScanResults: 'ScanResults',
 	htmlhintrules: 'htmlhintrules',
 	alertEmailAddresses: 'alertEmailAddresses',
-	UnscannableLinks: 'UnscannableLinks'
+	UnscannableLinks: 'UnscannableLinks',
+	HtmlRulesCustomOptions: 'HtmlRulesCustomOptions'
 };
 
 // blob storage names cannot have uppercase or numbers

api/functions/index.js

@@ -22,6 +22,7 @@ const {
 	addHTMLHintRulesForEachRun,
 	addAlertEmailAddresses,
 	removeAlertEmailAddress,
+	addCustomHtmlRuleOptions,
 } = require('./commands');
 const {
 	getPersonalSummary,
@@ -39,6 +40,7 @@ const {
 	getAllScanSummaryFromUrl,
 	getUnscannableLinks,
 	compareScans,
+	getCustomHtmlRuleOptions,
 } = require('./queries');
 const {
 	newGuid,
@@ -179,6 +181,14 @@ app.get('/unscannableLinks', async (req, res) => {
 	res.json(await getUnscannableLinks());
 });
 
+app.post('/config/getCustomHtmlRuleOptions/:api', async (req, res) => {
+	res.json(await getCustomHtmlRuleOptions(req.params.api, req.body.url));
+});
+
+app.post('/config/addCustomHtmlRuleOptions/:api', async (req, res) => {
+	res.json(await addCustomHtmlRuleOptions(req.params.api, req.body));
+});
+
 app.post('/scanresult/:api/:buildId', async (req, res) => {
 	const {
 		badUrls,
@@ -230,7 +240,6 @@ app.post('/scanresult/:api/:buildId', async (req, res) => {
 	const buildId = req.params.buildId;
 	const runId = newGuid();
 	const buildDate = new Date();
-	const unscannableLinks = await getUnscannableLinks();
 
 	const uid = await getUserIdFromApiKey(apikey);
 	if (!uid) {
@@ -314,7 +323,8 @@ app.post('/scanresult/:api/:buildId', async (req, res) => {
 						buildId,
 						runId,
 						brokenLinkData,
-						buildDate
+						buildDate,
+						url
 					);
 					cb(data);
 				}, {

api/functions/queries.js

@@ -72,7 +72,14 @@ exports.getConfig = (api) =>
 
 exports.getScanDetails = async (runId) => {
 	const scan = await exports.getSummaryById(runId);
-	const filter = odata`PartitionKey eq ${scan.partitionKey} and src ge ${scan.url} and src le ${incrementString(scan.url)}`;
+	let filter;
+
+	if (scan.scanResultVersion === 2) {
+		filter = `PartitionKey eq '${scan.partitionKey}-${slug(scan.url)}'`;
+	} else {
+		filter = odata`PartitionKey eq ${scan.partitionKey} and src ge ${scan.url} and src le ${incrementString(scan.url)}`;
+	}
+
     const entity = new TableClient(azureUrl, TABLE.ScanResults, credential).listEntities({
         queryOptions: { filter }
     });
@@ -208,7 +215,10 @@ exports.getAllPublicSummary = (showAll) =>
 			for await (const item of entity) {
 				result.push(item);
 			}
-			resolve(result)
+
+			resolve(result.filter((value, index, self) => {
+				return self.findIndex(v => v.runId === value.runId) === index;
+			}))
 		} else {
 			// Top 500 scans in last 24 months
 			var date = new Date();
@@ -218,24 +228,39 @@ exports.getAllPublicSummary = (showAll) =>
 				queryOptions: { filter: odata`isPrivate eq ${false} and buildDate gt datetime'${date.toISOString()}'` }
 			});
 			const iterator = entity.byPage({ maxPageSize: parseInt(process.env.MAX_SCAN_SIZE) });
+			let result = [];
 			for await (const item of iterator) {
-				resolve(item)
+				result = item;
 				break;
 			}
+
+			resolve(result.filter((value, index, self) => {
+				return self.findIndex(v => v.runId === value.runId) === index;
+			}))
 		}
 	});
 
 exports.getSummaryById = (runId) => 
 	getRun(runId).then((doc) =>
 		new Promise(async (resolve) => {
-			const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({
-				queryOptions: { filter: odata`PartitionKey eq ${doc.apikey} and runId eq ${doc.runId}` }
-			});
-			let result = []
-			for await (const item of entity) {
-				result.push(item);
+			const getSummary = async (filter) => {
+				const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({
+					queryOptions: { filter }
+				});
+				let result = []
+				for await (const item of entity) {
+					result.push(item);
+				}
+				return result[0];
+			};
+
+			let summary = await getSummary(`PartitionKey eq '${doc.apikey}-${doc.runId}'`);
+
+			if (!summary) {
+				summary = await getSummary(odata`PartitionKey eq ${doc.apikey} and runId eq ${doc.runId}`);
 			}
-			resolve(result[0] || {})
+
+			resolve(summary || {});
 		}));
 
 exports.getLatestSummaryFromUrlAndApi = (url, api) => 
@@ -272,19 +297,30 @@ exports.getAlertEmailAddressesFromTokenAndUrl = (api, url) =>
 
 exports.getAllScanSummaryFromUrl = (url, api) =>
 	new Promise(async (resolve) => {
-		const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({
-			queryOptions: { filter: odata`url eq ${url} and PartitionKey eq ${api}` }
-		});
-		const iterator = entity.byPage({ maxPageSize: 10 });
-		for await (const item of iterator) {
-			if (item[0]) {
-				const existing = await getExistingBrokenLinkCount(item[0].runId);
-				item[0].totalUniqueBrokenLinksExisting = existing;
+		const getSummary = async (filter) => {
+			const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({
+				queryOptions: { filter }
+			});
+			const iterator = entity.byPage({ maxPageSize: 10 });
+			let result;
+			for await (const item of iterator) {
+				if (item[0]) {
+					const existing = await getExistingBrokenLinkCount(item[0].runId);
+					item[0].totalUniqueBrokenLinksExisting = existing;
+				}
+				result = item;
+				break;
 			}
-
-			resolve(item);
-			break;
+			return result;
+		};
+
+		let summary = await getSummary(`PartitionKey eq '${api}-${slug(url)}'`);
+
+		if (!summary) {
+			summary = await getSummary(odata`url eq ${url} and PartitionKey eq ${api}`);
 		}
+
+		resolve(summary);
 	});
 
 exports.getUnscannableLinks = () =>
@@ -332,3 +368,15 @@ exports.compareScans = (api, url) =>
 		} 
 		resolve(isErrorUp)
 	});
+
+exports.getCustomHtmlRuleOptions = (api, url) => 
+	new Promise(async (resolve) => {
+		const entity = new TableClient(azureUrl, TABLE.HtmlRulesCustomOptions, credential).listEntities({
+			queryOptions: { filter: odata`PartitionKey eq ${api} and url eq ${url}` }
+		});
+		let result = []
+		for await (const item of entity) {
+			result.push(item);
+		}
+		resolve(result || [])
+	})

docker/api.js

@@ -125,6 +125,20 @@ exports.getAllScanSummaryFromUrl = (api, url) => {
   );
 };
 
+exports.getCustomHtmlRuleOptions = (api, url) => {
+  return fetch(`${endpoint}/api/config/getCustomHtmlRuleOptions/${api}`, {
+    method: "POST",
+    body: JSON.stringify({url}),
+    headers: { "Content-Type": "application/json" },
+  }).then((res) => {
+    if (res) {
+      return res.ok ? res.json() : [];
+    } else {
+      throw Error("Failed to get custom html rule options");
+    }
+  });
+};
+
 exports.htmlHintConfig = {
   "grammar-scrum-terms": true,
   "code-block-missing-language": true,

docker/customHtmlRules.js

@@ -1,7 +1,10 @@
+const { getCustomHtmlRuleOptions } = require("./api");
 const HTMLHint = require("htmlhint").default;
 const findPhoneNumbersInText = require('libphonenumber-js').findPhoneNumbersInText;
 
-exports.addCustomHtmlRule = () => {
+exports.addCustomHtmlRule = async (apiToken, url) => {
+  const customRuleOptions = await getCustomHtmlRuleOptions(apiToken, url);
+
   HTMLHint.addRule({
     id: "code-block-missing-language",
     description: "Code blocks must contain a language specifier.",
@@ -380,9 +383,15 @@ exports.addCustomHtmlRule = () => {
       parser.addListener("tagstart", (event) => {
         var tagName = event.tagName.toLowerCase(),
         mapAttrs = parser.getMapAttrs(event.attrs);
+        const ruleId = "detect-absolute-references-url-path-correctly";
         if (tagName === "a") {
           if (mapAttrs["href"]) {
-            if (mapAttrs["href"].startsWith("https://ssw.com.au/rules")) {
+            // Check if custom options exist in this rule
+            let optionValue = '';
+            if (customRuleOptions && customRuleOptions.length > 0 && customRuleOptions.filter(option => option.ruleId === ruleId).length > 0) {
+              optionValue = customRuleOptions.find(option => option.ruleId === ruleId).optionValue
+            }
+            if (mapAttrs["href"].startsWith(optionValue.length > 0 ? optionValue : "https://ssw.com.au/rules")) {
               if (!mapAttrs["href"].startsWith("/")) {
               reporter.warn(
                 "URLs must be formatted to direct to a url path correctly.",
@@ -433,17 +442,26 @@ exports.addCustomHtmlRule = () => {
       var self = this;
 
       parser.addListener("text", (event) => {
-        var spellings = [
-          "a.k.a",
-          "A.K.A",
-          "AKA",
-          "e-mail",
-          "EMail",
-          "can not",
-          "web site",
-          "user name",
-          "task bar"
-        ];
+        const ruleId = "common-spelling-mistakes";
+        let optionValue = [];
+        // Check if custom options exist in this rule
+        if (customRuleOptions && customRuleOptions.length > 0 && customRuleOptions.filter(option => option.ruleId === ruleId).length > 0) {
+          optionValue = customRuleOptions.find(option => option.ruleId === ruleId).optionValue.split(',');
+        }
+        var spellings = 
+          optionValue.length > 0 ? 
+          optionValue :
+          [
+            "a.k.a",
+            "A.K.A",
+            "AKA",
+            "e-mail",
+            "EMail",
+            "can not",
+            "web site",
+            "user name",
+            "task bar"
+          ];
 
         if (event.raw) {
           const pageContent = event.raw;
@@ -485,7 +503,12 @@ exports.addCustomHtmlRule = () => {
       init: function (parser, reporter) {
         const self = this;
         let isInCodeBlock = false;
+        let optionValue = '';
         parser.addListener("tagstart", (event) => {
+          // Check if custom options exist in this rule
+          if (customRuleOptions && customRuleOptions.length > 0 && customRuleOptions.filter(option => option.ruleId === ruleId).length > 0) {
+            optionValue = customRuleOptions.find(option => option.ruleId === ruleId).optionValue
+          }
           const tagName = event.tagName.toLowerCase();
           if (tagName === "code") {
             isInCodeBlock = true;
@@ -498,7 +521,9 @@ exports.addCustomHtmlRule = () => {
           }
         });
         parser.addListener("text", (event) => {
-          if (event.raw && event.lastEvent && findPhoneNumbersInText(event.raw, "AU").length) {
+          // Replace "." and "/" characters to avoid false positives when parsing phone numbers
+          const text = event.raw?.replace(/\.|\//g, "_");
+          if (text && event.lastEvent && findPhoneNumbersInText(text, optionValue.length > 0 ? optionValue : 'AU').length) {
             const pageContent = event.lastEvent.raw;
             if (pageContent && event.lastEvent.tagName) {
               const tagName = event.lastEvent.tagName.toLowerCase();