From ddb9c1ef0cbc22aa23ad151855e9dd383ff14c11 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Wed, 8 Nov 2023 09:14:55 +1100 Subject: [PATCH 1/8] Optimise getHTMLHintRulesByRunId by calling PartitionKey --- api/functions/queries.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/api/functions/queries.js b/api/functions/queries.js index b0f46447..2fc57d19 100644 --- a/api/functions/queries.js +++ b/api/functions/queries.js @@ -166,17 +166,17 @@ exports.getHTMLHintRules = (api, url, isGetAllRecords) => resolve(isGetAllRecords ? result : result[result.length - 1] || {}) }); -exports.getHTMLHintRulesByRunId = (runId) => - new Promise(async (resolve) => { - const entity = new TableClient(azureUrl, TABLE.htmlhintrules, credential).listEntities({ - queryOptions: { filter: odata`RowKey eq ${runId}` } - }); - let result = [] - for await (const item of entity) { - result.push(item); - } - resolve(result[0] || {}) +exports.getHTMLHintRulesByRunId = async (runId) => { + const doc = await getRun(runId); + const entity = new TableClient(azureUrl, TABLE.htmlhintrules, credential).listEntities({ + queryOptions: { filter: odata`PartitionKey eq ${doc.apikey} and RowKey eq ${runId}` } }); + let result = [] + for await (const item of entity) { + result.push(item); + } + return result[0] || {}; +} exports.getPersonalSummary = (api, showAll) => new Promise(async (resolve) => { From 71084afe5dc62d625a89b74aa004f20df879afbb Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Wed, 8 Nov 2023 10:15:43 +1100 Subject: [PATCH 2/8] Add error handling and response messages to viewsource --- api/functions/index.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/api/functions/index.js b/api/functions/index.js index 0a822ec9..37f65493 100644 --- a/api/functions/index.js +++ b/api/functions/index.js @@ -146,9 +146,15 @@ app.get('/allscans', async (req, res) => { }); app.get('/viewsource', async (req, res) => { - const resp = await fetch(req.query.url); - const source = await resp.text(); - res.send(source); + const resp = await fetch(req.query.url).catch((err) => { + res.send(`Failed to load source: ${err.message}`); + }); + if (resp.ok) { + const source = await resp.text(); + res.send(source); + } else { + res.send(`Failed to load source: ${resp.status} - ${resp.statusText}`); + } }); app.get('/run/:runId', async (req, res) => { From b29edc46844747c2980cbacf7f47a7d26fe5f1a5 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Wed, 8 Nov 2023 11:20:26 +1100 Subject: [PATCH 3/8] Resolve request forgery alert --- api/functions/index.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/api/functions/index.js b/api/functions/index.js index 37f65493..b1519918 100644 --- a/api/functions/index.js +++ b/api/functions/index.js @@ -146,7 +146,16 @@ app.get('/allscans', async (req, res) => { }); app.get('/viewsource', async (req, res) => { - const resp = await fetch(req.query.url).catch((err) => { + const target = new URL(req.query.url); + const functionHost = '-sswlinkauditor-c1131.cloudfunctions.net'; + + // Prevent fetching from same host to prevent request forgery + if (target.hostname.includes(functionHost) || target.hostname === 'localhost') { + res.send('Cannot fetch from internal URL'); + return; + } + + const resp = await fetch(target.href).catch((err) => { res.send(`Failed to load source: ${err.message}`); }); if (resp.ok) { From d4f4c41e7ade5030ce739569f4e5ef4de6f4da61 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Wed, 8 Nov 2023 11:24:01 +1100 Subject: [PATCH 4/8] Update text --- api/functions/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/functions/index.js b/api/functions/index.js index b1519918..67ce71d8 100644 --- a/api/functions/index.js +++ b/api/functions/index.js @@ -149,9 +149,9 @@ app.get('/viewsource', async (req, res) => { const target = new URL(req.query.url); const functionHost = '-sswlinkauditor-c1131.cloudfunctions.net'; - // Prevent fetching from same host to prevent request forgery + // Disallow fetching from same host to prevent request forgery if (target.hostname.includes(functionHost) || target.hostname === 'localhost') { - res.send('Cannot fetch from internal URL'); + res.send('Cannot fetch from internal host'); return; } From ca1ec6e023448caa69406249df3d03c3f315fd4b Mon Sep 17 00:00:00 2001 From: tombui99 Date: Thu, 9 Nov 2023 09:49:42 +1100 Subject: [PATCH 5/8] Updated custom option input UI from designer feedback --- ui/src/components/htmlhintcomponents/UpdateHTMLRules.svelte | 6 +++--- ui/src/utils/utils.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ui/src/components/htmlhintcomponents/UpdateHTMLRules.svelte b/ui/src/components/htmlhintcomponents/UpdateHTMLRules.svelte index f179a9c1..27d37ec4 100644 --- a/ui/src/components/htmlhintcomponents/UpdateHTMLRules.svelte +++ b/ui/src/components/htmlhintcomponents/UpdateHTMLRules.svelte @@ -358,9 +358,9 @@ on:keypress={undefined} > Edit -
+
{#if customHtmlRuleOptions && customHtmlRuleOptions.length > 0 && customHtmlRuleOptions.find(x => x.ruleId === rule.rule)?.optionValue.length > 0} -
+
Applied custom option value: @@ -370,7 +370,7 @@
{/if} {#if currSelectedCustomOption === index} -
+
{rule.customOptionsMessage}
diff --git a/ui/src/utils/utils.js b/ui/src/utils/utils.js index 621ce66a..b76fcac2 100644 --- a/ui/src/utils/utils.js +++ b/ui/src/utils/utils.js @@ -560,7 +560,7 @@ export const customHtmlHintRules = [ ruleLink: "https://www.ssw.com.au/rules/do-you-know-to-hyperlink-your-phone-numbers", type: RuleType.Warning, isEnableCustomOptions: true, - customOptionsMessage: 'Please choose your country code:', + customOptionsMessage: 'Please choose the country code:', customOptionInputType: customOptionInputType.dropDown, customOptionDropdownValues: countryCodes }, @@ -588,7 +588,7 @@ export const customHtmlHintRules = [ ruleLink: "https://ssw.com.au/rules/avoid-absolute-internal-links/", type: RuleType.Warning, isEnableCustomOptions: true, - customOptionsMessage: 'Please enter your website internal URL:', + customOptionsMessage: 'Please enter the website internal URL:', customOptionInputType: customOptionInputType.singleTextBox, customOptionInputValueType: 'url' }, From 145380445e06c36c2db1e91e4c0054c3bf9e3fe3 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Thu, 9 Nov 2023 10:24:16 +1100 Subject: [PATCH 6/8] Set a scan result cap --- api/functions/queries.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/functions/queries.js b/api/functions/queries.js index 8fcaaf15..bb0c1532 100644 --- a/api/functions/queries.js +++ b/api/functions/queries.js @@ -73,11 +73,14 @@ exports.getConfig = (api) => exports.getScanDetails = async (runId) => { const scan = await exports.getSummaryById(runId); let filter; + const filterDays = 90; + const startDate = new Date(scan.buildDate); + startDate.setDate(startDate.getDate() - filterDays); if (scan.scanResultVersion === 2) { - filter = `PartitionKey eq '${scan.apiKey}-${slug(scan.url)}'`; + filter = `PartitionKey eq '${scan.apiKey}-${slug(scan.url)}' and buildDate ge datetime'${startDate.toISOString()}' and buildDate le datetime'${scan.buildDate.toISOString()}'`; } else { - filter = odata`PartitionKey eq ${scan.partitionKey} and src ge ${scan.url} and src le ${incrementString(scan.url)}`; + filter = odata`PartitionKey eq ${scan.partitionKey} and src ge ${scan.url} and src le ${incrementString(scan.url)} and buildDate ge datetime'${startDate.toISOString()}' and buildDate le datetime'${scan.buildDate.toISOString()}'`; } const entity = new TableClient(azureUrl, TABLE.ScanResults, credential).listEntities({ @@ -256,7 +259,7 @@ exports.getSummaryById = (runId) => let summary = await getSummary(`PartitionKey eq '${doc.apikey}-${doc.runId}'`); - if (!summary) { + if (!summary || summary.scanResultVersion !== 2) { summary = await getSummary(odata`PartitionKey eq ${doc.apikey} and runId eq ${doc.runId}`); } From 976c716ab477fa18f9b2e71df7d85e536a37ecf7 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Thu, 9 Nov 2023 12:33:35 +1100 Subject: [PATCH 7/8] Filter and sort on all results --- api/functions/queries.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/api/functions/queries.js b/api/functions/queries.js index bb0c1532..e04abc8a 100644 --- a/api/functions/queries.js +++ b/api/functions/queries.js @@ -223,23 +223,25 @@ exports.getAllPublicSummary = (showAll) => return self.findIndex(v => v.runId === value.runId) === index; })) } else { - // Top 500 scans in last 24 months + // Top 500 scans in last 12 months var date = new Date(); date.setMonth(date.getMonth() - 12); const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({ queryOptions: { filter: odata`isPrivate eq ${false} and buildDate gt datetime'${date.toISOString()}'` } }); - const iterator = entity.byPage({ maxPageSize: parseInt(process.env.MAX_SCAN_SIZE) }); - let result = []; - for await (const item of iterator) { - result = item; - break; + let result = [] + for await (const item of entity) { + result.push(item); } - resolve(result.filter((value, index, self) => { + const filteredResult = result.filter((value, index, self) => { return self.findIndex(v => v.runId === value.runId) === index; - })) + }) + .sort((a, b) => (a.rowKey > b.rowKey) ? 1 : -1) + .slice(0, parseInt(process.env.MAX_SCAN_SIZE)); + + resolve(filteredResult) } }); From 91c9f2d26865ab7a348f8e606dc3c8e7aff99547 Mon Sep 17 00:00:00 2001 From: Zachary Keeping Date: Thu, 9 Nov 2023 14:47:51 +1100 Subject: [PATCH 8/8] Optimisations --- api/functions/queries.js | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/api/functions/queries.js b/api/functions/queries.js index e04abc8a..78c7b35c 100644 --- a/api/functions/queries.js +++ b/api/functions/queries.js @@ -219,9 +219,16 @@ exports.getAllPublicSummary = (showAll) => result.push(item); } - resolve(result.filter((value, index, self) => { - return self.findIndex(v => v.runId === value.runId) === index; - })) + const seen = new Set(); + const filteredResult = result.filter(value => { + if (seen.has(value.runId)) { + return false; + } + seen.add(value.runId); + return true; + }) + + resolve(filteredResult); } else { // Top 500 scans in last 12 months var date = new Date(); @@ -235,13 +242,18 @@ exports.getAllPublicSummary = (showAll) => result.push(item); } - const filteredResult = result.filter((value, index, self) => { - return self.findIndex(v => v.runId === value.runId) === index; + const seen = new Set(); + const filteredResult = result.filter(value => { + if (seen.has(value.runId)) { + return false; + } + seen.add(value.runId); + return true; }) .sort((a, b) => (a.rowKey > b.rowKey) ? 1 : -1) .slice(0, parseInt(process.env.MAX_SCAN_SIZE)); - resolve(filteredResult) + resolve(filteredResult); } });