diff --git a/api/functions/commands.js b/api/functions/commands.js index a2a5d232..1aa6118b 100644 --- a/api/functions/commands.js +++ b/api/functions/commands.js @@ -16,7 +16,7 @@ const { const azure = require('azure-storage'); const slug = require('slug'); -exports.insertScanResult = (api, buildId, runId, data, buildDate) => { +exports.insertScanResult = async (api, buildId, runId, data, buildDate, url) => { const entGen = azure.TableUtilities.entityGenerator; let entity = { PartitionKey: entGen.String(api), @@ -24,7 +24,18 @@ exports.insertScanResult = (api, buildId, runId, data, buildDate) => { buildId: entGen.String(buildId), runId: entGen.String(runId), buildDate: entGen.DateTime(buildDate), + apiKey: entGen.String(api) }; + let entityRunIndexed = { + ...entity, + PartitionKey: entGen.String(`${api}-${runId}`), + }; + let entityUrlIndexed = { + ...entity, + PartitionKey: entGen.String(`${api}-${slug(url)}`), + }; + await insertEntity(TABLE.ScanResults, replaceProp(data, entityRunIndexed)); + await insertEntity(TABLE.ScanResults, replaceProp(data, entityUrlIndexed)); return insertEntity(TABLE.ScanResults, replaceProp(data, entity)); }; @@ -105,16 +116,28 @@ exports.addHTMLHintRulesForEachRun = (api, data) => { ); }; -exports.insertScanSummary = (api, buildId, runId, buildDate, data) => { +exports.insertScanSummary = async (api, buildId, runId, buildDate, data) => { var entGen = azure.TableUtilities.entityGenerator; // use Log tail pattern to get native sort from Table Storage - var entity = { + const entity = { PartitionKey: entGen.String(api), RowKey: entGen.String(getReversedTick()), buildId: entGen.String(buildId), runId: entGen.String(runId), buildDate: entGen.DateTime(buildDate), + scanResultVersion: entGen.Int32(2), + apiKey: entGen.String(api) + }; + const entityRunIndexed = { + ...entity, + PartitionKey: entGen.String(`${api}-${runId}`), + }; + let entityUrlIndexed = { + ...entity, + PartitionKey: entGen.String(`${api}-${slug(data.url)}`), }; + await insertEntity(TABLE.Scans, replaceProp(data, entityRunIndexed)); + await insertEntity(TABLE.Scans, replaceProp(data, entityUrlIndexed)); return insertEntity(TABLE.Scans, replaceProp(data, entity)); }; diff --git a/api/functions/index.js b/api/functions/index.js index a986e309..a7ca7604 100644 --- a/api/functions/index.js +++ b/api/functions/index.js @@ -230,7 +230,6 @@ app.post('/scanresult/:api/:buildId', async (req, res) => { const buildId = req.params.buildId; const runId = newGuid(); const buildDate = new Date(); - const unscannableLinks = await getUnscannableLinks(); const uid = await getUserIdFromApiKey(apikey); if (!uid) { @@ -314,7 +313,8 @@ app.post('/scanresult/:api/:buildId', async (req, res) => { buildId, runId, brokenLinkData, - buildDate + buildDate, + url ); cb(data); }, { diff --git a/api/functions/queries.js b/api/functions/queries.js index 89fe8601..ad5f47c5 100644 --- a/api/functions/queries.js +++ b/api/functions/queries.js @@ -72,7 +72,14 @@ exports.getConfig = (api) => exports.getScanDetails = async (runId) => { const scan = await exports.getSummaryById(runId); - const filter = odata`PartitionKey eq ${scan.partitionKey} and src ge ${scan.url} and src le ${incrementString(scan.url)}`; + let filter; + + if (scan.scanResultVersion === 2) { + filter = `PartitionKey eq '${scan.partitionKey}-${slug(scan.url)}'`; + } else { + filter = odata`PartitionKey eq ${scan.partitionKey} and src ge ${scan.url} and src le ${incrementString(scan.url)}`; + } + const entity = new TableClient(azureUrl, TABLE.ScanResults, credential).listEntities({ queryOptions: { filter } }); @@ -208,7 +215,10 @@ exports.getAllPublicSummary = (showAll) => for await (const item of entity) { result.push(item); } - resolve(result) + + resolve(result.filter((value, index, self) => { + return self.findIndex(v => v.runId === value.runId) === index; + })) } else { // Top 500 scans in last 24 months var date = new Date(); @@ -218,24 +228,39 @@ exports.getAllPublicSummary = (showAll) => queryOptions: { filter: odata`isPrivate eq ${false} and buildDate gt datetime'${date.toISOString()}'` } }); const iterator = entity.byPage({ maxPageSize: parseInt(process.env.MAX_SCAN_SIZE) }); + let result = []; for await (const item of iterator) { - resolve(item) + result = item; break; } + + resolve(result.filter((value, index, self) => { + return self.findIndex(v => v.runId === value.runId) === index; + })) } }); exports.getSummaryById = (runId) => getRun(runId).then((doc) => new Promise(async (resolve) => { - const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({ - queryOptions: { filter: odata`PartitionKey eq ${doc.apikey} and runId eq ${doc.runId}` } - }); - let result = [] - for await (const item of entity) { - result.push(item); + const getSummary = async (filter) => { + const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({ + queryOptions: { filter } + }); + let result = [] + for await (const item of entity) { + result.push(item); + } + return result[0]; + }; + + let summary = await getSummary(`PartitionKey eq '${doc.apikey}-${doc.runId}'`); + + if (!summary) { + summary = await getSummary(odata`PartitionKey eq ${doc.apikey} and runId eq ${doc.runId}`); } - resolve(result[0] || {}) + + resolve(summary || {}); })); exports.getLatestSummaryFromUrlAndApi = (url, api) => @@ -272,19 +297,30 @@ exports.getAlertEmailAddressesFromTokenAndUrl = (api, url) => exports.getAllScanSummaryFromUrl = (url, api) => new Promise(async (resolve) => { - const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({ - queryOptions: { filter: odata`url eq ${url} and PartitionKey eq ${api}` } - }); - const iterator = entity.byPage({ maxPageSize: 10 }); - for await (const item of iterator) { - if (item[0]) { - const existing = await getExistingBrokenLinkCount(item[0].runId); - item[0].totalUniqueBrokenLinksExisting = existing; + const getSummary = async (filter) => { + const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({ + queryOptions: { filter } + }); + const iterator = entity.byPage({ maxPageSize: 10 }); + let result; + for await (const item of iterator) { + if (item[0]) { + const existing = await getExistingBrokenLinkCount(item[0].runId); + item[0].totalUniqueBrokenLinksExisting = existing; + } + result = item; + break; } - - resolve(item); - break; + return result; + }; + + let summary = await getSummary(`PartitionKey eq '${api}-${slug(url)}'`); + + if (!summary) { + summary = await getSummary(odata`url eq ${url} and PartitionKey eq ${api}`); } + + resolve(summary); }); exports.getUnscannableLinks = () => diff --git a/ui/src/components/summaryitemcomponents/CardSummary.svelte b/ui/src/components/summaryitemcomponents/CardSummary.svelte index 95ebf030..e4aa1342 100644 --- a/ui/src/components/summaryitemcomponents/CardSummary.svelte +++ b/ui/src/components/summaryitemcomponents/CardSummary.svelte @@ -12,7 +12,7 @@ export let value; export let isHtmlHintComp = false; export let isLighthouseAudit = false; - + let showShareAlert; let previousScans = []; let sharedEmailAddresses = []; @@ -86,7 +86,7 @@