Skip to content

Commit

Permalink
Merge pull request #752 from SSWConsulting/staging
Browse files Browse the repository at this point in the history 
Merge Staging into Main
  • Loading branch information
@tombui99
tombui99 authored Nov 3, 2023
2 parents 4d48f74 + 9240f64 commit eac7952
Show file tree
Hide file tree
Showing 27 changed files with 697 additions and 83 deletions.
40 changes: 37 additions & 3 deletions api/functions/commands.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,26 @@ const {
const azure = require('azure-storage');
const slug = require('slug');

exports.insertScanResult = (api, buildId, runId, data, buildDate) => {
exports.insertScanResult = async (api, buildId, runId, data, buildDate, url) => {
const entGen = azure.TableUtilities.entityGenerator;
let entity = {
PartitionKey: entGen.String(api),
RowKey: entGen.String(`${api}-${newGuid()}`),
buildId: entGen.String(buildId),
runId: entGen.String(runId),
buildDate: entGen.DateTime(buildDate),
apiKey: entGen.String(api)
};
let entityRunIndexed = {
...entity,
PartitionKey: entGen.String(`${api}-${runId}`),
};
let entityUrlIndexed = {
...entity,
PartitionKey: entGen.String(`${api}-${slug(url)}`),
};
await insertEntity(TABLE.ScanResults, replaceProp(data, entityRunIndexed));
await insertEntity(TABLE.ScanResults, replaceProp(data, entityUrlIndexed));
return insertEntity(TABLE.ScanResults, replaceProp(data, entity));
};

Expand Down Expand Up @@ -105,16 +116,28 @@ exports.addHTMLHintRulesForEachRun = (api, data) => {
);
};

exports.insertScanSummary = (api, buildId, runId, buildDate, data) => {
exports.insertScanSummary = async (api, buildId, runId, buildDate, data) => {
var entGen = azure.TableUtilities.entityGenerator;
// use Log tail pattern to get native sort from Table Storage
var entity = {
const entity = {
PartitionKey: entGen.String(api),
RowKey: entGen.String(getReversedTick()),
buildId: entGen.String(buildId),
runId: entGen.String(runId),
buildDate: entGen.DateTime(buildDate),
scanResultVersion: entGen.Int32(2),
apiKey: entGen.String(api)
};
const entityRunIndexed = {
...entity,
PartitionKey: entGen.String(`${api}-${runId}`),
};
let entityUrlIndexed = {
...entity,
PartitionKey: entGen.String(`${api}-${slug(data.url)}`),
};
await insertEntity(TABLE.Scans, replaceProp(data, entityRunIndexed));
await insertEntity(TABLE.Scans, replaceProp(data, entityUrlIndexed));
return insertEntity(TABLE.Scans, replaceProp(data, entity));
};

Expand All @@ -139,6 +162,17 @@ exports.removeAlertEmailAddress = (api, rowkey) => {
return deleteEntity(TABLE.alertEmailAddresses, entity);
};

exports.addCustomHtmlRuleOptions = (api, data) => {
const entGen = azure.TableUtilities.entityGenerator;
return updateEntity(
TABLE.HtmlRulesCustomOptions,
replaceProp(data, {
PartitionKey: entGen.String(api),
RowKey: entGen.String(data.ruleId),
})
);
};

exports.uploadLighthouseReport = (runId, lhr) =>
uploadBlob(BLOB.lhr, `${runId}.json`, JSON.stringify(lhr));

Expand Down
3 changes: 2 additions & 1 deletion api/functions/consts.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ exports.TABLE = {
ScanResults: 'ScanResults',
htmlhintrules: 'htmlhintrules',
alertEmailAddresses: 'alertEmailAddresses',
UnscannableLinks: 'UnscannableLinks'
UnscannableLinks: 'UnscannableLinks',
HtmlRulesCustomOptions: 'HtmlRulesCustomOptions'
};

// blob storage names cannot have uppercase or numbers
Expand Down
14 changes: 12 additions & 2 deletions api/functions/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ const {
addHTMLHintRulesForEachRun,
addAlertEmailAddresses,
removeAlertEmailAddress,
addCustomHtmlRuleOptions,
} = require('./commands');
const {
getPersonalSummary,
Expand All @@ -39,6 +40,7 @@ const {
getAllScanSummaryFromUrl,
getUnscannableLinks,
compareScans,
getCustomHtmlRuleOptions,
} = require('./queries');
const {
newGuid,
Expand Down Expand Up @@ -179,6 +181,14 @@ app.get('/unscannableLinks', async (req, res) => {
res.json(await getUnscannableLinks());
});

app.post('/config/getCustomHtmlRuleOptions/:api', async (req, res) => {
res.json(await getCustomHtmlRuleOptions(req.params.api, req.body.url));
});

app.post('/config/addCustomHtmlRuleOptions/:api', async (req, res) => {
res.json(await addCustomHtmlRuleOptions(req.params.api, req.body));
});

app.post('/scanresult/:api/:buildId', async (req, res) => {
const {
badUrls,
Expand Down Expand Up @@ -230,7 +240,6 @@ app.post('/scanresult/:api/:buildId', async (req, res) => {
const buildId = req.params.buildId;
const runId = newGuid();
const buildDate = new Date();
const unscannableLinks = await getUnscannableLinks();

const uid = await getUserIdFromApiKey(apikey);
if (!uid) {
Expand Down Expand Up @@ -314,7 +323,8 @@ app.post('/scanresult/:api/:buildId', async (req, res) => {
buildId,
runId,
brokenLinkData,
buildDate
buildDate,
url
);
cb(data);
}, {
Expand Down
90 changes: 69 additions & 21 deletions api/functions/queries.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,14 @@ exports.getConfig = (api) =>

exports.getScanDetails = async (runId) => {
const scan = await exports.getSummaryById(runId);
const filter = odata`PartitionKey eq ${scan.partitionKey} and src ge ${scan.url} and src le ${incrementString(scan.url)}`;
let filter;

if (scan.scanResultVersion === 2) {
filter = `PartitionKey eq '${scan.partitionKey}-${slug(scan.url)}'`;
} else {
filter = odata`PartitionKey eq ${scan.partitionKey} and src ge ${scan.url} and src le ${incrementString(scan.url)}`;
}

const entity = new TableClient(azureUrl, TABLE.ScanResults, credential).listEntities({
queryOptions: { filter }
});
Expand Down Expand Up @@ -208,7 +215,10 @@ exports.getAllPublicSummary = (showAll) =>
for await (const item of entity) {
result.push(item);
}
resolve(result)

resolve(result.filter((value, index, self) => {
return self.findIndex(v => v.runId === value.runId) === index;
}))
} else {
// Top 500 scans in last 24 months
var date = new Date();
Expand All @@ -218,24 +228,39 @@ exports.getAllPublicSummary = (showAll) =>
queryOptions: { filter: odata`isPrivate eq ${false} and buildDate gt datetime'${date.toISOString()}'` }
});
const iterator = entity.byPage({ maxPageSize: parseInt(process.env.MAX_SCAN_SIZE) });
let result = [];
for await (const item of iterator) {
resolve(item)
result = item;
break;
}

resolve(result.filter((value, index, self) => {
return self.findIndex(v => v.runId === value.runId) === index;
}))
}
});

exports.getSummaryById = (runId) =>
getRun(runId).then((doc) =>
new Promise(async (resolve) => {
const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({
queryOptions: { filter: odata`PartitionKey eq ${doc.apikey} and runId eq ${doc.runId}` }
});
let result = []
for await (const item of entity) {
result.push(item);
const getSummary = async (filter) => {
const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({
queryOptions: { filter }
});
let result = []
for await (const item of entity) {
result.push(item);
}
return result[0];
};

let summary = await getSummary(`PartitionKey eq '${doc.apikey}-${doc.runId}'`);

if (!summary) {
summary = await getSummary(odata`PartitionKey eq ${doc.apikey} and runId eq ${doc.runId}`);
}
resolve(result[0] || {})

resolve(summary || {});
}));

exports.getLatestSummaryFromUrlAndApi = (url, api) =>
Expand Down Expand Up @@ -272,19 +297,30 @@ exports.getAlertEmailAddressesFromTokenAndUrl = (api, url) =>

exports.getAllScanSummaryFromUrl = (url, api) =>
new Promise(async (resolve) => {
const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({
queryOptions: { filter: odata`url eq ${url} and PartitionKey eq ${api}` }
});
const iterator = entity.byPage({ maxPageSize: 10 });
for await (const item of iterator) {
if (item[0]) {
const existing = await getExistingBrokenLinkCount(item[0].runId);
item[0].totalUniqueBrokenLinksExisting = existing;
const getSummary = async (filter) => {
const entity = new TableClient(azureUrl, TABLE.Scans, credential).listEntities({
queryOptions: { filter }
});
const iterator = entity.byPage({ maxPageSize: 10 });
let result;
for await (const item of iterator) {
if (item[0]) {
const existing = await getExistingBrokenLinkCount(item[0].runId);
item[0].totalUniqueBrokenLinksExisting = existing;
}
result = item;
break;
}

resolve(item);
break;
return result;
};

let summary = await getSummary(`PartitionKey eq '${api}-${slug(url)}'`);

if (!summary) {
summary = await getSummary(odata`url eq ${url} and PartitionKey eq ${api}`);
}

resolve(summary);
});

exports.getUnscannableLinks = () =>
Expand Down Expand Up @@ -332,3 +368,15 @@ exports.compareScans = (api, url) =>
}
resolve(isErrorUp)
});

exports.getCustomHtmlRuleOptions = (api, url) =>
new Promise(async (resolve) => {
const entity = new TableClient(azureUrl, TABLE.HtmlRulesCustomOptions, credential).listEntities({
queryOptions: { filter: odata`PartitionKey eq ${api} and url eq ${url}` }
});
let result = []
for await (const item of entity) {
result.push(item);
}
resolve(result || [])
})
14 changes: 14 additions & 0 deletions docker/api.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,20 @@ exports.getAllScanSummaryFromUrl = (api, url) => {
);
};

exports.getCustomHtmlRuleOptions = (api, url) => {
return fetch(`${endpoint}/api/config/getCustomHtmlRuleOptions/${api}`, {
method: "POST",
body: JSON.stringify({url}),
headers: { "Content-Type": "application/json" },
}).then((res) => {
if (res) {
return res.ok ? res.json() : [];
} else {
throw Error("Failed to get custom html rule options");
}
});
};

exports.htmlHintConfig = {
"grammar-scrum-terms": true,
"code-block-missing-language": true,
Expand Down
53 changes: 39 additions & 14 deletions docker/customHtmlRules.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
const { getCustomHtmlRuleOptions } = require("./api");
const HTMLHint = require("htmlhint").default;
const findPhoneNumbersInText = require('libphonenumber-js').findPhoneNumbersInText;

exports.addCustomHtmlRule = () => {
exports.addCustomHtmlRule = async (apiToken, url) => {
const customRuleOptions = await getCustomHtmlRuleOptions(apiToken, url);

HTMLHint.addRule({
id: "code-block-missing-language",
description: "Code blocks must contain a language specifier.",
Expand Down Expand Up @@ -380,9 +383,15 @@ exports.addCustomHtmlRule = () => {
parser.addListener("tagstart", (event) => {
var tagName = event.tagName.toLowerCase(),
mapAttrs = parser.getMapAttrs(event.attrs);
const ruleId = "detect-absolute-references-url-path-correctly";
if (tagName === "a") {
if (mapAttrs["href"]) {
if (mapAttrs["href"].startsWith("https://ssw.com.au/rules")) {
// Check if custom options exist in this rule
let optionValue = '';
if (customRuleOptions && customRuleOptions.length > 0 && customRuleOptions.filter(option => option.ruleId === ruleId).length > 0) {
optionValue = customRuleOptions.find(option => option.ruleId === ruleId).optionValue
}
if (mapAttrs["href"].startsWith(optionValue.length > 0 ? optionValue : "https://ssw.com.au/rules")) {
if (!mapAttrs["href"].startsWith("/")) {
reporter.warn(
"URLs must be formatted to direct to a url path correctly.",
Expand Down Expand Up @@ -433,17 +442,26 @@ exports.addCustomHtmlRule = () => {
var self = this;

parser.addListener("text", (event) => {
var spellings = [
"a.k.a",
"A.K.A",
"AKA",
"e-mail",
"EMail",
"can not",
"web site",
"user name",
"task bar"
];
const ruleId = "common-spelling-mistakes";
let optionValue = [];
// Check if custom options exist in this rule
if (customRuleOptions && customRuleOptions.length > 0 && customRuleOptions.filter(option => option.ruleId === ruleId).length > 0) {
optionValue = customRuleOptions.find(option => option.ruleId === ruleId).optionValue.split(',');
}
var spellings =
optionValue.length > 0 ?
optionValue :
[
"a.k.a",
"A.K.A",
"AKA",
"e-mail",
"EMail",
"can not",
"web site",
"user name",
"task bar"
];

if (event.raw) {
const pageContent = event.raw;
Expand Down Expand Up @@ -485,7 +503,12 @@ exports.addCustomHtmlRule = () => {
init: function (parser, reporter) {
const self = this;
let isInCodeBlock = false;
let optionValue = '';
parser.addListener("tagstart", (event) => {
// Check if custom options exist in this rule
if (customRuleOptions && customRuleOptions.length > 0 && customRuleOptions.filter(option => option.ruleId === ruleId).length > 0) {
optionValue = customRuleOptions.find(option => option.ruleId === ruleId).optionValue
}
const tagName = event.tagName.toLowerCase();
if (tagName === "code") {
isInCodeBlock = true;
Expand All @@ -498,7 +521,9 @@ exports.addCustomHtmlRule = () => {
}
});
parser.addListener("text", (event) => {
if (event.raw && event.lastEvent && findPhoneNumbersInText(event.raw, "AU").length) {
// Replace "." and "/" characters to avoid false positives when parsing phone numbers
const text = event.raw?.replace(/\.|\//g, "_");
if (text && event.lastEvent && findPhoneNumbersInText(text, optionValue.length > 0 ? optionValue : 'AU').length) {
const pageContent = event.lastEvent.raw;
if (pageContent && event.lastEvent.tagName) {
const tagName = event.lastEvent.tagName.toLowerCase();
Expand Down
Loading

0 comments on commit eac7952

Please sign in to comment.