Skip to content

Commit

Permalink
feat: psi-client, re-modularize
Browse files Browse the repository at this point in the history
  • Loading branch information
@solaris007
solaris007 committed Dec 3, 2023
1 parent 216a602 commit fd714ba
Show file tree
Hide file tree
Showing 6 changed files with 333 additions and 93 deletions.
154 changes: 85 additions & 69 deletions src/lhs/handler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,12 @@
* governing permissions and limitations under the License.
*/

import { createUrl, Response } from '@adobe/fetch';
import { hasText, isObject, isValidUrl } from '@adobe/spacecat-shared-utils';

import { ensureValidUrl, fetch } from '../support/utils.js';
import GithubClient from '../support/github-client.js';
import ContentClient from '../support/content-client.js';
import { extractAuditScores, extractThirdPartySummary, extractTotalBlockingTime } from '../utils/lhs.js';
import PSIClient from '../support/psi-client.js';

const AUDIT_TYPES = {
MOBILE: 'lhs-mobile',
Expand Down Expand Up @@ -71,10 +70,9 @@ const validateContext = (config) => {
*
* @param {Object} site - The site object containing information about the site.
* @param {Object} latestAudit - The latest audit for the site.
* @param {Object} psiData - The PageSpeed Insights data.
* @param {Object} lighthouseResult - The PageSpeed Insights data.
* @param {object} gitHubDiff - The GitHub diff object.
* @param {object} markdownContext - The markdown context object.
* @param {string} psiApiBaseUrl - The base URL for the PageSpeed Insights API.
* @param {string} fullAuditRef - The URL to the full audit results.
* @param {string} strategy - The strategy of the audit.
*
Expand All @@ -83,18 +81,17 @@ const validateContext = (config) => {
const createAuditData = (
site,
latestAudit,
psiData,
lighthouseResult,
gitHubDiff,
markdownContext,
psiApiBaseUrl,
fullAuditRef,
strategy,
) => {
const {
audits,
categories,
finalUrl,
} = psiData.lighthouseResult;
} = lighthouseResult;

const scores = extractAuditScores(categories);
const totalBlockingTime = extractTotalBlockingTime(audits);
Expand Down Expand Up @@ -138,32 +135,6 @@ const createSQSMessage = (auditContext, site, auditData) => ({
},
});

/**
* Fetches PageSpeed Insights data for the given URL and PSI strategy. The data is fetched from the
* PSI API URL provided in the configuration. The PSI API URL is expected to return
* a 302 redirect to the actual data. This is currently provided by the EaaS API.
*
* @async
* @param {string} psiApiBaseUrl - The base URL for the PageSpeed Insights API.
* @param {string} url - The URL of the site to fetch PSI data for.
* @param {string} strategy - The strategy of the audit.
* @throws {Error} - Throws an error if the expected HTTP responses are not received.
* @returns {Promise<Object>} - Returns an object containing PSI data and a task ID.
*/
const fetchPsiData = async (psiApiBaseUrl, url, strategy) => {
const urlToBeAudited = ensureValidUrl(url);
const psiUrl = createUrl(psiApiBaseUrl, { url: urlToBeAudited, strategy });

const response = await fetch(psiUrl);

if (response.status !== 200) {
throw new Error(`Expected a 200 status from PSI API, received ${response.status}`);
}

const psiData = await response.json();
return { psiData, fullAuditRef: response.url };
};

/**
* Fetches site data based on the given base URL. If no site is found for the given
* base URL, null is returned. Otherwise, the site object is returned. If an error
Expand Down Expand Up @@ -232,65 +203,52 @@ const sendMessageToSQS = async (sqs, queueUrl, message, log) => {
* and sending a message to SQS.
*
* @async
* @param {Object} dataAccess - The data access object for database operations.
* @param {Object} services - The services object containing the PSI client,
* content client, and more.
* @param {Object} site - The site which to audit.
* @param {Object} auditContext - The audit context object containing information about the audit.
* @param {string} queueUrl - The URL of the SQS queue.
* @param {Object} sqs - The SQS service object.
* @param {string} psiApiBaseUrl - The base URL for the PageSpeed Insights API.
* @param {string} url - The URL of the site to audit.
* @param {string} strategy - The strategy of the audit.
* @param {string} gitHubId - The GitHub ID.
* @param {string} gitHubSecret - The GitHub secret.
* @param {Object} log - The logging object.
*
* @throws {Error} - Throws an error if any step in the audit process fails.
*/
async function processAudit(
dataAccess,
services,
site,
auditContext,
queueUrl,
sqs,
psiApiBaseUrl,
url,
strategy,
gitHubId,
gitHubSecret,
log = console,
) {
const site = await retrieveSite(dataAccess, url, log);
if (!site) {
throw new Error('Site not found');
}
const {
dataAccess, contentClient, githubClient, psiClient, sqs,
} = services;

const { psiData, fullAuditRef } = await fetchPsiData(psiApiBaseUrl, url, strategy);
const baseURL = site.getBaseURL();
const latestAudit = await dataAccess.getLatestAuditForSite(site.getId(), `lhs-${strategy}`);

const contentClient = ContentClient(log);
const { lighthouseResult, fullAuditRef } = await psiClient.runAudit(baseURL, strategy);

const markdownContext = await contentClient.fetchMarkdownDiff(
site.getBaseURL(),
baseURL,
latestAudit,
psiData.lighthouseResult.finalUrl,
lighthouseResult.finalUrl,
);

const githubClient = new GithubClient({
baseUrl: site.getBaseURL(),
gitHubId,
gitHubSecret,
}, log);
const gitHubDiff = await githubClient.fetchGithubDiff(
site.getBaseURL(),
psiData.lighthouseResult.fetchTime,
baseURL,
lighthouseResult.fetchTime,
latestAudit?.getAuditedAt(),
site.getGitHubURL(),
);

const auditData = createAuditData(
site,
latestAudit,
psiData,
lighthouseResult,
markdownContext,
gitHubDiff,
psiApiBaseUrl,
fullAuditRef,
strategy,
);
Expand All @@ -301,6 +259,50 @@ async function processAudit(
await sendMessageToSQS(sqs, queueUrl, message, log);
}

/**
* Initializes the services used by the audit process.
*
* @param {Object} config - The configuration object.
* @param {Object} config.site - The site object containing information about the site.
* @param {string} config.psiApiKey - The PageSpeed Insights API key.
* @param {string} config.psiApiBaseUrl - The PageSpeed Insights API base URL.
* @param {string} config.gitHubId - The GitHub client ID.
* @param {string} config.gitHubSecret - The GitHub client secret.
* @param {Object} config.sqs - The SQS service object.
* @param {Object} config.dataAccess - The data access object for database operations.
* @param {Object} log - The logging object.
*
* @returns {Object} - Returns an object containing the services.
* @throws {Error} - Throws an error if any of the services cannot be initialized.
*/
function initServices(config, log = console) {
const {
site,
psiApiKey,
psiApiBaseUrl,
gitHubId,
gitHubSecret,
sqs,
dataAccess,
} = config;

const psiClient = PSIClient({ apiKey: psiApiKey, baseUrl: psiApiBaseUrl }, log);
const contentClient = ContentClient(log);
const githubClient = new GithubClient({
baseUrl: site.getBaseURL(),
gitHubId,
gitHubSecret,
}, log);

return {
dataAccess,
contentClient,
githubClient,
psiClient,
sqs,
};
}

/**
* The main function to handle audit requests. This function is invoked by the
* SpaceCat runtime when a message is received on the audit request queue.
Expand Down Expand Up @@ -335,33 +337,47 @@ export default async function audit(message, context) {
const { dataAccess, log, sqs } = context;
const {
PAGESPEED_API_BASE_URL: psiApiBaseUrl,
PAGESPEED_API_KEY: psiApiKey,
AUDIT_RESULTS_QUEUE_URL: queueUrl,
GITHUB_CLIENT_ID: gitHubId,
GITHUB_CLIENT_SECRET: gitHubSecret,
} = context.env;

try {
const strategy = typeToPSIStrategy(type);

const validationResults = validateContext({
dataAccess, psiApiBaseUrl, queueUrl, sqs,
});

if (validationResults !== true) {
return respondWithError(`Invalid configuration: ${validationResults.join(', ')}`, log);
}

log.info(`Received ${type} audit request for baseURL: ${url}`);

const site = await retrieveSite(dataAccess, url, log);
if (!site) {
return new Response('Site not found', { status: 404 });
}

const services = initServices({
site,
psiApiKey,
psiApiBaseUrl,
gitHubId,
gitHubSecret,
sqs,
dataAccess,
}, log);

const startTime = process.hrtime();
await processAudit(
dataAccess,
services,
site,
auditContext,
queueUrl,
sqs,
psiApiBaseUrl,
url,
strategy,
gitHubId,
gitHubSecret,
log,
);
const endTime = process.hrtime(startTime);
Expand All @@ -370,7 +386,7 @@ export default async function audit(message, context) {

log.info(`Audit for ${type} completed in ${formattedElapsed} seconds`);

return new Response('', { status: 204 });
return new Response('', { status: 200 });
} catch (e) {
return respondWithError('Unexpected error occurred', log, e);
}
Expand Down
23 changes: 16 additions & 7 deletions src/support/content-client.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,31 @@
* governing permissions and limitations under the License.
*/

import { createPatch } from 'diff';
import { isValidUrl } from '@adobe/spacecat-shared-utils';
import { createPatch } from 'diff';

import { fetch } from './utils.js';

const NOT_FOUND_STATUS = 404;

/**
* Represents a utility for calculating content differences from Markdown files fetched via HTTP.
*
* @param {Object} log - The logger object.
*/
function ContentClient(log = console) {
/**
* Returns the Markdown URL for a given content URL.
* @param {string} contentUrl - The URL of the content page used in the audit.
* @return {string} The Markdown URL.
*/
function getMarkdownUrl(contentUrl) {
const markdownUrl = new URL(contentUrl);
const { pathname: path } = markdownUrl;

markdownUrl.pathname = path.endsWith('/') ? `${path}index.md` : `${path}.md`;

return markdownUrl.toString();
}

/**
* Creates a diff patch between two strings. Helper function for testability.
* @param {string} url - The URL of the Markdown file.
Expand Down Expand Up @@ -64,10 +76,7 @@ function ContentClient(log = console) {
return null;
}

const markdownUrl = new URL(contentUrl);
const { pathname: path } = markdownUrl;

markdownUrl.pathname = path.endsWith('/') ? `${path}index.md` : `${path}.md`;
const markdownUrl = getMarkdownUrl(contentUrl);

try {
const response = await fetch(markdownUrl.toString());
Expand Down
Loading

0 comments on commit fd714ba

Please sign in to comment.