From 2f43d308b46506c8c33e46f2ee5149aa16d4ba51 Mon Sep 17 00:00:00 2001 From: Nidhi Work Date: Fri, 27 Oct 2023 18:19:11 +0100 Subject: [PATCH] feat(playwright): add trace support for playwright with otel reporter --- packages/artillery-engine-playwright/index.js | 20 +- .../lib/open-telemetry/index.js | 221 ++++++++++++++++-- 2 files changed, 212 insertions(+), 29 deletions(-) diff --git a/packages/artillery-engine-playwright/index.js b/packages/artillery-engine-playwright/index.js index aafe8629f8..a183e8d98f 100644 --- a/packages/artillery-engine-playwright/index.js +++ b/packages/artillery-engine-playwright/index.js @@ -11,6 +11,9 @@ class PlaywrightEngine { this.launchOptions = this.config.launchOptions || {}; this.contextOptions = this.config.contextOptions || {}; + this.tracing = (script.config?.plugins?.['publish-metrics'] || []).some((config) => { + return (config.type === 'open-telemetry') && config.traces}) + this.defaultNavigationTimeout = (parseInt(this.config.defaultNavigationTimeout, 10) || 30) * 1000; this.defaultTimeout = @@ -135,7 +138,8 @@ class PlaywrightEngine { events.emit( 'histogram', `browser.page.dominteractive.${getName(page.url())}`, - startToInteractive + startToInteractive, + {url: page.url(), vuId: initialContext.vars.$uuid} ); } catch (err) {} }); @@ -152,7 +156,8 @@ class PlaywrightEngine { events.emit( 'histogram', `browser.page.${name}.${getName(url)}`, - value + value, + {rating: metric.metric.rating, url, vuId: initialContext.vars.$uuid} ); } } catch (err) {} @@ -177,7 +182,8 @@ class PlaywrightEngine { events.emit( 'histogram', 'browser.memory_used_mb', - usedJSHeapSize / 1000 / 1000 + usedJSHeapSize / 1000 / 1000, + {url: page.url(), vuId: initialContext.vars.$uuid} ); } catch (err) {} }); @@ -197,7 +203,13 @@ class PlaywrightEngine { const test = { step }; - await fn(page, initialContext, events, test); + let traceScenario + if (self.tracing){ + traceScenario = self.processor[spec.traceFlowFunction] + await traceScenario(page, initialContext, events, fn, spec.name) + } else { + await fn(page, initialContext, events, test); + } await page.close(); diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/index.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/index.js index 654845ffab..4c5e250cbf 100644 --- a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/index.js +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/index.js @@ -20,21 +20,22 @@ const { } = require('@opentelemetry/semantic-conventions'); const { - AsyncHooksContextManager + AsyncLocalStorageContextManager } = require('@opentelemetry/context-async-hooks'); -const contextManager = new AsyncHooksContextManager(); +const contextManager = new AsyncLocalStorageContextManager(); contextManager.enable(); context.setGlobalContextManager(contextManager); class OTelReporter { constructor(config, events, script) { + this.config = config; this.script = script; this.events = events; if ( process.env.DEBUG && process.env.DEBUG === 'plugin:publish-metrics:open-telemetry' ) { - diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); + diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR); } this.metricExporters = { 'otlp-proto'(options) { @@ -122,6 +123,7 @@ class OTelReporter { }); } + // Traces if (config.traces) { this.traceConfig = config.traces; this.validateExporter( @@ -133,28 +135,53 @@ class OTelReporter { this.configureTrace(this.traceConfig); - attachScenarioHooks(script, [ - { - type: 'beforeRequest', - name: 'startOTelSpan', - hook: this.startHTTPRequestSpan.bind(this) - }, - { - type: 'afterResponse', - name: 'exportOTelSpan', - hook: this.endHTTPRequestSpan.bind(this) - }, - { - type: 'beforeScenario', - name: 'startScenarioSpan', - hook: this.startScenarioSpan('http').bind(this) - }, - { - type: 'afterScenario', - name: 'endScenarioSpan', - hook: this.endScenarioSpan('http').bind(this) - } - ]); + // Create set of all engines used in test -> even though we only support Playwright and HTTP engine for now this is future compatible + this.engines = new Set(); + const scenarios = this.script.scenarios || []; + scenarios.forEach((scenario) => { + scenario.engine + ? this.engines.add(scenario.engine) + : this.engines.add('http'); + }); + + if (this.engines.has('http')) { + attachScenarioHooks(script, [ + { + type: 'beforeRequest', + name: 'startOTelSpan', + hook: this.startHTTPRequestSpan.bind(this) + }, + { + type: 'afterResponse', + name: 'exportOTelSpan', + hook: this.endHTTPRequestSpan.bind(this) + }, + { + type: 'beforeScenario', + name: 'startScenarioSpan', + hook: this.startScenarioSpan('http').bind(this) + }, + { + type: 'afterScenario', + name: 'endScenarioSpan', + hook: this.endScenarioSpan('http').bind(this) + } + ]); + } + + if (this.engines.has('playwright')) { + // Create tracer for Playwright engine + this.playwrightTracer = trace.getTracer('artillery-playwright'); + + attachScenarioHooks(script, [ + { + engine: 'playwright', + type: 'traceFlowFunction', + name: 'runOtelTracingForPlaywright', + hook: this.runOtelTracingForPlaywright.bind(this) + } + ]); + } } } @@ -456,6 +483,150 @@ class OTelReporter { return done(); } + async runOtelTracingForPlaywright( + page, + vuContext, + events, + userFlowFunction, + specName + ) { + // Start scenarioSpan as a root span for the trace and set it as active context + return await this.playwrightTracer.startActiveSpan( + specName || 'Scenario execution', + { kind: SpanKind.CLIENT }, + async (scenarioSpan) => { + scenarioSpan.setAttribute('vu.uuid', vuContext.vars.$uuid); + // Set variables to track state and context + const ctx = context.active(); + let lastPageUrl; + let pageUrl; + let pageSpan; + + // Listen to histograms to capture web vitals and other metrics set by Playwright engine, set them as attributes and if they are web vitals, as events too + events.on('histogram', (name, value, metadata) => { + // vuId from event must match current vuId + if (!metadata || metadata.vuId !== vuContext.vars.$uuid) { + return; + } + + // only look for page metrics or memory_used_mb metric. step metrics are handled separately in the step helper itself + if ( + !name.startsWith('browser.page') && + name !== 'browser.memory_used_mb' + ) { + return; + } + + // associate only the metrics that belong to the page + if (metadata.url !== pageSpan.name.replace('Page: ', '')) { + return; + } + const webVitals = ['LCP', 'FCP', 'CLS', 'TTFB', 'INP', 'FID']; + + try { + const attrs = {}; + const metricName = + name === 'browser.memory_used_mb' ? name : name.split('.')[2]; + + if (webVitals.includes(metricName)) { + attrs[`web_vitals.${metricName}.value`] = value; + attrs[`web_vitals.${metricName}.rating`] = metadata.rating; + pageSpan.addEvent(metricName, attrs); + } else { + attrs[metricName] = value; + } + pageSpan.setAttributes(attrs); + } catch (err) { + throw new Error(err); + } + }); + + // Upon navigation to main frame, if the URL is different than existing page span, the existing page span is closed and new opened with new URL + page.on('framenavigated', (frame) => { + //only interested in mainframe navigations (not iframes, etc) + if (frame !== page.mainFrame()) { + return; + } + + pageUrl = page.url(); + + //only create a new span if the currently navigated page is different. + //this is because we can have multiple framenavigated for the same url, but we're only interested in navigation changes + if (pageUrl !== lastPageUrl) { + scenarioSpan.addEvent(`navigated to ${page.url()}`); + if (pageSpan) { + pageSpan.end(); + } + + pageSpan = this.playwrightTracer.startSpan( + 'Page: ' + pageUrl, + { kind: SpanKind.CLIENT }, + ctx + ); + pageSpan.setAttribute('vu.uuid', vuContext.vars.$uuid); + lastPageUrl = pageUrl; + } + }); + + try { + // Set the tracing this.step function to test object which is exposed to the user + const test = { + step: ( + await this.step(scenarioSpan, this.playwrightTracer, events, page) + ).bind(this) + }; + // Execute the user-provided processor function within the context of the new span + await userFlowFunction(page, vuContext, events, test); + } catch (err) { + scenarioSpan.recordException(err, Date.now()); + scenarioSpan.setStatus({ + code: SpanStatusCode.ERROR, + message: err.message + }); + throw err; + } finally { + scenarioSpan.end(); + } + } + ); + } + + async step(parent, tracer, events) { + return async function (stepName, callback, attributes) { + // Set the parent context to be scenarioSpan and within it we create step spans + return contextManager.with( + trace.setSpan(context.active(), parent), + async () => { + const span = tracer.startSpan( + stepName, + { kind: SpanKind.CLIENT }, + context.active() + ); + const startTime = Date.now(); + + try { + if (attributes) { + span.setAttributes(attributes); + } + + await callback(); + } catch (err) { + debug('There has been an error during step execution: ', err); + span.recordException(err, Date.now()); + span.setStatus({ + code: SpanStatusCode.ERROR, + message: err.message + }); + } finally { + const difference = Date.now() - startTime; + events.emit('histogram', `browser.step.${stepName}`, difference); + span.end(); + } + } + ); + }; + } + validateExporter(supportedExporters, exporter, type) { const supported = Object.keys(supportedExporters).reduce( (acc, k, i) =>