From 5b905bde27e48024f12fbf2b391735e9a9797b13 Mon Sep 17 00:00:00 2001 From: Thiago Perrotta Date: Tue, 23 May 2023 16:47:38 +0200 Subject: [PATCH] align network module with BiDi spec and scaffold responseStarted Bug: #765 Spec: https://w3c.github.io/webdriver-bidi/#module-network --- .../events/SubscriptionManager.spec.ts | 3 +- .../domains/network/networkProcessor.ts | 7 +- .../domains/network/networkRequest.ts | 303 +++++++++++------- src/protocol/protocol.ts | 34 +- .../network/combined/network_events.py.ini | 6 - .../response_started/response_started.py.ini | 3 - .../network/combined/network_events.py.ini | 6 - .../response_completed.py.ini | 3 + .../response_started/response_started.py.ini | 3 - 9 files changed, 217 insertions(+), 151 deletions(-) delete mode 100644 wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/combined/network_events.py.ini delete mode 100644 wpt-metadata/mapper/headless/webdriver/tests/bidi/network/combined/network_events.py.ini diff --git a/src/bidiMapper/domains/events/SubscriptionManager.spec.ts b/src/bidiMapper/domains/events/SubscriptionManager.spec.ts index 12e4339606..41e75db7a7 100644 --- a/src/bidiMapper/domains/events/SubscriptionManager.spec.ts +++ b/src/bidiMapper/domains/events/SubscriptionManager.spec.ts @@ -451,8 +451,9 @@ describe('unroll events', () => { it('all Network events', () => { expect(unrollEvents([Network.AllEvents])).to.deep.equal([ Network.EventNames.BeforeRequestSentEvent, - Network.EventNames.ResponseCompletedEvent, Network.EventNames.FetchErrorEvent, + Network.EventNames.ResponseStartedEvent, + Network.EventNames.ResponseCompletedEvent, ]); }); diff --git a/src/bidiMapper/domains/network/networkProcessor.ts b/src/bidiMapper/domains/network/networkProcessor.ts index 1ab82d0f12..2bfd35eb03 100644 --- a/src/bidiMapper/domains/network/networkProcessor.ts +++ b/src/bidiMapper/domains/network/networkProcessor.ts @@ -25,6 +25,7 @@ import Protocol from 'devtools-protocol'; import {CdpClient} from '../../CdpConnection.js'; import {IEventManager} from '../events/EventManager.js'; import {DefaultMap} from '../../../utils/DefaultMap.js'; +import {Network} from '../../../protocol/protocol.js'; import {NetworkRequest} from './networkRequest.js'; @@ -35,11 +36,11 @@ export class NetworkProcessor { * Map of request ID to NetworkRequest objects. Needed as long as information * about requests comes from different events. */ - readonly #requestMap: DefaultMap; + readonly #requestMap: DefaultMap; private constructor(eventManager: IEventManager) { this.#eventManager = eventManager; - this.#requestMap = new DefaultMap( + this.#requestMap = new DefaultMap( (requestId) => new NetworkRequest(requestId, this.#eventManager) ); } @@ -100,7 +101,7 @@ export class NetworkProcessor { return networkProcessor; } - #getOrCreateNetworkRequest(requestId: string): NetworkRequest { + #getOrCreateNetworkRequest(requestId: Network.Request): NetworkRequest { return this.#requestMap.get(requestId); } } diff --git a/src/bidiMapper/domains/network/networkRequest.ts b/src/bidiMapper/domains/network/networkRequest.ts index 9fbdcae411..bdc849d5e5 100644 --- a/src/bidiMapper/domains/network/networkRequest.ts +++ b/src/bidiMapper/domains/network/networkRequest.ts @@ -20,7 +20,6 @@ * @fileoverview `NetworkRequest` represents a single network request and keeps * track of all the related CDP events. */ - import Protocol from 'devtools-protocol'; import {Deferred} from '../../../utils/deferred.js'; @@ -29,101 +28,105 @@ import {Network} from '../../../protocol/protocol.js'; export class NetworkRequest { static #unknown = 'UNKNOWN'; - requestId: string; + + /** + * Each network request has an associated request id, which is a string + * uniquely identifying that request. + * + * The identifier for a request resulting from a redirect matches that of the + * request that initiated it. + */ + requestId: Network.Request; + #eventManager: IEventManager; - #requestWillBeSentEvent: Protocol.Network.RequestWillBeSentEvent | undefined; - #requestWillBeSentExtraInfoEvent: - | Protocol.Network.RequestWillBeSentExtraInfoEvent - | undefined; - #responseReceivedEvent: Protocol.Network.ResponseReceivedEvent | undefined; - #responseReceivedExtraInfoEvent: - | Protocol.Network.ResponseReceivedExtraInfoEvent - | undefined; + #requestWillBeSentEvent?: Protocol.Network.RequestWillBeSentEvent; + #requestWillBeSentExtraInfoEvent?: Protocol.Network.RequestWillBeSentExtraInfoEvent; + #responseReceivedEvent?: Protocol.Network.ResponseReceivedEvent; + #responseReceivedExtraInfoEvent?: Protocol.Network.ResponseReceivedExtraInfoEvent; #beforeRequestSentDeferred: Deferred; #responseReceivedDeferred: Deferred; - constructor(requestId: string, eventManager: IEventManager) { + constructor(requestId: Network.Request, eventManager: IEventManager) { this.requestId = requestId; this.#eventManager = eventManager; this.#beforeRequestSentDeferred = new Deferred(); this.#responseReceivedDeferred = new Deferred(); } - onRequestWillBeSentEvent( - requestWillBeSentEvent: Protocol.Network.RequestWillBeSentEvent - ) { + onRequestWillBeSentEvent(event: Protocol.Network.RequestWillBeSentEvent) { if (this.#requestWillBeSentEvent !== undefined) { // TODO: Handle redirect event, requestId is same for the redirect chain return; } + this.#requestWillBeSentEvent = event; - this.#requestWillBeSentEvent = requestWillBeSentEvent; if (this.#requestWillBeSentExtraInfoEvent !== undefined) { this.#beforeRequestSentDeferred.resolve(); } + this.#sendBeforeRequestEvent(); } onRequestWillBeSentExtraInfoEvent( - requestWillBeSentExtraInfoEvent: Protocol.Network.RequestWillBeSentExtraInfoEvent + event: Protocol.Network.RequestWillBeSentExtraInfoEvent ) { if (this.#requestWillBeSentExtraInfoEvent !== undefined) { // TODO: Handle redirect event, requestId is same for the redirect chain return; } - this.#requestWillBeSentExtraInfoEvent = requestWillBeSentExtraInfoEvent; + this.#requestWillBeSentExtraInfoEvent = event; + if (this.#requestWillBeSentEvent !== undefined) { this.#beforeRequestSentDeferred.resolve(); } } - onResponseReceivedEvent( - responseReceivedEvent: Protocol.Network.ResponseReceivedEvent + onResponseReceivedEventExtraInfo( + event: Protocol.Network.ResponseReceivedExtraInfoEvent ) { - if (this.#responseReceivedEvent !== undefined) { + if (this.#responseReceivedExtraInfoEvent !== undefined) { // TODO: Handle redirect event, requestId is same for the redirect chain return; } - this.#responseReceivedEvent = responseReceivedEvent; - if ( - !responseReceivedEvent.hasExtraInfo && - !this.#beforeRequestSentDeferred.isFinished - ) { - this.#beforeRequestSentDeferred.resolve(); - } - if ( - !responseReceivedEvent.hasExtraInfo || - this.#responseReceivedExtraInfoEvent !== undefined - ) { + this.#responseReceivedExtraInfoEvent = event; + + if (this.#responseReceivedEvent !== undefined) { this.#responseReceivedDeferred.resolve(); } - this.#sendResponseReceivedEvent(); + + this.#sendResponseStartedEvent(); } - onResponseReceivedEventExtraInfo( - responseReceivedExtraInfoEvent: Protocol.Network.ResponseReceivedExtraInfoEvent - ) { - if (this.#responseReceivedExtraInfoEvent !== undefined) { + onResponseReceivedEvent(event: Protocol.Network.ResponseReceivedEvent) { + if (this.#responseReceivedEvent !== undefined) { // TODO: Handle redirect event, requestId is same for the redirect chain return; } - this.#responseReceivedExtraInfoEvent = responseReceivedExtraInfoEvent; - if (this.#responseReceivedEvent !== undefined) { + this.#responseReceivedEvent = event; + + if (!event.hasExtraInfo && !this.#beforeRequestSentDeferred.isFinished) { + this.#beforeRequestSentDeferred.resolve(); + } + + if ( + !event.hasExtraInfo || + this.#responseReceivedExtraInfoEvent !== undefined + ) { this.#responseReceivedDeferred.resolve(); } + + this.#sendResponseReceivedEvent(); } - onLoadingFailedEvent( - loadingFailedEvent: Protocol.Network.LoadingFailedEvent - ) { + onLoadingFailedEvent(event: Protocol.Network.LoadingFailedEvent) { this.#beforeRequestSentDeferred.resolve(); - this.#responseReceivedDeferred.reject(loadingFailedEvent); + this.#responseReceivedDeferred.reject(event); const params: Network.FetchErrorParams = { ...this.#getBaseEventParams(), - errorText: loadingFailedEvent.errorText, + errorText: event.errorText, }; this.#eventManager.registerEvent( @@ -135,33 +138,7 @@ export class NetworkRequest { ); } - #sendBeforeRequestEvent() { - if (!this.#isIgnoredEvent()) { - this.#eventManager.registerPromiseEvent( - this.#beforeRequestSentDeferred.then(() => - this.#getBeforeRequestEvent() - ), - this.#requestWillBeSentEvent?.frameId ?? null, - Network.EventNames.BeforeRequestSentEvent - ); - } - } - - #getBeforeRequestEvent(): Network.BeforeRequestSentEvent { - if (this.#requestWillBeSentEvent === undefined) { - throw new Error('RequestWillBeSentEvent is not set'); - } - const params: Network.BeforeRequestSentParams = { - ...this.#getBaseEventParams(), - initiator: {type: this.#getInitiatorType()}, - }; - return { - method: Network.EventNames.BeforeRequestSentEvent, - params, - }; - } - - #getBaseEventParams(): Network.BaseEventParams { + #getBaseEventParams(): Network.BaseParameters { return { context: this.#requestWillBeSentEvent?.frameId ?? null, navigation: this.#requestWillBeSentEvent?.loaderId ?? null, @@ -232,53 +209,89 @@ export class NetworkRequest { }; } - #getInitiatorType(): Network.Initiator['type'] { - switch (this.#requestWillBeSentEvent?.initiator.type) { - case 'parser': - case 'script': - case 'preflight': - return this.#requestWillBeSentEvent.initiator.type; - default: - return 'other'; + #sendBeforeRequestEvent() { + if (!this.#isIgnoredEvent()) { + this.#eventManager.registerPromiseEvent( + this.#beforeRequestSentDeferred.then(() => + this.#getBeforeRequestEvent() + ), + this.#requestWillBeSentEvent?.frameId ?? null, + Network.EventNames.BeforeRequestSentEvent + ); } } - static #getCookiesSameSite( - cdpSameSiteValue: string | undefined - ): Network.Cookie['sameSite'] { - switch (cdpSameSiteValue) { - case 'Strict': - return 'strict'; - case 'Lax': - return 'lax'; - default: - return 'none'; + #getBeforeRequestEvent(): Network.BeforeRequestSentEvent { + if (this.#requestWillBeSentEvent === undefined) { + throw new Error('RequestWillBeSentEvent is not set'); } - } - static #getCookies( - associatedCookies: Protocol.Network.BlockedCookieWithReason[] - ): Network.Cookie[] { - return associatedCookies.map((cookieInfo) => { - return { - name: cookieInfo.cookie.name, - value: cookieInfo.cookie.value, - domain: cookieInfo.cookie.domain, - path: cookieInfo.cookie.path, - expires: cookieInfo.cookie.expires, - size: cookieInfo.cookie.size, - httpOnly: cookieInfo.cookie.httpOnly, - secure: cookieInfo.cookie.secure, - sameSite: NetworkRequest.#getCookiesSameSite( - cookieInfo.cookie.sameSite + const params: Network.BeforeRequestSentParams = { + ...this.#getBaseEventParams(), + initiator: { + type: NetworkRequest.#getInitiatorType( + this.#requestWillBeSentEvent.initiator.type ), - }; - }); + }, + }; + return { + method: Network.EventNames.BeforeRequestSentEvent, + params, + }; + } + + #sendResponseStartedEvent() { + if (!this.#isIgnoredEvent()) { + this.#eventManager.registerEvent( + this.#getResponseStartedEvent(), + this.#requestWillBeSentEvent?.frameId ?? null + ); + } + } + + #getResponseStartedEvent(): Network.ResponseStartedEvent { + if (this.#requestWillBeSentEvent === undefined) { + throw new Error('RequestWillBeSentEvent is not set'); + } + if (this.#responseReceivedExtraInfoEvent === undefined) { + throw new Error('responseReceivedExtraInfoEvent is not set'); + } + + return { + method: Network.EventNames.ResponseStartedEvent, + params: { + ...this.#getBaseEventParams(), + response: { + url: this.#requestWillBeSentEvent?.request.url, + protocol: this.#responseReceivedEvent?.response.protocol ?? '', + status: this.#responseReceivedEvent?.response.status ?? -1, + statusText: this.#responseReceivedEvent?.response.statusText ?? '', + fromCache: + (this.#responseReceivedEvent?.response.fromDiskCache || + this.#responseReceivedEvent?.response.fromPrefetchCache) ?? + false, + headers: NetworkRequest.#getHeaders( + this.#responseReceivedExtraInfoEvent.headers + ), + mimeType: this.#responseReceivedEvent?.response.mimeType ?? '', + bytesReceived: + this.#responseReceivedEvent?.response.encodedDataLength ?? -1, + headersSize: + this.#responseReceivedExtraInfoEvent.headersText?.length ?? -1, + // TODO: implement, + bodySize: -1, + // TODO: implement, + content: { + // TODO: implement, + size: -1, + }, + }, + }, + }; } #sendResponseReceivedEvent() { if (!this.#isIgnoredEvent()) { - // Wait for both ResponseReceived and ResponseReceivedExtraInfo events. this.#eventManager.registerPromiseEvent( this.#responseReceivedDeferred.then(() => this.#getResponseReceivedEvent() @@ -290,12 +303,12 @@ export class NetworkRequest { } #getResponseReceivedEvent(): Network.ResponseCompletedEvent { - if (this.#responseReceivedEvent === undefined) { - throw new Error('ResponseReceivedEvent is not set'); - } if (this.#requestWillBeSentEvent === undefined) { throw new Error('RequestWillBeSentEvent is not set'); } + if (this.#responseReceivedEvent === undefined) { + throw new Error('ResponseReceivedEvent is not set'); + } return { method: Network.EventNames.ResponseCompletedEvent, @@ -303,14 +316,14 @@ export class NetworkRequest { ...this.#getBaseEventParams(), response: { url: this.#responseReceivedEvent.response.url, - protocol: this.#responseReceivedEvent.response.protocol, + protocol: this.#responseReceivedEvent.response.protocol ?? '', status: this.#responseReceivedEvent.response.status, statusText: this.#responseReceivedEvent.response.statusText, fromCache: - this.#responseReceivedEvent.response.fromDiskCache || - this.#responseReceivedEvent.response.fromPrefetchCache, - // TODO: implement. - headers: this.#getHeaders( + (this.#responseReceivedEvent.response.fromDiskCache || + this.#responseReceivedEvent.response.fromPrefetchCache) ?? + false, + headers: NetworkRequest.#getHeaders( this.#responseReceivedEvent.response.headers ), mimeType: this.#responseReceivedEvent.response.mimeType, @@ -324,21 +337,67 @@ export class NetworkRequest { size: -1, }, }, - } as Network.ResponseCompletedParams, + }, }; } - #getHeaders(headers: Protocol.Network.Headers) { + #isIgnoredEvent(): boolean { + return ( + this.#requestWillBeSentEvent?.request.url.endsWith('/favicon.ico') ?? + false + ); + } + + static #getHeaders(headers: Protocol.Network.Headers): Network.Header[] { return Object.keys(headers).map((key) => ({ name: key, value: headers[key], })); } - #isIgnoredEvent(): boolean { - return ( - this.#requestWillBeSentEvent?.request.url.endsWith('/favicon.ico') ?? - false - ); + static #getInitiatorType( + initiatorType: Protocol.Network.Initiator['type'] + ): Network.Initiator['type'] { + switch (initiatorType) { + case 'parser': + case 'script': + case 'preflight': + return initiatorType; + default: + return 'other'; + } + } + + static #getCookies( + associatedCookies: Protocol.Network.BlockedCookieWithReason[] + ): Network.Cookie[] { + return associatedCookies.map((cookieInfo) => { + return { + name: cookieInfo.cookie.name, + value: cookieInfo.cookie.value, + domain: cookieInfo.cookie.domain, + path: cookieInfo.cookie.path, + expires: cookieInfo.cookie.expires, + size: cookieInfo.cookie.size, + httpOnly: cookieInfo.cookie.httpOnly, + secure: cookieInfo.cookie.secure, + sameSite: NetworkRequest.#getCookiesSameSite( + cookieInfo.cookie.sameSite + ), + }; + }); + } + + static #getCookiesSameSite( + cdpSameSiteValue?: string + ): Network.Cookie['sameSite'] { + switch (cdpSameSiteValue) { + case 'Strict': + return 'strict'; + case 'Lax': + return 'lax'; + default: + return 'none'; + } } } diff --git a/src/protocol/protocol.ts b/src/protocol/protocol.ts index bbfeffba99..3cbce31b88 100644 --- a/src/protocol/protocol.ts +++ b/src/protocol/protocol.ts @@ -32,6 +32,7 @@ export interface EventResponse { export type BiDiCommand = | BrowsingContext.Command | CDP.Command + | Network.Command | Script.Command | Session.Command; @@ -51,6 +52,7 @@ export namespace Message { export type CommandRequest = Pick & BiDiCommand; export type CommandResponse = Pick & ResultData; + export type EmptyCommand = never; export type EmptyParams = Record; export type EmptyResult = {result: Record}; @@ -60,6 +62,7 @@ export namespace Message { | BrowsingContext.Result | CDP.Result | ErrorResult + | Network.Result | Script.Result | Session.Result; // keep-sorted end @@ -1018,9 +1021,14 @@ export namespace Log { } export namespace Network { + export type Command = Message.EmptyCommand; + + export type Result = Message.EmptyResult; + export type Event = | BeforeRequestSentEvent | ResponseCompletedEvent + | ResponseStartedEvent | FetchErrorEvent; export type BeforeRequestSentEvent = EventResponse< @@ -1033,12 +1041,17 @@ export namespace Network { ResponseCompletedParams >; + export type ResponseStartedEvent = EventResponse< + EventNames.ResponseStartedEvent, + ResponseStartedParams + >; + export type FetchErrorEvent = EventResponse< EventNames.FetchErrorEvent, FetchErrorParams >; - type Header = { + export type Header = { name: string; value?: string; binaryValue?: number[]; @@ -1074,6 +1087,8 @@ export namespace Network { responseEnd: number; }; + export type Request = string; + export type RequestData = { request: string; url: string; @@ -1085,8 +1100,8 @@ export namespace Network { timings: FetchTimingInfo; }; - export type BaseEventParams = { - context: string | null; + export type BaseParameters = { + context: CommonDataTypes.BrowsingContext | null; navigation: BrowsingContext.Navigation | null; redirectCount: number; request: RequestData; @@ -1119,15 +1134,19 @@ export namespace Network { content: ResponseContent; }; - export type BeforeRequestSentParams = BaseEventParams & { + export type BeforeRequestSentParams = BaseParameters & { initiator: Initiator; }; - export type ResponseCompletedParams = BaseEventParams & { + export type ResponseCompletedParams = BaseParameters & { response: ResponseData; }; - export type FetchErrorParams = BaseEventParams & { + export type ResponseStartedParams = BaseParameters & { + response: ResponseData; + }; + + export type FetchErrorParams = BaseParameters & { errorText: string; }; @@ -1135,8 +1154,9 @@ export namespace Network { export enum EventNames { BeforeRequestSentEvent = 'network.beforeRequestSent', - ResponseCompletedEvent = 'network.responseCompleted', FetchErrorEvent = 'network.fetchError', + ResponseStartedEvent = 'network.responseStarted', + ResponseCompletedEvent = 'network.responseCompleted', } } diff --git a/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/combined/network_events.py.ini b/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/combined/network_events.py.ini deleted file mode 100644 index a004f69612..0000000000 --- a/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/combined/network_events.py.ini +++ /dev/null @@ -1,6 +0,0 @@ -[network_events.py] - [test_same_request_id] - expected: FAIL - - [test_subscribe_to_one_context] - expected: FAIL diff --git a/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini b/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini index 0d6acf346c..048fd743ea 100644 --- a/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini +++ b/wpt-metadata/chromedriver/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini @@ -110,9 +110,6 @@ [test_response_status[505-HTTP Version Not Supported\]] expected: FAIL - [test_response_headers] - expected: FAIL - [test_response_mime_type_file[/webdriver/tests/bidi/network/support/empty.html-text/html\]] expected: FAIL diff --git a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/combined/network_events.py.ini b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/combined/network_events.py.ini deleted file mode 100644 index a004f69612..0000000000 --- a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/combined/network_events.py.ini +++ /dev/null @@ -1,6 +0,0 @@ -[network_events.py] - [test_same_request_id] - expected: FAIL - - [test_subscribe_to_one_context] - expected: FAIL diff --git a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini index c7a5c747d6..9f7d306179 100644 --- a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini +++ b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_completed/response_completed.py.ini @@ -7,3 +7,6 @@ [test_redirect] expected: FAIL + + [test_response_headers] + expected: ERROR diff --git a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini index 0d6acf346c..048fd743ea 100644 --- a/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini +++ b/wpt-metadata/mapper/headless/webdriver/tests/bidi/network/response_started/response_started.py.ini @@ -110,9 +110,6 @@ [test_response_status[505-HTTP Version Not Supported\]] expected: FAIL - [test_response_headers] - expected: FAIL - [test_response_mime_type_file[/webdriver/tests/bidi/network/support/empty.html-text/html\]] expected: FAIL