From f21ec28e194d4af048b6a3bc535f561e0e945eae Mon Sep 17 00:00:00 2001 From: Garrett Stevens Date: Tue, 12 Sep 2023 04:08:48 +0000 Subject: [PATCH] Initial version of web workers calling main thread --- .../ApolloSequenceAdapter.ts | 205 +++++++++++------- .../src/BackendDrivers/BackendDriver.ts | 2 +- .../CollaborationServerDriver.ts | 38 +++- .../src/BackendDrivers/InMemoryFileDriver.ts | 33 ++- .../src/components/OpenLocalFile.tsx | 31 +-- packages/jbrowse-plugin-apollo/src/index.ts | 86 +++++++- 6 files changed, 290 insertions(+), 105 deletions(-) diff --git a/packages/jbrowse-plugin-apollo/src/ApolloSequenceAdapter/ApolloSequenceAdapter.ts b/packages/jbrowse-plugin-apollo/src/ApolloSequenceAdapter/ApolloSequenceAdapter.ts index b340d5dec..fe794fb84 100644 --- a/packages/jbrowse-plugin-apollo/src/ApolloSequenceAdapter/ApolloSequenceAdapter.ts +++ b/packages/jbrowse-plugin-apollo/src/ApolloSequenceAdapter/ApolloSequenceAdapter.ts @@ -3,12 +3,13 @@ import { BaseOptions, BaseSequenceAdapter, } from '@jbrowse/core/data_adapters/BaseAdapter' -import { getFetcher } from '@jbrowse/core/util/io' import { ObservableCreate } from '@jbrowse/core/util/rxjs' import SimpleFeature, { Feature } from '@jbrowse/core/util/simpleFeature' -import { NoAssemblyRegion, UriLocation } from '@jbrowse/core/util/types' +import { NoAssemblyRegion, Region } from '@jbrowse/core/util/types' +import { nanoid } from 'nanoid' -import { createFetchErrorMessage } from '../util' +import { BackendDriver } from '../BackendDrivers' +import { ApolloSessionModel } from '../session' export interface RefSeq { _id: string @@ -17,8 +18,26 @@ export interface RefSeq { length: number } +interface ApolloMessageData { + apollo: true + messageId: string + sequence: string + regions: Region[] +} + +function isApolloMessageData(data?: unknown): data is ApolloMessageData { + return ( + typeof data === 'object' && + data !== null && + 'apollo' in data && + data.apollo === true + ) +} + +const isInWebWorker = typeof sessionStorage === 'undefined' + export class ApolloSequenceAdapter extends BaseSequenceAdapter { - private refSeqs: Promise | undefined + private regions: NoAssemblyRegion[] | undefined get baseURL(): string { return readConfObject(this.config, 'baseURL').uri @@ -31,46 +50,63 @@ export class ApolloSequenceAdapter extends BaseSequenceAdapter { .internetAccountPreAuthorization } - protected async getRefSeqs({ signal }: BaseOptions) { - if (this.refSeqs) { - return this.refSeqs - } - const assemblyId = readConfObject(this.config, 'assemblyId') - const url = new URL('refSeqs', this.baseURL) - const searchParams = new URLSearchParams({ assembly: assemblyId }) - url.search = searchParams.toString() - const uri = url.toString() - const location: UriLocation = { locationType: 'UriLocation', uri } - if (this.internetAccountPreAuthorization) { - location.internetAccountPreAuthorization = - this.internetAccountPreAuthorization - } - const fetch = getFetcher(location, this.pluginManager) - const response = await fetch(uri, { signal }) - if (!response.ok) { - const errorMessage = await createFetchErrorMessage( - response, - 'Failed to fetch refSeqs', - ) - throw new Error(errorMessage) - } - const refSeqs = (await response.json()) as RefSeq[] - this.refSeqs = Promise.resolve(refSeqs) - return refSeqs - } - public async getRefNames(opts: BaseOptions) { - const refSeqs = await this.getRefSeqs(opts) - return refSeqs.map((refSeq) => refSeq.name) + const regions = await this.getRegions(opts) + return regions.map((regions) => regions.refName) } public async getRegions(opts: BaseOptions): Promise { - const refSeqs = await this.getRefSeqs(opts) - return refSeqs.map((refSeq) => ({ - refName: refSeq.name, - start: 0, - end: refSeq.length, - })) + if (this.regions) { + return this.regions + } + const assemblyId = readConfObject(this.config, 'assemblyId') + if (!isInWebWorker) { + const dataStore = ( + this.pluginManager?.rootModel?.session as ApolloSessionModel | undefined + )?.apolloDataStore + if (!dataStore) { + throw new Error('No Apollo data store found') + } + const backendDriver = dataStore.getBackendDriver( + assemblyId, + ) as BackendDriver + const regions = await backendDriver.getRegions(assemblyId) + this.regions = regions + return regions + } + const { signal } = opts + const regions = await new Promise( + ( + resolve: (sequence: Region[]) => void, + reject: (reason: string) => void, + ) => { + const timeoutId = setTimeout(() => { + reject('timeout') + }, 20_000) + const messageId = nanoid() + const messageListener = (event: MessageEvent) => { + const { data } = event + if (!isApolloMessageData(data)) { + return + } + if (data.messageId !== messageId) { + return + } + clearTimeout(timeoutId) + removeEventListener('message', messageListener) + resolve(data.regions) + } + addEventListener('message', messageListener, { signal }) + postMessage({ + apollo: true, + method: 'getRegions', + assembly: assemblyId, + messageId, + }) + }, + ) + this.regions = regions + return regions } /** @@ -78,49 +114,68 @@ export class ApolloSequenceAdapter extends BaseSequenceAdapter { * @param param - * @returns Observable of Feature objects in the region */ - public getFeatures( - { end, refName, start }: NoAssemblyRegion, - opts: BaseOptions, - ) { + public getFeatures(region: Region, opts: BaseOptions) { + const { signal } = opts + const { assemblyName, end, refName, start } = region return ObservableCreate(async (observer) => { - const refSeqs = await this.getRefSeqs(opts) - const refSeq = refSeqs.find((rs) => rs.name === refName) - if (!refSeq) { - return observer.error( - `Could not find refSeq that matched refName "${refName}"`, - ) - } - const url = new URL('refSeqs/getSequence', this.baseURL) - const searchParams = new URLSearchParams({ - refSeq: refSeq._id, - start: String(start), - end: String(end), - }) - url.search = searchParams.toString() - const uri = url.toString() - const location: UriLocation = { locationType: 'UriLocation', uri } - if (this.internetAccountPreAuthorization) { - location.internetAccountPreAuthorization = - this.internetAccountPreAuthorization - } - const fetch = getFetcher(location, this.pluginManager) - const response = await fetch(uri, { signal: opts.signal }) - if (!response.ok) { - const errorMessage = await createFetchErrorMessage( - response, - 'Failed to fetch refSeqs', - ) - throw new Error(errorMessage) - } - const seq = (await response.text()) as string - if (seq) { + if (!isInWebWorker) { + const dataStore = ( + this.pluginManager?.rootModel?.session as + | ApolloSessionModel + | undefined + )?.apolloDataStore + if (!dataStore) { + observer.error('No Apollo data store found') + } + const backendDriver = dataStore.getBackendDriver( + assemblyName, + ) as BackendDriver + const { seq } = await backendDriver.getSequence(region) observer.next( new SimpleFeature({ id: `${refName} ${start}-${end}`, data: { refName, start, end, seq }, }), ) + observer.complete() + return } + const seq = await new Promise( + ( + resolve: (sequence: string) => void, + reject: (reason: string) => void, + ) => { + const timeoutId = setTimeout(() => { + reject('timeout') + }, 20_000) + const messageId = nanoid() + const messageListener = (event: MessageEvent) => { + const { data } = event + if (!isApolloMessageData(data)) { + return + } + if (data.messageId !== messageId) { + return + } + clearTimeout(timeoutId) + removeEventListener('message', messageListener) + resolve(data.sequence) + } + addEventListener('message', messageListener, { signal }) + postMessage({ + apollo: true, + method: 'getSequence', + region, + messageId, + }) + }, + ) + observer.next( + new SimpleFeature({ + id: `${refName} ${start}-${end}`, + data: { refName, start, end, seq }, + }), + ) observer.complete() }) } diff --git a/packages/jbrowse-plugin-apollo/src/BackendDrivers/BackendDriver.ts b/packages/jbrowse-plugin-apollo/src/BackendDrivers/BackendDriver.ts index 6c6787c4c..fc00692d1 100644 --- a/packages/jbrowse-plugin-apollo/src/BackendDrivers/BackendDriver.ts +++ b/packages/jbrowse-plugin-apollo/src/BackendDrivers/BackendDriver.ts @@ -13,7 +13,7 @@ export abstract class BackendDriver { abstract getSequence(region: Region): Promise<{ seq: string; refSeq: string }> - abstract getRefSeqs(): Promise + abstract getRegions(assemblyName: string): Promise abstract getAssemblies(internetAccountConfigId?: string): Assembly[] diff --git a/packages/jbrowse-plugin-apollo/src/BackendDrivers/CollaborationServerDriver.ts b/packages/jbrowse-plugin-apollo/src/BackendDrivers/CollaborationServerDriver.ts index 6bf836c29..929028da6 100644 --- a/packages/jbrowse-plugin-apollo/src/BackendDrivers/CollaborationServerDriver.ts +++ b/packages/jbrowse-plugin-apollo/src/BackendDrivers/CollaborationServerDriver.ts @@ -201,9 +201,41 @@ export class CollaborationServerDriver extends BackendDriver { return { seq: await response.text(), refSeq } } - async getRefSeqs() { - throw new Error('getRefSeqs not yet implemented') - return [] + async getRegions(assemblyName: string): Promise { + const { assemblyManager } = getSession(this.clientStore) + const assembly = assemblyManager.get(assemblyName) + if (!assembly) { + throw new Error(`Could not find assembly with name "${assemblyName}"`) + } + const internetAccount = this.clientStore.getInternetAccount( + assemblyName, + ) as ApolloInternetAccount + const { baseURL } = internetAccount + const url = new URL('refSeqs', baseURL) + const searchParams = new URLSearchParams({ assembly: assemblyName }) + url.search = searchParams.toString() + const uri = url.toString() + + const response = await this.fetch(internetAccount, uri) + if (!response.ok) { + let errorMessage + try { + errorMessage = await response.text() + } catch { + errorMessage = '' + } + throw new Error( + `getRegions failed: ${response.status} (${response.statusText})${ + errorMessage ? ` (${errorMessage})` : '' + }`, + ) + } + const refSeqs = await response.json() + return refSeqs.map((refSeq: { name: string; length: number }) => ({ + refName: refSeq.name, + start: 0, + end: refSeq.length, + })) } getAssemblies(internetAccountId?: string) { diff --git a/packages/jbrowse-plugin-apollo/src/BackendDrivers/InMemoryFileDriver.ts b/packages/jbrowse-plugin-apollo/src/BackendDrivers/InMemoryFileDriver.ts index c2b4318fd..3ccc46fe7 100644 --- a/packages/jbrowse-plugin-apollo/src/BackendDrivers/InMemoryFileDriver.ts +++ b/packages/jbrowse-plugin-apollo/src/BackendDrivers/InMemoryFileDriver.ts @@ -1,5 +1,5 @@ import { getConf } from '@jbrowse/core/configuration' -import { getSession } from '@jbrowse/core/util' +import { Region, getSession } from '@jbrowse/core/util' import { AssemblySpecificChange, Change } from 'apollo-common' import { AnnotationFeatureSnapshot } from 'apollo-mst' import { ValidationResultSet } from 'apollo-shared' @@ -12,12 +12,35 @@ export class InMemoryFileDriver extends BackendDriver { return [] } - async getSequence() { - return { seq: '', refSeq: '' } + async getSequence(region: Region) { + const { assemblyName, end, refName, start } = region + const assembly = this.clientStore.assemblies.get(assemblyName) + if (!assembly) { + return { seq: '', refSeq: refName } + } + const refSeq = assembly.refSeqs.get(refName) + if (!refSeq) { + return { seq: '', refSeq: refName } + } + const seq = refSeq.getSequence(start, end) + return { seq, refSeq: refName } } - async getRefSeqs() { - return [] + async getRegions(assemblyName: string): Promise { + const assembly = this.clientStore.assemblies.get(assemblyName) + if (!assembly) { + return [] + } + const regions: Region[] = [] + for (const [, refSeq] of assembly.refSeqs) { + regions.push({ + assemblyName, + refName: refSeq.name, + start: refSeq.sequence[0].start, + end: refSeq.sequence[0].stop, + }) + } + return regions } getAssemblies() { diff --git a/packages/jbrowse-plugin-apollo/src/components/OpenLocalFile.tsx b/packages/jbrowse-plugin-apollo/src/components/OpenLocalFile.tsx index 6e096bad5..e64c1a87c 100644 --- a/packages/jbrowse-plugin-apollo/src/components/OpenLocalFile.tsx +++ b/packages/jbrowse-plugin-apollo/src/components/OpenLocalFile.tsx @@ -29,14 +29,6 @@ export interface RefSeqInterface { aliases?: string[] } -export interface SequenceAdapterFeatureInterface { - refName: string - uniqueId: string - start: number - end: number - seq: string -} - export function OpenLocalFile({ handleClose, session }: OpenLocalFileProps) { const { addApolloTrackConfig, @@ -100,7 +92,7 @@ export function OpenLocalFile({ handleClose, session }: OpenLocalFileProps) { const assemblyId = `${assemblyName}-${file.name}-${nanoid(8)}` - const sequenceAdapterFeatures: SequenceAdapterFeatureInterface[] = [] + let sequenceFeatureCount = 0 let assembly = session.apolloDataStore.assemblies.get(assemblyId) if (!assembly) { assembly = session.apolloDataStore.addAssembly(assemblyId) @@ -117,17 +109,20 @@ export function OpenLocalFile({ handleClose, session }: OpenLocalFileProps) { ref.addFeature(feature) } } else { + sequenceFeatureCount++ // sequence feature - sequenceAdapterFeatures.push({ - refName: seqLine.id, - uniqueId: `${assemblyId}-${seqLine.id}`, + let ref = assembly.refSeqs.get(seqLine.id) + if (!ref) { + ref = assembly.addRefSeq(seqLine.id, seqLine.id) + } + ref.addSequence({ start: 0, - end: seqLine.sequence.length, - seq: seqLine.sequence, + stop: seqLine.sequence.length, + sequence: seqLine.sequence, }) } } - if (sequenceAdapterFeatures.length === 0) { + if (sequenceFeatureCount === 0) { setErrorMessage('No embedded FASTA section found in GFF3') setSubmitted(false) return @@ -140,11 +135,7 @@ export function OpenLocalFile({ handleClose, session }: OpenLocalFileProps) { sequence: { trackId: `sequenceConfigId-${assemblyName}`, type: 'ReferenceSequenceTrack', - adapter: { - type: 'FromConfigSequenceAdapter', - assemblyId, - features: sequenceAdapterFeatures, - }, + adapter: { type: 'ApolloSequenceAdapter', assemblyId }, metadata: { apollo: true }, }, } diff --git a/packages/jbrowse-plugin-apollo/src/index.ts b/packages/jbrowse-plugin-apollo/src/index.ts index 758944731..12d420621 100644 --- a/packages/jbrowse-plugin-apollo/src/index.ts +++ b/packages/jbrowse-plugin-apollo/src/index.ts @@ -8,7 +8,11 @@ import { } from '@jbrowse/core/pluggableElementTypes' import Plugin from '@jbrowse/core/Plugin' import PluginManager from '@jbrowse/core/PluginManager' -import { AbstractSessionModel, isAbstractMenuManager } from '@jbrowse/core/util' +import { + AbstractSessionModel, + Region, + isAbstractMenuManager, +} from '@jbrowse/core/util' import { changeRegistry } from 'apollo-common' import { CoreValidation, @@ -29,6 +33,7 @@ import { configSchema as apolloSixFrameRendererConfigSchema, } from './ApolloSixFrameRenderer' import { installApolloTextSearchAdapter } from './ApolloTextSearchAdapter' +import { BackendDriver } from './BackendDrivers' import { DownloadGFF3, OpenLocalFile, ViewChangeLog } from './components' import ApolloPluginConfigurationSchema from './config' import { @@ -45,6 +50,26 @@ import { configSchemaFactory as sixFrameFeatureDisplayConfigSchemaFactory, } from './SixFrameFeatureDisplay' +interface ApolloMessageData { + apollo: true + messageId: string + method: string + region: Region + sequence: string + assembly: string +} + +function isApolloMessageData(data?: unknown): data is ApolloMessageData { + return ( + typeof data === 'object' && + data !== null && + 'apollo' in data && + data.apollo === true + ) +} + +const inWebWorker = 'WorkerGlobalScope' in globalThis + for (const [changeName, change] of Object.entries(changes)) { changeRegistry.registerChange(changeName, change) } @@ -138,6 +163,65 @@ export default class ApolloPlugin extends Plugin { 'Core-extendSession', extendSession.bind(this, pluginManager), ) + + if (!inWebWorker) { + pluginManager.addToExtensionPoint( + 'Core-extendWorker', + (handle: { workers: Worker[] }) => { + const [worker] = handle.workers + worker.addEventListener('message', async (event) => { + const { data } = event + if (!isApolloMessageData(data)) { + return + } + const { apollo, messageId, method } = data + switch (method) { + case 'getSequence': { + const { region } = data + const { assemblyName } = region + const dataStore = ( + pluginManager.rootModel?.session as + | ApolloSessionModel + | undefined + )?.apolloDataStore + if (!dataStore) { + break + } + const backendDriver = dataStore.getBackendDriver( + assemblyName, + ) as BackendDriver + const { seq: sequence } = await backendDriver.getSequence( + region, + ) + worker.postMessage({ apollo, messageId, sequence }) + break + } + case 'getRegions': { + const { assembly } = data + const dataStore = ( + pluginManager.rootModel?.session as + | ApolloSessionModel + | undefined + )?.apolloDataStore + if (!dataStore) { + break + } + const backendDriver = dataStore.getBackendDriver( + assembly, + ) as BackendDriver + const regions = await backendDriver.getRegions(assembly) + worker.postMessage({ apollo, messageId, regions }) + break + } + default: { + break + } + } + }) + return handle + }, + ) + } } configure(pluginManager: PluginManager) {