Merge branch 'main' into dev/glyph-click-drag

.devcontainer/devcontainer.json

@@ -22,7 +22,7 @@
   "forwardPorts": [3999, 27017],
 
   // Use 'postCreateCommand' to run commands after the container is created.
-  "postCreateCommand": "yarn install && mongosh --eval 'rs.initiate()'",
+  "postCreateCommand": "yarn install && mongosh --eval 'try {rs.initiate();} catch {}'",
 
   // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
   "remoteUser": "node"

packages/apollo-collaboration-server/.development.env

@@ -6,7 +6,7 @@
 URL=http://localhost
 
 # MongoDB connection
-MONGODB_URI=mongodb://localhost:27017/apolloDb
+MONGODB_URI=mongodb://127.0.0.1:27017/apolloDb
 # Alternatively, can be a path to a file with the URI
 # MONGODB_URI_FILE=/run/secrets/mongodb-uri
 

packages/apollo-collaboration-server/package.json

@@ -26,6 +26,7 @@
     "@emotion/react": "^11.10.6",
     "@emotion/styled": "^11.10.6",
     "@gmod/gff": "^1.2.0",
+    "@gmod/indexedfasta": "^2.0.4",
     "@jbrowse/core": "^2.3.4",
     "@mui/base": "^5.0.0-alpha.118",
     "@mui/material": "^5.11.10",
@@ -47,6 +48,7 @@
     "connect-mongodb-session": "^3.1.1",
     "express": "^4.18.0",
     "express-session": "^1.17.3",
+    "generic-filehandle": "^3.0.0",
     "joi": "^17.7.0",
     "material-ui-popup-state": "^5.0.4",
     "mobx": "^6.6.1",

packages/apollo-collaboration-server/src/assemblies/assemblies.module.ts

@@ -15,6 +15,6 @@ import { AssembliesService } from './assemblies.service'
     ]),
     forwardRef(() => OperationsModule),
   ],
-  exports: [MongooseModule],
+  exports: [MongooseModule, AssembliesService],
 })
 export class AssembliesModule {}

packages/apollo-collaboration-server/src/assemblies/assemblies.service.ts

@@ -29,7 +29,9 @@ export class AssembliesService {
   }
 
   async findOne(id: string) {
-    const assembly = await this.assemblyModel.findById({ id, status: 0 }).exec()
+    const assembly = await this.assemblyModel
+      .findOne({ _id: id, status: 0 })
+      .exec()
     if (!assembly) {
       throw new NotFoundException(`Assembly with id "${id}" not found`)
     }

packages/apollo-collaboration-server/src/refSeqChunks/refSeqChunks.module.ts

@@ -3,6 +3,7 @@ import { MongooseModule, getConnectionToken } from '@nestjs/mongoose'
 import { RefSeqChunk, RefSeqChunkSchema } from 'apollo-schemas'
 import idValidator from 'mongoose-id-validator'
 
+import { AssembliesModule } from '../assemblies/assemblies.module'
 import { RefSeqsModule } from '../refSeqs/refSeqs.module'
 import { RefSeqChunksController } from './refSeqChunks.controller'
 import { RefSeqChunksService } from './refSeqChunks.service'
@@ -22,6 +23,7 @@ import { RefSeqChunksService } from './refSeqChunks.service'
       },
     ]),
     forwardRef(() => RefSeqsModule),
+    forwardRef(() => AssembliesModule),
   ],
   exports: [MongooseModule, RefSeqChunksService],
 })

packages/apollo-collaboration-server/src/refSeqChunks/refSeqChunks.service.ts

@@ -1,3 +1,4 @@
+import { IndexedFasta } from '@gmod/indexedfasta'
 import { Injectable, Logger } from '@nestjs/common'
 import { InjectModel } from '@nestjs/mongoose'
 import {
@@ -6,8 +7,10 @@ import {
   RefSeqChunkDocument,
   RefSeqDocument,
 } from 'apollo-schemas'
+import { RemoteFile } from 'generic-filehandle'
 import { Model } from 'mongoose'
 
+import { AssembliesService } from '../assemblies/assemblies.service'
 import { CreateRefSeqChunkDto } from './dto/create-refSeqChunk.dto'
 import { GetSequenceDto } from './dto/get-sequence.dto'
 
@@ -18,6 +21,7 @@ export class RefSeqChunksService {
     private readonly refSeqChunkModel: Model<RefSeqChunkDocument>,
     @InjectModel(RefSeq.name)
     private readonly refSeqModel: Model<RefSeqDocument>,
+    private readonly assembliesService: AssembliesService,
   ) {}
 
   private readonly logger = new Logger(RefSeqChunksService.name)
@@ -31,7 +35,23 @@ export class RefSeqChunksService {
     if (!refSeq) {
       throw new Error(`RefSeq "${refSeqId}" not found`)
     }
-    const { chunkSize } = refSeq
+
+    const { assembly, chunkSize, name } = refSeq
+    const assemblyDoc = await this.assembliesService.findOne(
+      assembly.toString(),
+    )
+
+    if (assemblyDoc?.externalLocation) {
+      const { fa, fai } = assemblyDoc.externalLocation
+      this.logger.debug(`Fasta file URL = ${fa}, Fasta index file URL = ${fai}`)
+
+      const indexedFasta = new IndexedFasta({
+        fasta: new RemoteFile(fa, { fetch }),
+        fai: new RemoteFile(fai, { fetch }),
+      })
+      return indexedFasta.getSequence(name, start, end)
+    }
+
     const startChunk = Math.floor(start / chunkSize)
     const endChunk = Math.floor(end / chunkSize)
     const seq: string[] = []

packages/apollo-collaboration-server/test/data/volvox.fa.fai

@@ -0,0 +1,2 @@
+ctgA	50001	22	60	61
+ctgB	6079	50879	100	101

packages/apollo-schemas/src/assembly.schema.ts

@@ -22,6 +22,9 @@ export class Assembly {
 
   @Prop()
   user: string
+
+  @Prop({ type: { fa: String, fai: String } })
+  externalLocation: { fa: string; fai: string }
 }
 
 export const AssemblySchema = SchemaFactory.createForClass(Assembly)

packages/apollo-shared/package.json

@@ -9,11 +9,13 @@
   },
   "dependencies": {
     "@gmod/gff": "^1.2.0",
+    "@gmod/indexedfasta": "^2.0.4",
     "@jbrowse/core": "^2.3.4",
     "apollo-common": "workspace:^",
     "apollo-mst": "workspace:^",
     "apollo-schemas": "workspace:^",
     "bson-objectid": "^2.0.3",
+    "generic-filehandle": "^3.0.0",
     "jwt-decode": "^3.1.2",
     "regenerator-runtime": "^0.13.9",
     "socket.io-client": "^4.5.4",

packages/apollo-shared/src/Changes/AddAssemblyFromExternalChange.ts

@@ -0,0 +1,135 @@
+import { IndexedFasta } from '@gmod/indexedfasta'
+import {
+  AssemblySpecificChange,
+  ChangeOptions,
+  ClientDataStore,
+  LocalGFF3DataStore,
+  SerializedAssemblySpecificChange,
+  ServerDataStore,
+} from 'apollo-common'
+import { RemoteFile } from 'generic-filehandle'
+
+export interface SerializedAddAssemblyFromExternalChangeBase
+  extends SerializedAssemblySpecificChange {
+  typeName: 'AddAssemblyFromExternalChange'
+}
+
+export interface AddAssemblyFromExternalChangeDetails {
+  assemblyName: string
+  externalLocation: { fa: string; fai: string }
+}
+
+export interface SerializedAddAssemblyFromExternalChangeSingle
+  extends SerializedAddAssemblyFromExternalChangeBase,
+    AddAssemblyFromExternalChangeDetails {}
+
+export interface SerializedAddAssemblyFromExternalChangeMultiple
+  extends SerializedAddAssemblyFromExternalChangeBase {
+  changes: AddAssemblyFromExternalChangeDetails[]
+}
+
+export type SerializedAddAssemblyFromExternalChange =
+  | SerializedAddAssemblyFromExternalChangeSingle
+  | SerializedAddAssemblyFromExternalChangeMultiple
+
+export class AddAssemblyFromExternalChange extends AssemblySpecificChange {
+  typeName = 'AddAssemblyFromExternalChange' as const
+  changes: AddAssemblyFromExternalChangeDetails[]
+
+  constructor(
+    json: SerializedAddAssemblyFromExternalChange,
+    options?: ChangeOptions,
+  ) {
+    super(json, options)
+    this.changes = 'changes' in json ? json.changes : [json]
+  }
+
+  get notification(): string {
+    return `Assembly "${this.changes[0].assemblyName}" added successfully. To use it, please refresh the page.`
+  }
+
+  toJSON(): SerializedAddAssemblyFromExternalChange {
+    const { assembly, changes, typeName } = this
+    if (changes.length === 1) {
+      const [{ assemblyName, externalLocation }] = changes
+      return { typeName, assembly, assemblyName, externalLocation }
+    }
+    return { typeName, assembly, changes }
+  }
+
+  /**
+   * Applies the required change to database
+   * @param backend - parameters from backend
+   * @returns
+   */
+  async executeOnServer(backend: ServerDataStore) {
+    const { assemblyModel, refSeqModel, user } = backend
+    const { assembly, changes, logger } = this
+    const { CHUNK_SIZE } = process.env
+    const customChunkSize = CHUNK_SIZE && Number(CHUNK_SIZE)
+
+    for (const change of changes) {
+      const { assemblyName, externalLocation } = change
+      const { fa, fai } = externalLocation
+
+      const allSequenceSizes = await new IndexedFasta({
+        fasta: new RemoteFile(fa, { fetch }),
+        fai: new RemoteFile(fai, { fetch }),
+      }).getSequenceSizes()
+
+      if (!allSequenceSizes) {
+        throw new Error('No data read from indexed fasta getSequenceSizes')
+      }
+
+      const assemblyDoc = await assemblyModel
+        .findOne({ name: assemblyName })
+        .exec()
+      if (assemblyDoc) {
+        throw new Error(`Assembly "${assemblyName}" already exists`)
+      }
+      const [newAssemblyDoc] = await assemblyModel.create([
+        {
+          _id: assembly,
+          name: assemblyName,
+          user,
+          status: -1,
+          externalLocation,
+        },
+      ])
+      logger.debug?.(
+        `Added new assembly "${assemblyName}", docId "${newAssemblyDoc._id}"`,
+      )
+
+      for (const sequenceName in allSequenceSizes) {
+        const [newRefSeqDoc] = await refSeqModel.create([
+          {
+            name: sequenceName,
+            assembly: newAssemblyDoc._id,
+            length: allSequenceSizes[sequenceName],
+            ...(customChunkSize ? { chunkSize: customChunkSize } : null),
+            user,
+            status: -1,
+          },
+        ])
+        logger.debug?.(
+          `Added new refSeq "${sequenceName}", docId "${newRefSeqDoc._id}"`,
+        )
+      }
+    }
+  }
+
+  async executeOnLocalGFF3(_backend: LocalGFF3DataStore) {
+    throw new Error('executeOnLocalGFF3 not implemented')
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  async executeOnClient(_dataStore: ClientDataStore) {}
+
+  getInverse() {
+    const { assembly, changes, logger, typeName } = this
+    return new AddAssemblyFromExternalChange(
+      { typeName, changes, assembly },
+      { logger },
+    )
+  }
+}

packages/apollo-shared/src/Changes/index.ts

@@ -1,4 +1,5 @@
 import { AddAssemblyAndFeaturesFromFileChange } from './AddAssemblyAndFeaturesFromFileChange'
+import { AddAssemblyFromExternalChange } from './AddAssemblyFromExternalChange'
 import { AddAssemblyFromFileChange } from './AddAssemblyFromFileChange'
 import { AddFeatureChange } from './AddFeatureChange'
 import { AddFeaturesFromFileChange } from './AddFeaturesFromFileChange'
@@ -14,6 +15,7 @@ import { UserChange } from './UserChange'
 export const changes = {
   AddAssemblyAndFeaturesFromFileChange,
   AddAssemblyFromFileChange,
+  AddAssemblyFromExternalChange,
   AddFeatureChange,
   AddFeaturesFromFileChange,
   DeleteAssemblyChange,
@@ -28,6 +30,7 @@ export const changes = {
 
 export * from './AddAssemblyAndFeaturesFromFileChange'
 export * from './AddAssemblyFromFileChange'
+export * from './AddAssemblyFromExternalChange'
 export * from './AddFeatureChange'
 export * from './AddFeaturesFromFileChange'
 export * from './DeleteAssemblyChange'

packages/jbrowse-plugin-apollo/jbrowse_config.json

@@ -22,7 +22,7 @@
                 "name": "Gene Ontology",
                 "version": "full",
                 "source": {
-                    "uri": "http://release.geneontology.org/2023-06-11/ontology/go.json",
+                    "uri": "https://release.geneontology.org/2023-06-11/ontology/go.json",
                     "locationType": "UriLocation"
                 }
             },

packages/jbrowse-plugin-apollo/src/ApolloInternetAccount/components/AuthTypeSelector.tsx

@@ -1,14 +1,8 @@
-import {
-  Button,
-  Dialog,
-  DialogActions,
-  DialogContent,
-  DialogTitle,
-  Divider,
-} from '@mui/material'
+import { Button, DialogActions, DialogContent, Divider } from '@mui/material'
 import React from 'react'
 import { makeStyles } from 'tss-react/mui'
 
+import { Dialog } from '../../components/Dialog'
 import { GoogleButton, GuestButton, MicrosoftButton } from './LoginButtons'
 
 const useStyles = makeStyles()((theme) => ({
@@ -44,8 +38,13 @@ export const AuthTypeSelector = ({
   }
   // convert component to string useable in data-uri
   return (
-    <Dialog open maxWidth="xl" data-testid="login-apollo">
-      <DialogTitle>Log in to {name}</DialogTitle>
+    <Dialog
+      open
+      title={`Log in to ${name}`}
+      handleClose={handleClose}
+      maxWidth={false}
+      data-testid="login-apollo"
+    >
       <DialogContent
         style={{ display: 'flex', flexDirection: 'column', paddingTop: 8 }}
       >