Static Resource, Asset, CMS Backend Refactor (#10304)

* Remove sid field from static-resource table * move getCachedURL and getCacheDomain to storage provider interface, hiding cacheDomain property * resolve url * fix avatars and tsc errors * local upload changes * add more info to resource json and static resources * fixes * fix seeding * migration convert assets to static resources, fix reinit to wipe DB completely * fixes * fixes to seeder, avatar install and uploading local to storage provider * license * use static resource service instead of asset service * bug fixes * more simplification and refactoring * remove unused asset service * bug fixes, simplification * bug fixes for types * refactoring file browser * license * dynamically handle org name in file browser ui (#10354) * file upload and other fixes * tsc fix * fix merge conflict * test fixes * fix project upload * various fixes * file browser works with orgs * clean up * update resources json in hooks * bug fixes * fixes * fixes for update method and tests * improve file browser tests * bug fixes * fix rename scene * tsc err * fix rename * fix saving scenes * fix thumbnail gen * fix delete scene and rename scene * cleanup logs * optimize location query, improve export, add back removed delete for resource table * fix projectEventHooks * fix projectEventHooks * remove knex transaction from migration * remove console log * fixes * trying migrations * fix bug with migrating resources * optimize single static resource updates to resources.json * add thumbnail key field alongside thumbnail url * fix file browser tests * dont always generate resources json * fix for manifest * fix bugs with updating resources json * ensure new scenes have unique names and include thumbnail * fix pagination and reactivity * type fix --------- Co-authored-by: Aditya Mitra <[email protected]>
EtherealEngine · Jun 18, 2024 · abe0e11 · abe0e11
1 parent abf53cd
commit abe0e11
Show file tree

Hide file tree

Showing 96 changed files with 2,603 additions and 4,042 deletions.
diff --git a/packages/client-core/i18n/en/editor.json b/packages/client-core/i18n/en/editor.json
@@ -1167,6 +1167,7 @@
       "refresh": "Refresh",
       "back": "Back",
       "loadingFiles": "Loading files",
+      "loadingProjects": "Loading projects",
       "compress": "Compress",
       "convert": "Convert",
       "downloadProject": "Download Project",

diff --git a/packages/client-core/src/admin/common/Location/LocationDrawer.tsx b/packages/client-core/src/admin/common/Location/LocationDrawer.tsx
@@ -30,13 +30,12 @@ import InputSelect, { InputMenuItem } from '@etherealengine/client-core/src/comm
 import InputSwitch from '@etherealengine/client-core/src/common/components/InputSwitch'
 import InputText from '@etherealengine/client-core/src/common/components/InputText'
 import {
-  assetPath,
   LocationData,
   LocationID,
   locationPath,
-  LocationType
+  LocationType,
+  staticResourcePath
 } from '@etherealengine/common/src/schema.type.module'
-import { AssetType } from '@etherealengine/common/src/schemas/assets/asset.schema'
 import { getState, useHookstate } from '@etherealengine/hyperflux'
 import { useFind, useGet, useMutation } from '@etherealengine/spatial/src/common/functions/FeathersHooks'
 import Button from '@etherealengine/ui/src/primitives/mui/Button'
@@ -88,14 +87,14 @@ const LocationDrawer = ({ open, mode, selectedLocation, selectedScene, onClose }
   const editMode = useHookstate(false)
   const state = useHookstate({ ...defaultState })
 
-  const scenes = useFind(assetPath)
+  const scenes = useFind(staticResourcePath, { query: { type: 'scene' } })
   // const locationTypes = useFind(locationTypePath).data
 
   const locationMutation = useMutation(locationPath)
 
   const viewMode = mode === LocationDrawerMode.ViewEdit && !editMode.value
 
-  const selectedSceneData = useGet(assetPath, selectedScene!)
+  const selectedSceneData = useGet(staticResourcePath, selectedScene!)
 
   const editorState = getState(EditorState)
 
@@ -107,13 +106,13 @@ const LocationDrawer = ({ open, mode, selectedLocation, selectedScene, onClose }
     ? [
         {
           value: selectedSceneData.data.id,
-          label: selectedSceneData.data.assetURL
+          label: selectedSceneData.data.key
         }
       ]
-    : scenes.data.map((el: AssetType) => {
+    : scenes.data.map((el) => {
         return {
           value: el.id,
-          label: el.assetURL
+          label: el.key
         }
       })
 

diff --git a/packages/client-core/src/admin/components/locations/AddEditLocationModal.tsx b/packages/client-core/src/admin/components/locations/AddEditLocationModal.tsx
@@ -23,11 +23,11 @@ import { useTranslation } from 'react-i18next'
 
 import { PopoverState } from '@etherealengine/client-core/src/common/services/PopoverState'
 import {
-  assetPath,
   LocationData,
   LocationID,
   locationPath,
-  LocationType
+  LocationType,
+  staticResourcePath
 } from '@etherealengine/common/src/schema.type.module'
 import { useHookstate } from '@etherealengine/hyperflux'
 import { useFind, useMutation } from '@etherealengine/spatial/src/common/functions/FeathersHooks'
@@ -65,7 +65,7 @@ export default function AddEditLocationModal({ location }: { location?: Location
   const screenSharingEnabled = useHookstate<boolean>(location?.locationSetting.screenSharingEnabled || true)
   const locationType = useHookstate(location?.locationSetting.locationType || 'public')
 
-  const scenes = useFind(assetPath, {
+  const scenes = useFind(staticResourcePath, {
     query: {
       paginate: false
     }
@@ -160,8 +160,8 @@ export default function AddEditLocationModal({ location }: { location?: Location
               : [
                   { value: '', label: t('admin:components.location.selectScene'), disabled: true },
                   ...scenes.data.map((scene) => {
-                    const project = scene.projectName
-                    const name = scene.assetURL.split('/').pop()!.split('.').at(0)!
+                    const project = scene.project
+                    const name = scene.key.split('/').pop()!.split('.').at(0)!
                     return {
                       label: `${name} (${project})`,
                       value: scene.id

diff --git a/packages/client-core/src/admin/components/locations/LocationTable.tsx b/packages/client-core/src/admin/components/locations/LocationTable.tsx
@@ -78,9 +78,7 @@ export default function LocationTable({ search }: { search: string }) {
     rows.map((row) => ({
       name: <a href={`/location/${transformLink(row.name)}`}>{row.name}</a>,
       sceneId: (
-        <a href={`/studio?projectName=${row.sceneAsset.projectName}&scenePath=${row.sceneAsset.assetURL}`}>
-          {row.sceneId}
-        </a>
+        <a href={`/studio?projectName=${row.sceneAsset.project!}&scenePath=${row.sceneAsset.key}`}>{row.sceneId}</a>
       ),
       maxUsersPerInstance: row.maxUsersPerInstance.toString(),
       scene: row.slugifiedName,

diff --git a/packages/client-core/src/common/services/FileThumbnailJobState.tsx b/packages/client-core/src/common/services/FileThumbnailJobState.tsx
@@ -63,8 +63,6 @@ import { computeTransformMatrix } from '@etherealengine/spatial/src/transform/sy
 import React, { useEffect } from 'react'
 import { Color, Euler, MathUtils, Matrix4, Quaternion, Scene, Sphere, Vector3 } from 'three'
 
-import config from '@etherealengine/common/src/config'
-import { projectResourcesPath } from '@etherealengine/common/src/schemas/media/project-resource.schema'
 import { ErrorComponent } from '@etherealengine/engine/src/scene/components/ErrorComponent'
 import { ShadowComponent } from '@etherealengine/engine/src/scene/components/ShadowComponent'
 import { iterateEntityNode } from '@etherealengine/spatial/src/transform/components/EntityTree'
@@ -101,24 +99,20 @@ const drawToCanvas = (source: CanvasImageSource): Promise<HTMLCanvasElement | nu
   return Promise.resolve(canvas)
 }
 
-const uploadThumbnail = async (key: string, projectName: string, staticResourceId: string, blob: Blob | null) => {
+const uploadThumbnail = async (src: string, projectName: string, staticResourceId: string, blob: Blob | null) => {
   if (!blob) return
-  const thumbnailType = 'automatic'
-  const thumbnailKey = `${decodeURI(key.replace(/^.*?\/projects\//, ''))
+  const thumbnailMode = 'automatic'
+  const thumbnailKey = `${decodeURI(src.replace(/^.*?\/projects\//, ''))
     .replaceAll(/[^a-zA-Z0-9\.\-_\s]/g, '')
     .replaceAll(/\s/g, '-')}-thumbnail.png`
   const file = new File([blob], thumbnailKey)
-  const path = `projects/${projectName}/thumbnails`
-  const upload: Promise<string[]> = uploadToFeathersService(fileBrowserUploadPath, [file], {
+  await uploadToFeathersService(fileBrowserUploadPath, [file], {
     fileName: file.name,
-    path,
-    contentType: ''
+    project: projectName,
+    path: 'public/thumbnails/' + file.name,
+    contentType: file.type
   }).promise
-  const thumbnailURL = (await upload)[0]
-  await Engine.instance.api.service(staticResourcePath).patch(staticResourceId, { thumbnailURL, thumbnailType })
-  const urlPrefixRegex = new RegExp(`^${config.client.fileServer}\/`)
-  const resourceKey = key.replace(urlPrefixRegex, '')
-  await Engine.instance.api.service(projectResourcesPath).patch(staticResourceId, { project: projectName })
+  await Engine.instance.api.service(staticResourcePath).patch(staticResourceId, { thumbnailKey, thumbnailMode })
 }
 
 const seenThumbnails = new Set<string>()
@@ -144,13 +138,13 @@ export const FileThumbnailJobState = defineState({
             return
           }
           const resource = resources.data[0]
-          if (resource.thumbnailURL != null) {
+          if (resource.thumbnailKey != null) {
             return
           }
           getMutableState(FileThumbnailJobState).merge([
             {
               key: url,
-              project: resource.project,
+              project: resource.project!,
               id: resource.id
             }
           ])
@@ -201,13 +195,13 @@ export const FileThumbnailJobState = defineState({
           return
         }
         const resource = resources.data[0]
-        if (!forceRegenerate && resource.thumbnailURL != null) {
+        if (!forceRegenerate && resource.thumbnailKey != null) {
           return
         }
         getMutableState(FileThumbnailJobState).merge([
           {
             key: url,
-            project: resource.project,
+            project: resource.project!,
             id: resource.id
           }
         ])

diff --git a/packages/client-core/src/components/World/LoadLocationScene.tsx b/packages/client-core/src/components/World/LoadLocationScene.tsx
@@ -27,7 +27,7 @@ import { t } from 'i18next'
 import { useEffect } from 'react'
 
 import { LocationService, LocationState } from '@etherealengine/client-core/src/social/services/LocationService'
-import { assetPath } from '@etherealengine/common/src/schema.type.module'
+import { staticResourcePath } from '@etherealengine/common/src/schema.type.module'
 import { GLTFAssetState } from '@etherealengine/engine/src/gltf/GLTFState'
 import { getMutableState, useMutableState } from '@etherealengine/hyperflux'
 import { useFind, useGet } from '@etherealengine/spatial/src/common/functions/FeathersHooks'
@@ -37,7 +37,7 @@ import { WarningUIService } from '../../systems/WarningUISystem'
 
 export const useLoadLocation = (props: { locationName: string }) => {
   const locationState = useMutableState(LocationState)
-  const scene = useGet(assetPath, locationState.currentLocation.location.sceneId.value).data
+  const scene = useGet(staticResourcePath, locationState.currentLocation.location.sceneId.value).data
 
   useEffect(() => {
     LocationState.setLocationName(props.locationName)
@@ -80,18 +80,18 @@ export const useLoadLocation = (props: { locationName: string }) => {
       !scene
     )
       return
-    const sceneURL = scene.assetURL
+    const sceneURL = scene.url
     return GLTFAssetState.loadScene(sceneURL, scene.id)
   }, [locationState.currentLocation.location.sceneId, scene])
 }
 
 export const useLoadScene = (props: { projectName: string; sceneName: string }) => {
-  const sceneURL = `projects/${props.projectName}/${props.sceneName}`
-  const assetID = useFind(assetPath, { query: { assetURL: sceneURL } })
+  const sceneKey = `projects/${props.projectName}/${props.sceneName}`
+  const assetID = useFind(staticResourcePath, { query: { key: sceneKey, type: 'scene' } })
   useEffect(() => {
     if (!props.sceneName || !props.projectName) return
     if (!assetID.data.length) return
     getMutableState(LocationState).currentLocation.location.sceneId.set(assetID.data[0].id)
-    return GLTFAssetState.loadScene(sceneURL, assetID.data[0].id)
+    return GLTFAssetState.loadScene(assetID.data[0].url, assetID.data[0].id)
   }, [assetID.data.length])
 }
diff --git a/packages/client-core/src/util/upload.tsx b/packages/client-core/src/util/upload.tsx
@@ -26,23 +26,23 @@ Ethereal Engine. All Rights Reserved.
 import i18n from 'i18next'
 
 import config from '@etherealengine/common/src/config'
-import { uploadAssetPath } from '@etherealengine/common/src/schema.type.module'
 import { getMutableState } from '@etherealengine/hyperflux'
 
 import '@etherealengine/common/src/utils/jsonUtils'
 
+import { ServiceTypes } from '@etherealengine/common/declarations'
 import { AuthState } from '../user/services/AuthService'
 import { RethrownError } from './errors'
 
 export type CancelableUploadPromiseReturnType<T = any> = { cancel: () => void; promise: Promise<T | T[]> }
 export type CancelableUploadPromiseArrayReturnType<T = any> = { cancel: () => void; promises: Array<Promise<T | T[]>> }
 
 export const uploadToFeathersService = (
-  service = uploadAssetPath,
+  service: keyof ServiceTypes,
   files: Array<File>,
-  params: any = {},
+  params: ServiceTypes[typeof service]['create']['params'], // todo make this type work
   onUploadProgress?: (progress: number) => any
-): CancelableUploadPromiseReturnType => {
+): CancelableUploadPromiseReturnType<Awaited<ReturnType<ServiceTypes[typeof service]['create']['params']>>> => {
   const token = getMutableState(AuthState).authUser.accessToken.value
   const request = new XMLHttpRequest()
   request.timeout = 10 * 60 * 1000 // 10 minutes - need to support big files on slow connections

diff --git a/packages/common/src/interfaces/ManifestJson.ts b/packages/common/src/interfaces/ManifestJson.ts
@@ -45,11 +45,6 @@ export type ManifestJson = {
    * @example "https://example.com/thumbnail.jpg"
    */
   thumbnail?: string
-  /**
-   * project-relative path for scene GLTF files
-   * @example ["public/scenes/default.gltf"]
-   */
-  scenes?: string[]
   /**
    * The dependencies of this project. Specify other projects that are to be installed alongside this one.
    * @todo

diff --git a/.../schemas/media/project-resource.schema.ts → ...es/common/src/interfaces/ResourcesJson.ts b/.../schemas/media/project-resource.schema.ts → ...es/common/src/interfaces/ResourcesJson.ts
@@ -23,4 +23,17 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20
 Ethereal Engine. All Rights Reserved.
 */
 
-export const projectResourcesPath = 'project-resources'
+// key = /path/to/file.ext
+export type ResourcesJson = Record<
+  string,
+  {
+    type: string // 'scene' | 'asset' | 'file' | 'thumbnail' | 'avatar' | 'recording'
+    tags?: string[]
+    dependencies?: string[] // other keys
+    licensing?: string
+    description?: string
+    attribution?: string
+    thumbnailKey?: string
+    thumbnailMode?: string // 'automatic' | 'manual'
+  }
+>
diff --git a/packages/common/src/interfaces/UploadAssetInterface.ts b/packages/common/src/interfaces/UploadAssetInterface.ts
@@ -48,7 +48,6 @@ export type AdminAssetUploadArgumentsType = {
   name?: string
   project?: string
   hash?: string
-  stats?: any
 }
 
 export type AdminAssetUploadType = {
@@ -67,6 +66,6 @@ export interface UploadFile {
   originalname: string
   encoding?: string
   mimetype: string
-  buffer: Buffer | string
+  buffer: Buffer
   size: number
 }
diff --git a/packages/common/src/schema.type.module.ts b/packages/common/src/schema.type.module.ts
@@ -25,7 +25,6 @@ Ethereal Engine. All Rights Reserved.
 
 export type * from './schemas/analytics/analytics.schema'
 export type * from './schemas/assets/asset-library.schema'
-export type * from './schemas/assets/asset.schema'
 export type * from './schemas/assets/model-transform.schema'
 export type * from './schemas/bot/bot-command.schema'
 export type * from './schemas/bot/bot.schema'
@@ -62,7 +61,6 @@ export type * from './schemas/projects/project-permission-type.schema'
 export type * from './schemas/projects/project-permission.schema'
 export type * from './schemas/projects/project.schema'
 export type * from './schemas/projects/projects.schema'
-export type * from './schemas/projects/scene-data.schema'
 export type * from './schemas/recording/recording-resource-upload.schema'
 export type * from './schemas/recording/recording-resource.schema'
 export type * from './schemas/recording/recording.schema'
@@ -262,8 +260,6 @@ export const projectPath = 'project'
 
 export const projectsPath = 'projects'
 
-export const assetPath = 'asset'
-
 export const builderInfoPath = 'builder-info'
 
 export const projectCheckSourceDestinationMatchPath = 'project-check-source-destination-match'
@@ -276,8 +272,6 @@ export const projectPermissionTypePath = 'project-permission-type'
 
 export const projectDestinationCheckPath = 'project-destination-check'
 
-export const sceneDataPath = 'scene-data'
-
 export const spawnPointPath = 'spawn-point'
 
 export const projectCheckUnfetchedCommitPath = 'project-check-unfetched-commit'
@@ -301,3 +295,5 @@ export const migrationsInfoPath = 'knex_migrations'
 export const uploadAssetPath = 'upload-asset'
 
 export const invalidationPath = 'invalidation'
+
+export const imageConvertPath = 'image-convert'