Skip to content
This repository has been archived by the owner on Aug 21, 2024. It is now read-only.

Static Resource, Asset, CMS Backend Refactor #10304

Merged
merged 81 commits into from
Jun 18, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
81 commits
Select commit Hold shift + click to select a range
ccae33e
Remove sid field from static-resource table
HexaField Jun 3, 2024
1199c8d
move getCachedURL and getCacheDomain to storage provider interface, h…
HexaField Jun 3, 2024
6640ba2
resolve url
HexaField Jun 3, 2024
ded0117
fix avatars and tsc errors
HexaField Jun 3, 2024
bd63396
local upload changes
HexaField Jun 3, 2024
389a8e2
Merge branch 'dev' into cms-refactor
HexaField Jun 3, 2024
3f26fba
add more info to resource json and static resources
HexaField Jun 4, 2024
058df46
fixes
HexaField Jun 4, 2024
64482de
fix seeding
HexaField Jun 4, 2024
c7cdcb3
migration convert assets to static resources, fix reinit to wipe DB c…
HexaField Jun 4, 2024
b0bef50
Merge branch 'dev' into cms-refactor
HexaField Jun 4, 2024
f0babda
Merge branch 'dev' into cms-refactor
HexaField Jun 4, 2024
efaf2e1
fixes
HexaField Jun 5, 2024
2298c61
fixes to seeder, avatar install and uploading local to storage provider
HexaField Jun 6, 2024
446a63e
license
HexaField Jun 6, 2024
a32e3a2
Merge branch 'dev' into cms-refactor
HexaField Jun 6, 2024
76e4b55
use static resource service instead of asset service
HexaField Jun 6, 2024
68bc37f
bug fixes
HexaField Jun 7, 2024
40fa785
more simplification and refactoring
HexaField Jun 7, 2024
b0834a5
remove unused asset service
HexaField Jun 7, 2024
49e2998
bug fixes, simplification
HexaField Jun 7, 2024
0069e6f
bug fixes for types
HexaField Jun 7, 2024
436a161
refactoring file browser
HexaField Jun 10, 2024
85aad20
license
HexaField Jun 10, 2024
2144f74
Merge branch 'dev' into cms-refactor
HexaField Jun 10, 2024
29b03ae
dynamically handle org name in file browser ui (#10354)
aditya-mitra Jun 10, 2024
936db7b
file upload and other fixes
HexaField Jun 10, 2024
fe362f5
tsc fix
HexaField Jun 10, 2024
4a1d7f0
fix merge conflict
HexaField Jun 10, 2024
6546ae1
test fixes
HexaField Jun 10, 2024
c9fcf6f
fix project upload
HexaField Jun 11, 2024
72b9440
various fixes
HexaField Jun 11, 2024
71d68bf
file browser works with orgs
HexaField Jun 11, 2024
941035b
clean up
HexaField Jun 11, 2024
219d8ee
update resources json in hooks
HexaField Jun 11, 2024
da8f312
Merge branch 'dev' into cms-refactor
HexaField Jun 11, 2024
3c12617
Merge branch 'dev' into cms-refactor
HexaField Jun 11, 2024
972cd25
bug fixes
HexaField Jun 11, 2024
b6ead5f
fixes
HexaField Jun 11, 2024
979ddad
fixes for update method and tests
HexaField Jun 11, 2024
7b08b12
Merge branch 'dev' into cms-refactor
HexaField Jun 11, 2024
2080df2
improve file browser tests
HexaField Jun 12, 2024
7c0a6de
bug fixes
HexaField Jun 12, 2024
de17644
fix rename scene
HexaField Jun 12, 2024
e3eb963
Merge branch 'dev' into cms-refactor
HexaField Jun 12, 2024
877df44
Merge branch 'dev' into cms-refactor
HexaField Jun 12, 2024
399d925
tsc err
HexaField Jun 13, 2024
38c5187
fix rename
HexaField Jun 13, 2024
5ffdc16
Merge branch 'dev' into cms-refactor
HexaField Jun 13, 2024
6331c12
Merge branch 'dev' into cms-refactor
HexaField Jun 14, 2024
1438bd3
fix saving scenes
HexaField Jun 14, 2024
2093e28
fix thumbnail gen
HexaField Jun 14, 2024
24933d1
Merge branch 'dev' into cms-refactor
HexaField Jun 14, 2024
fd6ba2b
fix delete scene and rename scene
HexaField Jun 14, 2024
a73ff73
cleanup logs
HexaField Jun 14, 2024
6a3df40
Merge branch 'dev' into cms-refactor
HexaField Jun 14, 2024
767193a
Merge branch 'dev' into cms-refactor
HexaField Jun 14, 2024
f485257
optimize location query, improve export, add back removed delete for …
HexaField Jun 14, 2024
1cb6260
fix projectEventHooks
HexaField Jun 14, 2024
611139f
fix projectEventHooks
HexaField Jun 14, 2024
fd5eb9e
Merge branch 'dev' into cms-refactor
HexaField Jun 15, 2024
5cbc43f
remove knex transaction from migration
HexaField Jun 15, 2024
73211be
remove console log
HexaField Jun 15, 2024
63052dc
Merge branch 'dev' into cms-refactor
HexaField Jun 15, 2024
8825426
fixes
HexaField Jun 15, 2024
9c46994
trying migrations
HexaField Jun 15, 2024
63b83aa
fix bug with migrating resources
HexaField Jun 15, 2024
1242c75
optimize single static resource updates to resources.json
HexaField Jun 15, 2024
8d6ae12
Merge branch 'dev' into cms-refactor
HexaField Jun 15, 2024
15de684
Merge branch 'dev' into cms-refactor
HexaField Jun 17, 2024
fa0a473
add thumbnail key field alongside thumbnail url
HexaField Jun 17, 2024
7817ff5
fix file browser tests
HexaField Jun 17, 2024
f6110ab
Merge branch 'dev' into cms-refactor
HexaField Jun 17, 2024
6bf5d2f
dont always generate resources json
HexaField Jun 17, 2024
82617c0
fix for manifest
HexaField Jun 17, 2024
bed8ad1
Merge branch 'dev' into cms-refactor
HexaField Jun 17, 2024
6c23b25
fix bugs with updating resources json
HexaField Jun 17, 2024
cce7868
ensure new scenes have unique names and include thumbnail
HexaField Jun 18, 2024
343124b
fix pagination and reactivity
HexaField Jun 18, 2024
95e245c
type fix
HexaField Jun 18, 2024
be88424
Merge branch 'dev' into cms-refactor
HexaField Jun 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
various fixes
  • Loading branch information
HexaField committed Jun 11, 2024
commit 72b944004108c6a50edcbc8aedb431b04effa6a6
2 changes: 1 addition & 1 deletion packages/common/src/schemas/media/file-browser.schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ export const fileBrowserPatchSchema = Type.Intersect(
Type.Object({
path: Type.String(),
project: Type.String(),
body: Type.Any(),
body: Type.Any(), // Buffer | string
contentType: Type.Optional(Type.String()),
storageProviderName: Type.Optional(Type.String())
})
Expand Down
2 changes: 1 addition & 1 deletion packages/editor/src/components/EditorContainer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ const generateToolbarMenu = () => {
return [
{
name: t('editor:menubar.newScene'),
action: onNewScene
action: () => onNewScene()
},
{
name: t('editor:menubar.saveScene'),
Expand Down
2 changes: 1 addition & 1 deletion packages/editor/src/components/toolbar/Toolbar2.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ const generateToolbarMenu = () => {
return [
{
name: t('editor:menubar.newScene'),
action: onNewScene
action: () => onNewScene()
},
{
name: t('editor:menubar.saveScene'),
Expand Down
24 changes: 12 additions & 12 deletions packages/editor/src/functions/sceneFunctions.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import i18n from 'i18next'

import config from '@etherealengine/common/src/config'
import multiLogger from '@etherealengine/common/src/logger'
import { staticResourcePath } from '@etherealengine/common/src/schema.type.module'
import { fileBrowserPath, staticResourcePath } from '@etherealengine/common/src/schema.type.module'
import { cleanString } from '@etherealengine/common/src/utils/cleanString'
import { EntityUUID, UUIDComponent, UndefinedEntity } from '@etherealengine/ecs'
import { getComponent, getMutableComponent } from '@etherealengine/ecs/src/ComponentFunctions'
Expand All @@ -52,7 +52,7 @@ const logger = multiLogger.child({ component: 'editor:sceneFunctions' })
*/
export const deleteScene = async (sceneID: string): Promise<any> => {
try {
await Engine.instance.api.service(staticResourcePath).remove(sceneID)
await Engine.instance.api.service(fileBrowserPath).remove(sceneID)
} catch (error) {
logger.error(error, 'Error in deleting project')
throw error
Expand All @@ -62,9 +62,9 @@ export const deleteScene = async (sceneID: string): Promise<any> => {

export const renameScene = async (id: string, newKey: string, projectName: string, params?: Params) => {
try {
return await Engine.instance.api
.service(staticResourcePath)
.patch(id, { key: newKey, project: projectName }, params)
// return await Engine.instance.api
// .service(fileBrowserPath)
// .update(id, { key: newKey, project: projectName }, params)
} catch (error) {
logger.error(error, 'Error in renaming project')
throw error
Expand Down Expand Up @@ -136,26 +136,26 @@ export const saveSceneGLTF = async (
})
}

export const onNewScene = async () => {
export const onNewScene = async (
templateURL = config.client.fileServer + '/projects/default-project/public/scenes/default.gltf'
) => {
const { projectName } = getState(EditorState)
if (!projectName) return

try {
const sceneData = await Engine.instance.api.service(staticResourcePath).create({
const sceneData = await Engine.instance.api.service(fileBrowserPath).patch(null, {
project: projectName,
type: 'scene',
// TODO
// sourceURL: 'projects/default-project/public/scenes/default.gltf',
key: `projects/${projectName}/public/scenes/New-Scene.gltf`
body: templateURL,
path: 'public/scenes/New-Scene.gltf'
})
if (!sceneData) return
const sceneName = sceneData.key.split('/').pop()
const newProjectName = sceneData.project

getMutableState(EditorState).merge({
sceneName,
scenePath: sceneData.key,
projectName: newProjectName,
projectName: projectName,
sceneAssetID: sceneData.id
})
} catch (error) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,16 +54,18 @@ export class FileBrowserUploadService implements ServiceInterface<string[], any,
async create(data: FileBrowserUploadData, params: FileBrowserUploadParams) {
if (typeof data.args === 'string') data.args = JSON.parse(data.args)

const result = (await Promise.all(
params.files.map((file) =>
this.app.service(fileBrowserPath).patch(null, {
project: data.project,
path: data.path,
body: file.buffer as Buffer,
contentType: file.mimetype
})
const result = (
await Promise.all(
params.files.map((file) =>
this.app.service(fileBrowserPath).patch(null, {
project: data.project,
path: data.path,
body: file.buffer as Buffer,
contentType: file.mimetype
})
)
)
)) as string[]
).map((result) => result.url)

// Clear params otherwise all the files and auth details send back to client as response
for (const prop of Object.getOwnPropertyNames(params)) delete params[prop]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import { checkScope } from '@etherealengine/spatial/src/common/functions/checkSc

import { Application } from '../../../declarations'
import config from '../../appconfig'
import { updateProjectResourcesJson } from '../../projects/project/project-helper'
import { getContentType } from '../../util/fileUtils'
import { getIncrementalName } from '../FileUtil'
import { getStorageProvider } from '../storageprovider/storageprovider'
Expand All @@ -66,7 +67,7 @@ const ensureProjectsDirectory = (directory: string) => {
export class FileBrowserService
implements
ServiceInterface<
boolean | string | Paginated<FileBrowserContentType>,
boolean | StaticResourceType | Paginated<FileBrowserContentType>,
string | FileBrowserUpdate | FileBrowserPatch,
FileBrowserParams,
FileBrowserPatch
Expand Down Expand Up @@ -182,6 +183,8 @@ export class FileBrowserService

if (config.fsProjectSyncEnabled) fs.mkdirSync(path.resolve(projectsRootFolder, keyPath), { recursive: true })

// updateProjectResourcesJson(this.app, data.project)

return result
}

Expand Down Expand Up @@ -247,6 +250,8 @@ export class FileBrowserService
else fs.renameSync(oldNamePath, newNamePath)
}

// updateProjectResourcesJson(this.app, data.project)

return result
}

Expand All @@ -256,7 +261,17 @@ export class FileBrowserService
async patch(id: NullableId, data: FileBrowserPatch, params?: FileBrowserParams) {
const storageProviderName = data.storageProviderName
delete data.storageProviderName
const storageProvider = getStorageProvider(storageProviderName)

if (typeof data.body === 'string') {
const url = new URL(data.body)
try {
const response = await fetch(url)
const arr = await response.arrayBuffer()
data.body = Buffer.from(arr)
} catch (error) {
throw new Error('Invalid URL ' + url)
}
}
const key = path.join('projects', data.project, data.path)

/** @todo should we allow user-specific content types? Or standardize on the backend? */
Expand All @@ -267,7 +282,7 @@ export class FileBrowserService
})) as Paginated<StaticResourceType>
const existingResource = existingResourceQuery.data.length ? existingResourceQuery.data[0] : undefined

await uploadStaticResource(this.app, {
const staticResource = await uploadStaticResource(this.app, {
...data,
key,
contentType,
Expand All @@ -281,7 +296,9 @@ export class FileBrowserService
fs.writeFileSync(filePath, data.body)
}

return storageProvider.getCachedURL(key, params && params.provider == null)
updateProjectResourcesJson(this.app, data.project)

return staticResource
}

/**
Expand Down Expand Up @@ -317,6 +334,8 @@ export class FileBrowserService

if (config.fsProjectSyncEnabled) fs.rmSync(path.resolve(projectsRootFolder, key), { recursive: true })

// updateProjectResourcesJson(this.app, data.project)

return result
}
}
32 changes: 16 additions & 16 deletions packages/server-core/src/media/file-browser/file-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,14 @@ export const uploadStaticResource = async (app: Application, args: StaticResourc
project,
mimeType: contentType,
stats,
type: data?.type ?? undefined,
tags: data?.tags ?? [assetClass],
dependencies: data?.dependencies ?? undefined,
licensing: data?.licensing ?? undefined,
description: data?.description ?? undefined,
attribution: data?.attribution ?? undefined,
thumbnailURL: data?.thumbnailURL ?? undefined,
thumbnailMode: data?.thumbnailMode ?? undefined
type: data.type,
tags: data.tags ?? [assetClass],
dependencies: data.dependencies,
licensing: data.licensing,
description: data.description,
attribution: data.attribution,
thumbnailURL: data.thumbnailURL,
thumbnailMode: data.thumbnailMode
},
{ isInternal: true }
)
Expand All @@ -97,14 +97,14 @@ export const uploadStaticResource = async (app: Application, args: StaticResourc
mimeType: contentType,
project,
stats,
type: data?.type ?? undefined,
tags: data?.tags ?? [assetClass],
dependencies: data?.dependencies ?? undefined,
licensing: data?.licensing ?? undefined,
description: data?.description ?? undefined,
attribution: data?.attribution ?? undefined,
thumbnailURL: data?.thumbnailURL ?? undefined,
thumbnailMode: data?.thumbnailMode ?? undefined
type: data.type,
tags: data.tags ?? [assetClass],
dependencies: data.dependencies,
licensing: data.licensing,
description: data.description,
attribution: data.attribution,
thumbnailURL: data.thumbnailURL,
thumbnailMode: data.thumbnailMode
},
{ isInternal: true }
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ Original Code is the Ethereal Engine team.
All portions of the code written by the Ethereal Engine team are Copyright © 2021-2023
Ethereal Engine. All Rights Reserved.
*/
import { Forbidden } from '@feathersjs/errors'
import { BadRequest, Forbidden } from '@feathersjs/errors'
import { hooks as schemaHooks } from '@feathersjs/schema'
import { disallow, iff, isProvider } from 'feathers-hooks-common'

Expand All @@ -32,12 +32,11 @@ import { HookContext } from '../../../declarations'
import setLoggedinUserInBody from '../../hooks/set-loggedin-user-in-body'
import verifyScope from '../../hooks/verify-scope'
import { getStorageProvider } from '../storageprovider/storageprovider'
import { createStaticResourceHash } from '../upload-asset/upload-asset.service'
import { StaticResourceService } from './static-resource.class'
import {
staticResourceDataResolver,
staticResourceExternalResolver,
staticResourcePatchResolver,
staticResourceQueryResolver,
staticResourceResolver
} from './static-resource.resolvers'

Expand All @@ -60,34 +59,50 @@ const ensureResource = async (context: HookContext<StaticResourceService>) => {
}
}

const createHashIfNeeded = async (context: HookContext<StaticResourceService>) => {
if (!context.data || context.method !== 'create') {
throw new BadRequest(`${context.path} service only works for data in ${context.method}`)
}

if (Array.isArray(context.data)) throw new BadRequest('Batch create is not supported')

const data = context.data

if (!data.key) throw new BadRequest('key is required')

if (!data.hash) {
const storageProvider = getStorageProvider()
const [_, directory, file] = /(.*)\/([^\\\/]+$)/.exec(data.key)!
if (!(await storageProvider.doesExist(file, directory))) throw new BadRequest('File could not be found')
const result = await storageProvider.getObject(data.key)
const hash = createStaticResourceHash(result.Body)
context.data.hash = hash
}
}

export default {
around: {
all: [
schemaHooks.resolveExternal(staticResourceExternalResolver),
schemaHooks.resolveResult(staticResourceResolver)
]
all: [schemaHooks.resolveResult(staticResourceResolver)]
},

before: {
all: [
// schemaHooks.validateQuery(staticResourceQueryValidator),
schemaHooks.resolveQuery(staticResourceQueryResolver)
],
all: [],
find: [],
get: [],
create: [
iff(isProvider('external'), verifyScope('static_resource', 'write')),
iff(isProvider('external'), disallow()),
setLoggedinUserInBody('userId'),
// schemaHooks.validateData(staticResourceDataValidator),
schemaHooks.resolveData(staticResourceDataResolver)
schemaHooks.resolveData(staticResourceDataResolver),
createHashIfNeeded
],
update: [disallow()],
patch: [
iff(isProvider('external'), verifyScope('static_resource', 'write')),
iff(isProvider('external'), disallow()),
// schemaHooks.validateData(staticResourcePatchValidator),
schemaHooks.resolveData(staticResourcePatchResolver)
],
remove: [iff(isProvider('external'), verifyScope('static_resource', 'write')), ensureResource]
remove: [iff(isProvider('external'), disallow()), ensureResource]
},

after: {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import { v4 as uuidv4 } from 'uuid'

import {
StaticResourceDatabaseType,
StaticResourceQuery,
StaticResourceType
} from '@etherealengine/common/src/schemas/media/static-resource.schema'
import { fromDateTimeSql, getDateTimeSql } from '@etherealengine/common/src/utils/datetime-sql'
Expand Down Expand Up @@ -68,9 +67,9 @@ export const staticResourceResolver = resolve<StaticResourceType, HookContext>(
{
createdAt: virtual(async (staticResource) => fromDateTimeSql(staticResource.createdAt)),
updatedAt: virtual(async (staticResource) => fromDateTimeSql(staticResource.updatedAt)),
url: virtual(async (staticResource) => {
url: virtual(async (staticResource, context) => {
const storageProvider = getStorageProvider()
return storageProvider.getCachedURL(staticResource.key)
return storageProvider.getCachedURL(staticResource.key, context.params.isInternal)
})
},
{
Expand All @@ -81,8 +80,6 @@ export const staticResourceResolver = resolve<StaticResourceType, HookContext>(
}
)

export const staticResourceExternalResolver = resolve<StaticResourceType, HookContext>({})

export const staticResourceDataResolver = resolve<StaticResourceType, HookContext>(
{
id: async () => {
Expand Down Expand Up @@ -120,5 +117,3 @@ export const staticResourcePatchResolver = resolve<StaticResourceType, HookConte
}
}
)

export const staticResourceQueryResolver = resolve<StaticResourceQuery, HookContext>({})
Loading
Loading