From 33030f4566494dd0db9e67169210e9285ad911ba Mon Sep 17 00:00:00 2001 From: Josh Field Date: Mon, 24 Jun 2024 21:28:35 +1000 Subject: [PATCH] ir 2253 fix project zip archiver (#10437) * ir 2253 fix project zip archiver * console log * fix job args * remove storage arg --- .../src/schemas/media/archiver.schema.ts | 3 +- .../FileBrowser/FileBrowserContentPanel.tsx | 3 +- .../recursive-archiver/archiver.class.ts | 46 ++++++------------- .../recursive-archiver/archiver.hooks.ts | 30 ++++++++++-- .../src/projects/project/project-helper.ts | 14 ++---- .../editor/panels/Files/container/index.tsx | 5 +- scripts/archive-directory.ts | 7 ++- 7 files changed, 51 insertions(+), 57 deletions(-) diff --git a/packages/common/src/schemas/media/archiver.schema.ts b/packages/common/src/schemas/media/archiver.schema.ts index d3b6c89ddd..5c85747948 100644 --- a/packages/common/src/schemas/media/archiver.schema.ts +++ b/packages/common/src/schemas/media/archiver.schema.ts @@ -32,8 +32,7 @@ export const archiverPath = 'archiver' export const archiverMethods = ['get'] as const export const archiverQueryProperties = Type.Object({ - directory: Type.Optional(Type.String()), - storageProviderName: Type.Optional(Type.String()), + project: Type.Optional(Type.String()), isJob: Type.Optional(Type.Boolean()), jobId: Type.Optional(Type.String()) }) diff --git a/packages/editor/src/components/assets/FileBrowser/FileBrowserContentPanel.tsx b/packages/editor/src/components/assets/FileBrowser/FileBrowserContentPanel.tsx index b72e5f719d..6dc0ffd1dd 100644 --- a/packages/editor/src/components/assets/FileBrowser/FileBrowserContentPanel.tsx +++ b/packages/editor/src/components/assets/FileBrowser/FileBrowserContentPanel.tsx @@ -318,10 +318,9 @@ const FileBrowserContentPanel: React.FC = (props) const showBackButton = selectedDirectory.value !== originalPath const handleDownloadProject = async () => { - const url = selectedDirectory.value const data = await Engine.instance.api .service(archiverPath) - .get(null, { query: { directory: url } }) + .get(null, { query: { project: projectName } }) .catch((err: Error) => { NotificationService.dispatchNotify(err.message, { variant: 'warning' }) return null diff --git a/packages/server-core/src/media/recursive-archiver/archiver.class.ts b/packages/server-core/src/media/recursive-archiver/archiver.class.ts index c5ccae0c91..89c036ee15 100755 --- a/packages/server-core/src/media/recursive-archiver/archiver.class.ts +++ b/packages/server-core/src/media/recursive-archiver/archiver.class.ts @@ -48,29 +48,15 @@ const DIRECTORY_ARCHIVE_TIMEOUT = 60 * 10 //10 minutes export interface ArchiverParams extends KnexAdapterParams {} -const archive = async (app: Application, directory, params?: ArchiverParams): Promise => { - if (directory.at(0) === '/') directory = directory.slice(1) - if (!directory.startsWith('projects/') || ['projects', 'projects/'].includes(directory)) { - return Promise.reject(new Error('Cannot archive non-project directories')) - } - - const split = directory.split('/') - let projectName - if (split[split.length - 1].length === 0) projectName = split[split.length - 2] - else projectName = split[split.length - 1] - projectName = projectName.toLowerCase() - +const archive = async (app: Application, projectName: string, params?: ArchiverParams): Promise => { if (!params) params = {} if (!params.query) params.query = {} - const storageProviderName = params.query.storageProviderName?.toString() - delete params.query.storageProviderName + const storageProvider = getStorageProvider() - const storageProvider = getStorageProvider(storageProviderName) + logger.info(`Archiving ${projectName}`) - logger.info(`Archiving ${directory} using ${storageProviderName}`) - - const result = await storageProvider.listFolderContent(directory) + const result = await storageProvider.listFolderContent(`projects/${projectName}`) const zip = new JSZip() @@ -91,7 +77,7 @@ const archive = async (app: Application, directory, params?: ArchiverParams): Pr logger.info(`Added ${result[i].key} to archive`) - const dir = result[i].key.substring(result[i].key.indexOf('/') + 1) + const dir = result[i].key.replace(`projects/${projectName}/`, '') zip.file(dir, blobPromise) } @@ -107,7 +93,7 @@ const archive = async (app: Application, directory, params?: ArchiverParams): Pr ContentType: 'archive/zip' }) - logger.info(`Archived ${directory} to ${zipOutputDirectory}`) + logger.info(`Archived ${projectName} to ${zipOutputDirectory}`) if (params.query.jobId) { const date = await getDateTimeSql() @@ -131,17 +117,11 @@ export class ArchiverService implements ServiceInterface async get(id: NullableId, params?: ArchiverParams) { if (!params) throw new BadRequest('No directory specified') - const directory = params?.query?.directory!.toString() - delete params.query?.directory + const project = params?.query?.project!.toString()! + delete params.query?.project - if (!config.kubernetes.enabled || params?.query?.isJob) return archive(this.app, directory, params) + if (!config.kubernetes.enabled || params?.query?.isJob) return archive(this.app, project, params) else { - const split = directory!.split('/') - let projectName - if (split[split.length - 1].length === 0) projectName = split[split.length - 2] - else projectName = split[split.length - 1] - projectName = projectName.toLowerCase() - const date = await getDateTimeSql() const newJob = await this.app.service(apiJobPath).create({ name: '', @@ -150,11 +130,11 @@ export class ArchiverService implements ServiceInterface returnData: '', status: 'pending' }) - const jobBody = await getDirectoryArchiveJobBody(this.app, directory!, projectName, newJob.id) + const jobBody = await getDirectoryArchiveJobBody(this.app, project, newJob.id) await this.app.service(apiJobPath).patch(newJob.id, { name: jobBody.metadata!.name }) - const jobLabelSelector = `etherealengine/directoryField=${projectName},etherealengine/release=${process.env.RELEASE_NAME},etherealengine/directoryArchiver=true` + const jobLabelSelector = `etherealengine/projectField=${project},etherealengine/release=${process.env.RELEASE_NAME},etherealengine/directoryArchiver=true` const jobFinishedPromise = createExecutorJob( this.app, jobBody, @@ -166,11 +146,11 @@ export class ArchiverService implements ServiceInterface await jobFinishedPromise const job = await this.app.service(apiJobPath).get(newJob.id) - logger.info(`Archived ${directory} to ${job.returnData}`) + logger.info(`Archived ${project} to ${job.returnData}`) return job.returnData } catch (err) { - console.log('Error: Directory was not properly archived', directory, err) + console.log('Error: Directory was not properly archived', project, err) throw new BadRequest('Directory was not properly archived') } } diff --git a/packages/server-core/src/media/recursive-archiver/archiver.hooks.ts b/packages/server-core/src/media/recursive-archiver/archiver.hooks.ts index d94201342a..bcfc750f35 100755 --- a/packages/server-core/src/media/recursive-archiver/archiver.hooks.ts +++ b/packages/server-core/src/media/recursive-archiver/archiver.hooks.ts @@ -23,17 +23,41 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { disallow } from 'feathers-hooks-common' +import { disallow, discardQuery, iff, iffElse, isProvider } from 'feathers-hooks-common' import logRequest from '@etherealengine/server-core/src/hooks/log-request' +import { BadRequest } from '@feathersjs/errors' +import { HookContext } from '@feathersjs/feathers' +import checkScope from '../../hooks/check-scope' +import resolveProjectId from '../../hooks/resolve-project-id' +import setLoggedinUserInBody from '../../hooks/set-loggedin-user-in-body' +import verifyProjectPermission from '../../hooks/verify-project-permission' +import verifyScope from '../../hooks/verify-scope' +import { ArchiverService } from './archiver.class' -// Don't remove this comment. It's needed to format import lines nicely. +const ensureProject = async (context: HookContext) => { + if (context.method !== 'get') throw new BadRequest(`${context.path} service only works for data in get`) + + if (!context.params.query.project) throw new BadRequest('Project is required') +} export default { before: { all: [logRequest()], find: [disallow()], - get: [], + get: [ + ensureProject, + iff( + isProvider('external'), + iffElse( + checkScope('static_resource', 'write'), + [], + [verifyScope('editor', 'write'), resolveProjectId(), verifyProjectPermission(['owner', 'editor'])] + ) + ), + setLoggedinUserInBody('userId'), + discardQuery('projectId') + ], create: [disallow()], update: [disallow()], patch: [disallow()], diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index 11a675ed2b..1a670ab80d 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -1154,10 +1154,8 @@ export const getCronJobBody = (project: ProjectType, image: string): object => { export async function getDirectoryArchiveJobBody( app: Application, - directory: string, projectName: string, - jobId: string, - storageProviderName?: string + jobId: string ): Promise { const command = [ 'npx', @@ -1165,19 +1163,15 @@ export async function getDirectoryArchiveJobBody( 'ts-node', '--swc', 'scripts/archive-directory.ts', - `--directory`, - directory, + `--project`, + projectName, '--jobId', jobId ] - if (storageProviderName) { - command.push('--storageProviderName') - command.push(storageProviderName) - } const labels = { 'etherealengine/directoryArchiver': 'true', - 'etherealengine/directoryField': projectName, + 'etherealengine/projectField': projectName, 'etherealengine/release': process.env.RELEASE_NAME || '' } diff --git a/packages/ui/src/components/editor/panels/Files/container/index.tsx b/packages/ui/src/components/editor/panels/Files/container/index.tsx index 923589a8ac..28d8a6343a 100644 --- a/packages/ui/src/components/editor/panels/Files/container/index.tsx +++ b/packages/ui/src/components/editor/panels/Files/container/index.tsx @@ -141,7 +141,7 @@ export const useValidProjectForFileBrowser = (projectName: string) => { allowed: true } }) - return projects.data.find((project) => projectName.startsWith(`/projects/${project}/`))?.name ?? '' + return projects.data.find((project) => projectName.startsWith(`/projects/${project.name}/`))?.name ?? '' } function GeneratingThumbnailsProgress() { @@ -348,10 +348,9 @@ const FileBrowserContentPanel: React.FC = (props) const showBackButton = selectedDirectory.value.split('/').length > props.originalPath.split('/').length const handleDownloadProject = async () => { - const url = selectedDirectory.value const data = await Engine.instance.api .service(archiverPath) - .get(null, { query: { directory: url } }) + .get(null, { query: { project: projectName } }) .catch((err: Error) => { NotificationService.dispatchNotify(err.message, { variant: 'warning' }) return null diff --git a/scripts/archive-directory.ts b/scripts/archive-directory.ts index eedb7cfb09..ebe48afc48 100644 --- a/scripts/archive-directory.ts +++ b/scripts/archive-directory.ts @@ -51,8 +51,7 @@ db.url = process.env.MYSQL_URL ?? `mysql://${db.username}:${db.password}@${db.ho cli.enable('status') const options = cli.parse({ - directory: [false, 'Directory to archive', 'string'], - storageProviderName: [false, 'Storage Provider Name', 'string'], + project: [false, 'Project to archive', 'string'], jobId: [false, 'ID of Job record', 'string'] }) @@ -60,9 +59,9 @@ cli.main(async () => { try { const app = createFeathersKoaApp(ServerMode.API, serverJobPipe) await app.setup() - const { directory, jobId, storageProviderName } = options + const { project, jobId } = options await app.service(archiverPath).get(null, { - query: { storageProviderName: storageProviderName || undefined, isJob: true, directory, jobId } + query: { isJob: true, project, jobId } }) cli.exit(0) } catch (err) {