From 10e574757bcdd86a2a5aa9dade2ed34b5d0984f9 Mon Sep 17 00:00:00 2001 From: Alex Van Camp Date: Wed, 5 Jul 2023 09:58:20 -0500 Subject: [PATCH 1/3] feat: generate audio waveform images for audio-only files (and don't try to generate previews for them) --- .../windowsWorker/expectationHandlers/lib.ts | 13 ++++++++--- .../expectationHandlers/mediaFilePreview.ts | 22 +++++++++++++++++++ .../expectationHandlers/mediaFileThumbnail.ts | 13 ++++++++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts index cf8e9ae0..328adbfd 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts @@ -338,14 +338,21 @@ interface ThumbnailMetadata { } } /** Returns arguments for FFMpeg to generate a thumbnail image file */ -export function thumbnailFFMpegArguments(input: string, metadata: ThumbnailMetadata, seekTimeCode?: string): string[] { +export function thumbnailFFMpegArguments( + input: string, + metadata: ThumbnailMetadata, + seekTimeCode?: string, + hasVideoStream?: boolean +): string[] { return [ '-hide_banner', - seekTimeCode ? `-ss ${seekTimeCode}` : undefined, + hasVideoStream && seekTimeCode ? `-ss ${seekTimeCode}` : undefined, `-i "${input}"`, `-f image2`, '-frames:v 1', - `-vf ${!seekTimeCode ? 'thumbnail,' : ''}scale=${metadata.version.width}:${metadata.version.height}`, + hasVideoStream + ? `-vf ${!seekTimeCode ? 'thumbnail,' : ''}scale=${metadata.version.width}:${metadata.version.height}` // Creates a thumbnail of the video. + : '-filter_complex "showwavespic=s=640x240:split_channels=1"', // Creates an image of the audio waveform. '-threads 1', ].filter(Boolean) as string[] // remove undefined values } diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts index a7b27c84..353b4f49 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts @@ -28,6 +28,8 @@ import { } from './lib' import { FFMpegProcess, spawnFFMpeg } from './lib/ffmpeg' import { WindowsWorker } from '../windowsWorker' +import { scanWithFFProbe, FFProbeScanResult } from './lib/scan' +import { CancelablePromise } from '../../../lib/cancelablePromise' /** * Generates a low-res preview video of a source video file, and stores the resulting file into the target PackageContainer @@ -178,9 +180,11 @@ export const MediaFilePreview: ExpectationWindowsHandler = { throw new Error(`Target AccessHandler type is wrong`) let ffMpegProcess: FFMpegProcess | undefined + let ffProbeProcess: CancelablePromise | undefined const workInProgress = new WorkInProgress({ workLabel: 'Generating preview' }, async () => { // On cancel ffMpegProcess?.cancel() + ffProbeProcess?.cancel() }).do(async () => { const tryReadPackage = await sourceHandle.checkPackageReadAccess() if (!tryReadPackage.success) throw new Error(tryReadPackage.reason.tech) @@ -220,6 +224,24 @@ export const MediaFilePreview: ExpectationWindowsHandler = { throw new Error(`Unsupported Target AccessHandler`) } + // Scan with FFProbe: + ffProbeProcess = scanWithFFProbe(sourceHandle) + const ffProbeScan: FFProbeScanResult = await ffProbeProcess + ffProbeProcess = undefined + const hasVideoStream = + ffProbeScan.streams && ffProbeScan.streams.some((stream) => stream.codec_type === 'video') + if (!hasVideoStream) { + workInProgress._reportComplete( + actualSourceVersionHash, + { + user: `Preview generation skipped due to file having no video streams`, + tech: `Completed at ${Date.now()}`, + }, + undefined + ) + return + } + const args = previewFFMpegArguments(inputPath, true, metadata) const fileOperation = await targetHandle.prepareForOperation('Generate preview', lookupSource.handle) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts index ea6f9344..2a5020e4 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFileThumbnail.ts @@ -29,6 +29,8 @@ import { } from './lib' import { FFMpegProcess, spawnFFMpeg } from './lib/ffmpeg' import { WindowsWorker } from '../windowsWorker' +import { CancelablePromise } from '../../../lib/cancelablePromise' +import { scanWithFFProbe, FFProbeScanResult } from './lib/scan' /** * Generates a thumbnail image from a source video file, and stores the resulting file into the target PackageContainer @@ -148,9 +150,11 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { if (!lookupTarget.ready) throw new Error(`Can't start working due to target: ${lookupTarget.reason.tech}`) let ffMpegProcess: FFMpegProcess | undefined + let ffProbeProcess: CancelablePromise | undefined const workInProgress = new WorkInProgress({ workLabel: 'Generating thumbnail' }, async () => { // On cancel ffMpegProcess?.cancel() + ffProbeProcess?.cancel() }).do(async () => { if ( (lookupSource.accessor.type === Accessor.AccessType.LOCAL_FOLDER || @@ -220,8 +224,15 @@ export const MediaFileThumbnail: ExpectationWindowsHandler = { throw new Error(`Unsupported Target AccessHandler`) } + // Scan with FFProbe: + ffProbeProcess = scanWithFFProbe(sourceHandle) + const ffProbeScan: FFProbeScanResult = await ffProbeProcess + ffProbeProcess = undefined + const hasVideoStream = + ffProbeScan.streams && ffProbeScan.streams.some((stream) => stream.codec_type === 'video') + // Use FFMpeg to generate the thumbnail: - const args = thumbnailFFMpegArguments(inputPath, metadata, seekTimeCode) + const args = thumbnailFFMpegArguments(inputPath, metadata, seekTimeCode, hasVideoStream) const fileOperation = await targetHandle.prepareForOperation('Generate thumbnail', lookupSource.handle) From eec1da8ca5b03bc425f4a889adca15751276ae06 Mon Sep 17 00:00:00 2001 From: Alex Van Camp Date: Wed, 5 Jul 2023 10:35:06 -0500 Subject: [PATCH 2/3] chore: revert bug fix and focus only on feature --- .../expectationHandlers/mediaFilePreview.ts | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts index 353b4f49..a7b27c84 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/mediaFilePreview.ts @@ -28,8 +28,6 @@ import { } from './lib' import { FFMpegProcess, spawnFFMpeg } from './lib/ffmpeg' import { WindowsWorker } from '../windowsWorker' -import { scanWithFFProbe, FFProbeScanResult } from './lib/scan' -import { CancelablePromise } from '../../../lib/cancelablePromise' /** * Generates a low-res preview video of a source video file, and stores the resulting file into the target PackageContainer @@ -180,11 +178,9 @@ export const MediaFilePreview: ExpectationWindowsHandler = { throw new Error(`Target AccessHandler type is wrong`) let ffMpegProcess: FFMpegProcess | undefined - let ffProbeProcess: CancelablePromise | undefined const workInProgress = new WorkInProgress({ workLabel: 'Generating preview' }, async () => { // On cancel ffMpegProcess?.cancel() - ffProbeProcess?.cancel() }).do(async () => { const tryReadPackage = await sourceHandle.checkPackageReadAccess() if (!tryReadPackage.success) throw new Error(tryReadPackage.reason.tech) @@ -224,24 +220,6 @@ export const MediaFilePreview: ExpectationWindowsHandler = { throw new Error(`Unsupported Target AccessHandler`) } - // Scan with FFProbe: - ffProbeProcess = scanWithFFProbe(sourceHandle) - const ffProbeScan: FFProbeScanResult = await ffProbeProcess - ffProbeProcess = undefined - const hasVideoStream = - ffProbeScan.streams && ffProbeScan.streams.some((stream) => stream.codec_type === 'video') - if (!hasVideoStream) { - workInProgress._reportComplete( - actualSourceVersionHash, - { - user: `Preview generation skipped due to file having no video streams`, - tech: `Completed at ${Date.now()}`, - }, - undefined - ) - return - } - const args = previewFFMpegArguments(inputPath, true, metadata) const fileOperation = await targetHandle.prepareForOperation('Generate preview', lookupSource.handle) From 3f49920c21ace68bfe20c03caaf63b0e06534712 Mon Sep 17 00:00:00 2001 From: Alex Van Camp Date: Wed, 5 Jul 2023 15:35:26 -0500 Subject: [PATCH 3/3] chore: make the waveform image black and white --- .../src/worker/workers/windowsWorker/expectationHandlers/lib.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts index 328adbfd..7faf4c74 100644 --- a/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts +++ b/shared/packages/worker/src/worker/workers/windowsWorker/expectationHandlers/lib.ts @@ -352,7 +352,7 @@ export function thumbnailFFMpegArguments( '-frames:v 1', hasVideoStream ? `-vf ${!seekTimeCode ? 'thumbnail,' : ''}scale=${metadata.version.width}:${metadata.version.height}` // Creates a thumbnail of the video. - : '-filter_complex "showwavespic=s=640x240:split_channels=1"', // Creates an image of the audio waveform. + : '-filter_complex "showwavespic=s=640x240:split_channels=1:colors=white"', // Creates an image of the audio waveform. '-threads 1', ].filter(Boolean) as string[] // remove undefined values }