Merge commit '52043c03fd924f33c88e7e4771826e8e2ff435e5' into develop

# Conflicts: # apps/package-manager/packages/generic/package.json # apps/package-manager/packages/generic/src/packageManager.ts # shared/packages/expectationManager/src/evaluationRunner/evaluationRunner.ts # shared/packages/worker/package.json # shared/packages/worker/src/worker/accessorHandlers/fileShare.ts # shared/packages/worker/src/worker/accessorHandlers/lib/FileHandler.ts # shared/packages/worker/src/worker/accessorHandlers/localFolder.ts # tests/internal-tests/src/__tests__/issues.spec.ts # tests/internal-tests/src/__tests__/lib/lib.ts # yarn.lock
nrkno · Oct 12, 2023 · 39ced7f · 39ced7f
2 parents e213068 + 52043c0
commit 39ced7f
Show file tree

Hide file tree

Showing 26 changed files with 600 additions and 192 deletions.
diff --git a/apps/appcontainer-node/packages/generic/src/appContainer.ts b/apps/appcontainer-node/packages/generic/src/appContainer.ts
@@ -222,7 +222,7 @@ export class AppContainer {
 			this.initWorkForceApiPromise = { resolve, reject }
 		})
 
-		this.logger.info(`Initialized"`)
+		this.logger.info(`Initialized`)
 	}
 	/** Return the API-methods that the AppContainer exposes to the WorkerAgent */
 	private getWorkerAgentAPI(clientId: WorkerAgentId): AppContainerWorkerAgent.AppContainer {

diff --git a/apps/package-manager/packages/generic/package.json b/apps/package-manager/packages/generic/package.json
@@ -14,10 +14,10 @@
         "@sofie-automation/shared-lib": "*"
     },
     "dependencies": {
+        "@parcel/watcher": "^2.3.0",
         "@sofie-package-manager/api": "1.50.0-alpha.7",
         "@sofie-package-manager/expectation-manager": "1.50.0-alpha.7",
         "@sofie-package-manager/worker": "1.50.0-alpha.7",
-        "chokidar": "^3.5.1",
         "data-store": "^4.0.3",
         "deep-extend": "^0.6.0",
         "fast-clone": "^1.5.13",

diff --git a/apps/package-manager/packages/generic/src/connector.ts b/apps/package-manager/packages/generic/src/connector.ts
@@ -17,7 +17,6 @@ import {
 import { ExpectationManager, ExpectationManagerServerOptions } from '@sofie-package-manager/expectation-manager'
 import { CoreHandler, CoreConfig } from './coreHandler'
 import { PackageContainers, PackageManagerHandler } from './packageManager'
-import chokidar from 'chokidar'
 import fs from 'fs'
 import { promisify } from 'util'
 import path from 'path'
@@ -211,31 +210,22 @@ export class Connector {
 				'utf-8'
 			)
 		}
-
-		const watcher = chokidar.watch(fileName, { persistent: true })
-
-		this.logger.info(`Watching file "${fileName}"`)
-
-		watcher
-			.on('add', () => {
-				triggerReloadInput()
-			})
-			.on('change', () => {
-				triggerReloadInput()
-			})
-			.on('unlink', () => {
-				triggerReloadInput()
-			})
-			.on('error', (error) => {
-				this.logger.error(`Error emitter in Filewatcher: ${stringifyError(error)}`)
-			})
 		const triggerReloadInput = () => {
 			setTimeout(() => {
 				reloadInput().catch((error) => {
 					this.logger.error(`Error in reloadInput: ${stringifyError(error)}`)
 				})
 			}, 100)
 		}
+
+		this.logger.info(`Watching file "${fileName}"`)
+		fs.watchFile(fileName, { persistent: true }, (currStats, prevStats) => {
+			if (currStats.mtimeMs !== prevStats.mtimeMs) {
+				triggerReloadInput()
+			}
+		})
+		triggerReloadInput()
+
 		const reloadInput = async () => {
 			this.logger.info(`Change detected in ${fileName}`)
 			// Check that the file exists:

diff --git a/...age-manager/packages/generic/src/generateExpectations/nrk/packageContainerExpectations.ts b/...age-manager/packages/generic/src/generateExpectations/nrk/packageContainerExpectations.ts
@@ -31,7 +31,7 @@ export function getPackageContainerExpectations(
 					packages: {
 						label: 'Monitor Packages on source',
 						targetLayers: ['target0'],
-						ignore: '.bat',
+						ignore: ['.bat'],
 						// ignore: '',
 					},
 				},
@@ -50,9 +50,6 @@ export function getPackageContainerExpectations(
 					packages: {
 						label: 'Monitor for Smartbull',
 						targetLayers: ['source-smartbull'], // not used, since the layers of the original smartbull-package are used
-						usePolling: 2000,
-						awaitWriteFinishStabilityThreshold: 2000,
-						warningLimit: 3000, // We seem to get performance issues at around 9000 (when polling network drives), so 3000 should give us an early warning
 					},
 				},
 			}

diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts
@@ -228,18 +228,28 @@ export class PackageManagerHandler {
 		this._triggerUpdatedExpectedPackagesTimeout = setTimeout(() => {
 			this._triggerUpdatedExpectedPackagesTimeout = null
 
-			const expectedPackages: ExpectedPackageWrap[] = []
 			const packageContainers: PackageContainers = {}
+			const expectedPackageSources: {
+				sourceName: string
+				expectedPackages: ExpectedPackageWrap[]
+			}[] = []
 
 			let activePlaylist: PackageManagerActivePlaylist | null = null
 			let activeRundowns: PackageManagerActiveRundown[] = []
 
 			// Add from external data:
 			{
+				const expectedPackagesExternal: ExpectedPackageWrap[] = []
 				for (const expectedPackage of this.externalData.expectedPackages) {
-					expectedPackages.push(expectedPackage)
+					expectedPackagesExternal.push(expectedPackage)
 				}
 				Object.assign(packageContainers, this.externalData.packageContainers)
+				if (expectedPackagesExternal.length > 0) {
+					expectedPackageSources.push({
+						sourceName: 'external',
+						expectedPackages: expectedPackagesExternal,
+					})
+				}
 			}
 
 			if (!this.coreHandler.notUsingCore) {
@@ -268,21 +278,24 @@ export class PackageManagerHandler {
 				const expectedPackagesObjs = this.coreHandler
 					.getCollection<PackageManagerExpectedPackage>('packageManagerExpectedPackages')
 					.find()
+
+				const expectedPackagesCore: ExpectedPackageWrap[] = []
 				for (const expectedPackagesObj of expectedPackagesObjs) {
-					expectedPackages.push(expectedPackagesObj as any as ExpectedPackageWrap)
+					expectedPackagesCore.push(expectedPackagesObj as any as ExpectedPackageWrap)
 				}
 			}
 
 			// Add from Monitors:
 			{
-				for (const monitorExpectedPackages of this.monitoredPackages.values()) {
-					for (const expectedPackage of monitorExpectedPackages) {
-						expectedPackages.push(expectedPackage)
-					}
+				for (const [monitorId, monitorExpectedPackages] of this.monitoredPackages.entries()) {
+					expectedPackageSources.push({
+						sourceName: `monitor_${monitorId}`,
+						expectedPackages: monitorExpectedPackages,
+					})
 				}
 			}
 
-			this.handleExpectedPackages(packageContainers, activePlaylist, activeRundowns, expectedPackages)
+			this.handleExpectedPackages(packageContainers, activePlaylist, activeRundowns, expectedPackageSources)
 		}, 300)
 	}
 
@@ -291,8 +304,18 @@ export class PackageManagerHandler {
 		activePlaylist: PackageManagerActivePlaylist | null,
 		activeRundowns: PackageManagerActiveRundown[],
 
-		expectedPackages: ExpectedPackageWrap[]
+		expectedPackageSources: {
+			sourceName: string
+			expectedPackages: ExpectedPackageWrap[]
+		}[]
 	) {
+		const expectedPackages: ExpectedPackageWrap[] = []
+		for (const expectedPackageSource of expectedPackageSources) {
+			for (const exp of expectedPackageSource.expectedPackages) {
+				expectedPackages.push(exp)
+			}
+		}
+
 		// Step 0: Save local cache:
 		this.expectedPackageCache = new Map()
 		this.packageContainersCache = packageContainers
@@ -310,7 +333,12 @@ export class PackageManagerHandler {
 			}
 		}
 
-		this.logger.debug(`Has ${expectedPackages.length} expectedPackages`)
+		this.logger.debug(
+			`Has ${expectedPackages.length} expectedPackages (${expectedPackageSources
+				.map((s) => `${s.sourceName}: ${s.expectedPackages.length}`)
+				.join(', ')})`
+		)
+		this.logger.silly(JSON.stringify(expectedPackages, null, 2))
 		// this.logger.debug(JSON.stringify(expectedPackages, null, 2))
 
 		this.dataSnapshot.expectedPackages = expectedPackages
@@ -327,15 +355,19 @@ export class PackageManagerHandler {
 			this.settings
 		)
 		this.logger.debug(`Has ${objectSize(expectations)} expectations`)
+		this.logger.silly(JSON.stringify(expectations, null, 2))
 		// this.logger.debug(JSON.stringify(expectations, null, 2))
 		this.dataSnapshot.expectations = expectations
 
+		this.logger.debug(`Has ${Object.keys(this.packageContainersCache).length} packageContainers`)
+		this.logger.silly(JSON.stringify(this.packageContainersCache, null, 2))
 		const packageContainerExpectations = this.expectationGeneratorApi.getPackageContainerExpectations(
 			this.expectationManager.managerId,
 			this.packageContainersCache,
 			activePlaylist
 		)
 		this.logger.debug(`Has ${objectSize(packageContainerExpectations)} packageContainerExpectations`)
+		this.logger.silly(JSON.stringify(packageContainerExpectations, null, 2))
 		this.dataSnapshot.packageContainerExpectations = packageContainerExpectations
 		this.dataSnapshot.updated = Date.now()
 

diff --git a/apps/single-app/app/src/singleApp.ts b/apps/single-app/app/src/singleApp.ts
@@ -3,13 +3,26 @@ import * as QuantelHTTPTransformerProxy from '@quantel-http-transformer-proxy/ge
 import * as PackageManager from '@package-manager/generic'
 import * as Workforce from '@sofie-package-manager/workforce'
 import * as AppConatainerNode from '@appcontainer-node/generic'
-import { getSingleAppConfig, ProcessHandler, setupLogger, initializeLogger } from '@sofie-package-manager/api'
+import {
+	getSingleAppConfig,
+	ProcessHandler,
+	setupLogger,
+	initializeLogger,
+	setLogLevel,
+	isLogLevel,
+} from '@sofie-package-manager/api'
 
 export async function startSingleApp(): Promise<void> {
 	const config = await getSingleAppConfig()
 	initializeLogger(config)
 	const logger = setupLogger(config, 'single-app')
 	const baseLogger = setupLogger(config, '')
+
+	const logLevel = config.process.logLevel
+	if (logLevel && isLogLevel(logLevel)) {
+		logger.info(`Setting log level to ${logLevel}`)
+		setLogLevel(logLevel)
+	}
 	// Override some of the arguments, as they arent used in the single-app
 	config.packageManager.port = 0 // 0 = Set the packageManager port to whatever is available
 	config.packageManager.accessUrl = 'ws:127.0.0.1'

diff --git a/shared/packages/api/src/HelpfulEventEmitter.ts b/shared/packages/api/src/HelpfulEventEmitter.ts
@@ -3,24 +3,44 @@ import EventEmitter from 'events'
 
 /** An EventEmitter which does a check that you've remembered to listen to the 'error' event */
 export class HelpfulEventEmitter extends EventEmitter {
+	private _listenersToCheck: string[] = ['error']
+
 	constructor() {
 		super()
 		// Ensure that the error event is listened for:
 
 		const orgError = new Error('No error event listener registered')
 		setTimeout(() => {
-			if (!this.listenerCount('error')) {
-				// If no error event listener is registered, log a warning to let the developer
-				// know that they should do so:
-				console.error('WARNING: No error event listener registered')
-				console.error(`Stack: ${orgError.stack}`)
+			for (const event of this._listenersToCheck) {
+				if (!this.listenerCount(event)) {
+					// If no event listener is registered, log a warning to let the developer
+					// know that they should do so:
+					console.error(`WARNING: No "${event}" event listener registered`)
+					console.error(`Stack: ${orgError.stack}`)
 
-				// If we're running in Jest, it's better to make it a little more obvious that something is wrong:
-				if (process.env.JEST_WORKER_ID !== undefined) {
-					// Since no error listener is registered, this'll cause the process to exit and tests to fail:
-					this.emit('error', orgError)
+					// If we're running in Jest, it's better to make it a little more obvious that something is wrong:
+					if (process.env.JEST_WORKER_ID !== undefined) {
+						// Since no error listener is registered, this'll cause the process to exit and tests to fail:
+						this.emit('error', orgError)
+					}
 				}
 			}
+
+			// If we're running in Jest, it's better to make it a little more obvious that something is wrong:
+			if (process.env.JEST_WORKER_ID !== undefined && !this.listenerCount('error')) {
+				// Since no error listener is registered, this'll cause the process to exit and tests to fail:
+				this.emit('error', orgError)
+			}
 		}, 1)
 	}
+
+	/**
+	 * To be called in constructor.
+	 * Add an event that the HelpfulEventEmitter should check that it is being listened to
+	 */
+	protected addHelpfulEventCheck(event: string): void {
+		if (this._listenersToCheck.includes(event)) throw new Error(`Event "${event}" already added`)
+
+		this._listenersToCheck.push(event)
+	}
 }
diff --git a/shared/packages/api/src/config.ts b/shared/packages/api/src/config.ts
@@ -13,6 +13,7 @@ import { AppContainerId, WorkerAgentId } from './ids'
 /** Generic CLI-argument-definitions for any process */
 const processOptions = defineArguments({
 	logPath: { type: 'string', describe: 'Set to write logs to this file' },
+	logLevel: { type: 'string', describe: 'Set default log level. (Might be overwritten by Sofie Core)' },
 
 	unsafeSSL: {
 		type: 'boolean',
@@ -276,15 +277,22 @@ const quantelHTTPTransformerProxyConfigArguments = defineArguments({
 
 export interface ProcessConfig {
 	logPath: string | undefined
-	/** Will cause the Node applocation to blindly accept all certificates. Not recommenced unless in local, controlled networks. */
+	logLevel: string | undefined
+	/** Will cause the Node app to blindly accept all certificates. Not recommenced unless in local, controlled networks. */
 	unsafeSSL: boolean
 	/** Paths to certificates to load, for SSL-connections */
 	certificates: string[]
 }
-function getProcessConfig(argv: { logPath: string | undefined; unsafeSSL: boolean; certificates: string | undefined }) {
+function getProcessConfig(argv: {
+	logPath: string | undefined
+	logLevel: string | undefined
+	unsafeSSL: boolean
+	certificates: string | undefined
+}) {
 	const certs: string[] = (argv.certificates || process.env.CERTIFICATES || '').split(';') || []
 	return {
 		logPath: argv.logPath,
+		logLevel: argv.logLevel,
 		unsafeSSL: argv.unsafeSSL,
 		certificates: _.compact(certs),
 	}

diff --git a/shared/packages/api/src/logger.ts b/shared/packages/api/src/logger.ts
@@ -20,6 +20,9 @@ export enum LogLevel {
 	DEBUG = 'debug',
 	SILLY = 'silly',
 }
+export function isLogLevel(logLevel: string): logLevel is LogLevel {
+	return ['error', 'warn', 'info', 'verbose', 'debug', 'silly'].includes(logLevel)
+}
 
 export const DEFAULT_LOG_LEVEL = LogLevel.VERBOSE
 

diff --git a/shared/packages/api/src/packageContainerApi.ts b/shared/packages/api/src/packageContainerApi.ts
@@ -26,12 +26,12 @@ export interface PackageContainerExpectation extends PackageContainer {
 		/** Monitor the packages of a PackageContainer */
 		packages?: {
 			label: string
-			/** If set, ignore any files matching this. (Regular expression). */
-			ignore?: string
+			/** If set, ignore any files matching any of the patterns (Glob pattern). */
+			ignore?: string[]
 
 			/** If set, the monitoring will be using polling, at the given interval [ms] */
-			usePolling?: number | null
-			/** If set, will set the awaitWriteFinish.StabilityThreshold of chokidar */
+			// usePolling?: number | null
+			/** If set, will wait for the file being unchanged for the specified duration before considering it [ms] */
 			awaitWriteFinishStabilityThreshold?: number | null
 			/** If set, the monitor will warn if the monitored number of packages is greater than this */
 			warningLimit?: number

diff --git a/shared/packages/expectationManager/src/evaluationRunner/evaluationRunner.ts b/shared/packages/expectationManager/src/evaluationRunner/evaluationRunner.ts
@@ -638,6 +638,10 @@ export class EvaluationRunner {
 							if (monitorSetup.success) {
 								trackedPackageContainer.monitorIsSetup = true
 								for (const [monitorId, monitor] of objectEntries(monitorSetup.monitors)) {
+									this.logger.debug(
+										`Set up monitor "${monitor.label}" (${monitorId}) for PackageContainer ${trackedPackageContainer.id}`
+									)
+
 									if (trackedPackageContainer.status.monitors[monitorId]) {
 										// In case there no monitor status has been emitted yet:
 										this.tracker.trackedPackageContainerAPI.updateTrackedPackageContainerMonitorStatus(

diff --git a/shared/packages/worker/package.json b/shared/packages/worker/package.json
@@ -21,10 +21,10 @@
         "@types/tmp": "~0.2.2"
     },
     "dependencies": {
+        "@parcel/watcher": "^2.3.0",
         "@sofie-package-manager/api": "1.50.0-alpha.7",
         "abort-controller": "^3.0.0",
         "atem-connection": "^3.2.0",
-        "chokidar": "^3.5.1",
         "deep-diff": "^1.0.2",
         "form-data": "^4.0.0",
         "mkdirp": "^1.0.4",

diff --git a/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts b/shared/packages/worker/src/worker/accessorHandlers/fileShare.ts
@@ -362,8 +362,9 @@ export class FileShareAccessorHandle<Metadata> extends GenericFileAccessorHandle
 		for (const monitorIdStr of monitorIds) {
 			if (monitorIdStr === 'packages') {
 				// setup file monitor:
-				resultingMonitors[protectString<MonitorId>(monitorIdStr)] =
-					this.setupPackagesMonitor(packageContainerExp)
+				resultingMonitors[protectString<MonitorId>(monitorIdStr)] = await this.setupPackagesMonitor(
+					packageContainerExp
+				)
 			} else {
 				// Assert that cronjob is of type "never", to ensure that all types of monitors are handled:
 				assertNever(monitorIdStr)