From a5c35f25e13c1916a1c963734e644f859a3cb453 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 11 Oct 2023 12:44:58 +0200 Subject: [PATCH 1/7] chore: refactor build scripts --- scripts/build-win32.mjs | 13 ++++++--- scripts/prepare-for-build32.js | 50 ++++++++++++++++++---------------- 2 files changed, 36 insertions(+), 27 deletions(-) diff --git a/scripts/build-win32.mjs b/scripts/build-win32.mjs index 7977f57c..444960b9 100644 --- a/scripts/build-win32.mjs +++ b/scripts/build-win32.mjs @@ -93,18 +93,23 @@ if (!executableName) { const extraArgs = [] + const assets = [] + if (packageJson.name === '@single-app/app') { + assets.push(path.join(basePath, './node_modules/@sofie-automation/server-core-integration/package.json')) + assets.push(path.join(basePath, './package.json')) + } + + if (assets.length > 0) { extraArgs.push( '--assets', - [ - path.join(basePath, './node_modules/@sofie-automation/server-core-integration/package.json'), - path.join(basePath, './package.json'), - ].join(',') + assets.join(',') ) } await pkg.exec([ path.join(basePath, './dist/index.js'), + // '--debug', '--targets', 'node16-win-x64', '--output', diff --git a/scripts/prepare-for-build32.js b/scripts/prepare-for-build32.js index a28e07f5..5d5feca1 100644 --- a/scripts/prepare-for-build32.js +++ b/scripts/prepare-for-build32.js @@ -2,6 +2,7 @@ const promisify = require('util').promisify const cp = require('child_process') const path = require('path') +const os = require('os') // const nexe = require('nexe') const exec = promisify(cp.exec) @@ -23,37 +24,40 @@ const packageJson = require(path.join(basePath, '/package.json')) // } ;(async () => { - log(`Collecting dependencies for ${packageJson.name}...`) - // List all Lerna packages: - const list = await exec('yarn lerna list -a --json') - const str = list.stdout - .replace(/^yarn run .*$/gm, '') - .replace(/^\$.*$/gm, '') - .replace(/^Done in.*$/gm, '') + // Collecting dependencies + { + log(`Collecting dependencies for ${packageJson.name}...`) + // List all Lerna packages: + const list = await exec('yarn lerna list -a --json') + const str = list.stdout + .replace(/^yarn run .*$/gm, '') + .replace(/^\$.*$/gm, '') + .replace(/^Done in.*$/gm, '') - const packages = JSON.parse(str) + const packages = JSON.parse(str) - await mkdirp(path.join(basePath, 'node_modules')) + await mkdirp(path.join(basePath, 'node_modules')) - // Copy the packages into node_modules: - const copiedFolders = [] - let ps = [] - for (const package0 of packages) { - if (package0.name.match(/boilerplate/)) continue - if (package0.name.match(packageJson.name)) continue + // Copy the packages into node_modules: - const target = path.resolve(path.join(basePath, 'tmp_packages_for_build', package0.name)) - log(` Copying: ${package0.name} to ${target}`) + const copiedFolders = [] + let ps = [] + for (const package0 of packages) { + if (package0.name.match(/boilerplate/)) continue + if (package0.name.match(packageJson.name)) continue - // log(` ${package0.location} -> ${target}`) - ps.push(fseCopy(package0.location, target)) + const target = path.resolve(path.join(basePath, 'tmp_packages_for_build', package0.name)) + log(` Copying: ${package0.name} to ${target}`) - copiedFolders.push(target) - } + // log(` ${package0.location} -> ${target}`) + ps.push(fseCopy(package0.location, target)) - await Promise.all(ps) - ps = [] + copiedFolders.push(target) + } + await Promise.all(ps) + ps = [] + } log(`...done!`) })().catch(log) From 3f1fd7da092dee2d217f2e8c879a74c49e85441e Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Wed, 11 Oct 2023 15:04:43 +0200 Subject: [PATCH 2/7] fix: hack to make pkg include @parcel/watcher native dependencies --- scripts/cleanup-after-build32.mjs | 5 ++++- scripts/prepare-for-build32.js | 21 +++++++++++++++++++++ scripts/reset.js | 2 +- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/cleanup-after-build32.mjs b/scripts/cleanup-after-build32.mjs index 17102f5e..a2c3fadc 100644 --- a/scripts/cleanup-after-build32.mjs +++ b/scripts/cleanup-after-build32.mjs @@ -7,7 +7,7 @@ import { promisify } from 'util' const glob = promisify(glob0) -const basePath = process.cwd() +const basePath = '.' console.log(`Cleaning up...`) @@ -21,7 +21,10 @@ console.log(`Remove unused files...`) const copiedFiles = [ ...(await glob(`${basePath}/apps/*/app/node_modules/@*/app/*`)), ...(await glob(`${basePath}/apps/*/app/node_modules/@*/generic/*`)), + ...(await glob(`${basePath}/node_modules/@parcel/watcher/build/**/*`)), ] + +console.log('copiedFiles', copiedFiles) for (const file of copiedFiles) { console.log(`Removing file: "${file}"`) ps.push(fse.rm(file, { recursive: true })) diff --git a/scripts/prepare-for-build32.js b/scripts/prepare-for-build32.js index 5d5feca1..0f030a1c 100644 --- a/scripts/prepare-for-build32.js +++ b/scripts/prepare-for-build32.js @@ -58,6 +58,27 @@ const packageJson = require(path.join(basePath, '/package.json')) await Promise.all(ps) ps = [] } + + // Hack to make pkg include the native dependency @parcel/watcher: + { + log(`Hacking @parcel/watcher...`) + + /* + This hack exploits the line @parcel/watcher/index.js:21 : + binding = require('./build/Release/watcher.node'); + By copying the native module into that location, pkg will include it in the build. + */ + const arch = os.arch() + const platform = os.platform() + const prebuildType = process.argv[2] || `${platform}-${arch}` + + const source = path.join(basePath, `node_modules/@parcel/watcher-${prebuildType}`) // @parcel/watcher-win32-x64 + const target = path.join(basePath, 'node_modules/@parcel/watcher/build/Release') + + log(` Copying: ${source} to ${target}`) + await fse.copy(source, target) + } + log(`...done!`) })().catch(log) diff --git a/scripts/reset.js b/scripts/reset.js index 7c0a4cc6..5ff0d0df 100644 --- a/scripts/reset.js +++ b/scripts/reset.js @@ -7,7 +7,7 @@ const rimraf = promisify(require('rimraf')) Removing all /node_modules and /dist folders */ -const basePath = process.cwd() +const basePath = '.' ;(async () => { log('Gathering files to remove...') From 72f371c84aec555dd72de2723043d888e3be00b2 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Fri, 6 Oct 2023 07:23:12 +0200 Subject: [PATCH 3/7] chore: doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 689440a8..234455f1 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ Other useful commands: yarn start:single-app # Start the single-app in local-only mode, using packages from expectedPackages.json -yarn start:single-app -- -- --watchFiles=true --noCore=true +yarn start:single-app -- -- --watchFiles=true --noCore=true --logLevel=debug From 07443a774b71112020ca0bc31e8f9bf3a7d41c94 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 12 Oct 2023 08:27:08 +0200 Subject: [PATCH 4/7] chore: refactor: rename --- .../src/internalManager/internalManager.ts | 8 +++--- ...tusMonitor.ts => managerStatusWatchdog.ts} | 26 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) rename shared/packages/expectationManager/src/internalManager/lib/{managerStatusMonitor.ts => managerStatusWatchdog.ts} (70%) diff --git a/shared/packages/expectationManager/src/internalManager/internalManager.ts b/shared/packages/expectationManager/src/internalManager/internalManager.ts index f6ed20b9..6f78d911 100644 --- a/shared/packages/expectationManager/src/internalManager/internalManager.ts +++ b/shared/packages/expectationManager/src/internalManager/internalManager.ts @@ -13,7 +13,7 @@ import { WorkerAgentAPI } from '../workerAgentApi' import { getDefaultConstants } from '../lib/constants' import { ExpectationTracker } from '../expectationTracker/expectationTracker' import { TrackedWorkerAgents } from './lib/trackedWorkerAgents' -import { ManagerStatusMonitor } from './lib/managerStatusMonitor' +import { ManagerStatusWatchdog } from './lib/managerStatusWatchdog' import { ExpectationManagerCallbacks, ExpectationManagerOptions, @@ -38,7 +38,7 @@ export class InternalManager { public statuses: ManagerStatusReporter private enableChaosMonkey = false - private managerMonitor: ManagerStatusMonitor + private managerWatchdog: ManagerStatusWatchdog public statusReport: StatusReportCache @@ -74,7 +74,7 @@ export class InternalManager { this.tracker.on('error', (err) => this.logger.error(`ExpectationTracker error" ${stringifyError(err)}`)) this.workerAgents = new TrackedWorkerAgents(this.logger, this.tracker) this.statuses = new ManagerStatusReporter(this.callbacks) - this.managerMonitor = new ManagerStatusMonitor(this.logger, this.tracker, this.statuses) + this.managerWatchdog = new ManagerStatusWatchdog(this.logger, this.tracker, this.statuses) this.statusReport = new StatusReportCache(this) this.enableChaosMonkey = options?.chaosMonkey ?? false @@ -102,7 +102,7 @@ export class InternalManager { this.tracker.terminate() this.expectationManagerServer.terminate() this.workforceConnection.terminate() - this.managerMonitor.terminate() + this.managerWatchdog.terminate() } /** USED IN TESTS ONLY. Quickly reset the tracked work of the expectationManager. */ resetWork(): void { diff --git a/shared/packages/expectationManager/src/internalManager/lib/managerStatusMonitor.ts b/shared/packages/expectationManager/src/internalManager/lib/managerStatusWatchdog.ts similarity index 70% rename from shared/packages/expectationManager/src/internalManager/lib/managerStatusMonitor.ts rename to shared/packages/expectationManager/src/internalManager/lib/managerStatusWatchdog.ts index 647b7a05..5e80f345 100644 --- a/shared/packages/expectationManager/src/internalManager/lib/managerStatusMonitor.ts +++ b/shared/packages/expectationManager/src/internalManager/lib/managerStatusWatchdog.ts @@ -3,11 +3,11 @@ import { ExpectationTracker } from '../../expectationTracker/expectationTracker' import { ManagerStatusReporter } from './managerStatusReporter' /** Monitors the status of the ExpectationTracker and alerts if there's a problem */ -export class ManagerStatusMonitor { - private statusMonitorInterval: NodeJS.Timeout | null = null +export class ManagerStatusWatchdog { + private checkStatusInterval: NodeJS.Timeout | null = null /** Timestamp, used to determine how long the work-queue has been stuck */ - private monitorStatusWaiting: number | null = null + private stuckTimestamp: number | null = null private logger: LoggerInstance constructor( @@ -17,32 +17,32 @@ export class ManagerStatusMonitor { ) { this.logger = logger.category('StatusMonitor') - this.statusMonitorInterval = setInterval(() => { - this._monitorStatus() + this.checkStatusInterval = setInterval(() => { + this._checkStatus() }, 60 * 1000) } public terminate(): void { - if (this.statusMonitorInterval) { - clearInterval(this.statusMonitorInterval) - this.statusMonitorInterval = null + if (this.checkStatusInterval) { + clearInterval(this.checkStatusInterval) + this.checkStatusInterval = null } } - private _monitorStatus() { + private _checkStatus() { // If the work-queue is long (>10 items) and nothing has progressed for the past 10 minutes. const waitingExpectationCount = this.tracker.scaler.getWaitingExpectationCount() if (waitingExpectationCount > 10) { - if (!this.monitorStatusWaiting) { - this.monitorStatusWaiting = Date.now() + if (!this.stuckTimestamp) { + this.stuckTimestamp = Date.now() } } else { - this.monitorStatusWaiting = null + this.stuckTimestamp = null } - const stuckDuration: number = this.monitorStatusWaiting ? Date.now() - this.monitorStatusWaiting : 0 + const stuckDuration: number = this.stuckTimestamp ? Date.now() - this.stuckTimestamp : 0 if (stuckDuration > 10 * 60 * 1000) { this.logger.error(`_monitorStatus: Work Queue is Stuck for ${stuckDuration / 1000 / 60} minutes`) this.managerStatuses.update('work-queue-stuck', { From e31d5a7f11775f41929dbb77f13f8c8990fe9a00 Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 12 Oct 2023 08:27:35 +0200 Subject: [PATCH 5/7] chore: fix test --- tests/internal-tests/src/__tests__/lib/setupEnv.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/internal-tests/src/__tests__/lib/setupEnv.ts b/tests/internal-tests/src/__tests__/lib/setupEnv.ts index a476cbe9..ef897932 100644 --- a/tests/internal-tests/src/__tests__/lib/setupEnv.ts +++ b/tests/internal-tests/src/__tests__/lib/setupEnv.ts @@ -36,6 +36,7 @@ const defaultTestConfig: SingleAppConfig = { logPath: '', unsafeSSL: false, certificates: [], + logLevel: LogLevel.INFO, }, workforce: { port: null, @@ -192,6 +193,7 @@ export async function prepareTestEnviromnent(debugLogging: boolean): Promise Date: Thu, 12 Oct 2023 08:29:54 +0200 Subject: [PATCH 6/7] chore: remove noop code --- .../packages/generic/src/packageManager.ts | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index f4667778..c49fd11b 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -42,7 +42,6 @@ export class PackageManagerHandler { public expectationManager: ExpectationManager - private expectedPackageCache: { [id: string]: ExpectedPackageWrap } = {} public packageContainersCache: PackageContainers = {} private externalData: { packageContainers: PackageContainers; expectedPackages: ExpectedPackageWrap[] } = { @@ -295,19 +294,7 @@ export class PackageManagerHandler { } // Step 0: Save local cache: - this.expectedPackageCache = {} this.packageContainersCache = packageContainers - for (const exp of expectedPackages) { - // Note: There might be duplicates in expectedPackages - - const existing = this.expectedPackageCache[exp.expectedPackage._id] - if ( - !existing || - existing.priority > exp.priority // If the existing priority is lower (ie higher), replace it - ) { - this.expectedPackageCache[exp.expectedPackage._id] = exp - } - } this.logger.debug( `Has ${expectedPackages.length} expectedPackages (${expectedPackageSources @@ -401,7 +388,7 @@ export class PackageManagerHandler { return this.expectationManager.debugKillApp(appId) } - /** Ensures that the packageContainerExpectations containes the mandatory expectations */ + /** Ensures that the packageContainerExpectations contains the mandatory expectations */ private ensureMandatoryPackageContainerExpectations(packageContainerExpectations: { [id: string]: PackageContainerExpectation }): void { From 7d546d93c000cbc2f3db41c5bb79c8123934dfce Mon Sep 17 00:00:00 2001 From: Johan Nyman Date: Thu, 12 Oct 2023 08:42:05 +0200 Subject: [PATCH 7/7] chore: logging --- apps/package-manager/packages/generic/src/packageManager.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/package-manager/packages/generic/src/packageManager.ts b/apps/package-manager/packages/generic/src/packageManager.ts index c49fd11b..247b631c 100644 --- a/apps/package-manager/packages/generic/src/packageManager.ts +++ b/apps/package-manager/packages/generic/src/packageManager.ts @@ -189,6 +189,8 @@ export class PackageManagerHandler { this._triggerUpdatedExpectedPackagesTimeout = setTimeout(() => { this._triggerUpdatedExpectedPackagesTimeout = null + const startTime = Date.now() + const packageContainers: PackageContainers = {} const expectedPackageSources: { sourceName: string @@ -273,6 +275,8 @@ export class PackageManagerHandler { } this.handleExpectedPackages(packageContainers, activePlaylist, activeRundowns, expectedPackageSources) + + this.logger.debug(`Took ${Date.now() - startTime}ms to handle updated expectedPackages`) }, 300) }