Skip to content

Commit

Permalink
Add an option to control the resources available to workers.
Browse files Browse the repository at this point in the history
  • Loading branch information
iclanton committed Jan 22, 2025
1 parent 96116b4 commit 1008930
Show file tree
Hide file tree
Showing 6 changed files with 57 additions and 5 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@rushstack/module-minifier",
"comment": "Add a `workerResourceLimits` option to the `WorkerPoolMinifier` constructor to control the available resources to the workers.",
"type": "minor"
}
],
"packageName": "@rushstack/module-minifier"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"changes": [
{
"packageName": "@rushstack/worker-pool",
"comment": "Add a `workerResourceLimits` option to the `WorkerPool` constructor to control the available resources to the workers.",
"type": "minor"
}
],
"packageName": "@rushstack/worker-pool"
}
2 changes: 2 additions & 0 deletions common/reviews/api/module-minifier.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import { MinifyOptions } from 'terser';
import type { RawSourceMap } from 'source-map';
import type { ResourceLimits } from 'worker_threads';
import type * as WorkerThreads from 'worker_threads';

// @public
Expand Down Expand Up @@ -79,6 +80,7 @@ export interface IWorkerPoolMinifierOptions {
maxThreads?: number;
terserOptions?: MinifyOptions;
verbose?: boolean;
workerResourceLimits?: ResourceLimits;
}

// @public
Expand Down
2 changes: 2 additions & 0 deletions common/reviews/api/worker-pool.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

/// <reference types="node" />

import { ResourceLimits } from 'worker_threads';
import { Worker } from 'worker_threads';

// Warning: (ae-internal-missing-underscore) The name "IWorkerPoolOptions" should be prefixed with an underscore because the declaration is marked as @internal
Expand All @@ -17,6 +18,7 @@ export interface IWorkerPoolOptions {
onWorkerDestroyed?: () => void;
prepareWorker?: (worker: Worker) => void;
workerData?: unknown;
workerResourceLimits?: ResourceLimits;
workerScriptPath: string;
}

Expand Down
16 changes: 14 additions & 2 deletions libraries/module-minifier/src/WorkerPoolMinifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import { createHash } from 'crypto';
import { cpus } from 'os';
import type { ResourceLimits } from 'worker_threads';

import serialize from 'serialize-javascript';
import type { MinifyOptions } from 'terser';
Expand Down Expand Up @@ -36,6 +37,11 @@ export interface IWorkerPoolMinifierOptions {
* If true, log to the console about the minification results.
*/
verbose?: boolean;

/**
* Optional resource limits for the workers.
*/
workerResourceLimits?: ResourceLimits;
}

/**
Expand All @@ -55,15 +61,21 @@ export class WorkerPoolMinifier implements IModuleMinifier {
private readonly _activeRequests: Map<string, IModuleMinificationCallback[]>;

public constructor(options: IWorkerPoolMinifierOptions) {
const { maxThreads = cpus().length, terserOptions = {}, verbose = false } = options || {};
const {
maxThreads = cpus().length,
terserOptions = {},
verbose = false,
workerResourceLimits
} = options || {};

const activeRequests: Map<string, IModuleMinificationCallback[]> = new Map();
const resultCache: Map<string, IModuleMinificationResult> = new Map();
const terserPool: WorkerPool = new WorkerPool({
id: 'Minifier',
maxWorkers: maxThreads,
workerData: terserOptions,
workerScriptPath: require.resolve('./MinifierWorker')
workerScriptPath: require.resolve('./MinifierWorker'),
workerResourceLimits
});

const { version: terserVersion } = require('terser/package.json');
Expand Down
22 changes: 19 additions & 3 deletions libraries/worker-pool/src/WorkerPool.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
// See LICENSE in the project root for license information.

import { Worker } from 'worker_threads';
import { type ResourceLimits, Worker } from 'worker_threads';

/**
* Symbol to read the ID off of a worker
Expand Down Expand Up @@ -38,6 +38,11 @@ export interface IWorkerPoolOptions {
* Absolute path to the worker script.
*/
workerScriptPath: string;

/**
* Optional resource limits for the workers.
*/
workerResourceLimits?: ResourceLimits;
}

/**
Expand All @@ -60,9 +65,18 @@ export class WorkerPool {
private readonly _prepare: ((worker: Worker) => void) | undefined;
private readonly _workerData: unknown;
private readonly _workerScript: string;
private readonly _workerResourceLimits: ResourceLimits | undefined;

public constructor(options: IWorkerPoolOptions) {
const { id, maxWorkers, onWorkerDestroyed, prepareWorker, workerData, workerScriptPath } = options;
const {
id,
maxWorkers,
onWorkerDestroyed,
prepareWorker,
workerData,
workerScriptPath,
workerResourceLimits
} = options;

this.id = id;
this.maxWorkers = maxWorkers;
Expand All @@ -77,6 +91,7 @@ export class WorkerPool {
this._prepare = prepareWorker;
this._workerData = workerData;
this._workerScript = workerScriptPath;
this._workerResourceLimits = workerResourceLimits;
}

/**
Expand Down Expand Up @@ -193,7 +208,8 @@ export class WorkerPool {
[WORKER_ID_SYMBOL]?: string;
} = new Worker(this._workerScript, {
eval: false,
workerData: this._workerData
workerData: this._workerData,
resourceLimits: this._workerResourceLimits
});

const id: string = `${this.id}#${++this._nextId}`;
Expand Down

0 comments on commit 1008930

Please sign in to comment.