From 06220b2ffd5406988e6e6e484a1db8c7f131753f Mon Sep 17 00:00:00 2001 From: Aaron van Meerten Date: Mon, 6 Jan 2025 14:58:30 -0600 Subject: [PATCH] feat(instance_launcher): basic tests for launcher logic (#172) * feat(instance_launcher): basic tests for launcher logic * further launcher logic tests * log protected status during scaleUp action * fix: scale protection for new instances check backwards * better tests for redis * further tests * fix tests, update redis logging * ensure await before negating value --- src/cloud_manager.ts | 12 +- src/handlers.ts | 2 +- src/instance_group.ts | 10 +- src/instance_launcher.ts | 12 +- src/instance_tracker.ts | 2 +- src/redis.ts | 36 ++-- src/test/cloud_manager.ts | 119 +++++++++++++ src/test/instance_group.ts | 69 ++++++++ src/test/instance_launcher.ts | 324 ++++++++++++++++++++++++++++++++++ src/test/mock_store.ts | 25 +++ src/test/redis.ts | 143 +++++++++++++++ src/test/shutdown_manager.ts | 16 ++ 12 files changed, 738 insertions(+), 32 deletions(-) create mode 100644 src/test/cloud_manager.ts create mode 100644 src/test/instance_group.ts create mode 100644 src/test/instance_launcher.ts create mode 100644 src/test/redis.ts diff --git a/src/cloud_manager.ts b/src/cloud_manager.ts index 97b5f54..11d2856 100644 --- a/src/cloud_manager.ts +++ b/src/cloud_manager.ts @@ -15,6 +15,7 @@ export interface CloudManagerOptions extends CloudInstanceManagerSelectorOptions shutdownManager: ShutdownManager; instanceTracker: InstanceTracker; audit: Audit; + cloudInstanceManagerSelector?: CloudInstanceManagerSelector; } export interface CloudInstance { @@ -34,14 +35,15 @@ export default class CloudManager { constructor(options: CloudManagerOptions) { this.isDryRun = options.isDryRun; - this.cloudInstanceManagerSelector = new CloudInstanceManagerSelector(options); + if (options.cloudInstanceManagerSelector) { + this.cloudInstanceManagerSelector = options.cloudInstanceManagerSelector; + } else { + this.cloudInstanceManagerSelector = new CloudInstanceManagerSelector(options); + } this.instanceTracker = options.instanceTracker; this.shutdownManager = options.shutdownManager; this.audit = options.audit; - - this.scaleUp = this.scaleUp.bind(this); - this.scaleDown = this.scaleDown.bind(this); } async recordLaunch( @@ -88,7 +90,7 @@ export default class CloudManager { isScaleDownProtected: boolean, ): Promise { const groupName = group.name; - ctx.logger.info('[CloudManager] Scaling up', { groupName, quantity }); + ctx.logger.info('[CloudManager] Scaling up', { scaleUp: { groupName, quantity, isScaleDownProtected } }); const instanceManager = this.cloudInstanceManagerSelector.selectInstanceManager(group.cloud); if (!instanceManager) { diff --git a/src/handlers.ts b/src/handlers.ts index 8400efb..b46dda4 100644 --- a/src/handlers.ts +++ b/src/handlers.ts @@ -313,7 +313,7 @@ class Handlers { // found the group, so find the instances and act upon them // build the list of current instances const currentInventory = await this.instanceTracker.trimCurrent(req.context, req.params.name); - const instances = this.instanceTracker.mapToInstanceDetails(currentInventory); + const instances = InstanceTracker.mapToInstanceDetails(currentInventory); // set their reconfigure status to the current date try { await this.reconfigureManager.setReconfigureDate(req.context, instances); diff --git a/src/instance_group.ts b/src/instance_group.ts index 03604ee..d3e53c3 100644 --- a/src/instance_group.ts +++ b/src/instance_group.ts @@ -129,8 +129,9 @@ export default class InstanceGroupManager { await this.instanceStore.deleteInstanceGroup(ctx, groupName); } + // only allow autoscaling if the autoscale grace period has expired async allowAutoscaling(ctx: Context, group: string): Promise { - return this.instanceStore.checkValue(ctx, `autoScaleGracePeriod:${group}`); + return !(await this.instanceStore.checkValue(ctx, `autoScaleGracePeriod:${group}`)); } async setAutoScaleGracePeriod(ctx: Context, group: InstanceGroup): Promise { @@ -144,20 +145,23 @@ export default class InstanceGroupManager { return this.setValue(ctx, `isScaleDownProtected:${group.name}`, group.protectedTTLSec); } + // only show scale protection if value is set async isScaleDownProtected(ctx: Context, group: string): Promise { return this.instanceStore.checkValue(ctx, `isScaleDownProtected:${group}`); } + // only allow group jobs if the grace period has expired async isGroupJobsCreationAllowed(ctx: Context): Promise { - return this.instanceStore.checkValue(ctx, 'groupJobsCreationGracePeriod'); + return !(await this.instanceStore.checkValue(ctx, 'groupJobsCreationGracePeriod')); } async setGroupJobsCreationGracePeriod(ctx: Context): Promise { return this.setValue(ctx, `groupJobsCreationGracePeriod`, this.processingIntervalSeconds); } + // only allow sanity jobs if the grace period has expired async isSanityJobsCreationAllowed(ctx: Context): Promise { - return this.instanceStore.checkValue(ctx, 'sanityJobsCreationGracePeriod'); + return !(await this.instanceStore.checkValue(ctx, 'sanityJobsCreationGracePeriod')); } async setSanityJobsCreationGracePeriod(ctx: Context): Promise { diff --git a/src/instance_launcher.ts b/src/instance_launcher.ts index a124a5c..0e3caf5 100644 --- a/src/instance_launcher.ts +++ b/src/instance_launcher.ts @@ -56,8 +56,6 @@ export default class InstanceLauncher { if (options.maxThrottleThreshold) { this.maxThrottleThreshold = options.maxThrottleThreshold; } - - this.launchOrShutdownInstancesByGroup = this.launchOrShutdownInstancesByGroup.bind(this); } async launchOrShutdownInstancesByGroup(ctx: Context, groupName: string): Promise { @@ -350,7 +348,7 @@ export default class InstanceLauncher { instanceState.status.provisioning == true ); }); - return this.instanceTracker.mapToInstanceDetails(states); + return InstanceTracker.mapToInstanceDetails(states); } private getRunningInstances(instanceStates: InstanceState[]): InstanceDetails[] { @@ -364,7 +362,7 @@ export default class InstanceLauncher { instanceState.status.provisioning == false ); }); - return this.instanceTracker.mapToInstanceDetails(states); + return InstanceTracker.mapToInstanceDetails(states); } private getAvailableJibris(instanceStates: InstanceState[]): InstanceDetails[] { @@ -373,7 +371,7 @@ export default class InstanceLauncher { instanceState.status.jibriStatus && instanceState.status.jibriStatus.busyStatus == JibriStatusState.Idle ); }); - return this.instanceTracker.mapToInstanceDetails(states); + return InstanceTracker.mapToInstanceDetails(states); } private getExpiredJibris(instanceStates: InstanceState[]): InstanceDetails[] { @@ -383,7 +381,7 @@ export default class InstanceLauncher { instanceState.status.jibriStatus.busyStatus == JibriStatusState.Expired ); }); - return this.instanceTracker.mapToInstanceDetails(states); + return InstanceTracker.mapToInstanceDetails(states); } private getBusyJibris(instanceStates: InstanceState[]): InstanceDetails[] { @@ -392,6 +390,6 @@ export default class InstanceLauncher { instanceState.status.jibriStatus && instanceState.status.jibriStatus.busyStatus == JibriStatusState.Busy ); }); - return this.instanceTracker.mapToInstanceDetails(states); + return InstanceTracker.mapToInstanceDetails(states); } } diff --git a/src/instance_tracker.ts b/src/instance_tracker.ts index 795d5a1..aa765d9 100644 --- a/src/instance_tracker.ts +++ b/src/instance_tracker.ts @@ -464,7 +464,7 @@ export class InstanceTracker { return states.filter((_, index) => !statesShutdownStatus[index] && !shutdownConfirmations[index]); } - mapToInstanceDetails(states: InstanceState[]): InstanceDetails[] { + static mapToInstanceDetails(states: InstanceState[]): InstanceDetails[] { return states.map((response) => { return { instanceId: response.instanceId, diff --git a/src/redis.ts b/src/redis.ts index 15a0a5f..fcd393a 100644 --- a/src/redis.ts +++ b/src/redis.ts @@ -400,7 +400,7 @@ export default class RedisStore implements MetricsStore, InstanceStore { return res; } - async existsAtLeastOneGroup(_ctx: Context): Promise { + async existsAtLeastOneGroup(ctx: Context): Promise { let cursor = '0'; do { const result = await this.redisClient.hscan( @@ -411,21 +411,26 @@ export default class RedisStore implements MetricsStore, InstanceStore { 'COUNT', this.redisScanCount, ); - cursor = result[0]; - if (result[1].length > 0) { - const pipeline = this.redisClient.pipeline(); - result[1].forEach((key: string) => { - pipeline.hget(this.GROUPS_HASH_NAME, key); - }); + if (result) { + cursor = result[0]; + if (result[1].length > 0) { + const pipeline = this.redisClient.pipeline(); + result[1].forEach((key: string) => { + pipeline.hget(this.GROUPS_HASH_NAME, key); + }); - const items = await pipeline.exec(); - if (items) { - if (items.length > 0) { - return true; + const items = await pipeline.exec(); + if (items) { + if (items.length > 0) { + return true; + } + } else { + return false; } - } else { - return false; } + } else { + ctx.logger.error('Error scanning groups for existsAtLeastOneGroup'); + return false; } } while (cursor != '0'); @@ -433,7 +438,7 @@ export default class RedisStore implements MetricsStore, InstanceStore { } async upsertInstanceGroup(ctx: Context, group: InstanceGroup): Promise { - ctx.logger.info(`Storing ${group.name}`); + ctx.logger.info(`Storing ${group.name}`, { group }); await this.redisClient.hset(this.GROUPS_HASH_NAME, group.name, JSON.stringify(group)); return true; } @@ -507,8 +512,9 @@ export default class RedisStore implements MetricsStore, InstanceStore { async checkValue(_ctx: Context, key: string): Promise { const result = await this.redisClient.get(key); - return !(result !== null && result.length > 0); + return result !== null && result.length > 0; } + async setValue(_ctx: Context, key: string, value: string, ttl: number): Promise { const result = await this.redisClient.set(key, value, 'EX', ttl); if (result !== 'OK') { diff --git a/src/test/cloud_manager.ts b/src/test/cloud_manager.ts new file mode 100644 index 0000000..fe36069 --- /dev/null +++ b/src/test/cloud_manager.ts @@ -0,0 +1,119 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck + +import assert from 'node:assert'; +import test, { afterEach, describe, mock } from 'node:test'; + +import CloudManager from '../cloud_manager'; + +function log(msg, obj) { + console.log(msg, JSON.stringify(obj)); +} + +function initContext(): Context { + return { + logger: { + info: mock.fn(log), + debug: mock.fn(log), + error: mock.fn(log), + warn: mock.fn(log), + }, + }; +} + +describe('CloudManager', () => { + let context = initContext(); + + const shutdownManager = { + setScaleDownProtected: mock.fn(() => true), + areScaleDownProtected: mock.fn((arr) => arr.map(() => false)), + }; + + const instanceTracker = { + track: mock.fn(), + trimCurrent: mock.fn(), + }; + + const audit = { + saveLaunchEvent: mock.fn(), + }; + + const cloudInstanceManager = { + getInstances: mock.fn(() => []), + launchInstances: mock.fn((_ctx, _group, _cur, count) => { + for (let i = 0; i < count; i++) { + return [`mock-instance-${i}`]; + } + }), + }; + + const cloudInstanceManagerSelector = { + selectInstanceManager: mock.fn(() => cloudInstanceManager), + }; + + const cloudManager = new CloudManager({ + shutdownManager, + instanceTracker, + audit, + cloudInstanceManagerSelector, + cloudProviders: ['mock'], + isDryRun: false, + }); + + afterEach(() => { + context = initContext(); + cloudInstanceManager.launchInstances.mock.resetCalls(); + instanceTracker.track.mock.resetCalls(); + }); + + describe('cloudManager', () => { + test('should return empty for a group with no instances', async () => { + const result = await cloudManager.getInstances(context, 'group'); + assert.deepEqual(result, [], 'no instances'); + }); + + test('should return an item for a group with mock instances', async () => { + const instance = { instanceId: 'mock-instance', cloudStatus: 'running', displayName: 'mockery' }; + cloudInstanceManager.getInstances.mock.mockImplementation(() => [instance]); + const result = await cloudManager.getInstances(context, 'group'); + assert.deepEqual(result, [instance], 'mock instance'); + }); + + test('scaleUp should return success for mock group', async () => { + const group = { name: 'mock-group', cloud: 'mock', type: 'mock' }; + const currentCount = 0; + const quantity = 1; + const isScaleDownProtected = false; + const result = await cloudManager.scaleUp(context, group, currentCount, quantity, isScaleDownProtected); + assert.equal(result, 1, 'scale up success'); + assert.equal(cloudInstanceManager.launchInstances.mock.calls.length, 1, 'launch instances called'); + assert.equal(instanceTracker.track.mock.calls.length, 1, 'newly launched instance tracked'); + assert.equal( + shutdownManager.setScaleDownProtected.mock.calls.length, + 0, + 'launched instance is not scale protected', + ); + }); + + test('scaleUp should return success for mock group, including protected instance call', async () => { + const group = { name: 'mock-group-protected', cloud: 'mock', type: 'mock' }; + const currentCount = 0; + const quantity = 1; + const isScaleDownProtected = true; + const result = await cloudManager.scaleUp(context, group, currentCount, quantity, isScaleDownProtected); + assert.equal(result, 1, 'scale up success'); + assert.equal(cloudInstanceManager.launchInstances.mock.calls.length, 1, 'launch instances called'); + assert.equal(instanceTracker.track.mock.calls.length, 1, 'newly launched instance tracked'); + assert.equal( + shutdownManager.setScaleDownProtected.mock.calls.length, + 1, + 'launched instance is scale protected', + ); + assert.equal( + shutdownManager.setScaleDownProtected.mock.calls[0].arguments[2], + instanceTracker.track.mock.calls[0].arguments[1].instanceId, + 'protected instance id should match tracked instance id', + ); + }); + }); +}); diff --git a/src/test/instance_group.ts b/src/test/instance_group.ts new file mode 100644 index 0000000..04948a9 --- /dev/null +++ b/src/test/instance_group.ts @@ -0,0 +1,69 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck + +import assert from 'node:assert'; +import test, { afterEach, describe, mock } from 'node:test'; + +import { mockStore } from './mock_store'; + +import InstanceGroupManager from '../instance_group'; + +function log(msg, obj) { + console.log(msg, JSON.stringify(obj)); +} + +function initContext(): Context { + return { + logger: { + info: mock.fn(log), + debug: mock.fn(log), + error: mock.fn(log), + warn: mock.fn(log), + }, + }; +} + +describe('InstanceGroupManager', () => { + let context = initContext(); + const group = { + name: 'test', + type: 'test', + region: 'test', + environment: 'test', + cloud: 'test', + }; + + const instanceGroup = new InstanceGroupManager({ + instanceStore: mockStore, + initialGroupList: [group], + groupJobsCreationGracePeriod: 60, + sanityJobsCreationGracePeriod: 60, + }); + + afterEach(() => { + context = initContext(); + }); + + test('get initial set', async () => { + const groups = instanceGroup.getInitialGroups(); + assert.ok(groups, 'expect ok groups'); + assert.equal(groups.length, 1, 'expect groups to be 1'); + assert.deepEqual(groups[0], group, 'expect groups to be equal'); + }); + + test('check for any existing groups', async () => { + const res = await instanceGroup.existsAtLeastOneGroup(context); + assert.ok(res, 'expect ok result'); + }); + + test('set group as protected', async () => { + const preCheck = await instanceGroup.isScaleDownProtected(context, group.name); + assert.equal(preCheck, false, 'expect false result'); + + const res = await instanceGroup.setScaleDownProtected(context, group); + assert.ok(res, 'expect ok result'); + + const postCheck = await instanceGroup.isScaleDownProtected(context, group.name); + assert.ok(postCheck, 'expect group scale down protected'); + }); +}); diff --git a/src/test/instance_launcher.ts b/src/test/instance_launcher.ts new file mode 100644 index 0000000..e381468 --- /dev/null +++ b/src/test/instance_launcher.ts @@ -0,0 +1,324 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck + +import assert from 'node:assert'; +import test, { afterEach, describe, mock } from 'node:test'; + +import InstanceTracker from '../instance_tracker'; +import InstanceLauncher from '../instance_launcher'; + +function log(msg, obj) { + console.log(msg, JSON.stringify(obj)); +} + +function initContext() { + return { + logger: { + info: mock.fn(log), + debug: mock.fn(log), + error: mock.fn(log), + warn: mock.fn(log), + }, + }; +} + +describe('InstanceLauncher', () => { + let context = initContext(); + + const shutdownManager = { + shutdown: mock.fn(), + areScaleDownProtected: mock.fn(() => []), + }; + + const audit = { + updateLastLauncherRun: mock.fn(), + saveLauncherActionItem: mock.fn(), + }; + + const groupName = 'group'; + const groupDetails = { + name: groupName, + type: 'JVB', + region: 'default', + environment: 'test', + compartmentId: 'test', + instanceConfigurationId: 'test', + enableAutoScale: true, + enableLaunch: true, + gracePeriodTTLSec: 480, + protectedTTLSec: 600, + scalingOptions: { + minDesired: 1, + maxDesired: 1, + desiredCount: 1, + scaleUpQuantity: 1, + scaleDownQuantity: 1, + scaleUpThreshold: 0.8, + scaleDownThreshold: 0.3, + scalePeriod: 60, + scaleUpPeriodsCount: 2, + scaleDownPeriodsCount: 2, + }, + }; + + const inventory = [ + { + instanceId: 'i-deadbeef007', + metadata: { group: groupName }, + status: { provisioning: false, stats: { stress_level: 0.5, participants: 1 } }, + }, + ]; + + const instanceGroupManager = { + getInstanceGroup: mock.fn(() => groupDetails), + isScaleDownProtected: mock.fn(() => false), + }; + + const cloudManager = { + scaleUp: mock.fn(() => groupDetails.scalingOptions.scaleUpQuantity), + scaleDown: mock.fn(), + }; + + const instanceTracker = { + trimCurrent: mock.fn(() => inventory), + mapToInstanceDetails: mock.fn((i) => InstanceTracker.mapToInstanceDetails(i)), + }; + + const metricsLoop = { + updateMetrics: mock.fn(), + getUnTrackedCount: mock.fn(() => 0), + }; + + // now we can create an instance of the class + const instanceLauncher = new InstanceLauncher({ + instanceTracker, + instanceGroupManager, + cloudManager, + shutdownManager, + audit, + metricsLoop, + }); + + const groupDetailsDesired0 = { + ...groupDetails, + scalingOptions: { ...groupDetails.scalingOptions, desiredCount: 0, minDesired: 0 }, + }; + + afterEach(() => { + audit.updateLastLauncherRun.mock.resetCalls(); + instanceTracker.trimCurrent.mock.resetCalls(); + shutdownManager.areScaleDownProtected.mock.resetCalls(); + cloudManager.scaleDown.mock.resetCalls(); + cloudManager.scaleUp.mock.resetCalls(); + context = initContext(); + }); + + describe('instanceLauncher basic tests', () => { + // first test if disabled group exits correctly + test('launchOrShutdownInstancesByGroup should return false if group is disabled', async () => { + const groupDetailsDisabled = { ...groupDetails, enableLaunch: false }; + instanceGroupManager.getInstanceGroup.mock.mockImplementationOnce(() => groupDetailsDisabled); + const result = await instanceLauncher.launchOrShutdownInstancesByGroup(context, groupName); + assert.equal(result, false, 'skip disabled group'); + }); + + // now test if enable group does nothing with desired of 1 and inventory of 1 + test('launchOrShutdownInstancesByGroup should return true if desired is 1 and inventory is 1', async () => { + const result = await instanceLauncher.launchOrShutdownInstancesByGroup(context, groupName); + assert.equal(result, true, 'skip desired=1 and inventory=1'); + assert.equal(cloudManager.scaleUp.mock.calls.length, 0, 'no scaleUp'); + assert.equal(cloudManager.scaleDown.mock.calls.length, 0, 'no scaleDown'); + assert.equal(instanceTracker.trimCurrent.mock.calls.length, 1, 'trimCurrent called'); + assert.equal(audit.updateLastLauncherRun.mock.calls.length, 1, 'audit.updateLastLauncherRun called'); + assert.equal(context.logger.info.mock.calls.length, 1, 'logger.info called'); + assert.equal( + context.logger.info.mock.calls[0].arguments[0], + '[Launcher] No scaling activity needed for group group with 1 instances.', + ); + }); + + // now test if scaleDown occurs with desired of 0 and inventory of 1 + test('launchOrShutdownInstancesByGroup should return true if desired is 0 and inventory is 1', async () => { + instanceGroupManager.getInstanceGroup.mock.mockImplementationOnce(() => groupDetailsDesired0); + + const result = await instanceLauncher.launchOrShutdownInstancesByGroup(context, groupName); + assert.equal(result, true, 'scaleDown desired=0 and inventory=1'); + assert.equal(audit.updateLastLauncherRun.mock.calls.length, 1, 'audit.updateLastLauncherRun called'); + assert.equal(instanceTracker.trimCurrent.mock.calls.length, 1, 'trimCurrent called'); + assert.equal(cloudManager.scaleUp.mock.calls.length, 0, 'no scaleUp'); + assert.equal(cloudManager.scaleDown.mock.calls.length, 1, 'scaleDown called'); + assert.equal(shutdownManager.areScaleDownProtected.mock.calls.length, 1, 'areScaleDownProtected called'); + assert.equal(context.logger.info.mock.calls.length, 1, 'logger.info called'); + assert.equal( + context.logger.info.mock.calls[0].arguments[0], + '[Launcher] Will scale down to the desired count', + ); + }); + + // now test if scaleUp occurs with desired of 2 and inventory of 1 + test('launchOrShutdownInstancesByGroup should return true if desired is 2 and inventory is 1', async () => { + const groupDetailsDesired2 = { + ...groupDetails, + scalingOptions: { ...groupDetails.scalingOptions, desiredCount: 2, maxDesired: 2 }, + }; + instanceGroupManager.getInstanceGroup.mock.mockImplementationOnce(() => groupDetailsDesired2); + + const result = await instanceLauncher.launchOrShutdownInstancesByGroup(context, groupName); + assert.equal(result, true, 'scaleDown desired=0 and inventory=1'); + assert.equal(audit.updateLastLauncherRun.mock.calls.length, 1, 'audit.updateLastLauncherRun called'); + assert.equal(instanceTracker.trimCurrent.mock.calls.length, 1, 'trimCurrent called'); + assert.equal(cloudManager.scaleDown.mock.calls.length, 0, 'no scaleDown'); + assert.equal(cloudManager.scaleUp.mock.calls.length, 1, 'scaleUp called'); + assert.equal( + shutdownManager.areScaleDownProtected.mock.calls.length, + 0, + 'areScaleDownProtected not called', + ); + assert.equal(context.logger.info.mock.calls.length, 1, 'logger.info called'); + assert.equal( + context.logger.info.mock.calls[0].arguments[0], + '[Launcher] Will scale up to the desired count', + ); + }); + }); + + describe('instanceLauncher scaleUp protection tests', () => { + test('launchOrShutdownInstancesByGroup should launch an instance without protected mode if group is not protected', async () => { + instanceGroupManager.getInstanceGroup.mock.mockImplementationOnce(() => groupDetailsDesired2); + const groupDetailsDesired2 = { + ...groupDetails, + scalingOptions: { ...groupDetails.scalingOptions, desiredCount: 2, maxDesired: 2 }, + }; + const result = await instanceLauncher.launchOrShutdownInstancesByGroup(context, groupName); + assert.equal(result, true, 'launch unprotected instance'); + assert.equal(cloudManager.scaleUp.mock.calls.length, 1, 'scaleUp called'); + assert.equal(cloudManager.scaleUp.mock.calls[0].arguments[4], false, 'unprotected instance'); + }); + + test('launchOrShutdownInstancesByGroup should launch an instance in protected mode if group is protected', async () => { + instanceGroupManager.getInstanceGroup.mock.mockImplementationOnce(() => groupDetailsDesired2); + instanceGroupManager.isScaleDownProtected.mock.mockImplementationOnce(() => true); + const groupDetailsDesired2 = { + ...groupDetails, + scalingOptions: { ...groupDetails.scalingOptions, desiredCount: 2, maxDesired: 2 }, + }; + const result = await instanceLauncher.launchOrShutdownInstancesByGroup(context, groupName); + assert.equal(result, true, 'launch protected instance'); + assert.equal(cloudManager.scaleUp.mock.calls.length, 1, 'scaleUp called'); + assert.equal(cloudManager.scaleUp.mock.calls[0].arguments[4], true, 'protected instance'); + }); + }); + + describe('instanceLauncher scaleDown selection tests', () => { + test('getStatusMetricForScaleDown should give correct status', async () => { + const result = await instanceLauncher.getStatusMetricForScaleDown(inventory[0]); + assert.equal(result, inventory[0].status.stats.participants, 'participant count from inventory'); + }); + + test('getRunningInstances should return instance', async () => { + const result = await instanceLauncher.getRunningInstances(inventory); + assert.equal(result.length, inventory.length, 'all instances running'); + }); + + test('getInstancesForScaleDown should return empty array if no instances', async () => { + const result = await instanceLauncher.getInstancesForScaleDown(context, [], groupDetails); + assert.equal(result.length, 0, 'no instances to scale down'); + }); + + test('getInstancesForScaleDown should select inventory item if present', async () => { + const result = await instanceLauncher.getInstancesForScaleDown(context, inventory, groupDetailsDesired0); + assert.equal(result.length, 1, 'should select existing instance'); + }); + + test('filterOutProtectedInstances should return identical array by default', async () => { + const result = await instanceLauncher.filterOutProtectedInstances(context, groupDetailsDesired0, inventory); + assert.equal(result.length, inventory.length, 'no instances protected by default'); + }); + + test('filterOutProtectedInstances should return blank array if instance is protected', async () => { + shutdownManager.areScaleDownProtected.mock.mockImplementationOnce(() => [true]); + const result = await instanceLauncher.filterOutProtectedInstances(context, groupDetailsDesired0, inventory); + assert.equal(result.length, 0, 'no instances unprotected'); + }); + + test('getInstancesForScaleDown should return empty array if only instance is protected', async () => { + shutdownManager.areScaleDownProtected.mock.mockImplementationOnce(() => [true]); + const result = await instanceLauncher.getInstancesForScaleDown(context, inventory, groupDetailsDesired0); + assert.equal(result.length, 0, 'no instances unprotected'); + }); + + test('getInstancesForScaleDown should select the instance that has fewer participants', async () => { + const inventoryProvisioning = [ + ...inventory, + { + instanceId: 'i-deadbeef008', + metadata: { group: groupName }, + status: { provisioning: false, stats: { stress_level: 0, participants: 0 } }, + }, + ]; + const result = await instanceLauncher.getInstancesForScaleDown( + context, + inventoryProvisioning, + groupDetails, + ); + assert.equal(result.length, 1, '1 instance selected for shutdown'); + assert.equal(result[0].instanceId, 'i-deadbeef008', 'correct instance selected'); + }); + + test('getInstancesForScaleDown should select the instances that have fewer participants', async () => { + const inventoryProvisioning = [ + ...inventory, + { + instanceId: 'i-deadbeef008', + metadata: { group: groupName }, + status: { provisioning: false, stats: { stress_level: 0, participants: 0 } }, + }, + { + instanceId: 'i-deadbeef001', + metadata: { group: groupName }, + status: { provisioning: false, stats: { stress_level: 0.9, participants: 100 } }, + }, + ]; + const result = await instanceLauncher.getInstancesForScaleDown( + context, + inventoryProvisioning, + groupDetails, + ); + assert.equal(result.length, 2, '2 instance selected for shutdown'); + const instanceIds = result.map((v) => v.instanceId); + assert.ok(instanceIds.includes('i-deadbeef008'), 'one correct instance selected'); + assert.ok(instanceIds.includes('i-deadbeef007'), 'one correct instance selected'); + assert.ok(!instanceIds.includes('i-deadbeef001'), 'loaded instance not selected'); + }); + + test('getInstancesForScaleDown should skip protected but still select the instances that have fewer participants', async () => { + shutdownManager.areScaleDownProtected.mock.mockImplementationOnce((_ctx, _group, instanceIds) => { + return instanceIds.map((v) => v === 'i-deadbeef008'); + }); + + const inventoryProvisioning = [ + ...inventory, + { + instanceId: 'i-deadbeef008', + metadata: { group: groupName }, + status: { provisioning: false, stats: { stress_level: 0, participants: 0 } }, + }, + { + instanceId: 'i-deadbeef001', + metadata: { group: groupName }, + status: { provisioning: false, stats: { stress_level: 0.9, participants: 100 } }, + }, + ]; + const result = await instanceLauncher.getInstancesForScaleDown( + context, + inventoryProvisioning, + groupDetails, + ); + assert.equal(result.length, 2, '2 instance selected for shutdown'); + const instanceIds = result.map((v) => v.instanceId); + assert.ok(instanceIds.includes('i-deadbeef001'), 'one correct instance selected'); + assert.ok(instanceIds.includes('i-deadbeef007'), 'one correct instance selected'); + assert.ok(!instanceIds.includes('i-deadbeef008'), 'loaded instance not selected'); + }); + }); +}); diff --git a/src/test/mock_store.ts b/src/test/mock_store.ts index 554d7d7..1e93da0 100644 --- a/src/test/mock_store.ts +++ b/src/test/mock_store.ts @@ -1,7 +1,12 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck + import { mock } from 'node:test'; import { Context } from '../context'; import { InstanceState } from '../instance_store'; +const _values = {}; + export const mockStore = { fetchInstanceMetrics: mock.fn(() => [ { value: 0.5, instanceId: 'i-0a1b2c3d4e5f6g7h8', timestamp: Date.now() - 350 }, @@ -17,4 +22,24 @@ export const mockStore = { getShutdownStatus: mock.fn(() => false), getShutdownConfirmation: mock.fn(() => false), getShutdownConfirmations: mock.fn(() => [false]), + + setScaleDownProtected: mock.fn(() => true), + areScaleDownProtected: mock.fn((_ctx, _group, input) => { + return input.map(() => false); + }), + + existsAtLeastOneGroup: mock.fn(() => true), + upsertInstanceGroup: mock.fn(() => true), + + setValue: mock.fn((ctx: Context, key: string, value: string, ttl: number) => { + _values[key] = { value, ttl: Date.now() + ttl * 1000 }; + return Promise.resolve(true); + }), + + checkValue: mock.fn((_ctx, key) => { + if (_values[key]) { + return Promise.resolve(true); + } + return Promise.resolve(false); + }), }; diff --git a/src/test/redis.ts b/src/test/redis.ts new file mode 100644 index 0000000..381bd72 --- /dev/null +++ b/src/test/redis.ts @@ -0,0 +1,143 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck +import assert from 'node:assert'; +import test, { describe, mock, afterEach } from 'node:test'; +import RedisStore from '../redis'; +import Redis from 'ioredis'; +import { Context } from '../context'; + +function log(msg, obj) { + console.log(msg, JSON.stringify(obj)); +} + +function initContext(): Context { + return { + logger: { + info: mock.fn(log), + debug: mock.fn(log), + error: mock.fn(log), + warn: mock.fn(log), + }, + }; +} + +describe('RedisStore', () => { + let _keys = []; + let _hashes = {}; + let _values = {}; + + let context = initContext(); + + const mockPipeline = { + get: mock.fn((key) => _keys.push(key)), + set: mock.fn(), + exec: mock.fn(() => + Promise.resolve( + _keys.map((k, i) => { + return [i, _values[k]]; + }), + ), + ), + hget: mock.fn((key) => _keys.push(key)), + }; + + const mockClient = { + expire: mock.fn(), + zremrangebyscore: mock.fn(() => 0), + hget: mock.fn((_h, key) => { + return _hashes[key]; + }), + hgetall: mock.fn(), + hset: mock.fn((_h, key, value) => { + _hashes[key] = value; + }), + hscan: mock.fn(() => [0, []]), + hdel: mock.fn(), + del: mock.fn(), + scan: mock.fn(), + zrange: mock.fn(), + set: mock.fn((key, value) => { + _values[key] = value; + return 'OK'; + }), + get: mock.fn((key) => { + if (!_values[key]) { + return null; + } + return _values[key]; + }), + pipeline: mock.fn(() => mockPipeline), + }; + + const redisStore = new RedisStore({ + redisClient: mockClient, + redisScanCount: 100, + idleTTL: 60, + metricTTL: 60, + provisioningTTL: 60, + shutdownStatusTTL: 60, + groupRelatedDataTTL: 60, + serviceLevelMetricsTTL: 60, + }); + + afterEach(() => { + _keys = []; + _hashes = {}; + _values = {}; + context = initContext(); + }); + + test('redisStore checks for at least one group, finds none', async () => { + const res = await redisStore.existsAtLeastOneGroup(context); + assert.equal(res, false, 'expect no groups'); + }); + + test('redisStore checks for at least one group, finds one', async () => { + mockClient.hscan.mock.mockImplementationOnce(() => [0, ['group']]); + const res = await redisStore.existsAtLeastOneGroup(context); + assert.equal(res, true, 'expect at least one group'); + }); + + test('redisStore can store and retrieve a group', async () => { + const group = { + name: 'test', + type: 'test', + region: 'test', + environment: 'test', + enableScheduler: true, + tags: { + test: 'test', + }, + }; + + await redisStore.upsertInstanceGroup(context, group); + const res = await redisStore.getInstanceGroup(context, group.name); + assert.deepEqual(res, group, 'expect group to be stored and retrieved'); + }); + + test('redisStore can set protected status and check for it', async () => { + const setRes = await redisStore.setScaleDownProtected(context, 'test', 'instance-123a', 900); + assert.equal(setRes, true, 'expect set to succeed'); + const res = await redisStore.areScaleDownProtected(context, 'test', ['instance-123a']); + assert.deepEqual(res, [true], 'expect instance to be protected'); + }); + + test('redisStore does not find protected status for unknown instance', async () => { + const res = await redisStore.areScaleDownProtected(context, 'test', ['instance-321b']); + assert.deepEqual(res, [false], 'expect instance to be unprotected'); + }); + + test('setValue and checkValue return as expected', async () => { + const key = 'test-key'; + const value = 'test-value'; + const ttl = 60; + + const preCheckRes = await redisStore.checkValue(context, key); + assert.equal(preCheckRes, false, 'expect pre-check value to fail'); + + const res = await redisStore.setValue(context, key, value, ttl); + assert.equal(res, true, 'expect set value to succeed'); + const checkRes = await redisStore.checkValue(context, key); + assert.equal(checkRes, true, 'expect check value to succeed'); + }); +}); diff --git a/src/test/shutdown_manager.ts b/src/test/shutdown_manager.ts index e57fc98..7416494 100644 --- a/src/test/shutdown_manager.ts +++ b/src/test/shutdown_manager.ts @@ -41,6 +41,22 @@ describe('ShutdownManager', () => { mock.restoreAll(); }); + // these tests are for scale protection statuses + describe('scaleProtectionStatuses', () => { + test('areScaleDownProtected with no instances', async () => { + const result = await shutdownManager.areScaleDownProtected(context, 'group', []); + assert.ok(result, 'expect ok result'); + assert.equal(result.length, 0, 'expect result to be empty'); + }); + + test('areScaleDownProtected with default instances', async () => { + const result = await shutdownManager.areScaleDownProtected(context, 'group', ['instanceId']); + assert.ok(result, 'expect ok result'); + assert.equal(result.length, 1, 'expect result to be 1 false'); + assert.equal(result[0], false, 'expect result to be false'); + }); + }); + // these tests are for the shutdown confirmation statuses describe('shutdownConfirmationStatuses', () => { test('read non-existent shutdown confirmation status', async () => {