From b95e3dfbf2c1848b249e7b53262fc6d6f10cf6d3 Mon Sep 17 00:00:00 2001 From: selki Date: Tue, 19 Nov 2024 14:54:07 +0000 Subject: [PATCH 1/9] polling bandits only when needed --- src/configuration-requestor.ts | 57 +++++++++++++++++++++++++++------- src/http-client.ts | 9 +++++- src/interfaces.ts | 5 +++ 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/src/configuration-requestor.ts b/src/configuration-requestor.ts index 5d8be096..6d21d898 100644 --- a/src/configuration-requestor.ts +++ b/src/configuration-requestor.ts @@ -41,22 +41,57 @@ export default class ConfigurationRequestor { createdAt: configResponse.createdAt, }); - // TODO: different polling intervals for bandit parameters - const banditResponse = await this.httpClient.getBanditParameters(); - if (banditResponse?.bandits) { - if (!this.banditModelConfigurationStore) { - throw new Error('Bandit parameters fetched but no bandit configuration store provided'); - } + if (this.requiresBanditModelConfigurationStoreUpdate(configResponse.banditReferences)) { + const banditResponse = await this.httpClient.getBanditParameters(); + if (banditResponse?.bandits) { + if (!this.banditModelConfigurationStore) { + throw new Error('Bandit parameters fetched but no bandit configuration store provided'); + } - await this.hydrateConfigurationStore(this.banditModelConfigurationStore, { - entries: banditResponse.bandits, - environment: configResponse.environment, - createdAt: configResponse.createdAt, - }); + await this.hydrateConfigurationStore(this.banditModelConfigurationStore, { + entries: banditResponse.bandits, + environment: configResponse.environment, + createdAt: configResponse.createdAt, + }); + } } } } + private getLoadedBanditModelVersions( + banditModelConfigurationStore: IConfigurationStore | null, + ) { + if (banditModelConfigurationStore === null) { + return []; + } + return Object.values(banditModelConfigurationStore.entries()).map( + (banditParam: BanditParameters) => banditParam.modelVersion, + ); + } + + private requiresBanditModelConfigurationStoreUpdate( + banditReferences: Record, + ): boolean { + if (!this.banditModelConfigurationStore) { + throw new Error('Bandit parameters fetched but no bandit configuration store provided'); + } + const referencedModelVersions = Object.values(banditReferences).map( + (banditReference: BanditReference) => banditReference.modelVersion + ); + + const banditModelVersionsInStore = this.getLoadedBanditModelVersions( + this.banditModelConfigurationStore, + ); + + referencedModelVersions.forEach((modelVersion) => { + if (!banditModelVersionsInStore.includes(modelVersion)) { + return false; + } + }); + + return true; + } + private async hydrateConfigurationStore( configurationStore: IConfigurationStore | null, response: { diff --git a/src/http-client.ts b/src/http-client.ts index 6bb1ac6f..11ead305 100644 --- a/src/http-client.ts +++ b/src/http-client.ts @@ -1,5 +1,11 @@ import ApiEndpoints from './api-endpoints'; -import { BanditParameters, BanditVariation, Environment, Flag } from './interfaces'; +import { + BanditParameters, + BanditReference, + BanditVariation, + Environment, + Flag, +} from './interfaces'; export interface IQueryParams { apiKey: string; @@ -21,6 +27,7 @@ export interface IUniversalFlagConfigResponse { environment: Environment; flags: Record; bandits: Record; + banditReferences: Record; } export interface IBanditParametersResponse { diff --git a/src/interfaces.ts b/src/interfaces.ts index 0d89ebcd..fc9eb85b 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -99,6 +99,11 @@ export interface BanditVariation { variationValue: string; } +export interface BanditReference { + modelVersion: string; + flagVariation: BanditVariation[]; +} + export interface BanditParameters { banditKey: string; modelName: string; From 83025f38a56eaffb14ee214fec42e6592d5f01c6 Mon Sep 17 00:00:00 2001 From: selki Date: Tue, 19 Nov 2024 14:54:35 +0000 Subject: [PATCH 2/9] polling bandits only when needed --- src/configuration-requestor.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/configuration-requestor.ts b/src/configuration-requestor.ts index 6d21d898..b3896dd5 100644 --- a/src/configuration-requestor.ts +++ b/src/configuration-requestor.ts @@ -1,6 +1,12 @@ import { IConfigurationStore } from './configuration-store/configuration-store'; import { IHttpClient } from './http-client'; -import { BanditVariation, BanditParameters, Flag, Environment } from './interfaces'; +import { + BanditVariation, + BanditParameters, + Flag, + Environment, + BanditReference, +} from './interfaces'; type Entry = Flag | BanditVariation[] | BanditParameters; @@ -76,7 +82,7 @@ export default class ConfigurationRequestor { throw new Error('Bandit parameters fetched but no bandit configuration store provided'); } const referencedModelVersions = Object.values(banditReferences).map( - (banditReference: BanditReference) => banditReference.modelVersion + (banditReference: BanditReference) => banditReference.modelVersion, ); const banditModelVersionsInStore = this.getLoadedBanditModelVersions( From e38196366e3a2900fa6212abfbe9bb18d2888607 Mon Sep 17 00:00:00 2001 From: selki Date: Tue, 19 Nov 2024 17:20:50 +0000 Subject: [PATCH 3/9] polling bandits only when needed --- src/configuration-requestor.ts | 48 ++++++++++++++++++++-------------- src/interfaces.ts | 2 +- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/src/configuration-requestor.ts b/src/configuration-requestor.ts index b3896dd5..7c03f9bd 100644 --- a/src/configuration-requestor.ts +++ b/src/configuration-requestor.ts @@ -12,6 +12,8 @@ type Entry = Flag | BanditVariation[] | BanditParameters; // Requests AND stores flag configurations export default class ConfigurationRequestor { + private banditModelVersions: string[] = []; + constructor( private readonly httpClient: IHttpClient, private readonly flagConfigurationStore: IConfigurationStore, @@ -33,13 +35,13 @@ export default class ConfigurationRequestor { createdAt: configResponse.createdAt, }); - const flagsHaveBandits = Object.keys(configResponse.bandits ?? {}).length > 0; + const flagsHaveBandits = Object.keys(configResponse.banditReferences ?? {}).length > 0; const banditStoresProvided = Boolean( this.banditVariationConfigurationStore && this.banditModelConfigurationStore, ); if (flagsHaveBandits && banditStoresProvided) { // Map bandit flag associations by flag key for quick lookup (instead of bandit key as provided by the UFC) - const banditVariations = this.indexBanditVariationsByFlagKey(configResponse.bandits); + const banditVariations = this.indexBanditVariationsByFlagKey(configResponse.banditReferences); await this.hydrateConfigurationStore(this.banditVariationConfigurationStore, { entries: banditVariations, @@ -47,26 +49,34 @@ export default class ConfigurationRequestor { createdAt: configResponse.createdAt, }); - if (this.requiresBanditModelConfigurationStoreUpdate(configResponse.banditReferences)) { + if (!this.banditModelConfigurationStore) { + throw new Error('Bandit parameters fetched but no bandit configuration store provided'); + } + if ( + this.requiresBanditModelConfigurationStoreUpdate( + this.banditModelVersions, + configResponse.banditReferences, + ) + ) { const banditResponse = await this.httpClient.getBanditParameters(); if (banditResponse?.bandits) { - if (!this.banditModelConfigurationStore) { - throw new Error('Bandit parameters fetched but no bandit configuration store provided'); - } - await this.hydrateConfigurationStore(this.banditModelConfigurationStore, { entries: banditResponse.bandits, environment: configResponse.environment, createdAt: configResponse.createdAt, }); + + this.setBanditModelVersions( + this.getLoadedBanditModelVersionsFromStore(this.banditModelConfigurationStore), + ); } } } } - private getLoadedBanditModelVersions( + private getLoadedBanditModelVersionsFromStore( banditModelConfigurationStore: IConfigurationStore | null, - ) { + ): string[] { if (banditModelConfigurationStore === null) { return []; } @@ -75,22 +85,20 @@ export default class ConfigurationRequestor { ); } + private setBanditModelVersions(modelVersions: string[]) { + this.banditModelVersions = modelVersions; + } + private requiresBanditModelConfigurationStoreUpdate( + currentBanditModelVersions: string[], banditReferences: Record, ): boolean { - if (!this.banditModelConfigurationStore) { - throw new Error('Bandit parameters fetched but no bandit configuration store provided'); - } const referencedModelVersions = Object.values(banditReferences).map( (banditReference: BanditReference) => banditReference.modelVersion, ); - const banditModelVersionsInStore = this.getLoadedBanditModelVersions( - this.banditModelConfigurationStore, - ); - referencedModelVersions.forEach((modelVersion) => { - if (!banditModelVersionsInStore.includes(modelVersion)) { + if (!currentBanditModelVersions.includes(modelVersion)) { return false; } }); @@ -117,11 +125,11 @@ export default class ConfigurationRequestor { } private indexBanditVariationsByFlagKey( - banditVariationsByBanditKey: Record, + banditVariationsByBanditKey: Record, ): Record { const banditVariationsByFlagKey: Record = {}; - Object.values(banditVariationsByBanditKey).forEach((banditVariations) => { - banditVariations.forEach((banditVariation) => { + Object.values(banditVariationsByBanditKey).forEach((banditReference) => { + banditReference.flagVariations.forEach((banditVariation) => { let banditVariations = banditVariationsByFlagKey[banditVariation.flagKey]; if (!banditVariations) { banditVariations = []; diff --git a/src/interfaces.ts b/src/interfaces.ts index fc9eb85b..a1061281 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -101,7 +101,7 @@ export interface BanditVariation { export interface BanditReference { modelVersion: string; - flagVariation: BanditVariation[]; + flagVariations: BanditVariation[]; } export interface BanditParameters { From 401e1d3875640342a82160ab3723cb2d8323bf36 Mon Sep 17 00:00:00 2001 From: selki Date: Tue, 19 Nov 2024 18:11:36 +0000 Subject: [PATCH 4/9] added tests for polling bandits only when needed --- src/configuration-requestor.spec.ts | 8 ++++++++ src/configuration-requestor.ts | 13 +++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/configuration-requestor.spec.ts b/src/configuration-requestor.spec.ts index 746bb0f0..3e8bf341 100644 --- a/src/configuration-requestor.spec.ts +++ b/src/configuration-requestor.spec.ts @@ -203,5 +203,13 @@ describe('ConfigurationRequestor', () => { await configurationRequestor.fetchAndStoreConfigurations(); expect(fetchSpy).toHaveBeenCalledTimes(1); }); + + it('Requests bandits only when model versions are different', async () => { + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(2); // Once for UFC, another for bandits + + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(3); // Once just for UFC, bandits should be skipped + }); }); }); diff --git a/src/configuration-requestor.ts b/src/configuration-requestor.ts index 7c03f9bd..59e8432f 100644 --- a/src/configuration-requestor.ts +++ b/src/configuration-requestor.ts @@ -49,9 +49,6 @@ export default class ConfigurationRequestor { createdAt: configResponse.createdAt, }); - if (!this.banditModelConfigurationStore) { - throw new Error('Bandit parameters fetched but no bandit configuration store provided'); - } if ( this.requiresBanditModelConfigurationStoreUpdate( this.banditModelVersions, @@ -97,13 +94,9 @@ export default class ConfigurationRequestor { (banditReference: BanditReference) => banditReference.modelVersion, ); - referencedModelVersions.forEach((modelVersion) => { - if (!currentBanditModelVersions.includes(modelVersion)) { - return false; - } - }); - - return true; + return !referencedModelVersions.every((modelVersion) => + currentBanditModelVersions.includes(modelVersion), + ); } private async hydrateConfigurationStore( From 5e78633f9379ccfa0c0720456f4b6a10bd31d3ce Mon Sep 17 00:00:00 2001 From: selki Date: Wed, 20 Nov 2024 09:40:02 +0000 Subject: [PATCH 5/9] cleaned ufc response interface --- src/configuration-requestor.ts | 8 ++------ src/http-client.ts | 9 +-------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/configuration-requestor.ts b/src/configuration-requestor.ts index 59e8432f..f33abece 100644 --- a/src/configuration-requestor.ts +++ b/src/configuration-requestor.ts @@ -63,8 +63,8 @@ export default class ConfigurationRequestor { createdAt: configResponse.createdAt, }); - this.setBanditModelVersions( - this.getLoadedBanditModelVersionsFromStore(this.banditModelConfigurationStore), + this.banditModelVersions = this.getLoadedBanditModelVersionsFromStore( + this.banditModelConfigurationStore, ); } } @@ -82,10 +82,6 @@ export default class ConfigurationRequestor { ); } - private setBanditModelVersions(modelVersions: string[]) { - this.banditModelVersions = modelVersions; - } - private requiresBanditModelConfigurationStoreUpdate( currentBanditModelVersions: string[], banditReferences: Record, diff --git a/src/http-client.ts b/src/http-client.ts index 11ead305..21426b02 100644 --- a/src/http-client.ts +++ b/src/http-client.ts @@ -1,11 +1,5 @@ import ApiEndpoints from './api-endpoints'; -import { - BanditParameters, - BanditReference, - BanditVariation, - Environment, - Flag, -} from './interfaces'; +import { BanditParameters, BanditReference, Environment, Flag } from './interfaces'; export interface IQueryParams { apiKey: string; @@ -26,7 +20,6 @@ export interface IUniversalFlagConfigResponse { createdAt: string; // ISO formatted string environment: Environment; flags: Record; - bandits: Record; banditReferences: Record; } From f4704224831093e76035e40b8799f718302f9175 Mon Sep 17 00:00:00 2001 From: selki Date: Thu, 21 Nov 2024 15:49:17 +0000 Subject: [PATCH 6/9] beefed up tests for bandits polling --- src/configuration-requestor.spec.ts | 286 +++++++++++++++++++--------- 1 file changed, 201 insertions(+), 85 deletions(-) diff --git a/src/configuration-requestor.spec.ts b/src/configuration-requestor.spec.ts index 3e8bf341..90cc135d 100644 --- a/src/configuration-requestor.spec.ts +++ b/src/configuration-requestor.spec.ts @@ -9,8 +9,12 @@ import ApiEndpoints from './api-endpoints'; import ConfigurationRequestor from './configuration-requestor'; import { IConfigurationStore } from './configuration-store/configuration-store'; import { MemoryOnlyConfigurationStore } from './configuration-store/memory.store'; -import FetchHttpClient, { IHttpClient } from './http-client'; -import { BanditVariation, BanditParameters, Flag } from './interfaces'; +import FetchHttpClient, { + IBanditParametersResponse, + IHttpClient, + IUniversalFlagConfigResponse, +} from './http-client'; +import { BanditParameters, BanditVariation, Flag } from './interfaces'; describe('ConfigurationRequestor', () => { let flagStore: IConfigurationStore; @@ -111,13 +115,13 @@ describe('ConfigurationRequestor', () => { describe('Flags with bandits', () => { let fetchSpy: jest.Mock; - beforeAll(() => { + function initiateFetchSpy( + responseMockGenerator: ( + url: string, + ) => IUniversalFlagConfigResponse | IBanditParametersResponse, + ) { fetchSpy = jest.fn((url: string) => { - const responseFile = url.includes('bandits') - ? MOCK_BANDIT_MODELS_RESPONSE_FILE - : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; - const response = readMockUFCResponse(responseFile); - + const response = responseMockGenerator(url); return Promise.resolve({ ok: true, status: 200, @@ -125,91 +129,203 @@ describe('ConfigurationRequestor', () => { }); }) as jest.Mock; global.fetch = fetchSpy; - }); + } - it('Fetches and populates bandit parameters', async () => { - await configurationRequestor.fetchAndStoreConfigurations(); + function responseMockGenerator(url: string) { + const responseFile = url.includes('bandits') + ? MOCK_BANDIT_MODELS_RESPONSE_FILE + : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; + return readMockUFCResponse(responseFile); + } - expect(fetchSpy).toHaveBeenCalledTimes(2); // Once for UFC, another for bandits - - expect(flagStore.getKeys().length).toBeGreaterThanOrEqual(2); - expect(flagStore.get('banner_bandit_flag')).toBeDefined(); - expect(flagStore.get('cold_start_bandit')).toBeDefined(); - - expect(banditModelStore.getKeys().length).toBeGreaterThanOrEqual(2); - - const bannerBandit = banditModelStore.get('banner_bandit'); - expect(bannerBandit?.banditKey).toBe('banner_bandit'); - expect(bannerBandit?.modelName).toBe('falcon'); - expect(bannerBandit?.modelVersion).toBe('v123'); - const bannerModelData = bannerBandit?.modelData; - expect(bannerModelData?.gamma).toBe(1); - expect(bannerModelData?.defaultActionScore).toBe(0); - expect(bannerModelData?.actionProbabilityFloor).toBe(0); - const bannerCoefficients = bannerModelData?.coefficients || {}; - expect(Object.keys(bannerCoefficients).length).toBe(2); - - // Deep dive for the nike action - const nikeCoefficients = bannerCoefficients['nike']; - expect(nikeCoefficients.actionKey).toBe('nike'); - expect(nikeCoefficients.intercept).toBe(1); - expect(nikeCoefficients.actionNumericCoefficients).toHaveLength(1); - const nikeBrandAffinityCoefficient = nikeCoefficients.actionNumericCoefficients[0]; - expect(nikeBrandAffinityCoefficient.attributeKey).toBe('brand_affinity'); - expect(nikeBrandAffinityCoefficient.coefficient).toBe(1); - expect(nikeBrandAffinityCoefficient.missingValueCoefficient).toBe(-0.1); - expect(nikeCoefficients.actionCategoricalCoefficients).toHaveLength(2); - const nikeLoyaltyTierCoefficient = nikeCoefficients.actionCategoricalCoefficients[0]; - expect(nikeLoyaltyTierCoefficient.attributeKey).toBe('loyalty_tier'); - expect(nikeLoyaltyTierCoefficient.missingValueCoefficient).toBe(0); - expect(nikeLoyaltyTierCoefficient.valueCoefficients).toStrictEqual({ - gold: 4.5, - silver: 3.2, - bronze: 1.9, + describe('Fetching bandits', () => { + beforeAll(() => { + initiateFetchSpy(responseMockGenerator); }); - expect(nikeCoefficients.subjectNumericCoefficients).toHaveLength(1); - const nikeAccountAgeCoefficient = nikeCoefficients.subjectNumericCoefficients[0]; - expect(nikeAccountAgeCoefficient.attributeKey).toBe('account_age'); - expect(nikeAccountAgeCoefficient.coefficient).toBe(0.3); - expect(nikeAccountAgeCoefficient.missingValueCoefficient).toBe(0); - expect(nikeCoefficients.subjectCategoricalCoefficients).toHaveLength(1); - const nikeGenderIdentityCoefficient = nikeCoefficients.subjectCategoricalCoefficients[0]; - expect(nikeGenderIdentityCoefficient.attributeKey).toBe('gender_identity'); - expect(nikeGenderIdentityCoefficient.missingValueCoefficient).toBe(2.3); - expect(nikeGenderIdentityCoefficient.valueCoefficients).toStrictEqual({ - female: 0.5, - male: -0.5, + + it('Fetches and populates bandit parameters', async () => { + await configurationRequestor.fetchAndStoreConfigurations(); + + expect(fetchSpy).toHaveBeenCalledTimes(2); // Once for UFC, another for bandits + + expect(flagStore.getKeys().length).toBeGreaterThanOrEqual(2); + expect(flagStore.get('banner_bandit_flag')).toBeDefined(); + expect(flagStore.get('cold_start_bandit')).toBeDefined(); + + expect(banditModelStore.getKeys().length).toBeGreaterThanOrEqual(2); + + const bannerBandit = banditModelStore.get('banner_bandit'); + expect(bannerBandit?.banditKey).toBe('banner_bandit'); + expect(bannerBandit?.modelName).toBe('falcon'); + expect(bannerBandit?.modelVersion).toBe('v123'); + const bannerModelData = bannerBandit?.modelData; + expect(bannerModelData?.gamma).toBe(1); + expect(bannerModelData?.defaultActionScore).toBe(0); + expect(bannerModelData?.actionProbabilityFloor).toBe(0); + const bannerCoefficients = bannerModelData?.coefficients || {}; + expect(Object.keys(bannerCoefficients).length).toBe(2); + + // Deep dive for the nike action + const nikeCoefficients = bannerCoefficients['nike']; + expect(nikeCoefficients.actionKey).toBe('nike'); + expect(nikeCoefficients.intercept).toBe(1); + expect(nikeCoefficients.actionNumericCoefficients).toHaveLength(1); + const nikeBrandAffinityCoefficient = nikeCoefficients.actionNumericCoefficients[0]; + expect(nikeBrandAffinityCoefficient.attributeKey).toBe('brand_affinity'); + expect(nikeBrandAffinityCoefficient.coefficient).toBe(1); + expect(nikeBrandAffinityCoefficient.missingValueCoefficient).toBe(-0.1); + expect(nikeCoefficients.actionCategoricalCoefficients).toHaveLength(2); + const nikeLoyaltyTierCoefficient = nikeCoefficients.actionCategoricalCoefficients[0]; + expect(nikeLoyaltyTierCoefficient.attributeKey).toBe('loyalty_tier'); + expect(nikeLoyaltyTierCoefficient.missingValueCoefficient).toBe(0); + expect(nikeLoyaltyTierCoefficient.valueCoefficients).toStrictEqual({ + gold: 4.5, + silver: 3.2, + bronze: 1.9, + }); + expect(nikeCoefficients.subjectNumericCoefficients).toHaveLength(1); + const nikeAccountAgeCoefficient = nikeCoefficients.subjectNumericCoefficients[0]; + expect(nikeAccountAgeCoefficient.attributeKey).toBe('account_age'); + expect(nikeAccountAgeCoefficient.coefficient).toBe(0.3); + expect(nikeAccountAgeCoefficient.missingValueCoefficient).toBe(0); + expect(nikeCoefficients.subjectCategoricalCoefficients).toHaveLength(1); + const nikeGenderIdentityCoefficient = nikeCoefficients.subjectCategoricalCoefficients[0]; + expect(nikeGenderIdentityCoefficient.attributeKey).toBe('gender_identity'); + expect(nikeGenderIdentityCoefficient.missingValueCoefficient).toBe(2.3); + expect(nikeGenderIdentityCoefficient.valueCoefficients).toStrictEqual({ + female: 0.5, + male: -0.5, + }); + + // Just spot check the adidas parameters + expect(bannerCoefficients['adidas'].subjectNumericCoefficients).toHaveLength(0); + expect( + bannerCoefficients['adidas'].subjectCategoricalCoefficients[0].valueCoefficients[ + 'female' + ], + ).toBe(0); + + const coldStartBandit = banditModelStore.get('cold_start_bandit'); + expect(coldStartBandit?.banditKey).toBe('cold_start_bandit'); + expect(coldStartBandit?.modelName).toBe('falcon'); + expect(coldStartBandit?.modelVersion).toBe('cold start'); + const coldStartModelData = coldStartBandit?.modelData; + expect(coldStartModelData?.gamma).toBe(1); + expect(coldStartModelData?.defaultActionScore).toBe(0); + expect(coldStartModelData?.actionProbabilityFloor).toBe(0); + expect(coldStartModelData?.coefficients).toStrictEqual({}); }); - // Just spot check the adidas parameters - expect(bannerCoefficients['adidas'].subjectNumericCoefficients).toHaveLength(0); - expect( - bannerCoefficients['adidas'].subjectCategoricalCoefficients[0].valueCoefficients['female'], - ).toBe(0); - - const coldStartBandit = banditModelStore.get('cold_start_bandit'); - expect(coldStartBandit?.banditKey).toBe('cold_start_bandit'); - expect(coldStartBandit?.modelName).toBe('falcon'); - expect(coldStartBandit?.modelVersion).toBe('cold start'); - const coldStartModelData = coldStartBandit?.modelData; - expect(coldStartModelData?.gamma).toBe(1); - expect(coldStartModelData?.defaultActionScore).toBe(0); - expect(coldStartModelData?.actionProbabilityFloor).toBe(0); - expect(coldStartModelData?.coefficients).toStrictEqual({}); - }); + it('Will not fetch bandit parameters if there is no store', async () => { + configurationRequestor = new ConfigurationRequestor(httpClient, flagStore, null, null); + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); - it('Will not fetch bandit parameters if there is no store', async () => { - configurationRequestor = new ConfigurationRequestor(httpClient, flagStore, null, null); - await configurationRequestor.fetchAndStoreConfigurations(); - expect(fetchSpy).toHaveBeenCalledTimes(1); - }); + it('Should not fetch bandits if model version is un-changed', async () => { + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(2); // Once for UFC, another for bandits - it('Requests bandits only when model versions are different', async () => { - await configurationRequestor.fetchAndStoreConfigurations(); - expect(fetchSpy).toHaveBeenCalledTimes(2); // Once for UFC, another for bandits + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(3); // Once just for UFC, bandits should be skipped + }); - await configurationRequestor.fetchAndStoreConfigurations(); - expect(fetchSpy).toHaveBeenCalledTimes(3); // Once just for UFC, bandits should be skipped + it('Should fetch bandits if new bandit references model versions appeared', async () => { + let updateUFC = false; + await configurationRequestor.fetchAndStoreConfigurations(); + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(3); + + const customResponseMockGenerator = (url: string) => { + const responseFile = url.includes('bandits') + ? MOCK_BANDIT_MODELS_RESPONSE_FILE + : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; + + const response = readMockUFCResponse(responseFile); + + if (updateUFC === true) { + // this if is needed to appease linter + if (url.includes('config') && 'banditReferences' in response) { + response.banditReferences.warm_start = { + modelVersion: 'warm start', + flagVariations: [ + { + key: 'warm_start_bandit', + flagKey: 'warm_start_bandit_flag', + variationKey: 'warm_start_bandit', + variationValue: 'warm_start_bandit', + }, + ], + }; + } + + if (url.includes('bandits') && 'bandits' in response) { + response.bandits.warm_start = { + banditKey: 'warm_start_bandit', + modelName: 'pigeon', + modelVersion: 'warm start', + modelData: { + gamma: 1.0, + defaultActionScore: 0.0, + actionProbabilityFloor: 0.0, + coefficients: {}, + }, + }; + } + } + return response; + }; + updateUFC = true; + initiateFetchSpy(customResponseMockGenerator); + + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(2); // 2 because fetchSpy was re-initiated, 1UFC and 1bandits + + // let's check if warm start was hydrated properly! + const warm_start_bandit = banditModelStore.get('warm_start'); + expect(warm_start_bandit).toBeTruthy(); + expect(warm_start_bandit?.banditKey).toBe('warm_start_bandit'); + expect(warm_start_bandit?.modelVersion).toBe('warm start'); + expect(warm_start_bandit?.modelName).toBe('pigeon'); + expect(warm_start_bandit?.modelData.gamma).toBe(1); + expect(warm_start_bandit?.modelData.defaultActionScore).toBe(0); + expect(warm_start_bandit?.modelData.actionProbabilityFloor).toBe(0); + expect(warm_start_bandit?.modelData.coefficients).toStrictEqual({}); + }); + + it('Should not fetch bandits if bandit references model versions shrunk', async () => { + // Initial fetch + await configurationRequestor.fetchAndStoreConfigurations(); + + // Let's mock UFC response so that cold_start is no longer retrieved + const customResponseMockGenerator = (url: string) => { + const responseFile = url.includes('bandits') + ? MOCK_BANDIT_MODELS_RESPONSE_FILE + : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; + + const response = readMockUFCResponse(responseFile); + + if (url.includes('config') && 'banditReferences' in response) { + delete response.banditReferences.cold_start_bandit; + } + return response; + }; + + initiateFetchSpy(customResponseMockGenerator); + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(1); // only once for UFC + + // cold start should still be in memory + const warm_start_bandit = banditModelStore.get('cold_start_bandit'); + expect(warm_start_bandit).toBeTruthy(); + expect(warm_start_bandit?.banditKey).toBe('cold_start_bandit'); + expect(warm_start_bandit?.modelVersion).toBe('cold start'); + expect(warm_start_bandit?.modelName).toBe('falcon'); + expect(warm_start_bandit?.modelData.gamma).toBe(1); + expect(warm_start_bandit?.modelData.defaultActionScore).toBe(0); + expect(warm_start_bandit?.modelData.actionProbabilityFloor).toBe(0); + expect(warm_start_bandit?.modelData.coefficients).toStrictEqual({}); + }); }); }); }); From 8a304527c512f49dd02f0ee55aa3bb446a2fcde8 Mon Sep 17 00:00:00 2001 From: selki Date: Fri, 22 Nov 2024 12:03:45 +0000 Subject: [PATCH 7/9] more tests for bandits polling --- src/configuration-requestor.spec.ts | 184 +++++++++++++++++++++------- 1 file changed, 138 insertions(+), 46 deletions(-) diff --git a/src/configuration-requestor.spec.ts b/src/configuration-requestor.spec.ts index 7cfc5ed2..1984a888 100644 --- a/src/configuration-requestor.spec.ts +++ b/src/configuration-requestor.spec.ts @@ -230,6 +230,77 @@ describe('ConfigurationRequestor', () => { expect(fetchSpy).toHaveBeenCalledTimes(3); // Once just for UFC, bandits should be skipped }); + const warmStartBanditReference = { + modelVersion: 'warm start', + flagVariations: [ + { + key: 'warm_start_bandit', + flagKey: 'warm_start_bandit_flag', + variationKey: 'warm_start_bandit', + variationValue: 'warm_start_bandit', + }, + ], + }; + + const warmStartBanditParameters = { + banditKey: 'warm_start_bandit', + modelName: 'pigeon', + modelVersion: 'warm start', + modelData: { + gamma: 1.0, + defaultActionScore: 0.0, + actionProbabilityFloor: 0.0, + coefficients: {}, + }, + }; + + const coldStartBanditParameters = { + banditKey: 'cold_start_bandit', + modelName: 'falcon', + modelVersion: 'cold start', + modelData: { + gamma: 1.0, + defaultActionScore: 0.0, + actionProbabilityFloor: 0.0, + coefficients: {}, + }, + }; + + function expectBanditToBeInModelStore( + store: IConfigurationStore, + banditKey: string, + expectedBanditParameters: BanditParameters, + ) { + const bandit = store.get(banditKey); + expect(bandit).toBeTruthy(); + expect(bandit?.banditKey).toBe(expectedBanditParameters.banditKey); + expect(bandit?.modelVersion).toBe(expectedBanditParameters.modelVersion); + expect(bandit?.modelName).toBe(expectedBanditParameters.modelName); + expect(bandit?.modelData.gamma).toBe(expectedBanditParameters.modelData.gamma); + expect(bandit?.modelData.defaultActionScore).toBe( + expectedBanditParameters.modelData.defaultActionScore, + ); + expect(bandit?.modelData.actionProbabilityFloor).toBe( + expectedBanditParameters.modelData.actionProbabilityFloor, + ); + expect(bandit?.modelData.coefficients).toStrictEqual( + expectedBanditParameters.modelData.coefficients, + ); + } + + function injectWarmStartBanditToResponseByUrl( + url: string, + response: IUniversalFlagConfigResponse | IBanditParametersResponse, + ) { + if (url.includes('config') && 'banditReferences' in response) { + response.banditReferences.warm_start_bandit = warmStartBanditReference; + } + + if (url.includes('bandits') && 'bandits' in response) { + response.bandits.warm_start_bandit = warmStartBanditParameters; + } + } + it('Should fetch bandits if new bandit references model versions appeared', async () => { let updateUFC = false; await configurationRequestor.fetchAndStoreConfigurations(); @@ -244,34 +315,7 @@ describe('ConfigurationRequestor', () => { const response = readMockUFCResponse(responseFile); if (updateUFC === true) { - // this if is needed to appease linter - if (url.includes('config') && 'banditReferences' in response) { - response.banditReferences.warm_start = { - modelVersion: 'warm start', - flagVariations: [ - { - key: 'warm_start_bandit', - flagKey: 'warm_start_bandit_flag', - variationKey: 'warm_start_bandit', - variationValue: 'warm_start_bandit', - }, - ], - }; - } - - if (url.includes('bandits') && 'bandits' in response) { - response.bandits.warm_start = { - banditKey: 'warm_start_bandit', - modelName: 'pigeon', - modelVersion: 'warm start', - modelData: { - gamma: 1.0, - defaultActionScore: 0.0, - actionProbabilityFloor: 0.0, - coefficients: {}, - }, - }; - } + injectWarmStartBanditToResponseByUrl(url, response); } return response; }; @@ -282,15 +326,11 @@ describe('ConfigurationRequestor', () => { expect(fetchSpy).toHaveBeenCalledTimes(2); // 2 because fetchSpy was re-initiated, 1UFC and 1bandits // let's check if warm start was hydrated properly! - const warm_start_bandit = banditModelStore.get('warm_start'); - expect(warm_start_bandit).toBeTruthy(); - expect(warm_start_bandit?.banditKey).toBe('warm_start_bandit'); - expect(warm_start_bandit?.modelVersion).toBe('warm start'); - expect(warm_start_bandit?.modelName).toBe('pigeon'); - expect(warm_start_bandit?.modelData.gamma).toBe(1); - expect(warm_start_bandit?.modelData.defaultActionScore).toBe(0); - expect(warm_start_bandit?.modelData.actionProbabilityFloor).toBe(0); - expect(warm_start_bandit?.modelData.coefficients).toStrictEqual({}); + expectBanditToBeInModelStore( + banditModelStore, + 'warm_start_bandit', + warmStartBanditParameters, + ); }); it('Should not fetch bandits if bandit references model versions shrunk', async () => { @@ -316,15 +356,67 @@ describe('ConfigurationRequestor', () => { expect(fetchSpy).toHaveBeenCalledTimes(1); // only once for UFC // cold start should still be in memory - const warm_start_bandit = banditModelStore.get('cold_start_bandit'); - expect(warm_start_bandit).toBeTruthy(); - expect(warm_start_bandit?.banditKey).toBe('cold_start_bandit'); - expect(warm_start_bandit?.modelVersion).toBe('cold start'); - expect(warm_start_bandit?.modelName).toBe('falcon'); - expect(warm_start_bandit?.modelData.gamma).toBe(1); - expect(warm_start_bandit?.modelData.defaultActionScore).toBe(0); - expect(warm_start_bandit?.modelData.actionProbabilityFloor).toBe(0); - expect(warm_start_bandit?.modelData.coefficients).toStrictEqual({}); + expectBanditToBeInModelStore( + banditModelStore, + 'cold_start_bandit', + coldStartBanditParameters, + ); + }); + + /** + * 1. initial call - 1 fetch for ufc 1 for bandits + * 2. 2nd call - 1 fetch for ufc only; bandits unchanged + * 3. 3rd call - new bandit ref injected to UFC; 2 fetches, because new bandit appeared + * 4. 4th call - we remove a bandit from ufc; 1 fetch because there is no need to update. + * The bandit removed from UFC should still be in memory. + **/ + it('should fetch bandits based on banditReference change in UFC', async () => { + let injectWarmStart = false; + let removeColdStartBandit = false; + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(2); + + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(3); + + const customResponseMockGenerator = (url: string) => { + const responseFile = url.includes('bandits') + ? MOCK_BANDIT_MODELS_RESPONSE_FILE + : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; + const response = readMockUFCResponse(responseFile); + if (injectWarmStart === true) { + injectWarmStartBanditToResponseByUrl(url, response); + } else if ( + removeColdStartBandit === true && + 'banditReferences' in response && + url.includes('config') + ) { + delete response.banditReferences.cold_start_bandit; + } + return response; + }; + injectWarmStart = true; + initiateFetchSpy(customResponseMockGenerator); + + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(2); + expectBanditToBeInModelStore( + banditModelStore, + 'warm_start_bandit', + warmStartBanditParameters, + ); + + injectWarmStart = false; + removeColdStartBandit = true; + initiateFetchSpy(customResponseMockGenerator); + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(1); + + expectBanditToBeInModelStore( + banditModelStore, + 'cold_start_bandit', + coldStartBanditParameters, + ); }); }); }); From a91fa1ce4439ded784127719546940f8b0fdd956 Mon Sep 17 00:00:00 2001 From: selki Date: Mon, 25 Nov 2024 17:17:47 +0000 Subject: [PATCH 8/9] sdk version bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1005c46b..d1ae96c1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@eppo/js-client-sdk-common", - "version": "4.3.0", + "version": "4.5.0", "description": "Eppo SDK for client-side JavaScript applications (base for both web and react native)", "main": "dist/index.js", "files": [ From 99930721dd7ae51d5d7aaa7cc5ced0db7fd6cc44 Mon Sep 17 00:00:00 2001 From: selki Date: Mon, 25 Nov 2024 18:51:05 +0000 Subject: [PATCH 9/9] minor tests improvements for bandits polling --- package.json | 2 +- src/configuration-requestor.spec.ts | 360 ++++++++++++++-------------- 2 files changed, 184 insertions(+), 178 deletions(-) diff --git a/package.json b/package.json index d1ae96c1..c3e86451 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@eppo/js-client-sdk-common", - "version": "4.5.0", + "version": "4.4.1", "description": "Eppo SDK for client-side JavaScript applications (base for both web and react native)", "main": "dist/index.js", "files": [ diff --git a/src/configuration-requestor.spec.ts b/src/configuration-requestor.spec.ts index 1984a888..31d7f767 100644 --- a/src/configuration-requestor.spec.ts +++ b/src/configuration-requestor.spec.ts @@ -131,7 +131,7 @@ describe('ConfigurationRequestor', () => { global.fetch = fetchSpy; } - function responseMockGenerator(url: string) { + function defaultResponseMockGenerator(url: string) { const responseFile = url.includes('bandits') ? MOCK_BANDIT_MODELS_RESPONSE_FILE : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; @@ -140,7 +140,7 @@ describe('ConfigurationRequestor', () => { describe('Fetching bandits', () => { beforeAll(() => { - initiateFetchSpy(responseMockGenerator); + initiateFetchSpy(defaultResponseMockGenerator); }); it('Fetches and populates bandit parameters', async () => { @@ -230,193 +230,199 @@ describe('ConfigurationRequestor', () => { expect(fetchSpy).toHaveBeenCalledTimes(3); // Once just for UFC, bandits should be skipped }); - const warmStartBanditReference = { - modelVersion: 'warm start', - flagVariations: [ - { - key: 'warm_start_bandit', - flagKey: 'warm_start_bandit_flag', - variationKey: 'warm_start_bandit', - variationValue: 'warm_start_bandit', - }, - ], - }; - - const warmStartBanditParameters = { - banditKey: 'warm_start_bandit', - modelName: 'pigeon', - modelVersion: 'warm start', - modelData: { - gamma: 1.0, - defaultActionScore: 0.0, - actionProbabilityFloor: 0.0, - coefficients: {}, - }, - }; - - const coldStartBanditParameters = { - banditKey: 'cold_start_bandit', - modelName: 'falcon', - modelVersion: 'cold start', - modelData: { - gamma: 1.0, - defaultActionScore: 0.0, - actionProbabilityFloor: 0.0, - coefficients: {}, - }, - }; - - function expectBanditToBeInModelStore( - store: IConfigurationStore, - banditKey: string, - expectedBanditParameters: BanditParameters, - ) { - const bandit = store.get(banditKey); - expect(bandit).toBeTruthy(); - expect(bandit?.banditKey).toBe(expectedBanditParameters.banditKey); - expect(bandit?.modelVersion).toBe(expectedBanditParameters.modelVersion); - expect(bandit?.modelName).toBe(expectedBanditParameters.modelName); - expect(bandit?.modelData.gamma).toBe(expectedBanditParameters.modelData.gamma); - expect(bandit?.modelData.defaultActionScore).toBe( - expectedBanditParameters.modelData.defaultActionScore, - ); - expect(bandit?.modelData.actionProbabilityFloor).toBe( - expectedBanditParameters.modelData.actionProbabilityFloor, - ); - expect(bandit?.modelData.coefficients).toStrictEqual( - expectedBanditParameters.modelData.coefficients, - ); - } - - function injectWarmStartBanditToResponseByUrl( - url: string, - response: IUniversalFlagConfigResponse | IBanditParametersResponse, - ) { - if (url.includes('config') && 'banditReferences' in response) { - response.banditReferences.warm_start_bandit = warmStartBanditReference; - } - - if (url.includes('bandits') && 'bandits' in response) { - response.bandits.warm_start_bandit = warmStartBanditParameters; - } - } - - it('Should fetch bandits if new bandit references model versions appeared', async () => { - let updateUFC = false; - await configurationRequestor.fetchAndStoreConfigurations(); - await configurationRequestor.fetchAndStoreConfigurations(); - expect(fetchSpy).toHaveBeenCalledTimes(3); - - const customResponseMockGenerator = (url: string) => { - const responseFile = url.includes('bandits') - ? MOCK_BANDIT_MODELS_RESPONSE_FILE - : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; - - const response = readMockUFCResponse(responseFile); - - if (updateUFC === true) { - injectWarmStartBanditToResponseByUrl(url, response); - } - return response; + describe('Bandits polling', () => { + const warmStartBanditReference = { + modelVersion: 'warm start', + flagVariations: [ + { + key: 'warm_start_bandit', + flagKey: 'warm_start_bandit_flag', + variationKey: 'warm_start_bandit', + variationValue: 'warm_start_bandit', + }, + ], }; - updateUFC = true; - initiateFetchSpy(customResponseMockGenerator); - await configurationRequestor.fetchAndStoreConfigurations(); - expect(fetchSpy).toHaveBeenCalledTimes(2); // 2 because fetchSpy was re-initiated, 1UFC and 1bandits - - // let's check if warm start was hydrated properly! - expectBanditToBeInModelStore( - banditModelStore, - 'warm_start_bandit', - warmStartBanditParameters, - ); - }); + const warmStartBanditParameters = { + banditKey: 'warm_start_bandit', + modelName: 'pigeon', + modelVersion: 'warm start', + modelData: { + gamma: 1.0, + defaultActionScore: 0.0, + actionProbabilityFloor: 0.0, + coefficients: {}, + }, + }; - it('Should not fetch bandits if bandit references model versions shrunk', async () => { - // Initial fetch - await configurationRequestor.fetchAndStoreConfigurations(); + const coldStartBanditParameters = { + banditKey: 'cold_start_bandit', + modelName: 'falcon', + modelVersion: 'cold start', + modelData: { + gamma: 1.0, + defaultActionScore: 0.0, + actionProbabilityFloor: 0.0, + coefficients: {}, + }, + }; - // Let's mock UFC response so that cold_start is no longer retrieved - const customResponseMockGenerator = (url: string) => { - const responseFile = url.includes('bandits') - ? MOCK_BANDIT_MODELS_RESPONSE_FILE - : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; + afterAll(() => { + initiateFetchSpy(defaultResponseMockGenerator); + }); - const response = readMockUFCResponse(responseFile); + function expectBanditToBeInModelStore( + store: IConfigurationStore, + banditKey: string, + expectedBanditParameters: BanditParameters, + ) { + const bandit = store.get(banditKey); + expect(bandit).toBeTruthy(); + expect(bandit?.banditKey).toBe(expectedBanditParameters.banditKey); + expect(bandit?.modelVersion).toBe(expectedBanditParameters.modelVersion); + expect(bandit?.modelName).toBe(expectedBanditParameters.modelName); + expect(bandit?.modelData.gamma).toBe(expectedBanditParameters.modelData.gamma); + expect(bandit?.modelData.defaultActionScore).toBe( + expectedBanditParameters.modelData.defaultActionScore, + ); + expect(bandit?.modelData.actionProbabilityFloor).toBe( + expectedBanditParameters.modelData.actionProbabilityFloor, + ); + expect(bandit?.modelData.coefficients).toStrictEqual( + expectedBanditParameters.modelData.coefficients, + ); + } + function injectWarmStartBanditToResponseByUrl( + url: string, + response: IUniversalFlagConfigResponse | IBanditParametersResponse, + ) { if (url.includes('config') && 'banditReferences' in response) { - delete response.banditReferences.cold_start_bandit; + response.banditReferences.warm_start_bandit = warmStartBanditReference; } - return response; - }; - - initiateFetchSpy(customResponseMockGenerator); - await configurationRequestor.fetchAndStoreConfigurations(); - expect(fetchSpy).toHaveBeenCalledTimes(1); // only once for UFC - - // cold start should still be in memory - expectBanditToBeInModelStore( - banditModelStore, - 'cold_start_bandit', - coldStartBanditParameters, - ); - }); - /** - * 1. initial call - 1 fetch for ufc 1 for bandits - * 2. 2nd call - 1 fetch for ufc only; bandits unchanged - * 3. 3rd call - new bandit ref injected to UFC; 2 fetches, because new bandit appeared - * 4. 4th call - we remove a bandit from ufc; 1 fetch because there is no need to update. - * The bandit removed from UFC should still be in memory. - **/ - it('should fetch bandits based on banditReference change in UFC', async () => { - let injectWarmStart = false; - let removeColdStartBandit = false; - await configurationRequestor.fetchAndStoreConfigurations(); - expect(fetchSpy).toHaveBeenCalledTimes(2); - - await configurationRequestor.fetchAndStoreConfigurations(); - expect(fetchSpy).toHaveBeenCalledTimes(3); - - const customResponseMockGenerator = (url: string) => { - const responseFile = url.includes('bandits') - ? MOCK_BANDIT_MODELS_RESPONSE_FILE - : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; - const response = readMockUFCResponse(responseFile); - if (injectWarmStart === true) { - injectWarmStartBanditToResponseByUrl(url, response); - } else if ( - removeColdStartBandit === true && - 'banditReferences' in response && - url.includes('config') - ) { - delete response.banditReferences.cold_start_bandit; + if (url.includes('bandits') && 'bandits' in response) { + response.bandits.warm_start_bandit = warmStartBanditParameters; } - return response; - }; - injectWarmStart = true; - initiateFetchSpy(customResponseMockGenerator); + } - await configurationRequestor.fetchAndStoreConfigurations(); - expect(fetchSpy).toHaveBeenCalledTimes(2); - expectBanditToBeInModelStore( - banditModelStore, - 'warm_start_bandit', - warmStartBanditParameters, - ); - - injectWarmStart = false; - removeColdStartBandit = true; - initiateFetchSpy(customResponseMockGenerator); - await configurationRequestor.fetchAndStoreConfigurations(); - expect(fetchSpy).toHaveBeenCalledTimes(1); + it('Should fetch bandits if new bandit references model versions appeared', async () => { + let updateUFC = false; + await configurationRequestor.fetchAndStoreConfigurations(); + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(3); + + const customResponseMockGenerator = (url: string) => { + const responseFile = url.includes('bandits') + ? MOCK_BANDIT_MODELS_RESPONSE_FILE + : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; + + const response = readMockUFCResponse(responseFile); + + if (updateUFC === true) { + injectWarmStartBanditToResponseByUrl(url, response); + } + return response; + }; + updateUFC = true; + initiateFetchSpy(customResponseMockGenerator); + + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(2); // 2 because fetchSpy was re-initiated, 1UFC and 1bandits + + // let's check if warm start was hydrated properly! + expectBanditToBeInModelStore( + banditModelStore, + 'warm_start_bandit', + warmStartBanditParameters, + ); + }); + + it('Should not fetch bandits if bandit references model versions shrunk', async () => { + // Initial fetch + await configurationRequestor.fetchAndStoreConfigurations(); + + // Let's mock UFC response so that cold_start is no longer retrieved + const customResponseMockGenerator = (url: string) => { + const responseFile = url.includes('bandits') + ? MOCK_BANDIT_MODELS_RESPONSE_FILE + : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; + + const response = readMockUFCResponse(responseFile); + + if (url.includes('config') && 'banditReferences' in response) { + delete response.banditReferences.cold_start_bandit; + } + return response; + }; + + initiateFetchSpy(customResponseMockGenerator); + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(1); // only once for UFC + + // cold start should still be in memory + expectBanditToBeInModelStore( + banditModelStore, + 'cold_start_bandit', + coldStartBanditParameters, + ); + }); - expectBanditToBeInModelStore( - banditModelStore, - 'cold_start_bandit', - coldStartBanditParameters, - ); + /** + * 1. initial call - 1 fetch for ufc 1 for bandits + * 2. 2nd call - 1 fetch for ufc only; bandits unchanged + * 3. 3rd call - new bandit ref injected to UFC; 2 fetches, because new bandit appeared + * 4. 4th call - we remove a bandit from ufc; 1 fetch because there is no need to update. + * The bandit removed from UFC should still be in memory. + **/ + it('should fetch bandits based on banditReference change in UFC', async () => { + let injectWarmStart = false; + let removeColdStartBandit = false; + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(2); + + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(3); + + const customResponseMockGenerator = (url: string) => { + const responseFile = url.includes('bandits') + ? MOCK_BANDIT_MODELS_RESPONSE_FILE + : MOCK_FLAGS_WITH_BANDITS_RESPONSE_FILE; + const response = readMockUFCResponse(responseFile); + if (injectWarmStart === true) { + injectWarmStartBanditToResponseByUrl(url, response); + } else if ( + removeColdStartBandit === true && + 'banditReferences' in response && + url.includes('config') + ) { + delete response.banditReferences.cold_start_bandit; + } + return response; + }; + injectWarmStart = true; + initiateFetchSpy(customResponseMockGenerator); + + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(2); + expectBanditToBeInModelStore( + banditModelStore, + 'warm_start_bandit', + warmStartBanditParameters, + ); + + injectWarmStart = false; + removeColdStartBandit = true; + initiateFetchSpy(customResponseMockGenerator); + await configurationRequestor.fetchAndStoreConfigurations(); + expect(fetchSpy).toHaveBeenCalledTimes(1); + + expectBanditToBeInModelStore( + banditModelStore, + 'cold_start_bandit', + coldStartBanditParameters, + ); + }); }); }); });