diff --git a/package-lock.json b/package-lock.json index bd6659760..93964454a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,8 +17,6 @@ "@sentry/node": "^7.12.1", "@types/basic-auth": "^1.1.3", "@types/bluebird": "^3.5.36", - "@types/cache-manager": "^4.0.2", - "@types/cache-manager-ioredis": "^2.0.2", "@types/common-tags": "^1.8.1", "@types/compressible": "^2.0.0", "@types/compression-next": "^1.0.0", @@ -53,7 +51,7 @@ "bluebird": "^3.7.2", "body-parser": "^1.20.0", "cache-manager": "^5.0.0", - "cache-manager-ioredis": "^2.1.0", + "cache-manager-ioredis-yet": "^1.0.0", "common-tags": "^1.8.2", "compressible": "^2.0.18", "compression": "^1.7.4", @@ -800,20 +798,6 @@ "@types/node": "*" } }, - "node_modules/@types/cache-manager": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/cache-manager/-/cache-manager-4.0.2.tgz", - "integrity": "sha512-fT5FMdzsiSX0AbgnS5gDvHl2Nco0h5zYyjwDQy4yPC7Ww6DeGMVKPRqIZtg9HOXDV2kkc18SL1B0N8f0BecrCA==" - }, - "node_modules/@types/cache-manager-ioredis": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@types/cache-manager-ioredis/-/cache-manager-ioredis-2.0.2.tgz", - "integrity": "sha512-mcn1k5bCJ/iAIymLRq4d6KSvEdrdspuo8KEIIPkPEl5WP+2CC7NWtRJmgFBoNqCgNxr++B62kOmIH1HYXAiBdA==", - "dependencies": { - "@types/cache-manager": "*", - "@types/ioredis": "*" - } - }, "node_modules/@types/caseless": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", @@ -953,14 +937,6 @@ "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.15.6.tgz", "integrity": "sha512-pYVNNJ+winC4aek+lZp93sIKxnXt5qMkuKmaqS3WGuTq0Bw1ZDYNBgzG5kkdtwcv+GmYJGo3yEg6z2cKKAiEdw==" }, - "node_modules/@types/ioredis": { - "version": "4.28.10", - "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz", - "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/json-schema": { "version": "7.0.11", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", @@ -2029,56 +2005,16 @@ "lru-cache": "^7.14.0" } }, - "node_modules/cache-manager-ioredis": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cache-manager-ioredis/-/cache-manager-ioredis-2.1.0.tgz", - "integrity": "sha512-TCxbp9ceuFveTKWuNaCX8QjoC41rAlHen4s63u9Yd+iXlw3efYmimc/u935PKPxSdhkXpnMes4mxtK3/yb0L4g==", + "node_modules/cache-manager-ioredis-yet": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cache-manager-ioredis-yet/-/cache-manager-ioredis-yet-1.0.0.tgz", + "integrity": "sha512-iUjTvYGchVOEQPI2yyGnUbFHZd2G5ltrFcO94i3ThrISybBul3P15ZLCwplUwvJjbgQ/sj8FIn+ppliOv4afOA==", "dependencies": { - "ioredis": "^4.14.1" + "cache-manager": "^5.1.0", + "ioredis": "^5.2.3" }, "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/cache-manager-ioredis/node_modules/denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==", - "engines": { - "node": ">=0.10" - } - }, - "node_modules/cache-manager-ioredis/node_modules/ioredis": { - "version": "4.28.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", - "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", - "dependencies": { - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.1", - "denque": "^1.1.0", - "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", - "lodash.isarguments": "^3.1.0", - "p-map": "^2.1.0", - "redis-commands": "1.7.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/ioredis" - } - }, - "node_modules/cache-manager-ioredis/node_modules/p-map": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", - "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", - "engines": { - "node": ">=6" + "node": ">= 16.17.0" } }, "node_modules/call-bind": { @@ -4823,11 +4759,6 @@ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" }, - "node_modules/lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" - }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -9442,20 +9373,6 @@ "@types/node": "*" } }, - "@types/cache-manager": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/cache-manager/-/cache-manager-4.0.2.tgz", - "integrity": "sha512-fT5FMdzsiSX0AbgnS5gDvHl2Nco0h5zYyjwDQy4yPC7Ww6DeGMVKPRqIZtg9HOXDV2kkc18SL1B0N8f0BecrCA==" - }, - "@types/cache-manager-ioredis": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@types/cache-manager-ioredis/-/cache-manager-ioredis-2.0.2.tgz", - "integrity": "sha512-mcn1k5bCJ/iAIymLRq4d6KSvEdrdspuo8KEIIPkPEl5WP+2CC7NWtRJmgFBoNqCgNxr++B62kOmIH1HYXAiBdA==", - "requires": { - "@types/cache-manager": "*", - "@types/ioredis": "*" - } - }, "@types/caseless": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz", @@ -9595,14 +9512,6 @@ "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.15.6.tgz", "integrity": "sha512-pYVNNJ+winC4aek+lZp93sIKxnXt5qMkuKmaqS3WGuTq0Bw1ZDYNBgzG5kkdtwcv+GmYJGo3yEg6z2cKKAiEdw==" }, - "@types/ioredis": { - "version": "4.28.10", - "resolved": "https://registry.npmjs.org/@types/ioredis/-/ioredis-4.28.10.tgz", - "integrity": "sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==", - "requires": { - "@types/node": "*" - } - }, "@types/json-schema": { "version": "7.0.11", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz", @@ -10522,42 +10431,13 @@ "lru-cache": "^7.14.0" } }, - "cache-manager-ioredis": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cache-manager-ioredis/-/cache-manager-ioredis-2.1.0.tgz", - "integrity": "sha512-TCxbp9ceuFveTKWuNaCX8QjoC41rAlHen4s63u9Yd+iXlw3efYmimc/u935PKPxSdhkXpnMes4mxtK3/yb0L4g==", + "cache-manager-ioredis-yet": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cache-manager-ioredis-yet/-/cache-manager-ioredis-yet-1.0.0.tgz", + "integrity": "sha512-iUjTvYGchVOEQPI2yyGnUbFHZd2G5ltrFcO94i3ThrISybBul3P15ZLCwplUwvJjbgQ/sj8FIn+ppliOv4afOA==", "requires": { - "ioredis": "^4.14.1" - }, - "dependencies": { - "denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==" - }, - "ioredis": { - "version": "4.28.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.28.5.tgz", - "integrity": "sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==", - "requires": { - "cluster-key-slot": "^1.1.0", - "debug": "^4.3.1", - "denque": "^1.1.0", - "lodash.defaults": "^4.2.0", - "lodash.flatten": "^4.4.0", - "lodash.isarguments": "^3.1.0", - "p-map": "^2.1.0", - "redis-commands": "1.7.0", - "redis-errors": "^1.2.0", - "redis-parser": "^3.0.0", - "standard-as-callback": "^2.1.0" - } - }, - "p-map": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", - "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==" - } + "cache-manager": "^5.1.0", + "ioredis": "^5.2.3" } }, "call-bind": { @@ -12614,11 +12494,6 @@ "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==" }, - "lodash.flatten": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", - "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==" - }, "lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", diff --git a/package.json b/package.json index 5e8e346ae..6c0b1cb18 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,6 @@ "@sentry/node": "^7.12.1", "@types/basic-auth": "^1.1.3", "@types/bluebird": "^3.5.36", - "@types/cache-manager": "^4.0.2", - "@types/cache-manager-ioredis": "^2.0.2", "@types/common-tags": "^1.8.1", "@types/compressible": "^2.0.0", "@types/compression-next": "^1.0.0", @@ -75,7 +73,7 @@ "bluebird": "^3.7.2", "body-parser": "^1.20.0", "cache-manager": "^5.0.0", - "cache-manager-ioredis": "^2.1.0", + "cache-manager-ioredis-yet": "^1.0.0", "common-tags": "^1.8.2", "compressible": "^2.0.18", "compression": "^1.7.4", diff --git a/src/infra/cache/multi-level-memoizee.ts b/src/infra/cache/multi-level-memoizee.ts index 6486ed099..83a69591b 100644 --- a/src/infra/cache/multi-level-memoizee.ts +++ b/src/infra/cache/multi-level-memoizee.ts @@ -104,7 +104,7 @@ export function multiCacheMemoizee< }; const multiCacheOpts: Parameters[1] = { - default: { ...convertToMultiStoreOpts(opts), isCacheableValue: () => true }, + default: { ...convertToMultiStoreOpts(opts), isCacheable: () => true }, global: convertToMultiStoreOpts({ ...opts, ...sharedCacheOpts }), }; @@ -146,7 +146,16 @@ function multiCache Promise>( const valueToCache = await fn(...args); // Some caches (eg redis) cannot handle caching undefined/null so we convert it to the `undefinedAs` proxy value // which will be used when storing in the cache and then convert it back to undefined when retrieving from the cache - return valueToCache === undefined ? undefinedAs : valueToCache; + if (valueToCache === undefined) { + // Just to make TS happy, since the typings should already be protecting us. + if (undefinedAs === undefined) { + throw new Error( + 'Multilevel cache detected undefined value while the undefinedAs was not defined', + ); + } + return undefinedAs; + } + return valueToCache; }); return valueFromCache === undefinedAs ? undefined : valueFromCache; }; diff --git a/src/infra/cache/multi-level-store.ts b/src/infra/cache/multi-level-store.ts index 0b03ddf9c..f15485629 100644 --- a/src/infra/cache/multi-level-store.ts +++ b/src/infra/cache/multi-level-store.ts @@ -1,13 +1,21 @@ import * as _ from 'lodash'; import * as cacheManager from 'cache-manager'; -import redisStore = require('cache-manager-ioredis'); +import { redisStore } from 'cache-manager-ioredis-yet'; import { version } from '../../lib/config'; import { Defined } from '.'; import { getRedisOptions } from '../redis/config'; const usedCacheKeys: Dictionary = {}; -export type MultiStoreOpt = Pick & { +export interface CacheOptions extends Pick { + /** @deprecated User isCacheable instead */ + isCacheableValue?(value: unknown): boolean; +} + +export type MultiStoreOpt = ( + | Required> + | Required> +) & { refreshThreshold?: number; }; @@ -20,9 +28,9 @@ export type MultiStoreOpt = Pick & { export function createMultiLevelStore( cacheKey: string, opts: - | (MultiStoreOpt & cacheManager.CacheOptions) + | (MultiStoreOpt & CacheOptions) | { - default: MultiStoreOpt & cacheManager.CacheOptions; + default: MultiStoreOpt & CacheOptions; local?: MultiStoreOpt | false; /** * The global store will ignore the `max` anyway, so avoiding passing it in will help reduce confusion @@ -34,7 +42,7 @@ export function createMultiLevelStore( get: (key: string) => Promise; set: (key: string, value: T) => Promise; delete: (key: string) => Promise; - wrap: (key: string, fn: () => T) => Promise; + wrap: (key: string, fn: () => Promise) => Promise; } { if (usedCacheKeys[cacheKey] === true) { throw new Error(`Cache key '${cacheKey}' has already been taken`); @@ -44,36 +52,40 @@ export function createMultiLevelStore( if (!('default' in opts)) { opts = { default: opts }; } - const { default: baseOpts, local, global } = opts; - const { isCacheableValue } = baseOpts; - const memoryCache = + const { + default: { + isCacheableValue: $isCacheableValue, + isCacheable: $isCacheable, + ...baseOpts + }, + local, + global, + } = opts; + const isCacheable = $isCacheable ?? $isCacheableValue; + const memoryCachePromise = local === false ? undefined - : cacheManager.caching({ ...baseOpts, ...local, store: 'memory' }); + : cacheManager.caching('memory', { ...baseOpts, ...local, isCacheable }); - let cacheOpts: cacheManager.StoreConfig & cacheManager.CacheOptions = { + const redisOpts = getRedisOptions(); + const cacheOpts: NonNullable[0]> = { ...baseOpts, ...global, - store: redisStore, - isCacheableValue: (v) => + ...('nodes' in redisOpts ? { clusterConfig: redisOpts } : { redisOpts }), + isCacheable: (v) => // redis cannot cache undefined/null values whilst others can, so we explicitly mark those as uncacheable - v != null && (isCacheableValue == null || isCacheableValue(v) === true), + v != null && (isCacheable == null || isCacheable(v) === true), }; - const redisOpts = getRedisOptions(); - - if ('nodes' in redisOpts) { - // @ts-expect-error: This shouldn't really need to be passed here but due to a quirk of cache-manager-ioredis it expects - // all the store opts to be stored on the created redis instance - redisOpts.options.isCacheableValue = cacheOpts.isCacheableValue; - cacheOpts.clusterConfig = redisOpts; - } else { - cacheOpts = { ...cacheOpts, ...redisOpts }; - } - const redisCache = cacheManager.caching(cacheOpts); - const cache = memoryCache - ? cacheManager.multiCaching([memoryCache, redisCache]) - : redisCache; + const cachePromise = (async () => { + const [memoryCache, redisCache] = await Promise.all([ + memoryCachePromise, + cacheManager.caching(redisStore, cacheOpts), + ]); + return memoryCache + ? cacheManager.multiCaching([memoryCache, redisCache]) + : redisCache; + })(); let keyPrefix: string; const getKey = (key: string) => { @@ -87,19 +99,19 @@ export function createMultiLevelStore( return { get: async (key) => { const fullKey = getKey(key); - return await cache.get(fullKey); + return await (await cachePromise).get(fullKey); }, set: async (key, value) => { const fullKey = getKey(key); - await cache.set(fullKey, value); + await (await cachePromise).set(fullKey, value); }, delete: async (key) => { const fullKey = getKey(key); - await cache.del(fullKey); + await (await cachePromise).del(fullKey); }, wrap: async (key, fn) => { const fullKey = getKey(key); - return await cache.wrap(fullKey, fn); + return await (await cachePromise).wrap(fullKey, fn); }, }; }