From e7fa79222ad5f602365c0772dc4e8a4e62e419cb Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Wed, 28 Feb 2024 09:53:59 +0100 Subject: [PATCH 01/23] Initial migration of Onyx.js --- lib/{Onyx.js => Onyx-old.js} | 1 - lib/Onyx.d.ts | 315 ----- lib/Onyx.ts | 1641 ++++++++++++++++++++++++ lib/storage/__mocks__/index.ts | 3 - lib/storage/providers/IDBKeyVal.ts | 2 - lib/storage/providers/SQLiteStorage.ts | 3 - lib/storage/providers/types.ts | 7 +- 7 files changed, 1642 insertions(+), 330 deletions(-) rename lib/{Onyx.js => Onyx-old.js} (99%) delete mode 100644 lib/Onyx.d.ts create mode 100644 lib/Onyx.ts diff --git a/lib/Onyx.js b/lib/Onyx-old.js similarity index 99% rename from lib/Onyx.js rename to lib/Onyx-old.js index 6a8a4eb0..a45c5235 100644 --- a/lib/Onyx.js +++ b/lib/Onyx-old.js @@ -1625,7 +1625,6 @@ const Onyx = { removeFromEvictionBlockList, isSafeEvictionKey, METHOD, - setMemoryOnlyKeys, tryGetCachedValue, hasPendingMergeForKey, }; diff --git a/lib/Onyx.d.ts b/lib/Onyx.d.ts deleted file mode 100644 index b1c0cbe4..00000000 --- a/lib/Onyx.d.ts +++ /dev/null @@ -1,315 +0,0 @@ -import {Component} from 'react'; -import * as Logger from './Logger'; -import {CollectionKey, CollectionKeyBase, DeepRecord, KeyValueMapping, NullishDeep, OnyxCollection, OnyxEntry, OnyxKey} from './types'; - -/** - * Represents a mapping object where each `OnyxKey` maps to either a value of its corresponding type in `KeyValueMapping` or `null`. - * - * It's very similar to `KeyValueMapping` but this type accepts using `null` as well. - */ -type NullableKeyValueMapping = { - [TKey in OnyxKey]: OnyxEntry; -}; - -/** - * Represents the base options used in `Onyx.connect()` method. - */ -type BaseConnectOptions = { - statePropertyName?: string; - withOnyxInstance?: Component; - initWithStoredValues?: boolean; -}; - -/** - * Represents the options used in `Onyx.connect()` method. - * The type is built from `BaseConnectOptions` and extended to handle key/callback related options. - * It includes two different forms, depending on whether we are waiting for a collection callback or not. - * - * If `waitForCollectionCallback` is `true`, it expects `key` to be a Onyx collection key and `callback` will be triggered with the whole collection - * and will pass `value` as an `OnyxCollection`. - * - * - * If `waitForCollectionCallback` is `false` or not specified, the `key` can be any Onyx key and `callback` will be triggered with updates of each collection item - * and will pass `value` as an `OnyxEntry`. - */ -type ConnectOptions = BaseConnectOptions & - ( - | { - key: TKey extends CollectionKey ? TKey : never; - callback?: (value: OnyxCollection) => void; - waitForCollectionCallback: true; - } - | { - key: TKey; - callback?: (value: OnyxEntry, key: TKey) => void; - waitForCollectionCallback?: false; - } - ); - -/** - * Represents a mapping between Onyx collection keys and their respective values. - * - * It helps to enforce that a Onyx collection key should not be without suffix (e.g. should always be of the form `${TKey}${string}`), - * and to map each Onyx collection key with suffix to a value of type `TValue`. - * - * Also, the `TMap` type is inferred automatically in `mergeCollection()` method and represents - * the object of collection keys/values specified in the second parameter of the method. - */ -type Collection = { - [MapK in keyof TMap]: MapK extends `${TKey}${string}` - ? MapK extends `${TKey}` - ? never // forbids empty id - : TValue - : never; -}; - -/** - * Represents different kinds of updates that can be passed to `Onyx.update()` method. It is a discriminated union of - * different update methods (`SET`, `MERGE`, `MERGE_COLLECTION`), each with their own key and value structure. - */ -type OnyxUpdate = - | { - [TKey in OnyxKey]: - | { - onyxMethod: typeof METHOD.SET; - key: TKey; - value: OnyxEntry; - } - | { - onyxMethod: typeof METHOD.MERGE; - key: TKey; - value: OnyxEntry>; - }; - }[OnyxKey] - | { - [TKey in CollectionKeyBase]: { - onyxMethod: typeof METHOD.MERGE_COLLECTION; - key: TKey; - value: Record<`${TKey}${string}`, NullishDeep>; - }; - }[CollectionKeyBase]; - -/** - * Represents the options used in `Onyx.init()` method. - */ -type InitOptions = { - keys?: DeepRecord; - initialKeyStates?: Partial; - safeEvictionKeys?: OnyxKey[]; - maxCachedKeysCount?: number; - captureMetrics?: boolean; - shouldSyncMultipleInstances?: boolean; - debugSetState?: boolean; -}; - -declare const METHOD: { - readonly SET: 'set'; - readonly MERGE: 'merge'; - readonly MERGE_COLLECTION: 'mergecollection'; - readonly MULTI_SET: 'multiset'; - readonly CLEAR: 'clear'; -}; - -/** - * Returns current key names stored in persisted storage - */ -declare function getAllKeys(): Promise>; - -/** - * Checks to see if this key has been flagged as - * safe for removal. - */ -declare function isSafeEvictionKey(testKey: OnyxKey): boolean; - -/** - * Removes a key previously added to this list - * which will enable it to be deleted again. - */ -declare function removeFromEvictionBlockList(key: OnyxKey, connectionID: number): void; - -/** - * Keys added to this list can never be deleted. - */ -declare function addToEvictionBlockList(key: OnyxKey, connectionID: number): void; - -/** - * Subscribes a react component's state directly to a store key - * - * @example - * const connectionID = Onyx.connect({ - * key: ONYXKEYS.SESSION, - * callback: onSessionChange, - * }); - * - * @param mapping the mapping information to connect Onyx to the components state - * @param mapping.key ONYXKEY to subscribe to - * @param [mapping.statePropertyName] the name of the property in the state to connect the data to - * @param [mapping.withOnyxInstance] whose setState() method will be called with any changed data - * This is used by React components to connect to Onyx - * @param [mapping.callback] a method that will be called with changed data - * This is used by any non-React code to connect to Onyx - * @param [mapping.initWithStoredValues] If set to false, then no data will be prefilled into the - * component - * @param [mapping.waitForCollectionCallback] If set to true, it will return the entire collection to the callback as a single object - * @returns an ID to use when calling disconnect - */ -declare function connect(mapping: ConnectOptions): number; - -/** - * Remove the listener for a react component - * @example - * Onyx.disconnect(connectionID); - * - * @param connectionID unique id returned by call to Onyx.connect() - */ -declare function disconnect(connectionID: number, keyToRemoveFromEvictionBlocklist?: OnyxKey): void; - -/** - * Write a value to our store with the given key - * - * @param key ONYXKEY to set - * @param value value to store - */ -declare function set(key: TKey, value: OnyxEntry): Promise; - -/** - * Sets multiple keys and values - * - * @example Onyx.multiSet({'key1': 'a', 'key2': 'b'}); - * - * @param data object keyed by ONYXKEYS and the values to set - */ -declare function multiSet(data: Partial): Promise; - -/** - * Merge a new value into an existing value at a key. - * - * The types of values that can be merged are `Object` and `Array`. To set another type of value use `Onyx.set()`. Merge - * behavior uses lodash/merge under the hood for `Object` and simple concatenation for `Array`. However, it's important - * to note that if you have an array value property on an `Object` that the default behavior of lodash/merge is not to - * concatenate. See here: https://github.com/lodash/lodash/issues/2872 - * - * Calls to `Onyx.merge()` are batched so that any calls performed in a single tick will stack in a queue and get - * applied in the order they were called. Note: `Onyx.set()` calls do not work this way so use caution when mixing - * `Onyx.merge()` and `Onyx.set()`. - * - * @example - * Onyx.merge(ONYXKEYS.EMPLOYEE_LIST, ['Joe']); // -> ['Joe'] - * Onyx.merge(ONYXKEYS.EMPLOYEE_LIST, ['Jack']); // -> ['Joe', 'Jack'] - * Onyx.merge(ONYXKEYS.POLICY, {id: 1}); // -> {id: 1} - * Onyx.merge(ONYXKEYS.POLICY, {name: 'My Workspace'}); // -> {id: 1, name: 'My Workspace'} - * - * @param key ONYXKEYS key - * @param value Object or Array value to merge - */ -declare function merge(key: TKey, value: OnyxEntry>): Promise; - -/** - * Clear out all the data in the store - * - * Note that calling Onyx.clear() and then Onyx.set() on a key with a default - * key state may store an unexpected value in Storage. - * - * E.g. - * Onyx.clear(); - * Onyx.set(ONYXKEYS.DEFAULT_KEY, 'default'); - * Storage.getItem(ONYXKEYS.DEFAULT_KEY) - * .then((storedValue) => console.log(storedValue)); - * null is logged instead of the expected 'default' - * - * Onyx.set() might call Storage.setItem() before Onyx.clear() calls - * Storage.setItem(). Use Onyx.merge() instead if possible. Onyx.merge() calls - * Onyx.get(key) before calling Storage.setItem() via Onyx.set(). - * Storage.setItem() from Onyx.clear() will have already finished and the merged - * value will be saved to storage after the default value. - * - * @param keysToPreserve is a list of ONYXKEYS that should not be cleared with the rest of the data - */ -declare function clear(keysToPreserve?: OnyxKey[]): Promise; - -/** - * Merges a collection based on their keys - * - * Note that both `TKey` and `TMap` types are inferred automatically, `TKey` being the - * collection key specified in the first parameter and `TMap` being the object of - * collection keys/values specified in the second parameter. - * - * @example - * - * Onyx.mergeCollection(ONYXKEYS.COLLECTION.REPORT, { - * [`${ONYXKEYS.COLLECTION.REPORT}1`]: report1, - * [`${ONYXKEYS.COLLECTION.REPORT}2`]: report2, - * }); - * - * @param collectionKey e.g. `ONYXKEYS.COLLECTION.REPORT` - * @param collection Object collection keyed by individual collection member keys and values - */ -declare function mergeCollection(collectionKey: TKey, collection: Collection>): Promise; - -/** - * Insert API responses and lifecycle data into Onyx - * - * @param data An array of update objects - * @returns resolves when all operations are complete - */ -declare function update(data: OnyxUpdate[]): Promise; - -/** - * Initialize the store with actions and listening for storage events - * - * @param [options={}] config object - * @param [options.keys={}] `ONYXKEYS` constants object - * @param [options.initialKeyStates={}] initial data to set when `init()` and `clear()` is called - * @param [options.safeEvictionKeys=[]] This is an array of keys - * (individual or collection patterns) that when provided to Onyx are flagged - * as "safe" for removal. Any components subscribing to these keys must also - * implement a canEvict option. See the README for more info. - * @param [options.maxCachedKeysCount=55] Sets how many recent keys should we try to keep in cache - * Setting this to 0 would practically mean no cache - * We try to free cache when we connect to a safe eviction key - * @param [options.captureMetrics] Enables Onyx benchmarking and exposes the get/print/reset functions - * @param [options.shouldSyncMultipleInstances] Auto synchronize storage events between multiple instances - * of Onyx running in different tabs/windows. Defaults to true for platforms that support local storage (web/desktop) - * @param [options.debugSetState] Enables debugging setState() calls to connected components. - * @example - * Onyx.init({ - * keys: ONYXKEYS, - * initialKeyStates: { - * [ONYXKEYS.SESSION]: {loading: false}, - * }, - * }); - */ -declare function init(config?: InitOptions): void; - -/** - * @private - */ -declare function hasPendingMergeForKey(key: OnyxKey): boolean; - -/** - * When set these keys will not be persisted to storage - */ -declare function setMemoryOnlyKeys(keyList: OnyxKey[]): void; - -declare const Onyx: { - connect: typeof connect; - disconnect: typeof disconnect; - set: typeof set; - multiSet: typeof multiSet; - merge: typeof merge; - mergeCollection: typeof mergeCollection; - hasPendingMergeForKey: typeof hasPendingMergeForKey; - update: typeof update; - clear: typeof clear; - getAllKeys: typeof getAllKeys; - init: typeof init; - registerLogger: typeof Logger.registerLogger; - addToEvictionBlockList: typeof addToEvictionBlockList; - removeFromEvictionBlockList: typeof removeFromEvictionBlockList; - isSafeEvictionKey: typeof isSafeEvictionKey; - METHOD: typeof METHOD; - setMemoryOnlyKeys: typeof setMemoryOnlyKeys; -}; - -export default Onyx; -export {OnyxUpdate, ConnectOptions}; diff --git a/lib/Onyx.ts b/lib/Onyx.ts new file mode 100644 index 00000000..46e70a3f --- /dev/null +++ b/lib/Onyx.ts @@ -0,0 +1,1641 @@ +/* eslint-disable no-continue */ +import type {Component} from 'react'; +import {deepEqual} from 'fast-equals'; +import _ from 'underscore'; +import * as Logger from './Logger'; +import cache from './OnyxCache'; +import * as Str from './Str'; +import createDeferredTask from './createDeferredTask'; +import * as PerformanceUtils from './metrics/PerformanceUtils'; +import Storage from './storage'; +import utils from './utils'; +import unstable_batchedUpdates from './batch'; +import DevTools from './DevTools'; +import type {CollectionKeyBase, DeepRecord, KeyValueMapping, NullishDeep, OnyxCollection, OnyxEntry, OnyxKey, Selector} from './types'; + +/** + * Represents a mapping object where each `OnyxKey` maps to either a value of its corresponding type in `KeyValueMapping` or `null`. + * + * It's very similar to `KeyValueMapping` but this type accepts using `null` as well. + */ +type NullableKeyValueMapping = { + [TKey in OnyxKey]: OnyxEntry; +}; + +/** + * Represents a mapping between Onyx collection keys and their respective values. + * + * It helps to enforce that a Onyx collection key should not be without suffix (e.g. should always be of the form `${TKey}${string}`), + * and to map each Onyx collection key with suffix to a value of type `TValue`. + * + * Also, the `TMap` type is inferred automatically in `mergeCollection()` method and represents + * the object of collection keys/values specified in the second parameter of the method. + */ +type Collection = { + [MapK in keyof TMap]: MapK extends `${TKey}${string}` + ? MapK extends `${TKey}` + ? never // forbids empty id + : TValue + : never; +}; + +/** Represents the base options used in `Onyx.connect()` method. */ +type BaseConnectOptions = { + statePropertyName?: string; + withOnyxInstance?: Component; + initWithStoredValues?: boolean; +}; + +/** + * Represents the options used in `Onyx.connect()` method. + * The type is built from `BaseConnectOptions` and extended to handle key/callback related options. + * It includes two different forms, depending on whether we are waiting for a collection callback or not. + * + * If `waitForCollectionCallback` is `true`, it expects `key` to be a Onyx collection key and `callback` will be triggered with the whole collection + * and will pass `value` as an `OnyxCollection`. + * + * + * If `waitForCollectionCallback` is `false` or not specified, the `key` can be any Onyx key and `callback` will be triggered with updates of each collection item + * and will pass `value` as an `OnyxEntry`. + */ +type ConnectOptions = BaseConnectOptions & + ( + | { + key: TKey extends CollectionKeyBase ? TKey : never; + callback?: (value: OnyxCollection) => void; + waitForCollectionCallback: true; + } + | { + key: TKey; + callback?: (value: OnyxEntry, key: TKey) => void; + waitForCollectionCallback?: false; + } + ); + +// Method constants +const METHOD = { + SET: 'set', + MERGE: 'merge', + MERGE_COLLECTION: 'mergecollection', + MULTI_SET: 'multiset', + CLEAR: 'clear', +} as const; + +// Key/value store of Onyx key and arrays of values to merge +const mergeQueue = {}; +const mergeQueuePromise = {}; + +// Keeps track of the last connectionID that was used so we can keep incrementing it +let lastConnectionID = 0; + +// Holds a mapping of all the react components that want their state subscribed to a store key +const callbackToStateMapping: Record> = {}; + +// Keeps a copy of the values of the onyx collection keys as a map for faster lookups +let onyxCollectionKeyMap = new Map(); + +// Holds a list of keys that have been directly subscribed to or recently modified from least to most recent +let recentlyAccessedKeys: OnyxKey[] = []; + +// Holds a list of keys that are safe to remove when we reach max storage. If a key does not match with +// whatever appears in this list it will NEVER be a candidate for eviction. +let evictionAllowList: OnyxKey[] = []; + +// Holds a map of keys and connectionID arrays whose keys will never be automatically evicted as +// long as we have at least one subscriber that returns false for the canEvict property. +const evictionBlocklist: Record = {}; + +// Optional user-provided key value states set when Onyx initializes or clears +let defaultKeyStates: Partial = {}; + +// Connections can be made before `Onyx.init`. They would wait for this task before resolving +const deferredInitTask = createDeferredTask(); + +let batchUpdatesPromise: Promise | null = null; +let batchUpdatesQueue: Array<() => void> = []; + +/** + * Sends an action to DevTools extension + * + * @param method - Onyx method from METHOD + * @param key - Onyx key that was changed + * @param value - contains the change that was made by the method + * @param mergedValue - (optional) value that was written in the storage after a merge method was executed. + */ +function sendActionToDevTools(method: string, key: OnyxKey, value: KeyValueMapping[OnyxKey], mergedValue: any = undefined) { + DevTools.registerAction(utils.formatActionName(method, key), value, key ? {[key]: mergedValue || value} : value); +} + +/** + * We are batching together onyx updates. This helps with use cases where we schedule onyx updates after each other. + * This happens for example in the Onyx.update function, where we process API responses that might contain a lot of + * update operations. Instead of calling the subscribers for each update operation, we batch them together which will + * cause react to schedule the updates at once instead of after each other. This is mainly a performance optimization. + */ +function maybeFlushBatchUpdates(): Promise { + if (batchUpdatesPromise) { + return batchUpdatesPromise; + } + + batchUpdatesPromise = new Promise((resolve) => { + /* We use (setTimeout, 0) here which should be called once native module calls are flushed (usually at the end of the frame) + * We may investigate if (setTimeout, 1) (which in React Native is equal to requestAnimationFrame) works even better + * then the batch will be flushed on next frame. + */ + setTimeout(() => { + const updatesCopy = batchUpdatesQueue; + batchUpdatesQueue = []; + batchUpdatesPromise = null; + unstable_batchedUpdates(() => { + updatesCopy.forEach((applyUpdates) => { + applyUpdates(); + }); + }); + + resolve(); + }, 0); + }); + return batchUpdatesPromise; +} + +function batchUpdates(updates: () => void): Promise { + batchUpdatesQueue.push(updates); + return maybeFlushBatchUpdates(); +} + +/** + * Uses a selector function to return a simplified version of sourceData + * @param {Mixed} sourceData + * @param {Function} selector Function that takes sourceData and returns a simplified version of it + * @param {Object} [withOnyxInstanceState] + * @returns {Mixed} + */ +const getSubsetOfData = (sourceData, selector, withOnyxInstanceState) => selector(sourceData, withOnyxInstanceState); + +/** + * Takes a collection of items (eg. {testKey_1:{a:'a'}, testKey_2:{b:'b'}}) + * and runs it through a reducer function to return a subset of the data according to a selector. + * The resulting collection will only contain items that are returned by the selector. + * @param {Object} collection + * @param {String|Function} selector (see method docs for getSubsetOfData() for full details) + * @param {Object} [withOnyxInstanceState] + * @returns {Object} + */ +const reduceCollectionWithSelector = (collection, selector, withOnyxInstanceState) => + _.reduce( + collection, + (finalCollection, item, key) => { + // eslint-disable-next-line no-param-reassign + finalCollection[key] = getSubsetOfData(item, selector, withOnyxInstanceState); + + return finalCollection; + }, + {}, + ); + +/** + * Get some data from the store + * + * @private + * @param {string} key + * @returns {Promise<*>} + */ +function get(key) { + // When we already have the value in cache - resolve right away + if (cache.hasCacheForKey(key)) { + return Promise.resolve(cache.getValue(key)); + } + + const taskName = `get:${key}`; + + // When a value retrieving task for this key is still running hook to it + if (cache.hasPendingTask(taskName)) { + return cache.getTaskPromise(taskName); + } + + // Otherwise retrieve the value from storage and capture a promise to aid concurrent usages + const promise = Storage.getItem(key) + .then((val) => { + cache.set(key, val); + return val; + }) + .catch((err) => Logger.logInfo(`Unable to get item from persistent storage. Key: ${key} Error: ${err}`)); + + return cache.captureTask(taskName, promise); +} + +/** Returns current key names stored in persisted storage */ +function getAllKeys(): Promise { + // When we've already read stored keys, resolve right away + const storedKeys = cache.getAllKeys(); + if (storedKeys.length > 0) { + return Promise.resolve(storedKeys); + } + + const taskName = 'getAllKeys'; + + // When a value retrieving task for all keys is still running hook to it + if (cache.hasPendingTask(taskName)) { + return cache.getTaskPromise(taskName); + } + + // Otherwise retrieve the keys from storage and capture a promise to aid concurrent usages + const promise = Storage.getAllKeys().then((keys) => { + keys.forEach((key) => cache.addKey(key)); + return keys; + }); + + return cache.captureTask(taskName, promise); +} + +/** + * Checks to see if the a subscriber's supplied key + * is associated with a collection of keys. + */ +function isCollectionKey(key: OnyxKey): key is CollectionKeyBase { + return onyxCollectionKeyMap.has(key); +} + +function isCollectionMemberKey(collectionKey: TCollectionKey, key: string): key is `${TCollectionKey}${string}` { + return Str.startsWith(key, collectionKey) && key.length > collectionKey.length; +} + +/** + * Checks to see if a provided key is the exact configured key of our connected subscriber + * or if the provided key is a collection member key (in case our configured key is a "collection key") + */ +function isKeyMatch(configKey: OnyxKey, key: OnyxKey): boolean { + return isCollectionKey(configKey) ? Str.startsWith(key, configKey) : configKey === key; +} + +/** Checks to see if this key has been flagged as safe for removal. */ +function isSafeEvictionKey(testKey: OnyxKey): boolean { + return evictionAllowList.some((key) => isKeyMatch(key, testKey)); +} + +/** + * Tries to get a value from the cache. If the value is not present in cache it will return the default value or undefined. + * If the requested key is a collection, it will return an object with all the collection members. + * + * @param {String} key + * @param {Object} mapping + * @returns {Mixed} + */ +function tryGetCachedValue(key, mapping = {}) { + let val = cache.getValue(key); + + if (isCollectionKey(key)) { + const allCacheKeys = cache.getAllKeys(); + + // It is possible we haven't loaded all keys yet so we do not know if the + // collection actually exists. + if (allCacheKeys.length === 0) { + return; + } + const matchingKeys = _.filter(allCacheKeys, (k) => k.startsWith(key)); + const values = _.reduce( + matchingKeys, + (finalObject, matchedKey) => { + const cachedValue = cache.getValue(matchedKey); + if (cachedValue) { + // This is permissible because we're in the process of constructing the final object in a reduce function. + // eslint-disable-next-line no-param-reassign + finalObject[matchedKey] = cachedValue; + } + return finalObject; + }, + {}, + ); + + val = values; + } + + if (mapping.selector) { + const state = mapping.withOnyxInstance ? mapping.withOnyxInstance.state : undefined; + if (isCollectionKey(key)) { + return reduceCollectionWithSelector(val, mapping.selector, state); + } + return getSubsetOfData(val, mapping.selector, state); + } + + return val; +} + +/** Remove a key from the recently accessed key list. */ +function removeLastAccessedKey(key: OnyxKey): void { + recentlyAccessedKeys = recentlyAccessedKeys.filter((recentlyAccessedKey) => recentlyAccessedKey !== key); +} + +/** + * Add a key to the list of recently accessed keys. The least + * recently accessed key should be at the head and the most + * recently accessed key at the tail. + */ +function addLastAccessedKey(key: OnyxKey): void { + // Only specific keys belong in this list since we cannot remove an entire collection. + if (isCollectionKey(key) || !isSafeEvictionKey(key)) { + return; + } + + removeLastAccessedKey(key); + recentlyAccessedKeys.push(key); +} + +/** + * Removes a key previously added to this list which will enable it to be deleted again. + */ +function removeFromEvictionBlockList(key: OnyxKey, connectionID: number): void { + evictionBlocklist[key] = evictionBlocklist[key]?.filter((evictionKey) => evictionKey !== connectionID); + + // Remove the key if there are no more subscribers + if (evictionBlocklist[key].length === 0) { + delete evictionBlocklist[key]; + } +} + +/** Keys added to this list can never be deleted. */ +function addToEvictionBlockList(key: OnyxKey, connectionID: number): void { + removeFromEvictionBlockList(key, connectionID); + + if (!evictionBlocklist[key]) { + evictionBlocklist[key] = []; + } + + evictionBlocklist[key].push(connectionID); +} + +/** + * Take all the keys that are safe to evict and add them to + * the recently accessed list when initializing the app. This + * enables keys that have not recently been accessed to be removed. + */ +function addAllSafeEvictionKeysToRecentlyAccessedList(): Promise { + return getAllKeys().then((keys) => { + evictionAllowList.forEach((safeEvictionKey) => { + keys.forEach((key) => { + if (!isKeyMatch(safeEvictionKey, key)) { + return; + } + addLastAccessedKey(key); + }); + }); + }); +} + +/** + * @private + * @param {String} collectionKey + * @returns {Object} + */ +function getCachedCollection(collectionKey) { + const collectionMemberKeys = _.filter(cache.getAllKeys(), (storedKey) => isCollectionMemberKey(collectionKey, storedKey)); + + return _.reduce( + collectionMemberKeys, + (prev, curr) => { + const cachedValue = cache.getValue(curr); + if (!cachedValue) { + return prev; + } + + // eslint-disable-next-line no-param-reassign + prev[curr] = cachedValue; + return prev; + }, + {}, + ); +} + +/** + * When a collection of keys change, search for any callbacks matching the collection key and trigger those callbacks + * + * @private + * @param {String} collectionKey + * @param {Object} partialCollection - a partial collection of grouped member keys + * @param {boolean} [notifyRegularSubscibers=true] + * @param {boolean} [notifyWithOnyxSubscibers=true] + */ +function keysChanged(collectionKey, partialCollection, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true) { + // We are iterating over all subscribers similar to keyChanged(). However, we are looking for subscribers who are subscribing to either a collection key or + // individual collection key member for the collection that is being updated. It is important to note that the collection parameter cane be a PARTIAL collection + // and does not represent all of the combined keys and values for a collection key. It is just the "new" data that was merged in via mergeCollection(). + const stateMappingKeys = _.keys(callbackToStateMapping); + for (let i = 0; i < stateMappingKeys.length; i++) { + const subscriber = callbackToStateMapping[stateMappingKeys[i]]; + if (!subscriber) { + continue; + } + + // Skip iteration if we do not have a collection key or a collection member key on this subscriber + if (!Str.startsWith(subscriber.key, collectionKey)) { + continue; + } + + /** + * e.g. Onyx.connect({key: ONYXKEYS.COLLECTION.REPORT, callback: ...}); + */ + const isSubscribedToCollectionKey = subscriber.key === collectionKey; + + /** + * e.g. Onyx.connect({key: `${ONYXKEYS.COLLECTION.REPORT}{reportID}`, callback: ...}); + */ + const isSubscribedToCollectionMemberKey = isCollectionMemberKey(collectionKey, subscriber.key); + + // We prepare the "cached collection" which is the entire collection + the new partial data that + // was merged in via mergeCollection(). + const cachedCollection = getCachedCollection(collectionKey); + + // Regular Onyx.connect() subscriber found. + if (typeof subscriber.callback === 'function') { + if (!notifyRegularSubscibers) { + continue; + } + + // If they are subscribed to the collection key and using waitForCollectionCallback then we'll + // send the whole cached collection. + if (isSubscribedToCollectionKey) { + if (subscriber.waitForCollectionCallback) { + subscriber.callback(cachedCollection); + continue; + } + + // If they are not using waitForCollectionCallback then we notify the subscriber with + // the new merged data but only for any keys in the partial collection. + const dataKeys = _.keys(partialCollection); + for (let j = 0; j < dataKeys.length; j++) { + const dataKey = dataKeys[j]; + subscriber.callback(cachedCollection[dataKey], dataKey); + } + continue; + } + + // And if the subscriber is specifically only tracking a particular collection member key then we will + // notify them with the cached data for that key only. + if (isSubscribedToCollectionMemberKey) { + subscriber.callback(cachedCollection[subscriber.key], subscriber.key); + continue; + } + + continue; + } + + // React component subscriber found. + if (subscriber.withOnyxInstance) { + if (!notifyWithOnyxSubscibers) { + continue; + } + + // We are subscribed to a collection key so we must update the data in state with the new + // collection member key values from the partial update. + if (isSubscribedToCollectionKey) { + // If the subscriber has a selector, then the component's state must only be updated with the data + // returned by the selector. + if (subscriber.selector) { + subscriber.withOnyxInstance.setStateProxy((prevState) => { + const previousData = prevState[subscriber.statePropertyName]; + const newData = reduceCollectionWithSelector(cachedCollection, subscriber.selector, subscriber.withOnyxInstance.state); + + if (!deepEqual(previousData, newData)) { + return { + [subscriber.statePropertyName]: newData, + }; + } + return null; + }); + continue; + } + + subscriber.withOnyxInstance.setStateProxy((prevState) => { + const finalCollection = _.clone(prevState[subscriber.statePropertyName] || {}); + const dataKeys = _.keys(partialCollection); + for (let j = 0; j < dataKeys.length; j++) { + const dataKey = dataKeys[j]; + finalCollection[dataKey] = cachedCollection[dataKey]; + } + + PerformanceUtils.logSetStateCall(subscriber, prevState[subscriber.statePropertyName], finalCollection, 'keysChanged', collectionKey); + return { + [subscriber.statePropertyName]: finalCollection, + }; + }); + continue; + } + + // If a React component is only interested in a single key then we can set the cached value directly to the state name. + if (isSubscribedToCollectionMemberKey) { + // However, we only want to update this subscriber if the partial data contains a change. + // Otherwise, we would update them with a value they already have and trigger an unnecessary re-render. + const dataFromCollection = partialCollection[subscriber.key]; + if (dataFromCollection === undefined) { + continue; + } + + // If the subscriber has a selector, then the component's state must only be updated with the data + // returned by the selector and the state should only change when the subset of data changes from what + // it was previously. + if (subscriber.selector) { + subscriber.withOnyxInstance.setStateProxy((prevState) => { + const prevData = prevState[subscriber.statePropertyName]; + const newData = getSubsetOfData(cachedCollection[subscriber.key], subscriber.selector, subscriber.withOnyxInstance.state); + if (!deepEqual(prevData, newData)) { + PerformanceUtils.logSetStateCall(subscriber, prevData, newData, 'keysChanged', collectionKey); + return { + [subscriber.statePropertyName]: newData, + }; + } + + return null; + }); + continue; + } + + subscriber.withOnyxInstance.setStateProxy((prevState) => { + const data = cachedCollection[subscriber.key]; + const previousData = prevState[subscriber.statePropertyName]; + + // Avoids triggering unnecessary re-renders when feeding empty objects + if (utils.isEmptyObject(data) && utils.isEmptyObject(previousData)) { + return null; + } + if (data === previousData) { + return null; + } + + PerformanceUtils.logSetStateCall(subscriber, previousData, data, 'keysChanged', collectionKey); + return { + [subscriber.statePropertyName]: data, + }; + }); + } + } + } +} + +/** + * When a key change happens, search for any callbacks matching the key or collection key and trigger those callbacks + * + * @example + * keyChanged(key, value, subscriber => subscriber.initWithStoredValues === false) + * + * @private + * @param {String} key + * @param {*} data + * @param {*} prevData + * @param {Function} [canUpdateSubscriber] only subscribers that pass this truth test will be updated + * @param {boolean} [notifyRegularSubscibers=true] + * @param {boolean} [notifyWithOnyxSubscibers=true] + */ +function keyChanged(key, data, prevData, canUpdateSubscriber = () => true, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true) { + // Add or remove this key from the recentlyAccessedKeys lists + if (data !== null) { + addLastAccessedKey(key); + } else { + removeLastAccessedKey(key); + } + + // We are iterating over all subscribers to see if they are interested in the key that has just changed. If the subscriber's key is a collection key then we will + // notify them if the key that changed is a collection member. Or if it is a regular key notify them when there is an exact match. Depending on whether the subscriber + // was connected via withOnyx we will call setState() directly on the withOnyx instance. If it is a regular connection we will pass the data to the provided callback. + const stateMappingKeys = _.keys(callbackToStateMapping); + for (let i = 0; i < stateMappingKeys.length; i++) { + const subscriber = callbackToStateMapping[stateMappingKeys[i]]; + if (!subscriber || !isKeyMatch(subscriber.key, key) || !canUpdateSubscriber(subscriber)) { + continue; + } + + // Subscriber is a regular call to connect() and provided a callback + if (typeof subscriber.callback === 'function') { + if (!notifyRegularSubscibers) { + continue; + } + if (isCollectionKey(subscriber.key) && subscriber.waitForCollectionCallback) { + const cachedCollection = getCachedCollection(subscriber.key); + cachedCollection[key] = data; + subscriber.callback(cachedCollection); + continue; + } + + subscriber.callback(data, key); + continue; + } + + // Subscriber connected via withOnyx() HOC + if (subscriber.withOnyxInstance) { + if (!notifyWithOnyxSubscibers) { + continue; + } + + // Check if we are subscribing to a collection key and overwrite the collection member key value in state + if (isCollectionKey(subscriber.key)) { + // If the subscriber has a selector, then the consumer of this data must only be given the data + // returned by the selector and only when the selected data has changed. + if (subscriber.selector) { + subscriber.withOnyxInstance.setStateProxy((prevState) => { + const prevWithOnyxData = prevState[subscriber.statePropertyName]; + const newWithOnyxData = { + [key]: getSubsetOfData(data, subscriber.selector, subscriber.withOnyxInstance.state), + }; + const prevDataWithNewData = { + ...prevWithOnyxData, + ...newWithOnyxData, + }; + if (!deepEqual(prevWithOnyxData, prevDataWithNewData)) { + PerformanceUtils.logSetStateCall(subscriber, prevWithOnyxData, newWithOnyxData, 'keyChanged', key); + return { + [subscriber.statePropertyName]: prevDataWithNewData, + }; + } + return null; + }); + continue; + } + + subscriber.withOnyxInstance.setStateProxy((prevState) => { + const collection = prevState[subscriber.statePropertyName] || {}; + const newCollection = { + ...collection, + [key]: data, + }; + PerformanceUtils.logSetStateCall(subscriber, collection, newCollection, 'keyChanged', key); + return { + [subscriber.statePropertyName]: newCollection, + }; + }); + continue; + } + + // If the subscriber has a selector, then the component's state must only be updated with the data + // returned by the selector and only if the selected data has changed. + if (subscriber.selector) { + subscriber.withOnyxInstance.setStateProxy(() => { + const previousValue = getSubsetOfData(prevData, subscriber.selector, subscriber.withOnyxInstance.state); + const newValue = getSubsetOfData(data, subscriber.selector, subscriber.withOnyxInstance.state); + + if (!deepEqual(previousValue, newValue)) { + return { + [subscriber.statePropertyName]: newValue, + }; + } + return null; + }); + continue; + } + + // If we did not match on a collection key then we just set the new data to the state property + subscriber.withOnyxInstance.setStateProxy((prevState) => { + const prevWithOnyxData = prevState[subscriber.statePropertyName]; + + // Avoids triggering unnecessary re-renders when feeding empty objects + if (utils.isEmptyObject(data) && utils.isEmptyObject(prevWithOnyxData)) { + return null; + } + if (prevWithOnyxData === data) { + return null; + } + + PerformanceUtils.logSetStateCall(subscriber, prevData, data, 'keyChanged', key); + return { + [subscriber.statePropertyName]: data, + }; + }); + continue; + } + + console.error('Warning: Found a matching subscriber to a key that changed, but no callback or withOnyxInstance could be found.'); + } +} + +/** + * Sends the data obtained from the keys to the connection. It either: + * - sets state on the withOnyxInstances + * - triggers the callback function + * + * @private + * @param {Object} mapping + * @param {Object} [mapping.withOnyxInstance] + * @param {String} [mapping.statePropertyName] + * @param {Function} [mapping.callback] + * @param {String} [mapping.selector] + * @param {*|null} val + * @param {String|undefined} matchedKey + * @param {Boolean} isBatched + */ +function sendDataToConnection(mapping, val, matchedKey, isBatched) { + // If the mapping no longer exists then we should not send any data. + // This means our subscriber disconnected or withOnyx wrapped component unmounted. + if (!callbackToStateMapping[mapping.connectionID]) { + return; + } + + if (mapping.withOnyxInstance) { + let newData = val; + + // If the mapping has a selector, then the component's state must only be updated with the data + // returned by the selector. + if (mapping.selector) { + if (isCollectionKey(mapping.key)) { + newData = reduceCollectionWithSelector(val, mapping.selector, mapping.withOnyxInstance.state); + } else { + newData = getSubsetOfData(val, mapping.selector, mapping.withOnyxInstance.state); + } + } + + PerformanceUtils.logSetStateCall(mapping, null, newData, 'sendDataToConnection'); + if (isBatched) { + batchUpdates(() => { + mapping.withOnyxInstance.setWithOnyxState(mapping.statePropertyName, newData); + }); + } else { + mapping.withOnyxInstance.setWithOnyxState(mapping.statePropertyName, newData); + } + return; + } + + if (typeof mapping.callback === 'function') { + mapping.callback(val, matchedKey); + } +} + +/** + * We check to see if this key is flagged as safe for eviction and add it to the recentlyAccessedKeys list so that when we + * run out of storage the least recently accessed key can be removed. + * + * @private + * @param {Object} mapping + */ +function addKeyToRecentlyAccessedIfNeeded(mapping) { + if (!isSafeEvictionKey(mapping.key)) { + return; + } + + // Try to free some cache whenever we connect to a safe eviction key + cache.removeLeastRecentlyUsedKeys(); + + if (mapping.withOnyxInstance && !isCollectionKey(mapping.key)) { + // All React components subscribing to a key flagged as a safe eviction key must implement the canEvict property. + if (mapping.canEvict === undefined) { + throw new Error(`Cannot subscribe to safe eviction key '${mapping.key}' without providing a canEvict value.`); + } + + addLastAccessedKey(mapping.key); + } +} + +/** + * Gets the data for a given an array of matching keys, combines them into an object, and sends the result back to the subscriber. + * + * @private + * @param {Array} matchingKeys + * @param {Object} mapping + */ +function getCollectionDataAndSendAsObject(matchingKeys, mapping) { + Promise.all(matchingKeys.map((key) => get(key))) + .then((values) => + _.reduce( + values, + (finalObject, value, i) => { + // eslint-disable-next-line no-param-reassign + finalObject[matchingKeys[i]] = value; + return finalObject; + }, + {}, + ), + ) + .then((val) => sendDataToConnection(mapping, val, undefined, true)); +} + +/** + * Subscribes a react component's state directly to a store key + * + * @example + * const connectionID = Onyx.connect({ + * key: ONYXKEYS.SESSION, + * callback: onSessionChange, + * }); + * + * @param mapping the mapping information to connect Onyx to the components state + * @param mapping.key ONYXKEY to subscribe to + * @param [mapping.statePropertyName] the name of the property in the state to connect the data to + * @param [mapping.withOnyxInstance] whose setState() method will be called with any changed data + * This is used by React components to connect to Onyx + * @param [mapping.callback] a method that will be called with changed data + * This is used by any non-React code to connect to Onyx + * @param [mapping.initWithStoredValues] If set to false, then no data will be prefilled into the + * component + * @param [mapping.waitForCollectionCallback] If set to true, it will return the entire collection to the callback as a single object + * @returns an ID to use when calling disconnect + */ +function connect(mapping: ConnectOptions): number { + const connectionID = lastConnectionID++; + callbackToStateMapping[connectionID] = mapping; + callbackToStateMapping[connectionID].connectionID = connectionID; + + if (mapping.initWithStoredValues === false) { + return connectionID; + } + + // Commit connection only after init passes + deferredInitTask.promise + .then(() => addKeyToRecentlyAccessedIfNeeded(mapping)) + .then(() => { + // Performance improvement + // If the mapping is connected to an onyx key that is not a collection + // we can skip the call to getAllKeys() and return an array with a single item + if (Boolean(mapping.key) && typeof mapping.key === 'string' && !mapping.key.endsWith('_') && cache.storageKeys.has(mapping.key)) { + return [mapping.key]; + } + return getAllKeys(); + }) + .then((keys) => { + // We search all the keys in storage to see if any are a "match" for the subscriber we are connecting so that we + // can send data back to the subscriber. Note that multiple keys can match as a subscriber could either be + // subscribed to a "collection key" or a single key. + const matchingKeys = _.filter(keys, (key) => isKeyMatch(mapping.key, key)); + + // If the key being connected to does not exist we initialize the value with null. For subscribers that connected + // directly via connect() they will simply get a null value sent to them without any information about which key matched + // since there are none matched. In withOnyx() we wait for all connected keys to return a value before rendering the child + // component. This null value will be filtered out so that the connected component can utilize defaultProps. + if (matchingKeys.length === 0) { + if (mapping.key && !isCollectionKey(mapping.key)) { + cache.set(mapping.key, null); + } + + // Here we cannot use batching because the null value is expected to be set immediately for default props + // or they will be undefined. + sendDataToConnection(mapping, null, undefined, false); + return; + } + + // When using a callback subscriber we will either trigger the provided callback for each key we find or combine all values + // into an object and just make a single call. The latter behavior is enabled by providing a waitForCollectionCallback key + // combined with a subscription to a collection key. + if (typeof mapping.callback === 'function') { + if (isCollectionKey(mapping.key)) { + if (mapping.waitForCollectionCallback) { + getCollectionDataAndSendAsObject(matchingKeys, mapping); + return; + } + + // We did not opt into using waitForCollectionCallback mode so the callback is called for every matching key. + for (let i = 0; i < matchingKeys.length; i++) { + get(matchingKeys[i]).then((val) => sendDataToConnection(mapping, val, matchingKeys[i], true)); + } + return; + } + + // If we are not subscribed to a collection key then there's only a single key to send an update for. + get(mapping.key).then((val) => sendDataToConnection(mapping, val, mapping.key, true)); + return; + } + + // If we have a withOnyxInstance that means a React component has subscribed via the withOnyx() HOC and we need to + // group collection key member data into an object. + if (mapping.withOnyxInstance) { + if (isCollectionKey(mapping.key)) { + getCollectionDataAndSendAsObject(matchingKeys, mapping); + return; + } + + // If the subscriber is not using a collection key then we just send a single value back to the subscriber + get(mapping.key).then((val) => sendDataToConnection(mapping, val, mapping.key, true)); + return; + } + + console.error('Warning: Onyx.connect() was found without a callback or withOnyxInstance'); + }); + + // The connectionID is returned back to the caller so that it can be used to clean up the connection when it's no longer needed + // by calling Onyx.disconnect(connectionID). + return connectionID; +} + +/** + * Remove the listener for a react component + * @example + * Onyx.disconnect(connectionID); + * + * @param connectionID unique id returned by call to Onyx.connect() + */ +function disconnect(connectionID: number, keyToRemoveFromEvictionBlocklist?: OnyxKey): void { + if (!callbackToStateMapping[connectionID]) { + return; + } + + // Remove this key from the eviction block list as we are no longer + // subscribing to it and it should be safe to delete again + if (keyToRemoveFromEvictionBlocklist) { + removeFromEvictionBlockList(keyToRemoveFromEvictionBlocklist, connectionID); + } + + delete callbackToStateMapping[connectionID]; +} + +/** + * Schedules an update that will be appended to the macro task queue (so it doesn't update the subscribers immediately). + * + * @example + * scheduleSubscriberUpdate(key, value, subscriber => subscriber.initWithStoredValues === false) + * + * @param {String} key + * @param {*} value + * @param {*} prevValue + * @param {Function} [canUpdateSubscriber] only subscribers that pass this truth test will be updated + * @returns {Promise} + */ +function scheduleSubscriberUpdate(key, value, prevValue, canUpdateSubscriber = () => true) { + const promise = Promise.resolve().then(() => keyChanged(key, value, prevValue, canUpdateSubscriber, true, false)); + batchUpdates(() => keyChanged(key, value, prevValue, canUpdateSubscriber, false, true)); + return Promise.all([maybeFlushBatchUpdates(), promise]); +} + +/** + * This method is similar to notifySubscribersOnNextTick but it is built for working specifically with collections + * so that keysChanged() is triggered for the collection and not keyChanged(). If this was not done, then the + * subscriber callbacks receive the data in a different format than they normally expect and it breaks code. + * + * @param {String} key + * @param {*} value + * @returns {Promise} + */ +function scheduleNotifyCollectionSubscribers(key, value) { + const promise = Promise.resolve().then(() => keysChanged(key, value, true, false)); + batchUpdates(() => keysChanged(key, value, false, true)); + return Promise.all([maybeFlushBatchUpdates(), promise]); +} + +/** + * Remove a key from Onyx and update the subscribers + * + * @private + * @param {String} key + * @return {Promise} + */ +function remove(key) { + const prevValue = cache.getValue(key, false); + cache.drop(key); + scheduleSubscriberUpdate(key, null, prevValue); + return Storage.removeItem(key); +} + +/** + * @private + * @returns {Promise} + */ +function reportStorageQuota() { + return Storage.getDatabaseSize() + .then(({bytesUsed, bytesRemaining}) => { + Logger.logInfo(`Storage Quota Check -- bytesUsed: ${bytesUsed} bytesRemaining: ${bytesRemaining}`); + }) + .catch((dbSizeError) => { + Logger.logAlert(`Unable to get database size. Error: ${dbSizeError}`); + }); +} + +/** + * If we fail to set or merge we must handle this by + * evicting some data from Onyx and then retrying to do + * whatever it is we attempted to do. + * + * @private + * @param {Error} error + * @param {Function} onyxMethod + * @param {...any} args + * @return {Promise} + */ +function evictStorageAndRetry(error, onyxMethod, ...args) { + Logger.logInfo(`Failed to save to storage. Error: ${error}. onyxMethod: ${onyxMethod.name}`); + + if (error && Str.startsWith(error.message, "Failed to execute 'put' on 'IDBObjectStore'")) { + Logger.logAlert('Attempted to set invalid data set in Onyx. Please ensure all data is serializable.'); + throw error; + } + + // Find the first key that we can remove that has no subscribers in our blocklist + const keyForRemoval = _.find(recentlyAccessedKeys, (key) => !evictionBlocklist[key]); + if (!keyForRemoval) { + // If we have no acceptable keys to remove then we are possibly trying to save mission critical data. If this is the case, + // then we should stop retrying as there is not much the user can do to fix this. Instead of getting them stuck in an infinite loop we + // will allow this write to be skipped. + Logger.logAlert('Out of storage. But found no acceptable keys to remove.'); + return reportStorageQuota(); + } + + // Remove the least recently viewed key that is not currently being accessed and retry. + Logger.logInfo(`Out of storage. Evicting least recently accessed key (${keyForRemoval}) and retrying.`); + reportStorageQuota(); + return remove(keyForRemoval).then(() => onyxMethod(...args)); +} + +/** + * Notifys subscribers and writes current value to cache + * + * @param {String} key + * @param {*} value + * @param {String} method + * @param {Boolean} hasChanged + * @param {Boolean} wasRemoved + * @returns {Promise} + */ +function broadcastUpdate(key, value, method, hasChanged, wasRemoved = false) { + // Logging properties only since values could be sensitive things we don't want to log + Logger.logInfo(`${method}() called for key: ${key}${typeof value === 'object' ? ` properties: ${_.keys(value).join(',')}` : ''}`); + const prevValue = cache.getValue(key, false); + + // Update subscribers if the cached value has changed, or when the subscriber specifically requires + // all updates regardless of value changes (indicated by initWithStoredValues set to false). + if (hasChanged && !wasRemoved) { + cache.set(key, value); + } else { + cache.addToAccessedKeys(key); + } + + return scheduleSubscriberUpdate(key, value, prevValue, (subscriber) => hasChanged || subscriber.initWithStoredValues === false); +} + +function hasPendingMergeForKey(key: OnyxKey): boolean { + return !!mergeQueue[key]; +} + +/** + * Removes a key from storage if the value is null. + * Otherwise removes all nested null values in objects and returns the object + * @param {String} key + * @param {Mixed} value + * @returns {Mixed} The value without null values and a boolean "wasRemoved", which indicates if the key got removed completely + */ +function removeNullValues(key, value) { + if (value === null) { + remove(key); + return {value, wasRemoved: true}; + } + + // We can remove all null values in an object by merging it with itself + // utils.fastMerge recursively goes through the object and removes all null values + // Passing two identical objects as source and target to fastMerge will not change it, but only remove the null values + return {value: utils.removeNestedNullValues(value), wasRemoved: false}; +} + +/** + * Write a value to our store with the given key + * + * @param key ONYXKEY to set + * @param value value to store + */ + +function set(key: TKey, value: OnyxEntry): Promise { + // If the value is null, we remove the key from storage + const {value: valueAfterRemoving, wasRemoved} = removeNullValues(key, value); + + if (hasPendingMergeForKey(key)) { + delete mergeQueue[key]; + } + + const hasChanged = cache.hasValueChanged(key, valueAfterRemoving); + + // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. + const updatePromise = broadcastUpdate(key, valueAfterRemoving, 'set', hasChanged, wasRemoved); + + // If the value has not changed or the key got removed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead. + if (!hasChanged || wasRemoved) { + return updatePromise; + } + + return Storage.setItem(key, valueAfterRemoving) + .catch((error) => evictStorageAndRetry(error, set, key, valueAfterRemoving)) + .then(() => { + sendActionToDevTools(METHOD.SET, key, valueAfterRemoving); + return updatePromise; + }); +} + +/** + * Storage expects array like: [["@MyApp_user", value_1], ["@MyApp_key", value_2]] + * This method transforms an object like {'@MyApp_user': myUserValue, '@MyApp_key': myKeyValue} + * to an array of key-value pairs in the above format and removes key-value pairs that are being set to null + * @private + * @param {Record} data + * @return {Array} an array of key - value pairs <[key, value]> + */ +function prepareKeyValuePairsForStorage(data) { + const keyValuePairs = []; + + data.forEach((value, key) => { + const {value: valueAfterRemoving, wasRemoved} = removeNullValues(key, value); + + if (wasRemoved) return; + + keyValuePairs.push([key, valueAfterRemoving]); + }); + + return keyValuePairs; +} + +/** + * Sets multiple keys and values + * + * @example Onyx.multiSet({'key1': 'a', 'key2': 'b'}); + * + * @param data object keyed by ONYXKEYS and the values to set + */ +function multiSet(data: Partial): Promise { + const keyValuePairs = prepareKeyValuePairsForStorage(data); + + const updatePromises = keyValuePairs.map(([key, value]) => { + const prevValue = cache.getValue(key, false); + + // Update cache and optimistically inform subscribers on the next tick + cache.set(key, value); + return scheduleSubscriberUpdate(key, value, prevValue); + }); + + return Storage.multiSet(keyValuePairs) + .catch((error) => evictStorageAndRetry(error, multiSet, data)) + .then(() => { + sendActionToDevTools(METHOD.MULTI_SET, undefined, data); + return Promise.all(updatePromises); + }); +} + +/** + * Merges an array of changes with an existing value + * + * @private + * @param {*} existingValue + * @param {Array<*>} changes Array of changes that should be applied to the existing value + * @param {Boolean} shouldRemoveNullObjectValues + * @returns {*} + */ +function applyMerge(existingValue, changes, shouldRemoveNullObjectValues) { + const lastChange = _.last(changes); + + if (Array.isArray(lastChange)) { + return lastChange; + } + + if (changes.some((change) => typeof change === 'object')) { + // Object values are then merged one after the other + return _.reduce(changes, (modifiedData, change) => utils.fastMerge(modifiedData, change, shouldRemoveNullObjectValues), existingValue || {}); + } + + // If we have anything else we can't merge it so we'll + // simply return the last value that was queued + return lastChange; +} + +/** + * Merge a new value into an existing value at a key. + * + * The types of values that can be merged are `Object` and `Array`. To set another type of value use `Onyx.set()`. + * Values of type `Object` get merged with the old value, whilst for `Array`'s we simply replace the current value with the new one. + * + * Calls to `Onyx.merge()` are batched so that any calls performed in a single tick will stack in a queue and get + * applied in the order they were called. Note: `Onyx.set()` calls do not work this way so use caution when mixing + * `Onyx.merge()` and `Onyx.set()`. + * + * @example + * Onyx.merge(ONYXKEYS.EMPLOYEE_LIST, ['Joe']); // -> ['Joe'] + * Onyx.merge(ONYXKEYS.EMPLOYEE_LIST, ['Jack']); // -> ['Joe', 'Jack'] + * Onyx.merge(ONYXKEYS.POLICY, {id: 1}); // -> {id: 1} + * Onyx.merge(ONYXKEYS.POLICY, {name: 'My Workspace'}); // -> {id: 1, name: 'My Workspace'} + */ +function merge(key: TKey, changes: OnyxEntry>): Promise { + // Top-level undefined values are ignored + // Therefore we need to prevent adding them to the merge queue + if (changes === undefined) { + return mergeQueue[key] ? mergeQueuePromise[key] : Promise.resolve(); + } + + // Merge attempts are batched together. The delta should be applied after a single call to get() to prevent a race condition. + // Using the initial value from storage in subsequent merge attempts will lead to an incorrect final merged value. + if (mergeQueue[key]) { + mergeQueue[key].push(changes); + return mergeQueuePromise[key]; + } + mergeQueue[key] = [changes]; + + mergeQueuePromise[key] = get(key).then((existingValue) => { + // Calls to Onyx.set after a merge will terminate the current merge process and clear the merge queue + if (mergeQueue[key] == null) return; + + try { + // We first only merge the changes, so we can provide these to the native implementation (SQLite uses only delta changes in "JSON_PATCH" to merge) + // We don't want to remove null values from the "batchedChanges", because SQLite uses them to remove keys from storage natively. + let batchedChanges = applyMerge(undefined, mergeQueue[key], false); + + // The presence of a `null` in the merge queue instructs us to drop the existing value. + // In this case, we can't simply merge the batched changes with the existing value, because then the null in the merge queue would have no effect + const shouldOverwriteExistingValue = _.includes(mergeQueue[key], null); + + // Clean up the write queue, so we don't apply these changes again + delete mergeQueue[key]; + delete mergeQueuePromise[key]; + + // If the batched changes equal null, we want to remove the key from storage, to reduce storage size + const {wasRemoved} = removeNullValues(key, batchedChanges); + + // After that we merge the batched changes with the existing value + // We can remove null values from the "modifiedData", because "null" implicates that the user wants to remove a value from storage. + // The "modifiedData" will be directly "set" in storage instead of being merged + const modifiedData = shouldOverwriteExistingValue ? batchedChanges : applyMerge(existingValue, [batchedChanges], true); + + // On native platforms we use SQLite which utilises JSON_PATCH to merge changes. + // JSON_PATCH generally removes null values from the stored object. + // When there is no existing value though, SQLite will just insert the changes as a new value and thus the null values won't be removed. + // Therefore we need to remove null values from the `batchedChanges` which are sent to the SQLite, if no existing value is present. + if (!existingValue) { + batchedChanges = applyMerge(undefined, [batchedChanges], true); + } + + const hasChanged = cache.hasValueChanged(key, modifiedData); + + // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. + const updatePromise = broadcastUpdate(key, modifiedData, 'merge', hasChanged, wasRemoved); + + // If the value has not changed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead. + if (!hasChanged || wasRemoved) { + return updatePromise; + } + + return Storage.mergeItem(key, batchedChanges, modifiedData).then(() => { + sendActionToDevTools(METHOD.MERGE, key, changes, modifiedData); + return updatePromise; + }); + } catch (error) { + Logger.logAlert(`An error occurred while applying merge for key: ${key}, Error: ${error}`); + return Promise.resolve(); + } + }); + + return mergeQueuePromise[key]; +} + +/** + * Merge user provided default key value pairs. + * @private + * @returns {Promise} + */ +function initializeWithDefaultKeyStates() { + return Storage.multiGet(_.keys(defaultKeyStates)).then((pairs) => { + const existingDataAsObject = _.object(pairs); + + const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates); + cache.merge(merged); + + merged.forEach((val, key) => keyChanged(key, val, existingDataAsObject)); + }); +} + +/** + * Clear out all the data in the store + * + * Note that calling Onyx.clear() and then Onyx.set() on a key with a default + * key state may store an unexpected value in Storage. + * + * E.g. + * Onyx.clear(); + * Onyx.set(ONYXKEYS.DEFAULT_KEY, 'default'); + * Storage.getItem(ONYXKEYS.DEFAULT_KEY) + * .then((storedValue) => console.log(storedValue)); + * null is logged instead of the expected 'default' + * + * Onyx.set() might call Storage.setItem() before Onyx.clear() calls + * Storage.setItem(). Use Onyx.merge() instead if possible. Onyx.merge() calls + * Onyx.get(key) before calling Storage.setItem() via Onyx.set(). + * Storage.setItem() from Onyx.clear() will have already finished and the merged + * value will be saved to storage after the default value. + * + * @param keysToPreserve is a list of ONYXKEYS that should not be cleared with the rest of the data + */ +function clear(keysToPreserve: OnyxKey[] = []): Promise { + return getAllKeys().then((keys) => { + const keysToBeClearedFromStorage = []; + const keyValuesToResetAsCollection = {}; + const keyValuesToResetIndividually = {}; + + // The only keys that should not be cleared are: + // 1. Anything specifically passed in keysToPreserve (because some keys like language preferences, offline + // status, or activeClients need to remain in Onyx even when signed out) + // 2. Any keys with a default state (because they need to remain in Onyx as their default, and setting them + // to null would cause unknown behavior) + keys.forEach((key) => { + const isKeyToPreserve = _.contains(keysToPreserve, key); + const isDefaultKey = _.has(defaultKeyStates, key); + + // If the key is being removed or reset to default: + // 1. Update it in the cache + // 2. Figure out whether it is a collection key or not, + // since collection key subscribers need to be updated differently + if (!isKeyToPreserve) { + const oldValue = cache.getValue(key); + const newValue = _.get(defaultKeyStates, key, null); + if (newValue !== oldValue) { + cache.set(key, newValue); + const collectionKey = key.substring(0, key.indexOf('_') + 1); + if (collectionKey) { + if (!keyValuesToResetAsCollection[collectionKey]) { + keyValuesToResetAsCollection[collectionKey] = {}; + } + keyValuesToResetAsCollection[collectionKey][key] = newValue; + } else { + keyValuesToResetIndividually[key] = newValue; + } + } + } + + if (isKeyToPreserve || isDefaultKey) { + return; + } + + // If it isn't preserved and doesn't have a default, we'll remove it + keysToBeClearedFromStorage.push(key); + }); + + const updatePromises = []; + + // Notify the subscribers for each key/value group so they can receive the new values + keyValuesToResetIndividually.forEach((value, key) => { + updatePromises.push(scheduleSubscriberUpdate(key, value, cache.getValue(key, false))); + }); + keyValuesToResetAsCollection.forEach((value, key) => { + updatePromises.push(scheduleNotifyCollectionSubscribers(key, value)); + }); + + const defaultKeyValuePairs = _.pairs(_.omit(defaultKeyStates, keysToPreserve)); + + // Remove only the items that we want cleared from storage, and reset others to default + keysToBeClearedFromStorage.forEach((key) => cache.drop(key)); + return Storage.removeItems(keysToBeClearedFromStorage) + .then(() => Storage.multiSet(defaultKeyValuePairs)) + .then(() => { + DevTools.clearState(keysToPreserve); + return Promise.all(updatePromises); + }); + }); +} + +/** + * Merges a collection based on their keys + * + * @example + * + * Onyx.mergeCollection(ONYXKEYS.COLLECTION.REPORT, { + * [`${ONYXKEYS.COLLECTION.REPORT}1`]: report1, + * [`${ONYXKEYS.COLLECTION.REPORT}2`]: report2, + * }); + * + * @param collectionKey e.g. `ONYXKEYS.COLLECTION.REPORT` + * @param collection Object collection keyed by individual collection member keys and values + */ +function mergeCollection(collectionKey: TKey, collection: Collection>): Promise { + if (typeof collection !== 'object' || Array.isArray(collection) || utils.isEmptyObject(collection)) { + Logger.logInfo('mergeCollection() called with invalid or empty value. Skipping this update.'); + return Promise.resolve(); + } + + // Confirm all the collection keys belong to the same parent + let hasCollectionKeyCheckFailed = false; + Object.keys(collection).forEach((dataKey) => { + if (isKeyMatch(collectionKey, dataKey)) { + return; + } + + if (process.env.NODE_ENV === 'development') { + throw new Error(`Provided collection doesn't have all its data belonging to the same parent. CollectionKey: ${collectionKey}, DataKey: ${dataKey}`); + } + + hasCollectionKeyCheckFailed = true; + Logger.logAlert(`Provided collection doesn't have all its data belonging to the same parent. CollectionKey: ${collectionKey}, DataKey: ${dataKey}`); + }); + + // Gracefully handle bad mergeCollection updates so it doesn't block the merge queue + if (hasCollectionKeyCheckFailed) { + return Promise.resolve(); + } + + return getAllKeys().then((persistedKeys) => { + // Split to keys that exist in storage and keys that don't + const [existingKeys, newKeys] = _.chain(collection) + .pick((value, key) => { + if (value === null) { + remove(key); + return false; + } + return true; + }) + .keys() + .partition((key) => persistedKeys.includes(key)) + .value(); + + const existingKeyCollection = _.pick(collection, existingKeys); + const newCollection = _.pick(collection, newKeys); + const keyValuePairsForExistingCollection = prepareKeyValuePairsForStorage(existingKeyCollection); + const keyValuePairsForNewCollection = prepareKeyValuePairsForStorage(newCollection); + + const promises = []; + + // New keys will be added via multiSet while existing keys will be updated using multiMerge + // This is because setting a key that doesn't exist yet with multiMerge will throw errors + if (keyValuePairsForExistingCollection.length > 0) { + promises.push(Storage.multiMerge(keyValuePairsForExistingCollection)); + } + + if (keyValuePairsForNewCollection.length > 0) { + promises.push(Storage.multiSet(keyValuePairsForNewCollection)); + } + + // Prefill cache if necessary by calling get() on any existing keys and then merge original data to cache + // and update all subscribers + const promiseUpdate = Promise.all(existingKeys.map(get)).then(() => { + cache.merge(collection); + return scheduleNotifyCollectionSubscribers(collectionKey, collection); + }); + + return Promise.all(promises) + .catch((error) => evictStorageAndRetry(error, mergeCollection, collection)) + .then(() => { + sendActionToDevTools(METHOD.MERGE_COLLECTION, undefined, collection); + return promiseUpdate; + }); + }); +} + +/** + * Represents different kinds of updates that can be passed to `Onyx.update()` method. It is a discriminated union of + * different update methods (`SET`, `MERGE`, `MERGE_COLLECTION`), each with their own key and value structure. + */ +type OnyxUpdate = + | { + [TKey in OnyxKey]: + | { + onyxMethod: typeof METHOD.SET; + key: TKey; + value: OnyxEntry; + } + | { + onyxMethod: typeof METHOD.MERGE; + key: TKey; + value: OnyxEntry>; + }; + }[OnyxKey] + | { + [TKey in CollectionKeyBase]: { + onyxMethod: typeof METHOD.MERGE_COLLECTION; + key: TKey; + value: Record<`${TKey}${string}`, NullishDeep>; + }; + }[CollectionKeyBase]; + +/** + * Insert API responses and lifecycle data into Onyx + * + * @param {Array} data An array of objects with shape {onyxMethod: oneOf('set', 'merge', 'mergeCollection', 'multiSet', 'clear'), key: string, value: *} + * @returns {Promise} resolves when all operations are complete + */ +function update(data: OnyxUpdate[]): Promise { + // First, validate the Onyx object is in the format we expect + data.forEach(({onyxMethod, key, value}) => { + if (!_.contains([METHOD.CLEAR, METHOD.SET, METHOD.MERGE, METHOD.MERGE_COLLECTION, METHOD.MULTI_SET], onyxMethod)) { + throw new Error(`Invalid onyxMethod ${onyxMethod} in Onyx update.`); + } + if (onyxMethod === METHOD.MULTI_SET) { + // For multiset, we just expect the value to be an object + if (typeof value !== 'object' || Array.isArray(value) || typeof value === 'function') { + throw new Error('Invalid value provided in Onyx multiSet. Onyx multiSet value must be of type object.'); + } + } else if (onyxMethod !== METHOD.CLEAR && typeof key !== 'string') { + throw new Error(`Invalid ${typeof key} key provided in Onyx update. Onyx key must be of type string.`); + } + }); + + const promises = []; + let clearPromise = Promise.resolve(); + + data.forEach(({onyxMethod, key, value}) => { + switch (onyxMethod) { + case METHOD.SET: + promises.push(() => set(key, value)); + break; + case METHOD.MERGE: + promises.push(() => merge(key, value)); + break; + case METHOD.MERGE_COLLECTION: + promises.push(() => mergeCollection(key, value)); + break; + case METHOD.MULTI_SET: + promises.push(() => multiSet(value)); + break; + case METHOD.CLEAR: + clearPromise = clear(); + break; + default: + break; + } + }); + + return clearPromise.then(() => Promise.all(promises.map((p) => p()))); +} + +/** + * Represents the options used in `Onyx.init()` method. + */ +type InitOptions = { + /** `ONYXKEYS` constants object */ + keys?: DeepRecord; + + /** initial data to set when `init()` and `clear()` is called */ + initialKeyStates?: Partial; + + /** + * This is an array of keys (individual or collection patterns) that when provided to Onyx are flagged + * as "safe" for removal. Any components subscribing to these keys must also implement a canEvict option. See the README for more info. + */ + safeEvictionKeys?: OnyxKey[]; + + /** + * Sets how many recent keys should we try to keep in cache + * Setting this to 0 would practically mean no cache + * We try to free cache when we connect to a safe eviction key + */ + maxCachedKeysCount?: number; + + /** + * Auto synchronize storage events between multiple instances + * of Onyx running in different tabs/windows. Defaults to true for platforms that support local storage (web/desktop) + */ + shouldSyncMultipleInstances?: boolean; + + /** Enables debugging setState() calls to connected components */ + debugSetState?: boolean; +}; + +/** Initialize the store with actions and listening for storage events */ +function init({ + keys = {}, + initialKeyStates = {}, + safeEvictionKeys = [], + maxCachedKeysCount = 1000, + shouldSyncMultipleInstances = Boolean(global.localStorage), + debugSetState = false, +}: InitOptions) { + if (debugSetState) { + PerformanceUtils.setShouldDebugSetState(true); + } + + if (maxCachedKeysCount > 0) { + cache.setRecentKeysLimit(maxCachedKeysCount); + } + + // We need the value of the collection keys later for checking if a + // key is a collection. We store it in a map for faster lookup. + const collectionValues = Object.values(keys.COLLECTION); + onyxCollectionKeyMap = _.reduce( + collectionValues, + (acc, val) => { + acc.set(val, true); + return acc; + }, + new Map(), + ); + + // Set our default key states to use when initializing and clearing Onyx data + defaultKeyStates = initialKeyStates; + + DevTools.initState(initialKeyStates); + + // Let Onyx know about which keys are safe to evict + evictionAllowList = safeEvictionKeys; + + // Initialize all of our keys with data provided then give green light to any pending connections + Promise.all([addAllSafeEvictionKeysToRecentlyAccessedList(), initializeWithDefaultKeyStates()]).then(deferredInitTask.resolve); + + if (shouldSyncMultipleInstances && typeof Storage.keepInstancesSync === 'function') { + Storage.keepInstancesSync((key, value) => { + const prevValue = cache.getValue(key, false); + cache.set(key, value); + keyChanged(key, value, prevValue); + }); + } +} + +const Onyx = { + connect, + disconnect, + set, + multiSet, + merge, + mergeCollection, + update, + clear, + getAllKeys, + init, + registerLogger: Logger.registerLogger, + addToEvictionBlockList, + removeFromEvictionBlockList, + isSafeEvictionKey, + METHOD, + tryGetCachedValue, + hasPendingMergeForKey, +} as const; + +export default Onyx; diff --git a/lib/storage/__mocks__/index.ts b/lib/storage/__mocks__/index.ts index e202c0f6..3a2f167c 100644 --- a/lib/storage/__mocks__/index.ts +++ b/lib/storage/__mocks__/index.ts @@ -58,8 +58,6 @@ const idbKeyvalMock: StorageProvider = { getDatabaseSize() { return Promise.resolve({bytesRemaining: 0, bytesUsed: 99999}); }, - // eslint-disable-next-line @typescript-eslint/no-empty-function - setMemoryOnlyKeys() {}, }; const idbKeyvalMockSpy = { @@ -79,7 +77,6 @@ const idbKeyvalMockSpy = { storageMapInternal = data; }), getDatabaseSize: jest.fn(idbKeyvalMock.getDatabaseSize), - setMemoryOnlyKeys: jest.fn(idbKeyvalMock.setMemoryOnlyKeys), }; export default idbKeyvalMockSpy; diff --git a/lib/storage/providers/IDBKeyVal.ts b/lib/storage/providers/IDBKeyVal.ts index bc429bc2..cd9d3ba3 100644 --- a/lib/storage/providers/IDBKeyVal.ts +++ b/lib/storage/providers/IDBKeyVal.ts @@ -39,8 +39,6 @@ const provider: StorageProvider = { }, multiSet: (pairs) => setMany(pairs, getCustomStore()), clear: () => clear(getCustomStore()), - // eslint-disable-next-line @typescript-eslint/no-empty-function - setMemoryOnlyKeys: () => {}, getAllKeys: () => keys(getCustomStore()), getItem: (key) => get(key, getCustomStore()) diff --git a/lib/storage/providers/SQLiteStorage.ts b/lib/storage/providers/SQLiteStorage.ts index 6011277b..36eb3622 100644 --- a/lib/storage/providers/SQLiteStorage.ts +++ b/lib/storage/providers/SQLiteStorage.ts @@ -93,9 +93,6 @@ const provider: StorageProvider = { }); }, - // eslint-disable-next-line @typescript-eslint/no-empty-function - setMemoryOnlyKeys: () => {}, - // eslint-disable-next-line @typescript-eslint/no-empty-function keepInstancesSync: () => {}, }; diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index 3f811372..b2be36e3 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -1,7 +1,7 @@ import type {BatchQueryResult, QueryResult} from 'react-native-quick-sqlite'; type Key = string; -type Value = IDBValidKey; +type Value = IDBValidKey | null; type KeyValuePair = [Key, Value]; type KeyList = Key[]; type KeyValuePairList = KeyValuePair[]; @@ -61,11 +61,6 @@ type StorageProvider = { */ clear: () => Promise; - /** - * Sets memory only keys - */ - setMemoryOnlyKeys: () => void; - /** * Gets the total bytes of the database file */ From c8aec7b20a31c1c1d2718ed934c3830af0a873ff Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Wed, 28 Feb 2024 16:42:03 +0100 Subject: [PATCH 02/23] Continue migrating --- .eslintrc.js | 1 + lib/Onyx.ts | 322 +++++++++++++++++++++------------------------------ 2 files changed, 132 insertions(+), 191 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 07636534..4542f0da 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -38,6 +38,7 @@ module.exports = { project: './tsconfig.json', }, rules: { + '@typescript-eslint/prefer-for-of': 'off', 'rulesdir/prefer-underscore-method': 'off', 'react/jsx-props-no-spreading': 'off', 'react/require-default-props': 'off', diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 46e70a3f..f236f089 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -1,7 +1,9 @@ +import {Onyx} from 'react-native-onyx'; /* eslint-disable no-continue */ import type {Component} from 'react'; import {deepEqual} from 'fast-equals'; import _ from 'underscore'; +import type {ValueOf} from 'type-fest'; import * as Logger from './Logger'; import cache from './OnyxCache'; import * as Str from './Str'; @@ -11,7 +13,7 @@ import Storage from './storage'; import utils from './utils'; import unstable_batchedUpdates from './batch'; import DevTools from './DevTools'; -import type {CollectionKeyBase, DeepRecord, KeyValueMapping, NullishDeep, OnyxCollection, OnyxEntry, OnyxKey, Selector} from './types'; +import type {CollectionKeyBase, DeepRecord, KeyValueMapping, NullishDeep, OnyxCollection, OnyxEntry, OnyxKey, Selector, WithOnyxInstanceState} from './types'; /** * Represents a mapping object where each `OnyxKey` maps to either a value of its corresponding type in `KeyValueMapping` or `null`. @@ -81,15 +83,17 @@ const METHOD = { CLEAR: 'clear', } as const; +type OnyxMethod = ValueOf; + // Key/value store of Onyx key and arrays of values to merge -const mergeQueue = {}; -const mergeQueuePromise = {}; +const mergeQueue: Record = {}; +const mergeQueuePromise: Record> = {}; // Keeps track of the last connectionID that was used so we can keep incrementing it let lastConnectionID = 0; // Holds a mapping of all the react components that want their state subscribed to a store key -const callbackToStateMapping: Record> = {}; +const callbackToStateMapping: Record> = {}; // Keeps a copy of the values of the onyx collection keys as a map for faster lookups let onyxCollectionKeyMap = new Map(); @@ -122,7 +126,7 @@ let batchUpdatesQueue: Array<() => void> = []; * @param value - contains the change that was made by the method * @param mergedValue - (optional) value that was written in the storage after a merge method was executed. */ -function sendActionToDevTools(method: string, key: OnyxKey, value: KeyValueMapping[OnyxKey], mergedValue: any = undefined) { +function sendActionToDevTools(method: OnyxMethod, key: OnyxKey, value: KeyValueMapping[OnyxKey], mergedValue: any = undefined) { DevTools.registerAction(utils.formatActionName(method, key), value, key ? {[key]: mergedValue || value} : value); } @@ -163,44 +167,25 @@ function batchUpdates(updates: () => void): Promise { return maybeFlushBatchUpdates(); } -/** - * Uses a selector function to return a simplified version of sourceData - * @param {Mixed} sourceData - * @param {Function} selector Function that takes sourceData and returns a simplified version of it - * @param {Object} [withOnyxInstanceState] - * @returns {Mixed} - */ -const getSubsetOfData = (sourceData, selector, withOnyxInstanceState) => selector(sourceData, withOnyxInstanceState); - /** * Takes a collection of items (eg. {testKey_1:{a:'a'}, testKey_2:{b:'b'}}) * and runs it through a reducer function to return a subset of the data according to a selector. * The resulting collection will only contain items that are returned by the selector. - * @param {Object} collection - * @param {String|Function} selector (see method docs for getSubsetOfData() for full details) - * @param {Object} [withOnyxInstanceState] - * @returns {Object} */ -const reduceCollectionWithSelector = (collection, selector, withOnyxInstanceState) => - _.reduce( - collection, - (finalCollection, item, key) => { - // eslint-disable-next-line no-param-reassign - finalCollection[key] = getSubsetOfData(item, selector, withOnyxInstanceState); - - return finalCollection; - }, - {}, - ); +function reduceCollectionWithSelector( + collection: Collection>, + selector: Selector, + withOnyxInstanceState: WithOnyxInstanceState, +) { + return Object.values(collection).reduce((finalCollection, item, key) => { + finalCollection[key] = selector(item, withOnyxInstanceState); + + return finalCollection; + }, {}); +} -/** - * Get some data from the store - * - * @private - * @param {string} key - * @returns {Promise<*>} - */ -function get(key) { +/** Get some data from the store */ +function get(key: OnyxKey): Promise | undefined { // When we already have the value in cache - resolve right away if (cache.hasCacheForKey(key)) { return Promise.resolve(cache.getValue(key)); @@ -273,15 +258,17 @@ function isSafeEvictionKey(testKey: OnyxKey): boolean { return evictionAllowList.some((key) => isKeyMatch(key, testKey)); } +type Mapping = { + selector?: Selector; + withOnyxInstance?: Component; + connectionID: number; +}; + /** * Tries to get a value from the cache. If the value is not present in cache it will return the default value or undefined. * If the requested key is a collection, it will return an object with all the collection members. - * - * @param {String} key - * @param {Object} mapping - * @returns {Mixed} */ -function tryGetCachedValue(key, mapping = {}) { +function tryGetCachedValue(key: TKey, mapping: Mapping) { let val = cache.getValue(key); if (isCollectionKey(key)) { @@ -292,20 +279,16 @@ function tryGetCachedValue(key, mapping = {}) { if (allCacheKeys.length === 0) { return; } - const matchingKeys = _.filter(allCacheKeys, (k) => k.startsWith(key)); - const values = _.reduce( - matchingKeys, - (finalObject, matchedKey) => { - const cachedValue = cache.getValue(matchedKey); - if (cachedValue) { - // This is permissible because we're in the process of constructing the final object in a reduce function. - // eslint-disable-next-line no-param-reassign - finalObject[matchedKey] = cachedValue; - } - return finalObject; - }, - {}, - ); + const matchingKeys = allCacheKeys.filter((k) => k.startsWith(key)); + const values = matchingKeys.reduce((finalObject, matchedKey) => { + const cachedValue = cache.getValue(matchedKey); + if (cachedValue) { + // This is permissible because we're in the process of constructing the final object in a reduce function. + // eslint-disable-next-line no-param-reassign + finalObject[matchedKey] = cachedValue; + } + return finalObject; + }, {}); val = values; } @@ -315,7 +298,7 @@ function tryGetCachedValue(key, mapping = {}) { if (isCollectionKey(key)) { return reduceCollectionWithSelector(val, mapping.selector, state); } - return getSubsetOfData(val, mapping.selector, state); + return mapping.selector(val, state); } return val; @@ -382,44 +365,32 @@ function addAllSafeEvictionKeysToRecentlyAccessedList(): Promise { }); } -/** - * @private - * @param {String} collectionKey - * @returns {Object} - */ -function getCachedCollection(collectionKey) { - const collectionMemberKeys = _.filter(cache.getAllKeys(), (storedKey) => isCollectionMemberKey(collectionKey, storedKey)); - - return _.reduce( - collectionMemberKeys, - (prev, curr) => { - const cachedValue = cache.getValue(curr); - if (!cachedValue) { - return prev; - } +function getCachedCollection(collectionKey: TKey): Record> { + const collectionMemberKeys = cache.getAllKeys().filter((storedKey) => isCollectionMemberKey(collectionKey, storedKey)); - // eslint-disable-next-line no-param-reassign - prev[curr] = cachedValue; + return collectionMemberKeys.reduce((prev: Record>, key) => { + const cachedValue = cache.getValue(key); + if (!cachedValue) { return prev; - }, - {}, - ); + } + + // eslint-disable-next-line no-param-reassign + prev[key] = cachedValue; + return prev; + }, {}); } -/** - * When a collection of keys change, search for any callbacks matching the collection key and trigger those callbacks - * - * @private - * @param {String} collectionKey - * @param {Object} partialCollection - a partial collection of grouped member keys - * @param {boolean} [notifyRegularSubscibers=true] - * @param {boolean} [notifyWithOnyxSubscibers=true] - */ -function keysChanged(collectionKey, partialCollection, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true) { +/** When a collection of keys change, search for any callbacks matching the collection key and trigger those callbacks */ +function keysChanged( + collectionKey: TKey, + partialCollection: Collection>, + notifyRegularSubscibers = true, + notifyWithOnyxSubscibers = true, +) { // We are iterating over all subscribers similar to keyChanged(). However, we are looking for subscribers who are subscribing to either a collection key or // individual collection key member for the collection that is being updated. It is important to note that the collection parameter cane be a PARTIAL collection // and does not represent all of the combined keys and values for a collection key. It is just the "new" data that was merged in via mergeCollection(). - const stateMappingKeys = _.keys(callbackToStateMapping); + const stateMappingKeys = Object.keys(callbackToStateMapping); for (let i = 0; i < stateMappingKeys.length; i++) { const subscriber = callbackToStateMapping[stateMappingKeys[i]]; if (!subscriber) { @@ -461,7 +432,7 @@ function keysChanged(collectionKey, partialCollection, notifyRegularSubscibers = // If they are not using waitForCollectionCallback then we notify the subscriber with // the new merged data but only for any keys in the partial collection. - const dataKeys = _.keys(partialCollection); + const dataKeys = Object.keys(partialCollection); for (let j = 0; j < dataKeys.length; j++) { const dataKey = dataKeys[j]; subscriber.callback(cachedCollection[dataKey], dataKey); @@ -507,7 +478,7 @@ function keysChanged(collectionKey, partialCollection, notifyRegularSubscibers = subscriber.withOnyxInstance.setStateProxy((prevState) => { const finalCollection = _.clone(prevState[subscriber.statePropertyName] || {}); - const dataKeys = _.keys(partialCollection); + const dataKeys = Object.keys(partialCollection); for (let j = 0; j < dataKeys.length; j++) { const dataKey = dataKeys[j]; finalCollection[dataKey] = cachedCollection[dataKey]; @@ -536,7 +507,7 @@ function keysChanged(collectionKey, partialCollection, notifyRegularSubscibers = if (subscriber.selector) { subscriber.withOnyxInstance.setStateProxy((prevState) => { const prevData = prevState[subscriber.statePropertyName]; - const newData = getSubsetOfData(cachedCollection[subscriber.key], subscriber.selector, subscriber.withOnyxInstance.state); + const newData = subscriber.selector(cachedCollection[subscriber.key], subscriber.withOnyxInstance.state); if (!deepEqual(prevData, newData)) { PerformanceUtils.logSetStateCall(subscriber, prevData, newData, 'keysChanged', collectionKey); return { @@ -577,15 +548,16 @@ function keysChanged(collectionKey, partialCollection, notifyRegularSubscibers = * @example * keyChanged(key, value, subscriber => subscriber.initWithStoredValues === false) * - * @private - * @param {String} key - * @param {*} data - * @param {*} prevData * @param {Function} [canUpdateSubscriber] only subscribers that pass this truth test will be updated - * @param {boolean} [notifyRegularSubscibers=true] - * @param {boolean} [notifyWithOnyxSubscibers=true] */ -function keyChanged(key, data, prevData, canUpdateSubscriber = () => true, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true) { +function keyChanged( + key: TKey, + data: KeyValueMapping[TKey], + prevData: KeyValueMapping[TKey], + canUpdateSubscriber = () => true, + notifyRegularSubscibers = true, + notifyWithOnyxSubscibers = true, +) { // Add or remove this key from the recentlyAccessedKeys lists if (data !== null) { addLastAccessedKey(key); @@ -596,7 +568,7 @@ function keyChanged(key, data, prevData, canUpdateSubscriber = () => true, notif // We are iterating over all subscribers to see if they are interested in the key that has just changed. If the subscriber's key is a collection key then we will // notify them if the key that changed is a collection member. Or if it is a regular key notify them when there is an exact match. Depending on whether the subscriber // was connected via withOnyx we will call setState() directly on the withOnyx instance. If it is a regular connection we will pass the data to the provided callback. - const stateMappingKeys = _.keys(callbackToStateMapping); + const stateMappingKeys = Object.keys(callbackToStateMapping); for (let i = 0; i < stateMappingKeys.length; i++) { const subscriber = callbackToStateMapping[stateMappingKeys[i]]; if (!subscriber || !isKeyMatch(subscriber.key, key) || !canUpdateSubscriber(subscriber)) { @@ -633,7 +605,7 @@ function keyChanged(key, data, prevData, canUpdateSubscriber = () => true, notif subscriber.withOnyxInstance.setStateProxy((prevState) => { const prevWithOnyxData = prevState[subscriber.statePropertyName]; const newWithOnyxData = { - [key]: getSubsetOfData(data, subscriber.selector, subscriber.withOnyxInstance.state), + [key]: subscriber.selector(data, subscriber.withOnyxInstance.state), }; const prevDataWithNewData = { ...prevWithOnyxData, @@ -668,8 +640,8 @@ function keyChanged(key, data, prevData, canUpdateSubscriber = () => true, notif // returned by the selector and only if the selected data has changed. if (subscriber.selector) { subscriber.withOnyxInstance.setStateProxy(() => { - const previousValue = getSubsetOfData(prevData, subscriber.selector, subscriber.withOnyxInstance.state); - const newValue = getSubsetOfData(data, subscriber.selector, subscriber.withOnyxInstance.state); + const previousValue = subscriber.selector(prevData, subscriber.withOnyxInstance.state); + const newValue = subscriber.selector(data, subscriber.withOnyxInstance.state); if (!deepEqual(previousValue, newValue)) { return { @@ -709,18 +681,8 @@ function keyChanged(key, data, prevData, canUpdateSubscriber = () => true, notif * Sends the data obtained from the keys to the connection. It either: * - sets state on the withOnyxInstances * - triggers the callback function - * - * @private - * @param {Object} mapping - * @param {Object} [mapping.withOnyxInstance] - * @param {String} [mapping.statePropertyName] - * @param {Function} [mapping.callback] - * @param {String} [mapping.selector] - * @param {*|null} val - * @param {String|undefined} matchedKey - * @param {Boolean} isBatched */ -function sendDataToConnection(mapping, val, matchedKey, isBatched) { +function sendDataToConnection(mapping: Mapping, val: any, matchedKey: OnyxKey, isBatched: boolean) { // If the mapping no longer exists then we should not send any data. // This means our subscriber disconnected or withOnyx wrapped component unmounted. if (!callbackToStateMapping[mapping.connectionID]) { @@ -734,9 +696,9 @@ function sendDataToConnection(mapping, val, matchedKey, isBatched) { // returned by the selector. if (mapping.selector) { if (isCollectionKey(mapping.key)) { - newData = reduceCollectionWithSelector(val, mapping.selector, mapping.withOnyxInstance.state); + newData = mapping.selector(val, mapping.withOnyxInstance.state); } else { - newData = getSubsetOfData(val, mapping.selector, mapping.withOnyxInstance.state); + newData = mapping.selector(val, mapping.withOnyxInstance.state); } } @@ -759,11 +721,8 @@ function sendDataToConnection(mapping, val, matchedKey, isBatched) { /** * We check to see if this key is flagged as safe for eviction and add it to the recentlyAccessedKeys list so that when we * run out of storage the least recently accessed key can be removed. - * - * @private - * @param {Object} mapping */ -function addKeyToRecentlyAccessedIfNeeded(mapping) { +function addKeyToRecentlyAccessedIfNeeded(mapping: Mapping) { if (!isSafeEvictionKey(mapping.key)) { return; } @@ -788,18 +747,14 @@ function addKeyToRecentlyAccessedIfNeeded(mapping) { * @param {Array} matchingKeys * @param {Object} mapping */ -function getCollectionDataAndSendAsObject(matchingKeys, mapping) { +function getCollectionDataAndSendAsObject(matchingKeys: OnyxKey[], mapping: Mapping) { Promise.all(matchingKeys.map((key) => get(key))) .then((values) => - _.reduce( - values, - (finalObject, value, i) => { - // eslint-disable-next-line no-param-reassign - finalObject[matchingKeys[i]] = value; - return finalObject; - }, - {}, - ), + values.reduce((finalObject, value, i) => { + // eslint-disable-next-line no-param-reassign + finalObject[matchingKeys[i]] = value; + return finalObject; + }, {}), ) .then((val) => sendDataToConnection(mapping, val, undefined, true)); } @@ -825,7 +780,7 @@ function getCollectionDataAndSendAsObject(matchingKeys, mapping) { * @param [mapping.waitForCollectionCallback] If set to true, it will return the entire collection to the callback as a single object * @returns an ID to use when calling disconnect */ -function connect(mapping: ConnectOptions): number { +function connect(mapping: Mapping): number { const connectionID = lastConnectionID++; callbackToStateMapping[connectionID] = mapping; callbackToStateMapping[connectionID].connectionID = connectionID; @@ -850,7 +805,7 @@ function connect(mapping: ConnectOptions): number { // We search all the keys in storage to see if any are a "match" for the subscriber we are connecting so that we // can send data back to the subscriber. Note that multiple keys can match as a subscriber could either be // subscribed to a "collection key" or a single key. - const matchingKeys = _.filter(keys, (key) => isKeyMatch(mapping.key, key)); + const matchingKeys = keys.filter((key) => isKeyMatch(mapping.key, key)); // If the key being connected to does not exist we initialize the value with null. For subscribers that connected // directly via connect() they will simply get a null value sent to them without any information about which key matched @@ -936,14 +891,8 @@ function disconnect(connectionID: number, keyToRemoveFromEvictionBlocklist?: Ony * * @example * scheduleSubscriberUpdate(key, value, subscriber => subscriber.initWithStoredValues === false) - * - * @param {String} key - * @param {*} value - * @param {*} prevValue - * @param {Function} [canUpdateSubscriber] only subscribers that pass this truth test will be updated - * @returns {Promise} */ -function scheduleSubscriberUpdate(key, value, prevValue, canUpdateSubscriber = () => true) { +function scheduleSubscriberUpdate(key: TKey, value: KeyValueMapping[TKey], prevValue: KeyValueMapping[TKey], canUpdateSubscriber = () => true) { const promise = Promise.resolve().then(() => keyChanged(key, value, prevValue, canUpdateSubscriber, true, false)); batchUpdates(() => keyChanged(key, value, prevValue, canUpdateSubscriber, false, true)); return Promise.all([maybeFlushBatchUpdates(), promise]); @@ -958,7 +907,7 @@ function scheduleSubscriberUpdate(key, value, prevValue, canUpdateSubscriber = ( * @param {*} value * @returns {Promise} */ -function scheduleNotifyCollectionSubscribers(key, value) { +function scheduleNotifyCollectionSubscribers(key: TKey, value: KeyValueMapping[TKey]) { const promise = Promise.resolve().then(() => keysChanged(key, value, true, false)); batchUpdates(() => keysChanged(key, value, false, true)); return Promise.all([maybeFlushBatchUpdates(), promise]); @@ -971,7 +920,7 @@ function scheduleNotifyCollectionSubscribers(key, value) { * @param {String} key * @return {Promise} */ -function remove(key) { +function remove(key: TKey) { const prevValue = cache.getValue(key, false); cache.drop(key); scheduleSubscriberUpdate(key, null, prevValue); @@ -996,14 +945,8 @@ function reportStorageQuota() { * If we fail to set or merge we must handle this by * evicting some data from Onyx and then retrying to do * whatever it is we attempted to do. - * - * @private - * @param {Error} error - * @param {Function} onyxMethod - * @param {...any} args - * @return {Promise} */ -function evictStorageAndRetry(error, onyxMethod, ...args) { +function evictStorageAndRetry(error: Error, onyxMethod: TMethod, ...args: Parameters) { Logger.logInfo(`Failed to save to storage. Error: ${error}. onyxMethod: ${onyxMethod.name}`); if (error && Str.startsWith(error.message, "Failed to execute 'put' on 'IDBObjectStore'")) { @@ -1012,7 +955,7 @@ function evictStorageAndRetry(error, onyxMethod, ...args) { } // Find the first key that we can remove that has no subscribers in our blocklist - const keyForRemoval = _.find(recentlyAccessedKeys, (key) => !evictionBlocklist[key]); + const keyForRemoval = recentlyAccessedKeys.find((key) => !evictionBlocklist[key]); if (!keyForRemoval) { // If we have no acceptable keys to remove then we are possibly trying to save mission critical data. If this is the case, // then we should stop retrying as there is not much the user can do to fix this. Instead of getting them stuck in an infinite loop we @@ -1037,9 +980,9 @@ function evictStorageAndRetry(error, onyxMethod, ...args) { * @param {Boolean} wasRemoved * @returns {Promise} */ -function broadcastUpdate(key, value, method, hasChanged, wasRemoved = false) { +function broadcastUpdate(key: TKey, value: KeyValueMapping[TKey], method: string, hasChanged: boolean, wasRemoved = false) { // Logging properties only since values could be sensitive things we don't want to log - Logger.logInfo(`${method}() called for key: ${key}${typeof value === 'object' ? ` properties: ${_.keys(value).join(',')}` : ''}`); + Logger.logInfo(`${method}() called for key: ${key}${typeof value === 'object' ? ` properties: ${Object.keys(value).join(',')}` : ''}`); const prevValue = cache.getValue(key, false); // Update subscribers if the cached value has changed, or when the subscriber specifically requires @@ -1113,14 +1056,13 @@ function set(key: TKey, value: OnyxEntry */ -function prepareKeyValuePairsForStorage(data) { - const keyValuePairs = []; +function prepareKeyValuePairsForStorage(data: Partial): Array<[string, unknown]> { + const keyValuePairs: Array<[string, unknown]> = []; - data.forEach((value, key) => { + Object.entries(data).forEach(([key, value]) => { const {value: valueAfterRemoving, wasRemoved} = removeNullValues(key, value); if (wasRemoved) return; @@ -1160,14 +1102,10 @@ function multiSet(data: Partial): Promise { /** * Merges an array of changes with an existing value * - * @private - * @param {*} existingValue * @param {Array<*>} changes Array of changes that should be applied to the existing value - * @param {Boolean} shouldRemoveNullObjectValues - * @returns {*} */ -function applyMerge(existingValue, changes, shouldRemoveNullObjectValues) { - const lastChange = _.last(changes); +function applyMerge(existingValue: unknown, changes: [], shouldRemoveNullObjectValues: boolean) { + const lastChange = changes?.at(-1); if (Array.isArray(lastChange)) { return lastChange; @@ -1175,7 +1113,7 @@ function applyMerge(existingValue, changes, shouldRemoveNullObjectValues) { if (changes.some((change) => typeof change === 'object')) { // Object values are then merged one after the other - return _.reduce(changes, (modifiedData, change) => utils.fastMerge(modifiedData, change, shouldRemoveNullObjectValues), existingValue || {}); + return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, shouldRemoveNullObjectValues), existingValue || {}); } // If we have anything else we can't merge it so we'll @@ -1225,7 +1163,7 @@ function merge(key: TKey, changes: OnyxEntry(key: TKey, changes: OnyxEntry { - const existingDataAsObject = _.object(pairs); + return Storage.multiGet(Object.keys(defaultKeyStates)).then((pairs) => { + const existingDataAsObject = Object.fromEntries(pairs); const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates); cache.merge(merged); @@ -1319,8 +1257,8 @@ function clear(keysToPreserve: OnyxKey[] = []): Promise { // 2. Any keys with a default state (because they need to remain in Onyx as their default, and setting them // to null would cause unknown behavior) keys.forEach((key) => { - const isKeyToPreserve = _.contains(keysToPreserve, key); - const isDefaultKey = _.has(defaultKeyStates, key); + const isKeyToPreserve = keysToPreserve.includes(key); + const isDefaultKey = key in defaultKeyStates; // If the key is being removed or reset to default: // 1. Update it in the cache @@ -1328,7 +1266,7 @@ function clear(keysToPreserve: OnyxKey[] = []): Promise { // since collection key subscribers need to be updated differently if (!isKeyToPreserve) { const oldValue = cache.getValue(key); - const newValue = _.get(defaultKeyStates, key, null); + const newValue = defaultKeyStates[key] ?? null; if (newValue !== oldValue) { cache.set(key, newValue); const collectionKey = key.substring(0, key.indexOf('_') + 1); @@ -1415,20 +1353,26 @@ function mergeCollection(collectionKey: TK return getAllKeys().then((persistedKeys) => { // Split to keys that exist in storage and keys that don't - const [existingKeys, newKeys] = _.chain(collection) - .pick((value, key) => { - if (value === null) { - remove(key); - return false; - } - return true; - }) - .keys() - .partition((key) => persistedKeys.includes(key)) - .value(); - - const existingKeyCollection = _.pick(collection, existingKeys); - const newCollection = _.pick(collection, newKeys); + const keys = Object.keys(collection).filter((key) => { + if (collection[key] === null) { + remove(key); + return false; + } + return true; + }); + + const existingKeys = keys.filter((key) => persistedKeys.includes(key)); + const newKeys = keys.filter((key) => !persistedKeys.includes(key)); + + const existingKeyCollection = existingKeys.reduce((obj, key) => { + obj[key] = collection[key]; + return obj; + }, {}); + + const newCollection = newKeys.reduce((obj, key) => { + obj[key] = collection[key]; + return obj; + }, {}); const keyValuePairsForExistingCollection = prepareKeyValuePairsForStorage(existingKeyCollection); const keyValuePairsForNewCollection = prepareKeyValuePairsForStorage(newCollection); @@ -1495,7 +1439,7 @@ type OnyxUpdate = function update(data: OnyxUpdate[]): Promise { // First, validate the Onyx object is in the format we expect data.forEach(({onyxMethod, key, value}) => { - if (!_.contains([METHOD.CLEAR, METHOD.SET, METHOD.MERGE, METHOD.MERGE_COLLECTION, METHOD.MULTI_SET], onyxMethod)) { + if (![METHOD.CLEAR, METHOD.SET, METHOD.MERGE, METHOD.MERGE_COLLECTION, METHOD.MULTI_SET].includes(onyxMethod)) { throw new Error(`Invalid onyxMethod ${onyxMethod} in Onyx update.`); } if (onyxMethod === METHOD.MULTI_SET) { @@ -1589,14 +1533,10 @@ function init({ // We need the value of the collection keys later for checking if a // key is a collection. We store it in a map for faster lookup. const collectionValues = Object.values(keys.COLLECTION); - onyxCollectionKeyMap = _.reduce( - collectionValues, - (acc, val) => { - acc.set(val, true); - return acc; - }, - new Map(), - ); + onyxCollectionKeyMap = collectionValues.reduce((acc, val) => { + acc.set(val, true); + return acc; + }, new Map()); // Set our default key states to use when initializing and clearing Onyx data defaultKeyStates = initialKeyStates; From 0950f527ef397c75727a57b9973dd8f0d76f79f8 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Wed, 28 Feb 2024 17:10:02 +0100 Subject: [PATCH 03/23] Rename types.d.ts to types.ts --- lib/Onyx.ts | 41 +++++++++++++++++++++--------- lib/OnyxCache.ts | 4 +-- lib/storage/__mocks__/index.ts | 13 +++++++--- lib/storage/providers/IDBKeyVal.ts | 2 +- lib/storage/providers/types.ts | 5 ++-- lib/{types.d.ts => types.ts} | 27 +++++++++++++++++--- 6 files changed, 69 insertions(+), 23 deletions(-) rename lib/{types.d.ts => types.ts} (92%) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index f236f089..c1561c40 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -1,8 +1,7 @@ -import {Onyx} from 'react-native-onyx'; /* eslint-disable no-continue */ import type {Component} from 'react'; import {deepEqual} from 'fast-equals'; -import _ from 'underscore'; +import lodashClone from 'lodash/clone'; import type {ValueOf} from 'type-fest'; import * as Logger from './Logger'; import cache from './OnyxCache'; @@ -13,7 +12,7 @@ import Storage from './storage'; import utils from './utils'; import unstable_batchedUpdates from './batch'; import DevTools from './DevTools'; -import type {CollectionKeyBase, DeepRecord, KeyValueMapping, NullishDeep, OnyxCollection, OnyxEntry, OnyxKey, Selector, WithOnyxInstanceState} from './types'; +import type {CollectionKeyBase, DeepRecord, KeyValueMapping, NullishDeep, OnyxCollection, OnyxEntry, OnyxKey, OnyxValue, Selector, WithOnyxInstanceState} from './types'; /** * Represents a mapping object where each `OnyxKey` maps to either a value of its corresponding type in `KeyValueMapping` or `null`. @@ -86,7 +85,7 @@ const METHOD = { type OnyxMethod = ValueOf; // Key/value store of Onyx key and arrays of values to merge -const mergeQueue: Record = {}; +const mergeQueue: Record>> = {}; const mergeQueuePromise: Record> = {}; // Keeps track of the last connectionID that was used so we can keep incrementing it @@ -96,7 +95,7 @@ let lastConnectionID = 0; const callbackToStateMapping: Record> = {}; // Keeps a copy of the values of the onyx collection keys as a map for faster lookups -let onyxCollectionKeyMap = new Map(); +let onyxCollectionKeyMap = new Map(); // Holds a list of keys that have been directly subscribed to or recently modified from least to most recent let recentlyAccessedKeys: OnyxKey[] = []; @@ -126,7 +125,8 @@ let batchUpdatesQueue: Array<() => void> = []; * @param value - contains the change that was made by the method * @param mergedValue - (optional) value that was written in the storage after a merge method was executed. */ -function sendActionToDevTools(method: OnyxMethod, key: OnyxKey, value: KeyValueMapping[OnyxKey], mergedValue: any = undefined) { +function sendActionToDevTools(method: OnyxMethod, key: OnyxKey, value: OnyxValue, mergedValue: any = undefined) { + // @ts-expect-error Migrate DevTools DevTools.registerAction(utils.formatActionName(method, key), value, key ? {[key]: mergedValue || value} : value); } @@ -185,7 +185,7 @@ function reduceCollectionWithSelector | undefined { +function get(key: OnyxKey): Promise { // When we already have the value in cache - resolve right away if (cache.hasCacheForKey(key)) { return Promise.resolve(cache.getValue(key)); @@ -477,7 +477,7 @@ function keysChanged( } subscriber.withOnyxInstance.setStateProxy((prevState) => { - const finalCollection = _.clone(prevState[subscriber.statePropertyName] || {}); + const finalCollection = lodashClone(prevState[subscriber.statePropertyName] || {}); const dataKeys = Object.keys(partialCollection); for (let j = 0; j < dataKeys.length; j++) { const dataKey = dataKeys[j]; @@ -1104,7 +1104,7 @@ function multiSet(data: Partial): Promise { * * @param {Array<*>} changes Array of changes that should be applied to the existing value */ -function applyMerge(existingValue: unknown, changes: [], shouldRemoveNullObjectValues: boolean) { +function applyMerge(existingValue: unknown, changes: Array>, shouldRemoveNullObjectValues: boolean) { const lastChange = changes?.at(-1); if (Array.isArray(lastChange)) { @@ -1299,7 +1299,14 @@ function clear(keysToPreserve: OnyxKey[] = []): Promise { updatePromises.push(scheduleNotifyCollectionSubscribers(key, value)); }); - const defaultKeyValuePairs = _.pairs(_.omit(defaultKeyStates, keysToPreserve)); + const defaultKeyValuePairs = Object.entries( + Object.keys(defaultKeyStates) + .filter((key) => !keysToPreserve.includes(key)) + .reduce((obj, key) => { + obj[key] = defaultKeyStates[key]; + return obj; + }, {}), + ); // Remove only the items that we want cleared from storage, and reset others to default keysToBeClearedFromStorage.forEach((key) => cache.drop(key)); @@ -1420,6 +1427,16 @@ type OnyxUpdate = onyxMethod: typeof METHOD.MERGE; key: TKey; value: OnyxEntry>; + } + | { + onyxMethod: typeof METHOD.MULTI_SET; + key: TKey; + value: Partial; + } + | { + onyxMethod: typeof METHOD.CLEAR; + key: TKey; + value?: undefined; }; }[OnyxKey] | { @@ -1436,7 +1453,7 @@ type OnyxUpdate = * @param {Array} data An array of objects with shape {onyxMethod: oneOf('set', 'merge', 'mergeCollection', 'multiSet', 'clear'), key: string, value: *} * @returns {Promise} resolves when all operations are complete */ -function update(data: OnyxUpdate[]): Promise { +function update(data: OnyxUpdate[]): Promise { // First, validate the Onyx object is in the format we expect data.forEach(({onyxMethod, key, value}) => { if (![METHOD.CLEAR, METHOD.SET, METHOD.MERGE, METHOD.MERGE_COLLECTION, METHOD.MULTI_SET].includes(onyxMethod)) { @@ -1452,7 +1469,7 @@ function update(data: OnyxUpdate[]): Promise { } }); - const promises = []; + const promises: Array<() => Promise> = []; let clearPromise = Promise.resolve(); data.forEach(({onyxMethod, key, value}) => { diff --git a/lib/OnyxCache.ts b/lib/OnyxCache.ts index 14908d46..2fdeb3d0 100644 --- a/lib/OnyxCache.ts +++ b/lib/OnyxCache.ts @@ -130,8 +130,8 @@ class OnyxCache { * provided from this function * @param taskName - unique name given for the task */ - getTaskPromise(taskName: string): Promise | undefined { - return this.pendingPromises.get(taskName); + getTaskPromise(taskName: string): Promise { + return this.pendingPromises.get(taskName) ?? Promise.resolve(); } /** diff --git a/lib/storage/__mocks__/index.ts b/lib/storage/__mocks__/index.ts index 3a2f167c..3f13484a 100644 --- a/lib/storage/__mocks__/index.ts +++ b/lib/storage/__mocks__/index.ts @@ -15,20 +15,27 @@ const idbKeyvalMock: StorageProvider = { }, multiSet(pairs) { const setPromises = pairs.map(([key, value]) => this.setItem(key, value)); - return new Promise((resolve) => Promise.all(setPromises).then(() => resolve(storageMapInternal))); + return new Promise((resolve) => { + Promise.all(setPromises).then(() => resolve(storageMapInternal)); + }); }, getItem(key) { return Promise.resolve(storageMapInternal[key]); }, multiGet(keys) { - const getPromises = keys.map((key) => new Promise((resolve) => this.getItem(key).then((value) => resolve([key, value])))); + const getPromises = keys.map( + (key) => + new Promise((resolve) => { + this.getItem(key).then((value) => resolve([key, value])); + }), + ); return Promise.all(getPromises) as Promise; }, multiMerge(pairs) { pairs.forEach(([key, value]) => { const existingValue = storageMapInternal[key]; // eslint-disable-next-line @typescript-eslint/no-explicit-any - const newValue = utils.fastMerge(existingValue as any, value); + const newValue = utils.fastMerge(existingValue as any, value as any); set(key, newValue); }); diff --git a/lib/storage/providers/IDBKeyVal.ts b/lib/storage/providers/IDBKeyVal.ts index cd9d3ba3..80b04185 100644 --- a/lib/storage/providers/IDBKeyVal.ts +++ b/lib/storage/providers/IDBKeyVal.ts @@ -27,7 +27,7 @@ const provider: StorageProvider = { const upsertMany = pairs.map(([key, value], index) => { const prev = values[index]; // eslint-disable-next-line @typescript-eslint/no-explicit-any - const newValue = utils.fastMerge(prev as any, value); + const newValue = utils.fastMerge(prev as any, value as any); return promisifyRequest(store.put(newValue, key)); }); return Promise.all(upsertMany); diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index b2be36e3..6d800cfd 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -1,7 +1,8 @@ import type {BatchQueryResult, QueryResult} from 'react-native-quick-sqlite'; +import type {KeyValueMapping, OnyxCollection, OnyxEntry, OnyxKey} from '../../types'; -type Key = string; -type Value = IDBValidKey | null; +type Key = OnyxKey; +type Value = OnyxEntry | OnyxCollection; type KeyValuePair = [Key, Value]; type KeyList = Key[]; type KeyValuePairList = KeyValuePair[]; diff --git a/lib/types.d.ts b/lib/types.ts similarity index 92% rename from lib/types.d.ts rename to lib/types.ts index 6d1f517a..01446474 100644 --- a/lib/types.d.ts +++ b/lib/types.ts @@ -1,5 +1,5 @@ -import {Merge} from 'type-fest'; -import {BuiltIns} from 'type-fest/source/internal'; +import type {Merge} from 'type-fest'; +import type {BuiltIns} from 'type-fest/source/internal'; /** * Represents a deeply nested record. It maps keys to values, @@ -76,6 +76,7 @@ type TypeOptions = Merge< * } * ``` */ +// eslint-disable-next-line @typescript-eslint/no-empty-interface interface CustomTypeOptions {} /** @@ -99,6 +100,11 @@ type CollectionKey = `${CollectionKeyBase}${string}`; */ type OnyxKey = Key | CollectionKey; +/** + * Represents a value type of an Onyx key. + */ +type OnyxValue = TypeOptions['values'][OnyxKey]; + /** * Represents a mapping of Onyx keys to values, where keys are either normal or collection Onyx keys * and values are the corresponding values in Onyx's state. @@ -185,6 +191,7 @@ type OnyxCollection = OnyxEntry>; type NonTransformableTypes = | BuiltIns + // eslint-disable-next-line @typescript-eslint/no-explicit-any | ((...args: any[]) => unknown) | Map | Set @@ -226,4 +233,18 @@ type NullishObjectDeep = { */ type WithOnyxInstanceState = (TOnyxProps & {loading: boolean}) | undefined; -export {CollectionKey, CollectionKeyBase, CustomTypeOptions, DeepRecord, Key, KeyValueMapping, OnyxCollection, OnyxEntry, OnyxKey, Selector, NullishDeep, WithOnyxInstanceState}; +export type { + CollectionKey, + CollectionKeyBase, + OnyxValue, + CustomTypeOptions, + DeepRecord, + Key, + KeyValueMapping, + OnyxCollection, + OnyxEntry, + OnyxKey, + Selector, + NullishDeep, + WithOnyxInstanceState, +}; From 5ace7b020a897e4f157f4499b100f7833c6140dd Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Wed, 28 Feb 2024 17:30:24 +0100 Subject: [PATCH 04/23] Clear a lot of errors --- lib/Onyx.ts | 47 +++++++++++++++--------------- lib/OnyxCache.ts | 34 ++++++++++----------- lib/storage/providers/IDBKeyVal.ts | 4 +-- lib/storage/providers/types.ts | 20 ++++++------- 4 files changed, 50 insertions(+), 55 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index c1561c40..029003ca 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -41,10 +41,12 @@ type Collection = { }; /** Represents the base options used in `Onyx.connect()` method. */ -type BaseConnectOptions = { +type BaseConnectOptions = { statePropertyName?: string; withOnyxInstance?: Component; initWithStoredValues?: boolean; + selector?: Selector; + connectionID: number; }; /** @@ -59,7 +61,7 @@ type BaseConnectOptions = { * If `waitForCollectionCallback` is `false` or not specified, the `key` can be any Onyx key and `callback` will be triggered with updates of each collection item * and will pass `value` as an `OnyxEntry`. */ -type ConnectOptions = BaseConnectOptions & +type Mapping = BaseConnectOptions & ( | { key: TKey extends CollectionKeyBase ? TKey : never; @@ -85,14 +87,14 @@ const METHOD = { type OnyxMethod = ValueOf; // Key/value store of Onyx key and arrays of values to merge -const mergeQueue: Record>> = {}; +const mergeQueue: Record = {}; const mergeQueuePromise: Record> = {}; // Keeps track of the last connectionID that was used so we can keep incrementing it let lastConnectionID = 0; // Holds a mapping of all the react components that want their state subscribed to a store key -const callbackToStateMapping: Record> = {}; +const callbackToStateMapping: Record> = {}; // Keeps a copy of the values of the onyx collection keys as a map for faster lookups let onyxCollectionKeyMap = new Map(); @@ -125,7 +127,7 @@ let batchUpdatesQueue: Array<() => void> = []; * @param value - contains the change that was made by the method * @param mergedValue - (optional) value that was written in the storage after a merge method was executed. */ -function sendActionToDevTools(method: OnyxMethod, key: OnyxKey, value: OnyxValue, mergedValue: any = undefined) { +function sendActionToDevTools(method: OnyxMethod, key: OnyxKey | undefined, value: OnyxValue, mergedValue: any = undefined) { // @ts-expect-error Migrate DevTools DevTools.registerAction(utils.formatActionName(method, key), value, key ? {[key]: mergedValue || value} : value); } @@ -185,7 +187,7 @@ function reduceCollectionWithSelector { +function get(key: OnyxKey): Promise { // When we already have the value in cache - resolve right away if (cache.hasCacheForKey(key)) { return Promise.resolve(cache.getValue(key)); @@ -258,12 +260,6 @@ function isSafeEvictionKey(testKey: OnyxKey): boolean { return evictionAllowList.some((key) => isKeyMatch(key, testKey)); } -type Mapping = { - selector?: Selector; - withOnyxInstance?: Component; - connectionID: number; -}; - /** * Tries to get a value from the cache. If the value is not present in cache it will return the default value or undefined. * If the requested key is a collection, it will return an object with all the collection members. @@ -682,7 +678,7 @@ function keyChanged( * - sets state on the withOnyxInstances * - triggers the callback function */ -function sendDataToConnection(mapping: Mapping, val: any, matchedKey: OnyxKey, isBatched: boolean) { +function sendDataToConnection(mapping: Mapping, val: OnyxValue, matchedKey: OnyxKey | undefined, isBatched: boolean) { // If the mapping no longer exists then we should not send any data. // This means our subscriber disconnected or withOnyx wrapped component unmounted. if (!callbackToStateMapping[mapping.connectionID]) { @@ -722,7 +718,7 @@ function sendDataToConnection(mapping: Mapping, val: * We check to see if this key is flagged as safe for eviction and add it to the recentlyAccessedKeys list so that when we * run out of storage the least recently accessed key can be removed. */ -function addKeyToRecentlyAccessedIfNeeded(mapping: Mapping) { +function addKeyToRecentlyAccessedIfNeeded(mapping: Mapping) { if (!isSafeEvictionKey(mapping.key)) { return; } @@ -780,7 +776,9 @@ function getCollectionDataAndSendAsObject(matchi * @param [mapping.waitForCollectionCallback] If set to true, it will return the entire collection to the callback as a single object * @returns an ID to use when calling disconnect */ -function connect(mapping: Mapping): number { +function connect(mappingWithoutConnectionID: Omit, 'connectionID'>): number { + const mapping = mappingWithoutConnectionID as Mapping; + const connectionID = lastConnectionID++; callbackToStateMapping[connectionID] = mapping; callbackToStateMapping[connectionID].connectionID = connectionID; @@ -967,6 +965,8 @@ function evictStorageAndRetry onyxMethod(...args)); } @@ -1059,8 +1059,8 @@ function set(key: TKey, value: OnyxEntry */ -function prepareKeyValuePairsForStorage(data: Partial): Array<[string, unknown]> { - const keyValuePairs: Array<[string, unknown]> = []; +function prepareKeyValuePairsForStorage(data: Partial): Array<[OnyxKey, OnyxValue]> { + const keyValuePairs: Array<[OnyxKey, OnyxValue]> = []; Object.entries(data).forEach(([key, value]) => { const {value: valueAfterRemoving, wasRemoved} = removeNullValues(key, value); @@ -1102,9 +1102,9 @@ function multiSet(data: Partial): Promise { /** * Merges an array of changes with an existing value * - * @param {Array<*>} changes Array of changes that should be applied to the existing value + * @param changes Array of changes that should be applied to the existing value */ -function applyMerge(existingValue: unknown, changes: Array>, shouldRemoveNullObjectValues: boolean) { +function applyMerge(existingValue: OnyxValue, changes: Array>, shouldRemoveNullObjectValues: boolean) { const lastChange = changes?.at(-1); if (Array.isArray(lastChange)) { @@ -1113,7 +1113,8 @@ function applyMerge(existingValue: unknown, changes: Array> if (changes.some((change) => typeof change === 'object')) { // Object values are then merged one after the other - return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, shouldRemoveNullObjectValues), existingValue || {}); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData as any, change as any, shouldRemoveNullObjectValues), existingValue || {}); } // If we have anything else we can't merge it so we'll @@ -1210,10 +1211,8 @@ function merge(key: TKey, changes: OnyxEntry { return Storage.multiGet(Object.keys(defaultKeyStates)).then((pairs) => { const existingDataAsObject = Object.fromEntries(pairs); @@ -1403,7 +1402,7 @@ function mergeCollection(collectionKey: TK }); return Promise.all(promises) - .catch((error) => evictStorageAndRetry(error, mergeCollection, collection)) + .catch((error) => evictStorageAndRetry(error, mergeCollection, collectionKey, collection)) .then(() => { sendActionToDevTools(METHOD.MERGE_COLLECTION, undefined, collection); return promiseUpdate; diff --git a/lib/OnyxCache.ts b/lib/OnyxCache.ts index 2fdeb3d0..e5d15ec1 100644 --- a/lib/OnyxCache.ts +++ b/lib/OnyxCache.ts @@ -1,9 +1,7 @@ import {deepEqual} from 'fast-equals'; import bindAll from 'lodash/bindAll'; -import type {Key, Value} from './storage/providers/types'; import utils from './utils'; - -type StorageMap = Record; +import type {OnyxKey, OnyxValue} from './types'; /** * In memory cache providing data by reference @@ -11,19 +9,19 @@ type StorageMap = Record; */ class OnyxCache { /** Cache of all the storage keys available in persistent storage */ - private storageKeys: Set; + storageKeys: Set; /** Unique list of keys maintained in access order (most recent at the end) */ - private recentKeys: Set; + private recentKeys: Set; /** A map of cached values */ - private storageMap: StorageMap; + private storageMap: Record; /** * Captured pending tasks for already running storage methods * Using a map yields better performance on operations such a delete */ - private pendingPromises: Map>; + private pendingPromises: Map>; /** Maximum size of the keys store din cache */ private maxRecentKeysSize = 0; @@ -53,7 +51,7 @@ class OnyxCache { } /** Get all the storage keys */ - getAllKeys(): Key[] { + getAllKeys(): OnyxKey[] { return Array.from(this.storageKeys); } @@ -61,7 +59,7 @@ class OnyxCache { * Get a cached value from storage * @param [shouldReindexCache] – This is an LRU cache, and by default accessing a value will make it become last in line to be evicted. This flag can be used to skip that and just access the value directly without side-effects. */ - getValue(key: Key, shouldReindexCache = true): Value { + getValue(key: OnyxKey, shouldReindexCache = true): OnyxValue { if (shouldReindexCache) { this.addToAccessedKeys(key); } @@ -69,14 +67,14 @@ class OnyxCache { } /** Check whether cache has data for the given key */ - hasCacheForKey(key: Key): boolean { + hasCacheForKey(key: OnyxKey): boolean { return this.storageMap[key] !== undefined; } /** Saves a key in the storage keys list * Serves to keep the result of `getAllKeys` up to date */ - addKey(key: Key): void { + addKey(key: OnyxKey): void { this.storageKeys.add(key); } @@ -84,7 +82,7 @@ class OnyxCache { * Set's a key value in cache * Adds the key to the storage keys list as well */ - set(key: Key, value: Value): Value { + set(key: OnyxKey, value: OnyxValue): OnyxValue { this.addKey(key); this.addToAccessedKeys(key); this.storageMap[key] = value; @@ -93,7 +91,7 @@ class OnyxCache { } /** Forget the cached value for the given key */ - drop(key: Key): void { + drop(key: OnyxKey): void { delete this.storageMap[key]; this.storageKeys.delete(key); this.recentKeys.delete(key); @@ -103,7 +101,7 @@ class OnyxCache { * Deep merge data to cache, any non existing keys will be created * @param data - a map of (cache) key - values */ - merge(data: StorageMap): void { + merge(data: Record): void { if (typeof data !== 'object' || Array.isArray(data)) { throw new Error('data passed to cache.merge() must be an Object of onyx key/value pairs'); } @@ -130,7 +128,7 @@ class OnyxCache { * provided from this function * @param taskName - unique name given for the task */ - getTaskPromise(taskName: string): Promise { + getTaskPromise(taskName: string): Promise { return this.pendingPromises.get(taskName) ?? Promise.resolve(); } @@ -139,7 +137,7 @@ class OnyxCache { * hook up to the promise if it's still pending * @param taskName - unique name for the task */ - captureTask(taskName: string, promise: Promise): Promise { + captureTask(taskName: string, promise: Promise): Promise { const returnPromise = promise.finally(() => { this.pendingPromises.delete(taskName); }); @@ -150,7 +148,7 @@ class OnyxCache { } /** Adds a key to the top of the recently accessed keys */ - private addToAccessedKeys(key: Key): void { + private addToAccessedKeys(key: OnyxKey): void { this.recentKeys.delete(key); this.recentKeys.add(key); } @@ -182,7 +180,7 @@ class OnyxCache { } /** Check if the value has changed */ - hasValueChanged(key: Key, value: Value): boolean { + hasValueChanged(key: OnyxKey, value: OnyxValue): boolean { return !deepEqual(this.storageMap[key], value); } } diff --git a/lib/storage/providers/IDBKeyVal.ts b/lib/storage/providers/IDBKeyVal.ts index 80b04185..fa3562e7 100644 --- a/lib/storage/providers/IDBKeyVal.ts +++ b/lib/storage/providers/IDBKeyVal.ts @@ -2,7 +2,7 @@ import type {UseStore} from 'idb-keyval'; import {set, keys, getMany, setMany, get, clear, del, delMany, createStore, promisifyRequest} from 'idb-keyval'; import utils from '../../utils'; import type StorageProvider from './types'; -import type {Value} from './types'; +import type {OnyxValue} from '../../types'; // We don't want to initialize the store while the JS bundle loads as idb-keyval will try to use global.indexedDB // which might not be available in certain environments that load the bundle (e.g. electron main process). @@ -21,7 +21,7 @@ const provider: StorageProvider = { getCustomStore()('readwrite', (store) => { // Note: we are using the manual store transaction here, to fit the read and update // of the items in one transaction to achieve best performance. - const getValues = Promise.all(pairs.map(([key]) => promisifyRequest(store.get(key)))); + const getValues = Promise.all(pairs.map(([key]) => promisifyRequest(store.get(key)))); return getValues.then((values) => { const upsertMany = pairs.map(([key, value], index) => { diff --git a/lib/storage/providers/types.ts b/lib/storage/providers/types.ts index 6d800cfd..6de42d14 100644 --- a/lib/storage/providers/types.ts +++ b/lib/storage/providers/types.ts @@ -1,19 +1,17 @@ import type {BatchQueryResult, QueryResult} from 'react-native-quick-sqlite'; -import type {KeyValueMapping, OnyxCollection, OnyxEntry, OnyxKey} from '../../types'; +import type {OnyxKey, OnyxValue} from '../../types'; -type Key = OnyxKey; -type Value = OnyxEntry | OnyxCollection; -type KeyValuePair = [Key, Value]; -type KeyList = Key[]; +type KeyValuePair = [OnyxKey, OnyxValue]; +type KeyList = OnyxKey[]; type KeyValuePairList = KeyValuePair[]; -type OnStorageKeyChanged = (key: Key, value: Value | null) => void; +type OnStorageKeyChanged = (key: OnyxKey, value: OnyxValue | null) => void; type StorageProvider = { /** * Gets the value of a given key or return `null` if it's not available in storage */ - getItem: (key: Key) => Promise; + getItem: (key: OnyxKey) => Promise; /** * Get multiple key-value pairs for the given array of keys in a batch @@ -23,7 +21,7 @@ type StorageProvider = { /** * Sets the value for a given key. The only requirement is that the value should be serializable to JSON string */ - setItem: (key: Key, value: Value) => Promise; + setItem: (key: OnyxKey, value: OnyxValue) => Promise; /** * Stores multiple key-value pairs in a batch @@ -40,7 +38,7 @@ type StorageProvider = { * @param changes - the delta for a specific key * @param modifiedData - the pre-merged data from `Onyx.applyMerge` */ - mergeItem: (key: Key, changes: Value, modifiedData: Value) => Promise; + mergeItem: (key: OnyxKey, changes: OnyxValue, modifiedData: OnyxValue) => Promise; /** * Returns all keys available in storage @@ -50,7 +48,7 @@ type StorageProvider = { /** * Removes given key and its value from storage */ - removeItem: (key: Key) => Promise; + removeItem: (key: OnyxKey) => Promise; /** * Removes given keys and their values from storage @@ -74,4 +72,4 @@ type StorageProvider = { }; export default StorageProvider; -export type {Value, Key, KeyList, KeyValuePairList}; +export type {OnyxValue as Value, OnyxKey as Key, KeyList, KeyValuePairList}; From a01ffd2832289223fa5c3b313679fbf907b41ff5 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Wed, 28 Feb 2024 17:52:16 +0100 Subject: [PATCH 05/23] Fix errors related to tests --- lib/Onyx.ts | 60 +++++++++++++++++++++-------------------------------- 1 file changed, 24 insertions(+), 36 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 029003ca..ab9b337d 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -223,7 +223,7 @@ function getAllKeys(): Promise { // When a value retrieving task for all keys is still running hook to it if (cache.hasPendingTask(taskName)) { - return cache.getTaskPromise(taskName); + return cache.getTaskPromise(taskName) as Promise; } // Otherwise retrieve the keys from storage and capture a promise to aid concurrent usages @@ -232,7 +232,7 @@ function getAllKeys(): Promise { return keys; }); - return cache.captureTask(taskName, promise); + return cache.captureTask(taskName, promise) as Promise; } /** @@ -327,7 +327,7 @@ function removeFromEvictionBlockList(key: OnyxKey, connectionID: number): void { evictionBlocklist[key] = evictionBlocklist[key]?.filter((evictionKey) => evictionKey !== connectionID); // Remove the key if there are no more subscribers - if (evictionBlocklist[key].length === 0) { + if (evictionBlocklist[key]?.length === 0) { delete evictionBlocklist[key]; } } @@ -377,12 +377,7 @@ function getCachedCollection(collectionKey: TKey } /** When a collection of keys change, search for any callbacks matching the collection key and trigger those callbacks */ -function keysChanged( - collectionKey: TKey, - partialCollection: Collection>, - notifyRegularSubscibers = true, - notifyWithOnyxSubscibers = true, -) { +function keysChanged(collectionKey: TKey, partialCollection: OnyxValue, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true) { // We are iterating over all subscribers similar to keyChanged(). However, we are looking for subscribers who are subscribing to either a collection key or // individual collection key member for the collection that is being updated. It is important to note that the collection parameter cane be a PARTIAL collection // and does not represent all of the combined keys and values for a collection key. It is just the "new" data that was merged in via mergeCollection(). @@ -428,7 +423,7 @@ function keysChanged( // If they are not using waitForCollectionCallback then we notify the subscriber with // the new merged data but only for any keys in the partial collection. - const dataKeys = Object.keys(partialCollection); + const dataKeys = partialCollection && typeof partialCollection === 'object' ? Object.keys(partialCollection) : []; for (let j = 0; j < dataKeys.length; j++) { const dataKey = dataKeys[j]; subscriber.callback(cachedCollection[dataKey], dataKey); @@ -890,7 +885,7 @@ function disconnect(connectionID: number, keyToRemoveFromEvictionBlocklist?: Ony * @example * scheduleSubscriberUpdate(key, value, subscriber => subscriber.initWithStoredValues === false) */ -function scheduleSubscriberUpdate(key: TKey, value: KeyValueMapping[TKey], prevValue: KeyValueMapping[TKey], canUpdateSubscriber = () => true) { +function scheduleSubscriberUpdate(key: TKey, value: KeyValueMapping[TKey], prevValue: KeyValueMapping[TKey], canUpdateSubscriber = () => true): Promise<[void, void]> { const promise = Promise.resolve().then(() => keyChanged(key, value, prevValue, canUpdateSubscriber, true, false)); batchUpdates(() => keyChanged(key, value, prevValue, canUpdateSubscriber, false, true)); return Promise.all([maybeFlushBatchUpdates(), promise]); @@ -913,23 +908,15 @@ function scheduleNotifyCollectionSubscribers(key: TKey, va /** * Remove a key from Onyx and update the subscribers - * - * @private - * @param {String} key - * @return {Promise} */ -function remove(key: TKey) { +function remove(key: TKey): Promise { const prevValue = cache.getValue(key, false); cache.drop(key); scheduleSubscriberUpdate(key, null, prevValue); - return Storage.removeItem(key); + return Storage.removeItem(key) as Promise; } -/** - * @private - * @returns {Promise} - */ -function reportStorageQuota() { +function reportStorageQuota(): Promise { return Storage.getDatabaseSize() .then(({bytesUsed, bytesRemaining}) => { Logger.logInfo(`Storage Quota Check -- bytesUsed: ${bytesUsed} bytesRemaining: ${bytesRemaining}`); @@ -982,7 +969,7 @@ function evictStorageAndRetry(key: TKey, value: KeyValueMapping[TKey], method: string, hasChanged: boolean, wasRemoved = false) { // Logging properties only since values could be sensitive things we don't want to log - Logger.logInfo(`${method}() called for key: ${key}${typeof value === 'object' ? ` properties: ${Object.keys(value).join(',')}` : ''}`); + Logger.logInfo(`${method}() called for key: ${key}${value && typeof value === 'object' ? ` properties: ${Object.keys(value).join(',')}` : ''}`); const prevValue = cache.getValue(key, false); // Update subscribers if the cached value has changed, or when the subscriber specifically requires @@ -1219,7 +1206,7 @@ function initializeWithDefaultKeyStates(): Promise { const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates); cache.merge(merged); - merged.forEach((val, key) => keyChanged(key, val, existingDataAsObject)); + Object.entries(merged).forEach(([key, value]) => keyChanged(key, value, existingDataAsObject)); }); } @@ -1244,11 +1231,11 @@ function initializeWithDefaultKeyStates(): Promise { * * @param keysToPreserve is a list of ONYXKEYS that should not be cleared with the rest of the data */ -function clear(keysToPreserve: OnyxKey[] = []): Promise { +function clear(keysToPreserve: OnyxKey[] = []): Promise> { return getAllKeys().then((keys) => { - const keysToBeClearedFromStorage = []; - const keyValuesToResetAsCollection = {}; - const keyValuesToResetIndividually = {}; + const keysToBeClearedFromStorage: OnyxKey[] = []; + const keyValuesToResetAsCollection: Record> = {}; + const keyValuesToResetIndividually: Record = {}; // The only keys that should not be cleared are: // 1. Anything specifically passed in keysToPreserve (because some keys like language preferences, offline @@ -1270,10 +1257,11 @@ function clear(keysToPreserve: OnyxKey[] = []): Promise { cache.set(key, newValue); const collectionKey = key.substring(0, key.indexOf('_') + 1); if (collectionKey) { - if (!keyValuesToResetAsCollection[collectionKey]) { - keyValuesToResetAsCollection[collectionKey] = {}; + let collection = keyValuesToResetAsCollection[collectionKey]; + if (!collection) { + collection = {}; } - keyValuesToResetAsCollection[collectionKey][key] = newValue; + collection[key] = newValue; } else { keyValuesToResetIndividually[key] = newValue; } @@ -1288,13 +1276,13 @@ function clear(keysToPreserve: OnyxKey[] = []): Promise { keysToBeClearedFromStorage.push(key); }); - const updatePromises = []; + const updatePromises: Array> = []; // Notify the subscribers for each key/value group so they can receive the new values - keyValuesToResetIndividually.forEach((value, key) => { + Object.entries(keyValuesToResetIndividually).forEach(([key, value]) => { updatePromises.push(scheduleSubscriberUpdate(key, value, cache.getValue(key, false))); }); - keyValuesToResetAsCollection.forEach((value, key) => { + Object.entries(keyValuesToResetAsCollection).forEach(([key, value]) => { updatePromises.push(scheduleNotifyCollectionSubscribers(key, value)); }); @@ -1469,7 +1457,7 @@ function update(data: OnyxUpdate[]): Promise { }); const promises: Array<() => Promise> = []; - let clearPromise = Promise.resolve(); + let clearPromise: Promise = Promise.resolve(); data.forEach(({onyxMethod, key, value}) => { switch (onyxMethod) { @@ -1548,7 +1536,7 @@ function init({ // We need the value of the collection keys later for checking if a // key is a collection. We store it in a map for faster lookup. - const collectionValues = Object.values(keys.COLLECTION); + const collectionValues = keys.COLLECTION ? Object.values(keys.COLLECTION) : []; onyxCollectionKeyMap = collectionValues.reduce((acc, val) => { acc.set(val, true); return acc; From 2a9aa58e469d126abc351d4d0720f91ff868a8e4 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Wed, 28 Feb 2024 17:52:35 +0100 Subject: [PATCH 06/23] Remove unneccessary metrics tests --- tests/unit/onyxMetricsDecorationTest.js | 36 ------------------------- 1 file changed, 36 deletions(-) diff --git a/tests/unit/onyxMetricsDecorationTest.js b/tests/unit/onyxMetricsDecorationTest.js index b123fa8c..0488f4b8 100644 --- a/tests/unit/onyxMetricsDecorationTest.js +++ b/tests/unit/onyxMetricsDecorationTest.js @@ -15,20 +15,6 @@ describe('Onyx', () => { Onyx = require('../../lib').default; }); - it('Should expose metrics methods when `captureMetrics` is true', () => { - // When Onyx is initialized with `captureMetrics: true` - Onyx.init({ - keys: ONYX_KEYS, - registerStorageEventListener: jest.fn(), - captureMetrics: true, - }); - - // Then Onyx should have statistic related methods - expect(Onyx.getMetrics).toEqual(expect.any(Function)); - expect(Onyx.printMetrics).toEqual(expect.any(Function)); - expect(Onyx.resetMetrics).toEqual(expect.any(Function)); - }); - it('Should not expose metrics methods when `captureMetrics` is false or not set', () => { // When Onyx is initialized without setting `captureMetrics` Onyx.init({ @@ -53,27 +39,5 @@ describe('Onyx', () => { expect(Onyx.printMetrics).not.toBeDefined(); expect(Onyx.resetMetrics).not.toBeDefined(); }); - - it('Should decorate exposed methods', () => { - // Given Onyx is initialized with `captureMetrics: true` - Onyx.init({ - keys: ONYX_KEYS, - registerStorageEventListener: jest.fn(), - captureMetrics: true, - }); - - // When calling decorated methods through Onyx[methodName] - const methods = ['set', 'multiSet', 'clear', 'merge', 'mergeCollection']; - methods.forEach((name) => Onyx[name]('mockKey', {mockKey: {mockValue: 'mockValue'}})); - - return waitForPromisesToResolve().then(() => { - // Then metrics should have captured data for each method - const summaries = Onyx.getMetrics().summaries; - - methods.forEach((name) => { - expect(summaries[`Onyx:${name}`].total).toBeGreaterThan(0); - }); - }); - }); }); }); From a7ed2b74031a5d879bf8c5916f3677b83a705770 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Mon, 4 Mar 2024 18:25:22 +0100 Subject: [PATCH 07/23] Adjust PerformanceUtils import --- lib/Onyx.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index ab9b337d..69d856b4 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -7,7 +7,7 @@ import * as Logger from './Logger'; import cache from './OnyxCache'; import * as Str from './Str'; import createDeferredTask from './createDeferredTask'; -import * as PerformanceUtils from './metrics/PerformanceUtils'; +import * as PerformanceUtils from './PerformanceUtils'; import Storage from './storage'; import utils from './utils'; import unstable_batchedUpdates from './batch'; From 6236ca0615d7d20ba93b05a1584c77cef9bb98b5 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Mon, 4 Mar 2024 18:33:04 +0100 Subject: [PATCH 08/23] Migrate index.js --- lib/index.d.ts | 6 ------ lib/index.js | 5 ----- lib/index.ts | 8 ++++++++ 3 files changed, 8 insertions(+), 11 deletions(-) delete mode 100644 lib/index.d.ts delete mode 100644 lib/index.js create mode 100644 lib/index.ts diff --git a/lib/index.d.ts b/lib/index.d.ts deleted file mode 100644 index 5a380070..00000000 --- a/lib/index.d.ts +++ /dev/null @@ -1,6 +0,0 @@ -import Onyx, {OnyxUpdate, ConnectOptions} from './Onyx'; -import {CustomTypeOptions, OnyxCollection, OnyxEntry, NullishDeep, KeyValueMapping, OnyxKey, Selector, WithOnyxInstanceState} from './types'; -import withOnyx from './withOnyx'; - -export default Onyx; -export {CustomTypeOptions, OnyxCollection, OnyxEntry, OnyxUpdate, withOnyx, ConnectOptions, NullishDeep, KeyValueMapping, OnyxKey, Selector, WithOnyxInstanceState}; diff --git a/lib/index.js b/lib/index.js deleted file mode 100644 index bb2bba7e..00000000 --- a/lib/index.js +++ /dev/null @@ -1,5 +0,0 @@ -import Onyx from './Onyx'; -import withOnyx from './withOnyx'; - -export default Onyx; -export {withOnyx}; diff --git a/lib/index.ts b/lib/index.ts new file mode 100644 index 00000000..a7b5cf63 --- /dev/null +++ b/lib/index.ts @@ -0,0 +1,8 @@ +import Onyx from './Onyx'; +import type {OnyxUpdate, Mapping} from './Onyx'; +import type {CustomTypeOptions, OnyxCollection, OnyxEntry, NullishDeep, KeyValueMapping, OnyxKey, Selector, WithOnyxInstanceState} from './types'; +import withOnyx from './withOnyx'; + +export default Onyx; +export {withOnyx}; +export type {CustomTypeOptions, OnyxCollection, OnyxEntry, OnyxUpdate, Mapping as ConnectOptions, NullishDeep, KeyValueMapping, OnyxKey, Selector, WithOnyxInstanceState}; From 6a9c50f7949fee63100d2c40eda33197483d03aa Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Mon, 4 Mar 2024 18:33:20 +0100 Subject: [PATCH 09/23] Migrate DevTools.js --- lib/DevTools.js | 71 --------------------------------- lib/DevTools.ts | 104 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 71 deletions(-) delete mode 100644 lib/DevTools.js create mode 100644 lib/DevTools.ts diff --git a/lib/DevTools.js b/lib/DevTools.js deleted file mode 100644 index 58f15a71..00000000 --- a/lib/DevTools.js +++ /dev/null @@ -1,71 +0,0 @@ -import _ from 'underscore'; - -const ERROR_LABEL = 'Onyx DevTools - Error: '; - -/* eslint-disable no-underscore-dangle */ -class DevTools { - constructor() { - this.remoteDev = this.connectViaExtension(); - this.state = {}; - this.defaultState = {}; - } - - connectViaExtension(options) { - try { - if ((options && options.remote) || typeof window === 'undefined' || !window.__REDUX_DEVTOOLS_EXTENSION__) { - return; - } - return window.__REDUX_DEVTOOLS_EXTENSION__.connect(options); - } catch (e) { - console.error(ERROR_LABEL, e); - } - } - - /** - * Registers an action that updated the current state of the storage - * - * @param {string} type - name of the action - * @param {any} payload - data written to the storage - * @param {object} stateChanges - partial state that got updated after the changes - */ - registerAction(type, payload = undefined, stateChanges = {}) { - try { - if (!this.remoteDev) { - return; - } - const newState = { - ...this.state, - ...stateChanges, - }; - this.remoteDev.send({type, payload}, newState); - this.state = newState; - } catch (e) { - console.error(ERROR_LABEL, e); - } - } - - initState(initialState = {}) { - try { - if (!this.remoteDev) { - return; - } - this.remoteDev.init(initialState); - this.state = initialState; - this.defaultState = initialState; - } catch (e) { - console.error(ERROR_LABEL, e); - } - } - - /** - * This clears the internal state of the DevTools, preserving the keys included in `keysToPreserve` - * - * @param {string[]} keysToPreserve - */ - clearState(keysToPreserve = []) { - const newState = _.mapObject(this.state, (value, key) => (keysToPreserve.includes(key) ? value : this.defaultState[key])); - this.registerAction('CLEAR', undefined, newState); - } -} - -export default new DevTools(); diff --git a/lib/DevTools.ts b/lib/DevTools.ts new file mode 100644 index 00000000..4155bc5b --- /dev/null +++ b/lib/DevTools.ts @@ -0,0 +1,104 @@ +type DevtoolsOptions = { + maxAge?: number; + name?: string; + postTimelineUpdate?: () => void; + preAction?: () => void; + logTrace?: boolean; + remote?: boolean; +}; + +type DevtoolsSubscriber = (message: {type: string; payload: Record; state: string}) => void; + +type DevtoolsConnection = { + send(data: Record, state: Record): void; + init(state: Record): void; + unsubscribe(): void; + subscribe(cb: DevtoolsSubscriber): () => void; +}; + +const ERROR_LABEL = 'Onyx DevTools - Error: '; + +type ReduxDevtools = { + connect(options?: DevtoolsOptions): DevtoolsConnection; +}; + +class DevTools { + private remoteDev?: DevtoolsConnection; + + private state: Record; + + private defaultState: Record; + + constructor() { + this.remoteDev = this.connectViaExtension(); + this.state = {}; + this.defaultState = {}; + } + + connectViaExtension(options?: DevtoolsOptions): DevtoolsConnection | undefined { + try { + // We don't want to augment the window type in a library code, so we use type assertion instead + // eslint-disable-next-line no-underscore-dangle, @typescript-eslint/no-explicit-any + const reduxDevtools: ReduxDevtools = (window as any).__REDUX_DEVTOOLS_EXTENSION__; + + if ((options && options.remote) || typeof window === 'undefined' || !reduxDevtools) { + return; + } + // eslint-disable-next-line no-underscore-dangle, @typescript-eslint/no-explicit-any + return reduxDevtools.connect(options); + } catch (e) { + console.error(ERROR_LABEL, e); + } + } + + /** + * Registers an action that updated the current state of the storage + * + * @param type - name of the action + * @param payload - data written to the storage + * @param stateChanges - partial state that got updated after the changes + */ + registerAction(type: string, payload: Record | undefined, stateChanges: Record = {}) { + try { + if (!this.remoteDev) { + return; + } + const newState = { + ...this.state, + ...stateChanges, + }; + this.remoteDev.send({type, payload}, newState); + this.state = newState; + } catch (e) { + console.error(ERROR_LABEL, e); + } + } + + initState(initialState: Record = {}) { + try { + if (!this.remoteDev) { + return; + } + this.remoteDev.init(initialState); + this.state = initialState; + this.defaultState = initialState; + } catch (e) { + console.error(ERROR_LABEL, e); + } + } + + /** + * This clears the internal state of the DevTools, preserving the keys included in `keysToPreserve` + */ + public clearState(keysToPreserve: string[] = []): void { + const newState = Object.entries(this.state).reduce((obj: Record, [key, value]) => { + // eslint-disable-next-line no-param-reassign + obj[key] = keysToPreserve.includes(key) ? value : this.defaultState[key]; + return obj; + }, {}); + + this.registerAction('CLEAR', undefined, newState); + } +} + +export default new DevTools(); From db462ca92e36175e31ccb4c284d391e7e69fbdb3 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Mon, 4 Mar 2024 18:33:31 +0100 Subject: [PATCH 10/23] Export types from Onyx.js --- lib/Onyx.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 69d856b4..250cbb41 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -1583,3 +1583,4 @@ const Onyx = { } as const; export default Onyx; +export type {OnyxUpdate, Mapping}; From 1741bab9ca2c224f1c3e439940015a3c8f034a3f Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Mon, 4 Mar 2024 18:38:54 +0100 Subject: [PATCH 11/23] Fix linter errors --- lib/storage/InstanceSync/index.web.ts | 10 +++++----- tests/unit/cacheEvictionTest.js | 7 ++++++- tests/utils/waitForPromisesToResolve.js | 5 ++++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/storage/InstanceSync/index.web.ts b/lib/storage/InstanceSync/index.web.ts index a06fcd57..8399e582 100644 --- a/lib/storage/InstanceSync/index.web.ts +++ b/lib/storage/InstanceSync/index.web.ts @@ -4,15 +4,15 @@ * when using LocalStorage APIs in the browser. These events are great because multiple tabs can listen for when * data changes and then stay up-to-date with everything happening in Onyx. */ -import type {KeyList, Key, OnStorageKeyChanged, Value} from '../providers/types'; +import type {OnyxKey, OnyxValue} from '../../types'; +import type {KeyList, OnStorageKeyChanged} from '../providers/types'; const SYNC_ONYX = 'SYNC_ONYX'; /** * Raise an event through `localStorage` to let other tabs know a value changed - * @param {String} onyxKey */ -function raiseStorageSyncEvent(onyxKey: Key) { +function raiseStorageSyncEvent(onyxKey: OnyxKey) { global.localStorage.setItem(SYNC_ONYX, onyxKey); global.localStorage.removeItem(SYNC_ONYX); } @@ -25,7 +25,7 @@ function raiseStorageSyncManyKeysEvent(onyxKeys: KeyList) { const InstanceSync = { /** - * @param {Function} onStorageKeyChanged Storage synchronization mechanism keeping all opened tabs in sync + * @param onStorageKeyChanged Storage synchronization mechanism keeping all opened tabs in sync */ init: (onStorageKeyChanged: OnStorageKeyChanged) => { // This listener will only be triggered by events coming from other tabs @@ -37,7 +37,7 @@ const InstanceSync = { const onyxKey = event.newValue; // @ts-expect-error `this` will be substituted later in actual function call - this.getItem(onyxKey).then((value: Value) => onStorageKeyChanged(onyxKey, value)); + this.getItem(onyxKey).then((value: OnyxValue) => onStorageKeyChanged(onyxKey, value)); }); }, setItem: raiseStorageSyncEvent, diff --git a/tests/unit/cacheEvictionTest.js b/tests/unit/cacheEvictionTest.js index 59d4c253..ed3d77be 100644 --- a/tests/unit/cacheEvictionTest.js +++ b/tests/unit/cacheEvictionTest.js @@ -43,7 +43,12 @@ test('Cache eviction', () => { // When we set a new key we want to add and force the first attempt to fail const originalSetItem = StorageMock.setItem; - const setItemMock = jest.fn(originalSetItem).mockImplementationOnce(() => new Promise((_resolve, reject) => reject())); + const setItemMock = jest.fn(originalSetItem).mockImplementationOnce( + () => + new Promise((_resolve, reject) => { + reject(); + }), + ); StorageMock.setItem = setItemMock; return Onyx.set(`${ONYX_KEYS.COLLECTION.TEST_KEY}${RECORD_TO_ADD}`, {test: 'add'}).then(() => { diff --git a/tests/utils/waitForPromisesToResolve.js b/tests/utils/waitForPromisesToResolve.js index 8ec5a247..be68f88b 100644 --- a/tests/utils/waitForPromisesToResolve.js +++ b/tests/utils/waitForPromisesToResolve.js @@ -1 +1,4 @@ -export default () => new Promise((resolve) => setTimeout(resolve, 0)); +export default () => + new Promise((resolve) => { + setTimeout(resolve, 0); + }); From d1836b495301fff3eea69796eb63ef2e6023a09b Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Mon, 4 Mar 2024 19:48:19 +0100 Subject: [PATCH 12/23] Continue migrating Onyx --- lib/DevTools.ts | 4 +- lib/Onyx.ts | 131 ++++++++++++++++++++-------------------- lib/OnyxCache.ts | 2 +- lib/PerformanceUtils.ts | 8 +-- lib/storage/index.ts | 3 - 5 files changed, 72 insertions(+), 76 deletions(-) diff --git a/lib/DevTools.ts b/lib/DevTools.ts index 4155bc5b..76be784e 100644 --- a/lib/DevTools.ts +++ b/lib/DevTools.ts @@ -7,7 +7,7 @@ type DevtoolsOptions = { remote?: boolean; }; -type DevtoolsSubscriber = (message: {type: string; payload: Record; state: string}) => void; +type DevtoolsSubscriber = (message: {type: string; payload: unknown; state: string}) => void; type DevtoolsConnection = { send(data: Record, state: Record): void; @@ -58,7 +58,7 @@ class DevTools { * @param payload - data written to the storage * @param stateChanges - partial state that got updated after the changes */ - registerAction(type: string, payload: Record | undefined, stateChanges: Record = {}) { + registerAction(type: string, payload: unknown, stateChanges: Record = {}) { try { if (!this.remoteDev) { return; diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 250cbb41..b4ce652e 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -40,13 +40,21 @@ type Collection = { : never; }; +type WithOnyxInstance = Component>> & { + setStateProxy: (cb: (state: Record) => OnyxValue) => void; + setWithOnyxState: (statePropertyName: OnyxKey, value: OnyxValue) => void; +}; + /** Represents the base options used in `Onyx.connect()` method. */ type BaseConnectOptions = { - statePropertyName?: string; - withOnyxInstance?: Component; + statePropertyName: string; + withOnyxInstance?: WithOnyxInstance; initWithStoredValues?: boolean; selector?: Selector; connectionID: number; + key: string; + displayName: string; + canEvict?: boolean; }; /** @@ -94,7 +102,7 @@ const mergeQueuePromise: Record> = {}; let lastConnectionID = 0; // Holds a mapping of all the react components that want their state subscribed to a store key -const callbackToStateMapping: Record> = {}; +const callbackToStateMapping: Record> = {}; // Keeps a copy of the values of the onyx collection keys as a map for faster lookups let onyxCollectionKeyMap = new Map(); @@ -127,9 +135,10 @@ let batchUpdatesQueue: Array<() => void> = []; * @param value - contains the change that was made by the method * @param mergedValue - (optional) value that was written in the storage after a merge method was executed. */ -function sendActionToDevTools(method: OnyxMethod, key: OnyxKey | undefined, value: OnyxValue, mergedValue: any = undefined) { - // @ts-expect-error Migrate DevTools - DevTools.registerAction(utils.formatActionName(method, key), value, key ? {[key]: mergedValue || value} : value); +function sendActionToDevTools(method: OnyxMethod, key: undefined, value: Record, mergedValue: OnyxValue): void; +function sendActionToDevTools(method: OnyxMethod, key: OnyxKey, value: OnyxValue, mergedValue: OnyxValue): void; +function sendActionToDevTools(method: OnyxMethod, key: OnyxKey | undefined, value: Record | OnyxValue, mergedValue: OnyxValue = undefined): void { + DevTools.registerAction(utils.formatActionName(method, key), value, key ? {[key]: mergedValue || value} : (value as Record)); } /** @@ -175,12 +184,13 @@ function batchUpdates(updates: () => void): Promise { * The resulting collection will only contain items that are returned by the selector. */ function reduceCollectionWithSelector( - collection: Collection>, - selector: Selector, - withOnyxInstanceState: WithOnyxInstanceState, -) { - return Object.values(collection).reduce((finalCollection, item, key) => { - finalCollection[key] = selector(item, withOnyxInstanceState); + collection: Record, + selector: Selector | undefined, + withOnyxInstanceState: WithOnyxInstanceState | undefined, +): Record { + return Object.values(collection).reduce((finalCollection: Record, item, key) => { + // eslint-disable-next-line no-param-reassign + finalCollection[key] = selector?.(item, withOnyxInstanceState); return finalCollection; }, {}); @@ -236,8 +246,7 @@ function getAllKeys(): Promise { } /** - * Checks to see if the a subscriber's supplied key - * is associated with a collection of keys. + * Checks to see if the a subscriber's supplied key is associated with a collection of keys. */ function isCollectionKey(key: OnyxKey): key is CollectionKeyBase { return onyxCollectionKeyMap.has(key); @@ -264,8 +273,8 @@ function isSafeEvictionKey(testKey: OnyxKey): boolean { * Tries to get a value from the cache. If the value is not present in cache it will return the default value or undefined. * If the requested key is a collection, it will return an object with all the collection members. */ -function tryGetCachedValue(key: TKey, mapping: Mapping) { - let val = cache.getValue(key); +function tryGetCachedValue(key: TKey, mapping: Mapping): OnyxValue { + let val: OnyxValue | Record = cache.getValue(key); if (isCollectionKey(key)) { const allCacheKeys = cache.getAllKeys(); @@ -275,8 +284,9 @@ function tryGetCachedValue(key: TKey, mapping: Mapping k.startsWith(key)); - const values = matchingKeys.reduce((finalObject, matchedKey) => { + const values = matchingKeys.reduce((finalObject: Record, matchedKey) => { const cachedValue = cache.getValue(matchedKey); if (cachedValue) { // This is permissible because we're in the process of constructing the final object in a reduce function. @@ -292,7 +302,7 @@ function tryGetCachedValue(key: TKey, mapping: Mapping, mapping.selector, state); } return mapping.selector(val, state); } @@ -306,9 +316,8 @@ function removeLastAccessedKey(key: OnyxKey): void { } /** - * Add a key to the list of recently accessed keys. The least - * recently accessed key should be at the head and the most - * recently accessed key at the tail. + * Add a key to the list of recently accessed keys. + * The least recently accessed key should be at the head and the most recently accessed key at the tail. */ function addLastAccessedKey(key: OnyxKey): void { // Only specific keys belong in this list since we cannot remove an entire collection. @@ -364,7 +373,7 @@ function addAllSafeEvictionKeysToRecentlyAccessedList(): Promise { function getCachedCollection(collectionKey: TKey): Record> { const collectionMemberKeys = cache.getAllKeys().filter((storedKey) => isCollectionMemberKey(collectionKey, storedKey)); - return collectionMemberKeys.reduce((prev: Record>, key) => { + return collectionMemberKeys.reduce((prev: Record, key) => { const cachedValue = cache.getValue(key); if (!cachedValue) { return prev; @@ -377,9 +386,9 @@ function getCachedCollection(collectionKey: TKey } /** When a collection of keys change, search for any callbacks matching the collection key and trigger those callbacks */ -function keysChanged(collectionKey: TKey, partialCollection: OnyxValue, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true) { +function keysChanged(collectionKey: TKey, partialCollection: OnyxCollection, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true) { // We are iterating over all subscribers similar to keyChanged(). However, we are looking for subscribers who are subscribing to either a collection key or - // individual collection key member for the collection that is being updated. It is important to note that the collection parameter cane be a PARTIAL collection + // individual collection key member for the collection that is being updated. It is important to note that the collection parameter can be a PARTIAL collection // and does not represent all of the combined keys and values for a collection key. It is just the "new" data that was merged in via mergeCollection(). const stateMappingKeys = Object.keys(callbackToStateMapping); for (let i = 0; i < stateMappingKeys.length; i++) { @@ -434,7 +443,7 @@ function keysChanged(collectionKey: TKey, partia // And if the subscriber is specifically only tracking a particular collection member key then we will // notify them with the cached data for that key only. if (isSubscribedToCollectionMemberKey) { - subscriber.callback(cachedCollection[subscriber.key], subscriber.key); + subscriber.callback(cachedCollection[subscriber.key] as OnyxCollection, subscriber.key); continue; } @@ -455,7 +464,7 @@ function keysChanged(collectionKey: TKey, partia if (subscriber.selector) { subscriber.withOnyxInstance.setStateProxy((prevState) => { const previousData = prevState[subscriber.statePropertyName]; - const newData = reduceCollectionWithSelector(cachedCollection, subscriber.selector, subscriber.withOnyxInstance.state); + const newData = reduceCollectionWithSelector(cachedCollection, subscriber.selector, subscriber.withOnyxInstance?.state); if (!deepEqual(previousData, newData)) { return { @@ -468,8 +477,8 @@ function keysChanged(collectionKey: TKey, partia } subscriber.withOnyxInstance.setStateProxy((prevState) => { - const finalCollection = lodashClone(prevState[subscriber.statePropertyName] || {}); - const dataKeys = Object.keys(partialCollection); + const finalCollection = lodashClone(prevState[subscriber.statePropertyName] || {}) as Record; + const dataKeys = Object.keys(partialCollection ?? {}); for (let j = 0; j < dataKeys.length; j++) { const dataKey = dataKeys[j]; finalCollection[dataKey] = cachedCollection[dataKey]; @@ -487,7 +496,7 @@ function keysChanged(collectionKey: TKey, partia if (isSubscribedToCollectionMemberKey) { // However, we only want to update this subscriber if the partial data contains a change. // Otherwise, we would update them with a value they already have and trigger an unnecessary re-render. - const dataFromCollection = partialCollection[subscriber.key]; + const dataFromCollection = partialCollection?.[subscriber.key]; if (dataFromCollection === undefined) { continue; } @@ -498,7 +507,7 @@ function keysChanged(collectionKey: TKey, partia if (subscriber.selector) { subscriber.withOnyxInstance.setStateProxy((prevState) => { const prevData = prevState[subscriber.statePropertyName]; - const newData = subscriber.selector(cachedCollection[subscriber.key], subscriber.withOnyxInstance.state); + const newData = subscriber.selector?.(cachedCollection[subscriber.key], subscriber.withOnyxInstance?.state); if (!deepEqual(prevData, newData)) { PerformanceUtils.logSetStateCall(subscriber, prevData, newData, 'keysChanged', collectionKey); return { @@ -539,13 +548,13 @@ function keysChanged(collectionKey: TKey, partia * @example * keyChanged(key, value, subscriber => subscriber.initWithStoredValues === false) * - * @param {Function} [canUpdateSubscriber] only subscribers that pass this truth test will be updated + * @param [canUpdateSubscriber] only subscribers that pass this truth test will be updated */ -function keyChanged( - key: TKey, - data: KeyValueMapping[TKey], - prevData: KeyValueMapping[TKey], - canUpdateSubscriber = () => true, +function keyChanged( + key: OnyxKey, + data: OnyxValue, + prevData: OnyxValue, + canUpdateSubscriber = (_subscriber: Mapping) => true, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true, ) { @@ -577,8 +586,7 @@ function keyChanged( subscriber.callback(cachedCollection); continue; } - - subscriber.callback(data, key); + subscriber.callback(data as Record, key); continue; } @@ -594,9 +602,9 @@ function keyChanged( // returned by the selector and only when the selected data has changed. if (subscriber.selector) { subscriber.withOnyxInstance.setStateProxy((prevState) => { - const prevWithOnyxData = prevState[subscriber.statePropertyName]; + const prevWithOnyxData = prevState[subscriber.statePropertyName] as Record; const newWithOnyxData = { - [key]: subscriber.selector(data, subscriber.withOnyxInstance.state), + [key]: subscriber.selector?.(data, subscriber.withOnyxInstance?.state), }; const prevDataWithNewData = { ...prevWithOnyxData, @@ -631,8 +639,8 @@ function keyChanged( // returned by the selector and only if the selected data has changed. if (subscriber.selector) { subscriber.withOnyxInstance.setStateProxy(() => { - const previousValue = subscriber.selector(prevData, subscriber.withOnyxInstance.state); - const newValue = subscriber.selector(data, subscriber.withOnyxInstance.state); + const previousValue = subscriber.selector?.(prevData, subscriber.withOnyxInstance?.state); + const newValue = subscriber.selector?.(data, subscriber.withOnyxInstance?.state); if (!deepEqual(previousValue, newValue)) { return { @@ -673,7 +681,7 @@ function keyChanged( * - sets state on the withOnyxInstances * - triggers the callback function */ -function sendDataToConnection(mapping: Mapping, val: OnyxValue, matchedKey: OnyxKey | undefined, isBatched: boolean) { +function sendDataToConnection(mapping: Mapping, val: OnyxValue, matchedKey: OnyxKey | undefined, isBatched: boolean) { // If the mapping no longer exists then we should not send any data. // This means our subscriber disconnected or withOnyx wrapped component unmounted. if (!callbackToStateMapping[mapping.connectionID]) { @@ -696,7 +704,7 @@ function sendDataToConnection(mapping: Mapping, val: PerformanceUtils.logSetStateCall(mapping, null, newData, 'sendDataToConnection'); if (isBatched) { batchUpdates(() => { - mapping.withOnyxInstance.setWithOnyxState(mapping.statePropertyName, newData); + mapping.withOnyxInstance?.setWithOnyxState(mapping.statePropertyName, newData); }); } else { mapping.withOnyxInstance.setWithOnyxState(mapping.statePropertyName, newData); @@ -705,7 +713,7 @@ function sendDataToConnection(mapping: Mapping, val: } if (typeof mapping.callback === 'function') { - mapping.callback(val, matchedKey); + mapping.callback(val as Record, matchedKey as string); } } @@ -733,15 +741,11 @@ function addKeyToRecentlyAccessedIfNeeded(mapping: Mapping) { /** * Gets the data for a given an array of matching keys, combines them into an object, and sends the result back to the subscriber. - * - * @private - * @param {Array} matchingKeys - * @param {Object} mapping */ -function getCollectionDataAndSendAsObject(matchingKeys: OnyxKey[], mapping: Mapping) { +function getCollectionDataAndSendAsObject(matchingKeys: CollectionKeyBase[], mapping: Mapping) { Promise.all(matchingKeys.map((key) => get(key))) .then((values) => - values.reduce((finalObject, value, i) => { + values.reduce((finalObject: Record, value, i) => { // eslint-disable-next-line no-param-reassign finalObject[matchingKeys[i]] = value; return finalObject; @@ -885,7 +889,12 @@ function disconnect(connectionID: number, keyToRemoveFromEvictionBlocklist?: Ony * @example * scheduleSubscriberUpdate(key, value, subscriber => subscriber.initWithStoredValues === false) */ -function scheduleSubscriberUpdate(key: TKey, value: KeyValueMapping[TKey], prevValue: KeyValueMapping[TKey], canUpdateSubscriber = () => true): Promise<[void, void]> { +function scheduleSubscriberUpdate( + key: TKey, + value: KeyValueMapping[TKey], + prevValue: KeyValueMapping[TKey], + canUpdateSubscriber = (_subscriber: Mapping) => true, +): Promise<[void, void]> { const promise = Promise.resolve().then(() => keyChanged(key, value, prevValue, canUpdateSubscriber, true, false)); batchUpdates(() => keyChanged(key, value, prevValue, canUpdateSubscriber, false, true)); return Promise.all([maybeFlushBatchUpdates(), promise]); @@ -895,12 +904,8 @@ function scheduleSubscriberUpdate(key: TKey, value: KeyVal * This method is similar to notifySubscribersOnNextTick but it is built for working specifically with collections * so that keysChanged() is triggered for the collection and not keyChanged(). If this was not done, then the * subscriber callbacks receive the data in a different format than they normally expect and it breaks code. - * - * @param {String} key - * @param {*} value - * @returns {Promise} */ -function scheduleNotifyCollectionSubscribers(key: TKey, value: KeyValueMapping[TKey]) { +function scheduleNotifyCollectionSubscribers(key: OnyxKey, value: OnyxCollection) { const promise = Promise.resolve().then(() => keysChanged(key, value, true, false)); batchUpdates(() => keysChanged(key, value, false, true)); return Promise.all([maybeFlushBatchUpdates(), promise]); @@ -990,11 +995,9 @@ function hasPendingMergeForKey(key: OnyxKey): boolean { /** * Removes a key from storage if the value is null. * Otherwise removes all nested null values in objects and returns the object - * @param {String} key - * @param {Mixed} value - * @returns {Mixed} The value without null values and a boolean "wasRemoved", which indicates if the key got removed completely + * @returns The value without null values and a boolean "wasRemoved", which indicates if the key got removed completely */ -function removeNullValues(key, value) { +function removeNullValues(key: OnyxKey, value: OnyxValue | Record | null) { if (value === null) { remove(key); return {value, wasRemoved: true}; @@ -1003,7 +1006,7 @@ function removeNullValues(key, value) { // We can remove all null values in an object by merging it with itself // utils.fastMerge recursively goes through the object and removes all null values // Passing two identical objects as source and target to fastMerge will not change it, but only remove the null values - return {value: utils.removeNestedNullValues(value), wasRemoved: false}; + return {value: utils.removeNestedNullValues(value as Record), wasRemoved: false}; } /** @@ -1013,7 +1016,7 @@ function removeNullValues(key, value) { * @param value value to store */ -function set(key: TKey, value: OnyxEntry): Promise { +function set(key: OnyxKey, value: OnyxValue | Record): Promise { // If the value is null, we remove the key from storage const {value: valueAfterRemoving, wasRemoved} = removeNullValues(key, value); @@ -1044,7 +1047,7 @@ function set(key: TKey, value: OnyxEntry + * @return an array of key - value pairs <[key, value]> */ function prepareKeyValuePairsForStorage(data: Partial): Array<[OnyxKey, OnyxValue]> { const keyValuePairs: Array<[OnyxKey, OnyxValue]> = []; diff --git a/lib/OnyxCache.ts b/lib/OnyxCache.ts index e5d15ec1..e7d9b6bc 100644 --- a/lib/OnyxCache.ts +++ b/lib/OnyxCache.ts @@ -148,7 +148,7 @@ class OnyxCache { } /** Adds a key to the top of the recently accessed keys */ - private addToAccessedKeys(key: OnyxKey): void { + addToAccessedKeys(key: OnyxKey): void { this.recentKeys.delete(key); this.recentKeys.add(key); } diff --git a/lib/PerformanceUtils.ts b/lib/PerformanceUtils.ts index b378eb28..4bad2268 100644 --- a/lib/PerformanceUtils.ts +++ b/lib/PerformanceUtils.ts @@ -1,5 +1,6 @@ import lodashTransform from 'lodash/transform'; import {deepEqual} from 'fast-equals'; +import type {Mapping} from './Onyx'; type UnknownObject = Record; @@ -10,11 +11,6 @@ type LogParams = { newValue?: unknown; }; -type Mapping = Record & { - key: string; - displayName: string; -}; - let debugSetState = false; function setShouldDebugSetState(debug: boolean) { @@ -44,7 +40,7 @@ function diffObject( /** * Provide insights into why a setState() call occurred by diffing the before and after values. */ -function logSetStateCall(mapping: Mapping, previousValue: unknown, newValue: unknown, caller: string, keyThatChanged: string) { +function logSetStateCall(mapping: Mapping, previousValue: unknown, newValue: unknown, caller: string, keyThatChanged?: string) { if (!debugSetState) { return; } diff --git a/lib/storage/index.ts b/lib/storage/index.ts index a38348fc..0696d1cd 100644 --- a/lib/storage/index.ts +++ b/lib/storage/index.ts @@ -109,9 +109,6 @@ const Storage: Storage = { return provider.clear(); }, - // This is a noop for now in order to keep clients from crashing see https://github.com/Expensify/Expensify/issues/312438 - setMemoryOnlyKeys: () => provider.setMemoryOnlyKeys(), - /** * Returns all available keys */ From 73fc03813bdf4ac36651c9a1555d0be1fdc4cd93 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 08:18:43 +0100 Subject: [PATCH 13/23] Remove MDTable --- lib/MDTable.js | 66 -------------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 lib/MDTable.js diff --git a/lib/MDTable.js b/lib/MDTable.js deleted file mode 100644 index 9a09132e..00000000 --- a/lib/MDTable.js +++ /dev/null @@ -1,66 +0,0 @@ -import AsciTable from 'ascii-table'; - -class MDTable extends AsciTable { - /** - * Create a CSV string from the table data - * @returns {string} - */ - toCSV() { - return [this.getTitle(), this.getHeading(), ...this.getRows()].join('\n'); - } - - /** - * Create a JSON string from the table data - * @returns {string} - */ - toJSON() { - return JSON.stringify(super.toJSON()); - } - - /** - * Create a MD string from the table data - * @returns {string} - */ - toString() { - // Ignore modifying the first |---| for titled tables - let idx = this.getTitle() ? -2 : -1; - const ascii = super.toString().replace(/-\|/g, () => { - /* we replace "----|" with "---:|" to align the data to the right in MD */ - idx++; - - if (idx < 0 || this.leftAlignedCols.includes(idx)) { - return '-|'; - } - - return ':|'; - }); - - // strip the top and the bottom row (----) to make an MD table - const md = ascii.split('\n').slice(1, -1).join('\n'); - return md; - } -} - -/** - * Table Factory helper - * @param {Object} options - * @param {string} [options.title] - optional title center above the table - * @param {string[]} options.heading - table column names - * @param {number[]} [options.leftAlignedCols=[]] - indexes of columns that should be left aligned - * Pass the columns that are non numeric here - the rest will be aligned to the right - * @param {Array} [options.rows] The table can be initialized with row. Rows can also be added by `addRow` - * @returns {MDTable} - */ -MDTable.factory = ({title, heading, leftAlignedCols = [], rows = []}) => { - const table = new MDTable({title, heading, rows}); - table.leftAlignedCols = leftAlignedCols; - - /* By default we want everything aligned to the right as most values are numbers - * we just override the columns that are not right aligned */ - heading.forEach((name, idx) => table.setAlign(idx, AsciTable.RIGHT)); - leftAlignedCols.forEach((idx) => table.setAlign(idx, AsciTable.LEFT)); - - return table; -}; - -export default MDTable; From 2b8630719bdc261aeb10fa83a46feba155817d44 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 09:10:48 +0100 Subject: [PATCH 14/23] Clear all errors --- lib/Onyx.ts | 119 ++++++++++++++++++++-------------------- lib/PerformanceUtils.ts | 3 +- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index b4ce652e..4891d518 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -3,6 +3,7 @@ import type {Component} from 'react'; import {deepEqual} from 'fast-equals'; import lodashClone from 'lodash/clone'; import type {ValueOf} from 'type-fest'; +import type {QueryResult} from 'react-native-quick-sqlite'; import * as Logger from './Logger'; import cache from './OnyxCache'; import * as Str from './Str'; @@ -40,8 +41,8 @@ type Collection = { : never; }; -type WithOnyxInstance = Component>> & { - setStateProxy: (cb: (state: Record) => OnyxValue) => void; +type WithOnyxInstance = Component>> & { + setStateProxy: (cb: (state: Record) => OnyxValue) => void; setWithOnyxState: (statePropertyName: OnyxKey, value: OnyxValue) => void; }; @@ -95,8 +96,8 @@ const METHOD = { type OnyxMethod = ValueOf; // Key/value store of Onyx key and arrays of values to merge -const mergeQueue: Record = {}; -const mergeQueuePromise: Record> = {}; +const mergeQueue: Record = {}; +const mergeQueuePromise: Record> = {}; // Keeps track of the last connectionID that was used so we can keep incrementing it let lastConnectionID = 0; @@ -119,7 +120,7 @@ let evictionAllowList: OnyxKey[] = []; const evictionBlocklist: Record = {}; // Optional user-provided key value states set when Onyx initializes or clears -let defaultKeyStates: Partial = {}; +let defaultKeyStates: Record = {}; // Connections can be made before `Onyx.init`. They would wait for this task before resolving const deferredInitTask = createDeferredTask(); @@ -135,8 +136,8 @@ let batchUpdatesQueue: Array<() => void> = []; * @param value - contains the change that was made by the method * @param mergedValue - (optional) value that was written in the storage after a merge method was executed. */ -function sendActionToDevTools(method: OnyxMethod, key: undefined, value: Record, mergedValue: OnyxValue): void; -function sendActionToDevTools(method: OnyxMethod, key: OnyxKey, value: OnyxValue, mergedValue: OnyxValue): void; +function sendActionToDevTools(method: OnyxMethod, key: undefined, value: Record, mergedValue?: OnyxValue): void; +function sendActionToDevTools(method: OnyxMethod, key: OnyxKey, value: OnyxValue, mergedValue?: OnyxValue): void; function sendActionToDevTools(method: OnyxMethod, key: OnyxKey | undefined, value: Record | OnyxValue, mergedValue: OnyxValue = undefined): void { DevTools.registerAction(utils.formatActionName(method, key), value, key ? {[key]: mergedValue || value} : (value as Record)); } @@ -184,11 +185,11 @@ function batchUpdates(updates: () => void): Promise { * The resulting collection will only contain items that are returned by the selector. */ function reduceCollectionWithSelector( - collection: Record, + collection: Record, selector: Selector | undefined, withOnyxInstanceState: WithOnyxInstanceState | undefined, -): Record { - return Object.values(collection).reduce((finalCollection: Record, item, key) => { +): Record { + return Object.values(collection).reduce((finalCollection: Record, item, key) => { // eslint-disable-next-line no-param-reassign finalCollection[key] = selector?.(item, withOnyxInstanceState); @@ -274,7 +275,7 @@ function isSafeEvictionKey(testKey: OnyxKey): boolean { * If the requested key is a collection, it will return an object with all the collection members. */ function tryGetCachedValue(key: TKey, mapping: Mapping): OnyxValue { - let val: OnyxValue | Record = cache.getValue(key); + let val: OnyxValue | Record = cache.getValue(key); if (isCollectionKey(key)) { const allCacheKeys = cache.getAllKeys(); @@ -286,7 +287,7 @@ function tryGetCachedValue(key: TKey, mapping: Mapping k.startsWith(key)); - const values = matchingKeys.reduce((finalObject: Record, matchedKey) => { + const values = matchingKeys.reduce((finalObject: Record, matchedKey) => { const cachedValue = cache.getValue(matchedKey); if (cachedValue) { // This is permissible because we're in the process of constructing the final object in a reduce function. @@ -302,7 +303,7 @@ function tryGetCachedValue(key: TKey, mapping: Mapping, mapping.selector, state); + return reduceCollectionWithSelector(val as Record, mapping.selector, state); } return mapping.selector(val, state); } @@ -370,10 +371,10 @@ function addAllSafeEvictionKeysToRecentlyAccessedList(): Promise { }); } -function getCachedCollection(collectionKey: TKey): Record> { +function getCachedCollection(collectionKey: TKey): Record { const collectionMemberKeys = cache.getAllKeys().filter((storedKey) => isCollectionMemberKey(collectionKey, storedKey)); - return collectionMemberKeys.reduce((prev: Record, key) => { + return collectionMemberKeys.reduce((prev: Record, key) => { const cachedValue = cache.getValue(key); if (!cachedValue) { return prev; @@ -477,7 +478,7 @@ function keysChanged(collectionKey: TKey, partia } subscriber.withOnyxInstance.setStateProxy((prevState) => { - const finalCollection = lodashClone(prevState[subscriber.statePropertyName] || {}) as Record; + const finalCollection = lodashClone(prevState[subscriber.statePropertyName] || {}) as Record; const dataKeys = Object.keys(partialCollection ?? {}); for (let j = 0; j < dataKeys.length; j++) { const dataKey = dataKeys[j]; @@ -554,7 +555,7 @@ function keyChanged( key: OnyxKey, data: OnyxValue, prevData: OnyxValue, - canUpdateSubscriber = (_subscriber: Mapping) => true, + canUpdateSubscriber = (_subscriber: Mapping) => true, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true, ) { @@ -586,7 +587,7 @@ function keyChanged( subscriber.callback(cachedCollection); continue; } - subscriber.callback(data as Record, key); + subscriber.callback(data as Record, key); continue; } @@ -602,7 +603,7 @@ function keyChanged( // returned by the selector and only when the selected data has changed. if (subscriber.selector) { subscriber.withOnyxInstance.setStateProxy((prevState) => { - const prevWithOnyxData = prevState[subscriber.statePropertyName] as Record; + const prevWithOnyxData = prevState[subscriber.statePropertyName] as Record; const newWithOnyxData = { [key]: subscriber.selector?.(data, subscriber.withOnyxInstance?.state), }; @@ -681,7 +682,7 @@ function keyChanged( * - sets state on the withOnyxInstances * - triggers the callback function */ -function sendDataToConnection(mapping: Mapping, val: OnyxValue, matchedKey: OnyxKey | undefined, isBatched: boolean) { +function sendDataToConnection(mapping: Mapping, val: OnyxValue, matchedKey: OnyxKey | undefined, isBatched: boolean) { // If the mapping no longer exists then we should not send any data. // This means our subscriber disconnected or withOnyx wrapped component unmounted. if (!callbackToStateMapping[mapping.connectionID]) { @@ -713,7 +714,7 @@ function sendDataToConnection(mapping: Mapping, val: OnyxValue, matchedK } if (typeof mapping.callback === 'function') { - mapping.callback(val as Record, matchedKey as string); + mapping.callback(val as Record, matchedKey as string); } } @@ -742,10 +743,10 @@ function addKeyToRecentlyAccessedIfNeeded(mapping: Mapping) { /** * Gets the data for a given an array of matching keys, combines them into an object, and sends the result back to the subscriber. */ -function getCollectionDataAndSendAsObject(matchingKeys: CollectionKeyBase[], mapping: Mapping) { +function getCollectionDataAndSendAsObject(matchingKeys: CollectionKeyBase[], mapping: Mapping) { Promise.all(matchingKeys.map((key) => get(key))) .then((values) => - values.reduce((finalObject: Record, value, i) => { + values.reduce((finalObject: Record, value, i) => { // eslint-disable-next-line no-param-reassign finalObject[matchingKeys[i]] = value; return finalObject; @@ -893,7 +894,7 @@ function scheduleSubscriberUpdate( key: TKey, value: KeyValueMapping[TKey], prevValue: KeyValueMapping[TKey], - canUpdateSubscriber = (_subscriber: Mapping) => true, + canUpdateSubscriber = (_subscriber: Mapping) => true, ): Promise<[void, void]> { const promise = Promise.resolve().then(() => keyChanged(key, value, prevValue, canUpdateSubscriber, true, false)); batchUpdates(() => keyChanged(key, value, prevValue, canUpdateSubscriber, false, true)); @@ -962,16 +963,7 @@ function evictStorageAndRetry onyxMethod(...args)); } -/** - * Notifys subscribers and writes current value to cache - * - * @param {String} key - * @param {*} value - * @param {String} method - * @param {Boolean} hasChanged - * @param {Boolean} wasRemoved - * @returns {Promise} - */ +/** Notifies subscribers and writes current value to cache */ function broadcastUpdate(key: TKey, value: KeyValueMapping[TKey], method: string, hasChanged: boolean, wasRemoved = false) { // Logging properties only since values could be sensitive things we don't want to log Logger.logInfo(`${method}() called for key: ${key}${value && typeof value === 'object' ? ` properties: ${Object.keys(value).join(',')}` : ''}`); @@ -997,7 +989,7 @@ function hasPendingMergeForKey(key: OnyxKey): boolean { * Otherwise removes all nested null values in objects and returns the object * @returns The value without null values and a boolean "wasRemoved", which indicates if the key got removed completely */ -function removeNullValues(key: OnyxKey, value: OnyxValue | Record | null) { +function removeNullValues(key: OnyxKey, value: OnyxValue | Record | null) { if (value === null) { remove(key); return {value, wasRemoved: true}; @@ -1006,7 +998,7 @@ function removeNullValues(key: OnyxKey, value: OnyxValue | Record), wasRemoved: false}; + return {value: utils.removeNestedNullValues(value as Record), wasRemoved: false}; } /** @@ -1016,7 +1008,7 @@ function removeNullValues(key: OnyxKey, value: OnyxValue | Record): Promise { +function set(key: OnyxKey, value: OnyxValue | Record): Promise { // If the value is null, we remove the key from storage const {value: valueAfterRemoving, wasRemoved} = removeNullValues(key, value); @@ -1049,7 +1041,7 @@ function set(key: OnyxKey, value: OnyxValue | Record): Promis * * @return an array of key - value pairs <[key, value]> */ -function prepareKeyValuePairsForStorage(data: Partial): Array<[OnyxKey, OnyxValue]> { +function prepareKeyValuePairsForStorage(data: Record): Array<[OnyxKey, OnyxValue]> { const keyValuePairs: Array<[OnyxKey, OnyxValue]> = []; Object.entries(data).forEach(([key, value]) => { @@ -1070,7 +1062,7 @@ function prepareKeyValuePairsForStorage(data: Partial): * * @param data object keyed by ONYXKEYS and the values to set */ -function multiSet(data: Partial): Promise { +function multiSet(data: Partial): Promise> { const keyValuePairs = prepareKeyValuePairsForStorage(data); const updatePromises = keyValuePairs.map(([key, value]) => { @@ -1128,7 +1120,7 @@ function applyMerge(existingValue: OnyxValue, changes: Array {id: 1} * Onyx.merge(ONYXKEYS.POLICY, {name: 'My Workspace'}); // -> {id: 1, name: 'My Workspace'} */ -function merge(key: TKey, changes: OnyxEntry>): Promise { +function merge(key: TKey, changes: OnyxEntry>): Promise { // Top-level undefined values are ignored // Therefore we need to prevent adding them to the merge queue if (changes === undefined) { @@ -1145,7 +1137,9 @@ function merge(key: TKey, changes: OnyxEntry { // Calls to Onyx.set after a merge will terminate the current merge process and clear the merge queue - if (mergeQueue[key] == null) return; + if (mergeQueue[key] == null) { + return undefined; + } try { // We first only merge the changes, so we can provide these to the native implementation (SQLite uses only delta changes in "JSON_PATCH" to merge) @@ -1183,7 +1177,7 @@ function merge(key: TKey, changes: OnyxEntry; } return Storage.mergeItem(key, batchedChanges, modifiedData).then(() => { @@ -1292,7 +1286,8 @@ function clear(keysToPreserve: OnyxKey[] = []): Promise> { const defaultKeyValuePairs = Object.entries( Object.keys(defaultKeyStates) .filter((key) => !keysToPreserve.includes(key)) - .reduce((obj, key) => { + .reduce((obj: Record, key) => { + // eslint-disable-next-line no-param-reassign obj[key] = defaultKeyStates[key]; return obj; }, {}), @@ -1322,15 +1317,16 @@ function clear(keysToPreserve: OnyxKey[] = []): Promise> { * @param collectionKey e.g. `ONYXKEYS.COLLECTION.REPORT` * @param collection Object collection keyed by individual collection member keys and values */ -function mergeCollection(collectionKey: TKey, collection: Collection>): Promise { +function mergeCollection(collectionKey: TKey, collection: Collection>): Promise { if (typeof collection !== 'object' || Array.isArray(collection) || utils.isEmptyObject(collection)) { Logger.logInfo('mergeCollection() called with invalid or empty value. Skipping this update.'); return Promise.resolve(); } + const mergedCollection: Record = collection; // Confirm all the collection keys belong to the same parent let hasCollectionKeyCheckFailed = false; - Object.keys(collection).forEach((dataKey) => { + Object.keys(mergedCollection).forEach((dataKey) => { if (isKeyMatch(collectionKey, dataKey)) { return; } @@ -1350,8 +1346,8 @@ function mergeCollection(collectionKey: TK return getAllKeys().then((persistedKeys) => { // Split to keys that exist in storage and keys that don't - const keys = Object.keys(collection).filter((key) => { - if (collection[key] === null) { + const keys = Object.keys(mergedCollection).filter((key) => { + if (mergedCollection[key] === null) { remove(key); return false; } @@ -1361,13 +1357,15 @@ function mergeCollection(collectionKey: TK const existingKeys = keys.filter((key) => persistedKeys.includes(key)); const newKeys = keys.filter((key) => !persistedKeys.includes(key)); - const existingKeyCollection = existingKeys.reduce((obj, key) => { - obj[key] = collection[key]; + const existingKeyCollection = existingKeys.reduce((obj: Record, key) => { + // eslint-disable-next-line no-param-reassign + obj[key] = mergedCollection[key]; return obj; }, {}); - const newCollection = newKeys.reduce((obj, key) => { - obj[key] = collection[key]; + const newCollection = newKeys.reduce((obj: Record, key) => { + // eslint-disable-next-line no-param-reassign + obj[key] = mergedCollection[key]; return obj; }, {}); const keyValuePairsForExistingCollection = prepareKeyValuePairsForStorage(existingKeyCollection); @@ -1388,14 +1386,14 @@ function mergeCollection(collectionKey: TK // Prefill cache if necessary by calling get() on any existing keys and then merge original data to cache // and update all subscribers const promiseUpdate = Promise.all(existingKeys.map(get)).then(() => { - cache.merge(collection); - return scheduleNotifyCollectionSubscribers(collectionKey, collection); + cache.merge(mergedCollection); + return scheduleNotifyCollectionSubscribers(collectionKey, mergedCollection); }); return Promise.all(promises) - .catch((error) => evictStorageAndRetry(error, mergeCollection, collectionKey, collection)) + .catch((error) => evictStorageAndRetry(error, mergeCollection, collectionKey, mergedCollection)) .then(() => { - sendActionToDevTools(METHOD.MERGE_COLLECTION, undefined, collection); + sendActionToDevTools(METHOD.MERGE_COLLECTION, undefined, mergedCollection); return promiseUpdate; }); }); @@ -1440,10 +1438,10 @@ type OnyxUpdate = /** * Insert API responses and lifecycle data into Onyx * - * @param {Array} data An array of objects with shape {onyxMethod: oneOf('set', 'merge', 'mergeCollection', 'multiSet', 'clear'), key: string, value: *} - * @returns {Promise} resolves when all operations are complete + * @param data An array of objects with shape {onyxMethod: oneOf('set', 'merge', 'mergeCollection', 'multiSet', 'clear'), key: string, value: *} + * @returns resolves when all operations are complete */ -function update(data: OnyxUpdate[]): Promise { +function update(data: OnyxUpdate[]): Promise { // First, validate the Onyx object is in the format we expect data.forEach(({onyxMethod, key, value}) => { if (![METHOD.CLEAR, METHOD.SET, METHOD.MERGE, METHOD.MERGE_COLLECTION, METHOD.MULTI_SET].includes(onyxMethod)) { @@ -1459,7 +1457,7 @@ function update(data: OnyxUpdate[]): Promise { } }); - const promises: Array<() => Promise> = []; + const promises: Array<() => Promise> = []; let clearPromise: Promise = Promise.resolve(); data.forEach(({onyxMethod, key, value}) => { @@ -1471,7 +1469,8 @@ function update(data: OnyxUpdate[]): Promise { promises.push(() => merge(key, value)); break; case METHOD.MERGE_COLLECTION: - promises.push(() => mergeCollection(key, value)); + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- We validated that the value is a collection + promises.push(() => mergeCollection(key, value as any)); break; case METHOD.MULTI_SET: promises.push(() => multiSet(value)); diff --git a/lib/PerformanceUtils.ts b/lib/PerformanceUtils.ts index 4bad2268..8ac0b0aa 100644 --- a/lib/PerformanceUtils.ts +++ b/lib/PerformanceUtils.ts @@ -1,6 +1,7 @@ import lodashTransform from 'lodash/transform'; import {deepEqual} from 'fast-equals'; import type {Mapping} from './Onyx'; +import type {OnyxKey} from './types'; type UnknownObject = Record; @@ -40,7 +41,7 @@ function diffObject( /** * Provide insights into why a setState() call occurred by diffing the before and after values. */ -function logSetStateCall(mapping: Mapping, previousValue: unknown, newValue: unknown, caller: string, keyThatChanged?: string) { +function logSetStateCall(mapping: Mapping, previousValue: unknown, newValue: unknown, caller: string, keyThatChanged?: string) { if (!debugSetState) { return; } From ac07d88a5d84828c66f2e34882921d3c2eee4dab Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 09:31:11 +0100 Subject: [PATCH 15/23] Fix error with clear() --- lib/Onyx.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 4891d518..ee634e17 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -1254,11 +1254,10 @@ function clear(keysToPreserve: OnyxKey[] = []): Promise> { cache.set(key, newValue); const collectionKey = key.substring(0, key.indexOf('_') + 1); if (collectionKey) { - let collection = keyValuesToResetAsCollection[collectionKey]; - if (!collection) { - collection = {}; + if (!keyValuesToResetAsCollection[collectionKey]) { + keyValuesToResetAsCollection[collectionKey] = {}; } - collection[key] = newValue; + keyValuesToResetAsCollection[collectionKey]![key] = newValue; } else { keyValuesToResetIndividually[key] = newValue; } From d2fe5c77504663697ed8341702b642e449de1144 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 09:56:10 +0100 Subject: [PATCH 16/23] Fix error with OnyxCache --- lib/OnyxCache.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/OnyxCache.ts b/lib/OnyxCache.ts index e7d9b6bc..b0df21e7 100644 --- a/lib/OnyxCache.ts +++ b/lib/OnyxCache.ts @@ -128,8 +128,8 @@ class OnyxCache { * provided from this function * @param taskName - unique name given for the task */ - getTaskPromise(taskName: string): Promise { - return this.pendingPromises.get(taskName) ?? Promise.resolve(); + getTaskPromise(taskName: string): Promise | Promise | undefined { + return this.pendingPromises.get(taskName); } /** From f717d4645aa064913581748a335804902e7609b7 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 09:57:49 +0100 Subject: [PATCH 17/23] Make a change that is already on main --- lib/Onyx.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index ee634e17..cc186148 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -1527,6 +1527,16 @@ function init({ shouldSyncMultipleInstances = Boolean(global.localStorage), debugSetState = false, }: InitOptions) { + Storage.init(); + + if (shouldSyncMultipleInstances && typeof Storage.keepInstancesSync === 'function') { + Storage.keepInstancesSync((key, value) => { + const prevValue = cache.getValue(key, false); + cache.set(key, value); + keyChanged(key, value, prevValue); + }); + } + if (debugSetState) { PerformanceUtils.setShouldDebugSetState(true); } @@ -1553,14 +1563,6 @@ function init({ // Initialize all of our keys with data provided then give green light to any pending connections Promise.all([addAllSafeEvictionKeysToRecentlyAccessedList(), initializeWithDefaultKeyStates()]).then(deferredInitTask.resolve); - - if (shouldSyncMultipleInstances && typeof Storage.keepInstancesSync === 'function') { - Storage.keepInstancesSync((key, value) => { - const prevValue = cache.getValue(key, false); - cache.set(key, value); - keyChanged(key, value, prevValue); - }); - } } const Onyx = { From 556f183ee9d417e01663373d1b52a087bd59064e Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 10:08:50 +0100 Subject: [PATCH 18/23] Adjust getCollectionDataAndSendAsObject --- lib/Onyx.ts | 78 ++++++++++++++++++++++++++++++++++++++++++------ lib/OnyxCache.ts | 4 +-- 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index cc186148..728cbadb 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -744,15 +744,75 @@ function addKeyToRecentlyAccessedIfNeeded(mapping: Mapping) { * Gets the data for a given an array of matching keys, combines them into an object, and sends the result back to the subscriber. */ function getCollectionDataAndSendAsObject(matchingKeys: CollectionKeyBase[], mapping: Mapping) { - Promise.all(matchingKeys.map((key) => get(key))) - .then((values) => - values.reduce((finalObject: Record, value, i) => { - // eslint-disable-next-line no-param-reassign - finalObject[matchingKeys[i]] = value; - return finalObject; - }, {}), - ) - .then((val) => sendDataToConnection(mapping, val, undefined, true)); + // Keys that are not in the cache + const missingKeys: OnyxKey[] = []; + // Tasks that are pending + const pendingTasks: Array> = []; + // Keys for the tasks that are pending + const pendingKeys: OnyxKey[] = []; + + // We are going to combine all the data from the matching keys into a single object + const data: Record = {}; + + /** + * We are going to iterate over all the matching keys and check if we have the data in the cache. + * If we do then we add it to the data object. If we do not then we check if there is a pending task + * for the key. If there is then we add the promise to the pendingTasks array and the key to the pendingKeys + * array. If there is no pending task then we add the key to the missingKeys array. + * + * These missingKeys will be later to use to multiGet the data from the storage. + */ + matchingKeys.forEach((key) => { + const cacheValue = cache.getValue(key); + if (cacheValue) { + data[key] = cacheValue; + return; + } + + const pendingKey = `get:${key}`; + if (cache.hasPendingTask(pendingKey)) { + pendingTasks.push(cache.getTaskPromise(pendingKey)); + pendingKeys.push(key); + } else { + missingKeys.push(key); + } + }); + + Promise.all(pendingTasks) + // We are going to wait for all the pending tasks to resolve and then add the data to the data object. + .then((values) => { + values.forEach((value, index) => { + data[pendingKeys[index]] = value; + }); + + return Promise.resolve(); + }) + // We are going to get the missing keys using multiGet from the storage. + .then(() => { + if (missingKeys.length === 0) { + return Promise.resolve(undefined); + } + return Storage.multiGet(missingKeys); + }) + // We are going to add the data from the missing keys to the data object and also merge it to the cache. + .then((values) => { + if (!values || values.length === 0) { + return Promise.resolve(); + } + + // temp object is used to merge the missing data into the cache + const temp: Record = {}; + values.forEach((value) => { + data[value[0]] = value[1]; + temp[value[0]] = value[1]; + }); + cache.merge(temp); + return Promise.resolve(); + }) + // We are going to send the data to the subscriber. + .finally(() => { + sendDataToConnection(mapping, data, undefined, true); + }); } /** diff --git a/lib/OnyxCache.ts b/lib/OnyxCache.ts index b0df21e7..d12618ef 100644 --- a/lib/OnyxCache.ts +++ b/lib/OnyxCache.ts @@ -128,8 +128,8 @@ class OnyxCache { * provided from this function * @param taskName - unique name given for the task */ - getTaskPromise(taskName: string): Promise | Promise | undefined { - return this.pendingPromises.get(taskName); + getTaskPromise(taskName: string): Promise { + return this.pendingPromises.get(taskName) as Promise; } /** From f003e696110ae2c602651a2d79e6169c99e7a90c Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 10:34:38 +0100 Subject: [PATCH 19/23] Fix last failing test --- lib/Onyx.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 728cbadb..d1288d4b 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -189,7 +189,7 @@ function reduceCollectionWithSelector | undefined, withOnyxInstanceState: WithOnyxInstanceState | undefined, ): Record { - return Object.values(collection).reduce((finalCollection: Record, item, key) => { + return Object.entries(collection ?? {}).reduce((finalCollection: Record, [key, item]) => { // eslint-disable-next-line no-param-reassign finalCollection[key] = selector?.(item, withOnyxInstanceState); @@ -696,7 +696,7 @@ function sendDataToConnection(mapping: Mapping, val: OnyxValue, matched // returned by the selector. if (mapping.selector) { if (isCollectionKey(mapping.key)) { - newData = mapping.selector(val, mapping.withOnyxInstance.state); + newData = reduceCollectionWithSelector(val, mapping.selector, mapping.withOnyxInstance.state); } else { newData = mapping.selector(val, mapping.withOnyxInstance.state); } From 437fdbc217a4ba5434a859c9852032ed1f181bfa Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 10:57:45 +0100 Subject: [PATCH 20/23] Add back setMemoryOnlyKeys --- lib/Onyx-old.js | 1676 ----------------------------------------------- lib/Onyx.ts | 7 + 2 files changed, 7 insertions(+), 1676 deletions(-) delete mode 100644 lib/Onyx-old.js diff --git a/lib/Onyx-old.js b/lib/Onyx-old.js deleted file mode 100644 index 9d9669c5..00000000 --- a/lib/Onyx-old.js +++ /dev/null @@ -1,1676 +0,0 @@ -/* eslint-disable no-continue */ -import {deepEqual} from 'fast-equals'; -import _ from 'underscore'; -import * as Logger from './Logger'; -import cache from './OnyxCache'; -import * as Str from './Str'; -import createDeferredTask from './createDeferredTask'; -import * as PerformanceUtils from './PerformanceUtils'; -import Storage from './storage'; -import utils from './utils'; -import unstable_batchedUpdates from './batch'; -import DevTools from './DevTools'; - -// Method constants -const METHOD = { - SET: 'set', - MERGE: 'merge', - MERGE_COLLECTION: 'mergecollection', - MULTI_SET: 'multiset', - CLEAR: 'clear', -}; - -// Key/value store of Onyx key and arrays of values to merge -const mergeQueue = {}; -const mergeQueuePromise = {}; - -// Keeps track of the last connectionID that was used so we can keep incrementing it -let lastConnectionID = 0; - -// Holds a mapping of all the react components that want their state subscribed to a store key -const callbackToStateMapping = {}; - -// Keeps a copy of the values of the onyx collection keys as a map for faster lookups -let onyxCollectionKeyMap = new Map(); - -// Holds a list of keys that have been directly subscribed to or recently modified from least to most recent -let recentlyAccessedKeys = []; - -// Holds a list of keys that are safe to remove when we reach max storage. If a key does not match with -// whatever appears in this list it will NEVER be a candidate for eviction. -let evictionAllowList = []; - -// Holds a map of keys and connectionID arrays whose keys will never be automatically evicted as -// long as we have at least one subscriber that returns false for the canEvict property. -const evictionBlocklist = {}; - -// Optional user-provided key value states set when Onyx initializes or clears -let defaultKeyStates = {}; - -// Connections can be made before `Onyx.init`. They would wait for this task before resolving -const deferredInitTask = createDeferredTask(); - -let batchUpdatesPromise = null; -let batchUpdatesQueue = []; - -/** - * Sends an action to DevTools extension - * - * @param {string} method - Onyx method from METHOD - * @param {string} key - Onyx key that was changed - * @param {any} value - contains the change that was made by the method - * @param {any} mergedValue - (optional) value that was written in the storage after a merge method was executed. - */ -function sendActionToDevTools(method, key, value, mergedValue = undefined) { - DevTools.registerAction(utils.formatActionName(method, key), value, key ? {[key]: mergedValue || value} : value); -} - -/** - * We are batching together onyx updates. This helps with use cases where we schedule onyx updates after each other. - * This happens for example in the Onyx.update function, where we process API responses that might contain a lot of - * update operations. Instead of calling the subscribers for each update operation, we batch them together which will - * cause react to schedule the updates at once instead of after each other. This is mainly a performance optimization. - * @returns {Promise} - */ -function maybeFlushBatchUpdates() { - if (batchUpdatesPromise) { - return batchUpdatesPromise; - } - - batchUpdatesPromise = new Promise((resolve) => { - /* We use (setTimeout, 0) here which should be called once native module calls are flushed (usually at the end of the frame) - * We may investigate if (setTimeout, 1) (which in React Native is equal to requestAnimationFrame) works even better - * then the batch will be flushed on next frame. - */ - setTimeout(() => { - const updatesCopy = batchUpdatesQueue; - batchUpdatesQueue = []; - batchUpdatesPromise = null; - unstable_batchedUpdates(() => { - updatesCopy.forEach((applyUpdates) => { - applyUpdates(); - }); - }); - - resolve(); - }, 0); - }); - return batchUpdatesPromise; -} - -function batchUpdates(updates) { - batchUpdatesQueue.push(updates); - return maybeFlushBatchUpdates(); -} - -/** - * Uses a selector function to return a simplified version of sourceData - * @param {Mixed} sourceData - * @param {Function} selector Function that takes sourceData and returns a simplified version of it - * @param {Object} [withOnyxInstanceState] - * @returns {Mixed} - */ -const getSubsetOfData = (sourceData, selector, withOnyxInstanceState) => selector(sourceData, withOnyxInstanceState); - -/** - * Takes a collection of items (eg. {testKey_1:{a:'a'}, testKey_2:{b:'b'}}) - * and runs it through a reducer function to return a subset of the data according to a selector. - * The resulting collection will only contain items that are returned by the selector. - * @param {Object} collection - * @param {String|Function} selector (see method docs for getSubsetOfData() for full details) - * @param {Object} [withOnyxInstanceState] - * @returns {Object} - */ -const reduceCollectionWithSelector = (collection, selector, withOnyxInstanceState) => - _.reduce( - collection, - (finalCollection, item, key) => { - // eslint-disable-next-line no-param-reassign - finalCollection[key] = getSubsetOfData(item, selector, withOnyxInstanceState); - - return finalCollection; - }, - {}, - ); - -/** - * Get some data from the store - * - * @private - * @param {string} key - * @returns {Promise<*>} - */ -function get(key) { - // When we already have the value in cache - resolve right away - if (cache.hasCacheForKey(key)) { - return Promise.resolve(cache.getValue(key)); - } - - const taskName = `get:${key}`; - - // When a value retrieving task for this key is still running hook to it - if (cache.hasPendingTask(taskName)) { - return cache.getTaskPromise(taskName); - } - - // Otherwise retrieve the value from storage and capture a promise to aid concurrent usages - const promise = Storage.getItem(key) - .then((val) => { - cache.set(key, val); - return val; - }) - .catch((err) => Logger.logInfo(`Unable to get item from persistent storage. Key: ${key} Error: ${err}`)); - - return cache.captureTask(taskName, promise); -} - -/** - * Returns current key names stored in persisted storage - * @private - * @returns {Promise} - */ -function getAllKeys() { - // When we've already read stored keys, resolve right away - const storedKeys = cache.getAllKeys(); - if (storedKeys.length > 0) { - return Promise.resolve(storedKeys); - } - - const taskName = 'getAllKeys'; - - // When a value retrieving task for all keys is still running hook to it - if (cache.hasPendingTask(taskName)) { - return cache.getTaskPromise(taskName); - } - - // Otherwise retrieve the keys from storage and capture a promise to aid concurrent usages - const promise = Storage.getAllKeys().then((keys) => { - _.each(keys, (key) => cache.addKey(key)); - return keys; - }); - - return cache.captureTask(taskName, promise); -} - -/** - * Checks to see if the a subscriber's supplied key - * is associated with a collection of keys. - * - * @private - * @param {String} key - * @returns {Boolean} - */ -function isCollectionKey(key) { - return onyxCollectionKeyMap.has(key); -} - -/** - * @param {String} collectionKey - * @param {String} key - * @returns {Boolean} - */ -function isCollectionMemberKey(collectionKey, key) { - return Str.startsWith(key, collectionKey) && key.length > collectionKey.length; -} - -/** - * Checks to see if a provided key is the exact configured key of our connected subscriber - * or if the provided key is a collection member key (in case our configured key is a "collection key") - * - * @private - * @param {String} configKey - * @param {String} key - * @return {Boolean} - */ -function isKeyMatch(configKey, key) { - return isCollectionKey(configKey) ? Str.startsWith(key, configKey) : configKey === key; -} - -/** - * Checks to see if this key has been flagged as - * safe for removal. - * - * @private - * @param {String} testKey - * @returns {Boolean} - */ -function isSafeEvictionKey(testKey) { - return _.some(evictionAllowList, (key) => isKeyMatch(key, testKey)); -} - -/** - * Tries to get a value from the cache. If the value is not present in cache it will return the default value or undefined. - * If the requested key is a collection, it will return an object with all the collection members. - * - * @param {String} key - * @param {Object} mapping - * @returns {Mixed} - */ -function tryGetCachedValue(key, mapping = {}) { - let val = cache.getValue(key); - - if (isCollectionKey(key)) { - const allCacheKeys = cache.getAllKeys(); - - // It is possible we haven't loaded all keys yet so we do not know if the - // collection actually exists. - if (allCacheKeys.length === 0) { - return; - } - const matchingKeys = _.filter(allCacheKeys, (k) => k.startsWith(key)); - const values = _.reduce( - matchingKeys, - (finalObject, matchedKey) => { - const cachedValue = cache.getValue(matchedKey); - if (cachedValue) { - // This is permissible because we're in the process of constructing the final object in a reduce function. - // eslint-disable-next-line no-param-reassign - finalObject[matchedKey] = cachedValue; - } - return finalObject; - }, - {}, - ); - - val = values; - } - - if (mapping.selector) { - const state = mapping.withOnyxInstance ? mapping.withOnyxInstance.state : undefined; - if (isCollectionKey(key)) { - return reduceCollectionWithSelector(val, mapping.selector, state); - } - return getSubsetOfData(val, mapping.selector, state); - } - - return val; -} - -/** - * Remove a key from the recently accessed key list. - * - * @private - * @param {String} key - */ -function removeLastAccessedKey(key) { - recentlyAccessedKeys = _.without(recentlyAccessedKeys, key); -} - -/** - * Add a key to the list of recently accessed keys. The least - * recently accessed key should be at the head and the most - * recently accessed key at the tail. - * - * @private - * @param {String} key - */ -function addLastAccessedKey(key) { - // Only specific keys belong in this list since we cannot remove an entire collection. - if (isCollectionKey(key) || !isSafeEvictionKey(key)) { - return; - } - - removeLastAccessedKey(key); - recentlyAccessedKeys.push(key); -} - -/** - * Removes a key previously added to this list - * which will enable it to be deleted again. - * - * @private - * @param {String} key - * @param {Number} connectionID - */ -function removeFromEvictionBlockList(key, connectionID) { - evictionBlocklist[key] = _.without(evictionBlocklist[key] || [], connectionID); - - // Remove the key if there are no more subscribers - if (evictionBlocklist[key].length === 0) { - delete evictionBlocklist[key]; - } -} - -/** - * Keys added to this list can never be deleted. - * - * @private - * @param {String} key - * @param {Number} connectionID - */ -function addToEvictionBlockList(key, connectionID) { - removeFromEvictionBlockList(key, connectionID); - - if (!evictionBlocklist[key]) { - evictionBlocklist[key] = []; - } - - evictionBlocklist[key].push(connectionID); -} - -/** - * Take all the keys that are safe to evict and add them to - * the recently accessed list when initializing the app. This - * enables keys that have not recently been accessed to be - * removed. - * - * @private - * @returns {Promise} - */ -function addAllSafeEvictionKeysToRecentlyAccessedList() { - return getAllKeys().then((keys) => { - _.each(evictionAllowList, (safeEvictionKey) => { - _.each(keys, (key) => { - if (!isKeyMatch(safeEvictionKey, key)) { - return; - } - addLastAccessedKey(key); - }); - }); - }); -} - -/** - * @private - * @param {String} collectionKey - * @returns {Object} - */ -function getCachedCollection(collectionKey) { - const collectionMemberKeys = _.filter(cache.getAllKeys(), (storedKey) => isCollectionMemberKey(collectionKey, storedKey)); - - return _.reduce( - collectionMemberKeys, - (prev, curr) => { - const cachedValue = cache.getValue(curr); - if (!cachedValue) { - return prev; - } - - // eslint-disable-next-line no-param-reassign - prev[curr] = cachedValue; - return prev; - }, - {}, - ); -} - -/** - * When a collection of keys change, search for any callbacks matching the collection key and trigger those callbacks - * - * @private - * @param {String} collectionKey - * @param {Object} partialCollection - a partial collection of grouped member keys - * @param {boolean} [notifyRegularSubscibers=true] - * @param {boolean} [notifyWithOnyxSubscibers=true] - */ -function keysChanged(collectionKey, partialCollection, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true) { - // We are iterating over all subscribers similar to keyChanged(). However, we are looking for subscribers who are subscribing to either a collection key or - // individual collection key member for the collection that is being updated. It is important to note that the collection parameter cane be a PARTIAL collection - // and does not represent all of the combined keys and values for a collection key. It is just the "new" data that was merged in via mergeCollection(). - const stateMappingKeys = _.keys(callbackToStateMapping); - for (let i = 0; i < stateMappingKeys.length; i++) { - const subscriber = callbackToStateMapping[stateMappingKeys[i]]; - if (!subscriber) { - continue; - } - - // Skip iteration if we do not have a collection key or a collection member key on this subscriber - if (!Str.startsWith(subscriber.key, collectionKey)) { - continue; - } - - /** - * e.g. Onyx.connect({key: ONYXKEYS.COLLECTION.REPORT, callback: ...}); - */ - const isSubscribedToCollectionKey = subscriber.key === collectionKey; - - /** - * e.g. Onyx.connect({key: `${ONYXKEYS.COLLECTION.REPORT}{reportID}`, callback: ...}); - */ - const isSubscribedToCollectionMemberKey = isCollectionMemberKey(collectionKey, subscriber.key); - - // We prepare the "cached collection" which is the entire collection + the new partial data that - // was merged in via mergeCollection(). - const cachedCollection = getCachedCollection(collectionKey); - - // Regular Onyx.connect() subscriber found. - if (_.isFunction(subscriber.callback)) { - if (!notifyRegularSubscibers) { - continue; - } - - // If they are subscribed to the collection key and using waitForCollectionCallback then we'll - // send the whole cached collection. - if (isSubscribedToCollectionKey) { - if (subscriber.waitForCollectionCallback) { - subscriber.callback(cachedCollection); - continue; - } - - // If they are not using waitForCollectionCallback then we notify the subscriber with - // the new merged data but only for any keys in the partial collection. - const dataKeys = _.keys(partialCollection); - for (let j = 0; j < dataKeys.length; j++) { - const dataKey = dataKeys[j]; - subscriber.callback(cachedCollection[dataKey], dataKey); - } - continue; - } - - // And if the subscriber is specifically only tracking a particular collection member key then we will - // notify them with the cached data for that key only. - if (isSubscribedToCollectionMemberKey) { - subscriber.callback(cachedCollection[subscriber.key], subscriber.key); - continue; - } - - continue; - } - - // React component subscriber found. - if (subscriber.withOnyxInstance) { - if (!notifyWithOnyxSubscibers) { - continue; - } - - // We are subscribed to a collection key so we must update the data in state with the new - // collection member key values from the partial update. - if (isSubscribedToCollectionKey) { - // If the subscriber has a selector, then the component's state must only be updated with the data - // returned by the selector. - if (subscriber.selector) { - subscriber.withOnyxInstance.setStateProxy((prevState) => { - const previousData = prevState[subscriber.statePropertyName]; - const newData = reduceCollectionWithSelector(cachedCollection, subscriber.selector, subscriber.withOnyxInstance.state); - - if (!deepEqual(previousData, newData)) { - return { - [subscriber.statePropertyName]: newData, - }; - } - return null; - }); - continue; - } - - subscriber.withOnyxInstance.setStateProxy((prevState) => { - const finalCollection = _.clone(prevState[subscriber.statePropertyName] || {}); - const dataKeys = _.keys(partialCollection); - for (let j = 0; j < dataKeys.length; j++) { - const dataKey = dataKeys[j]; - finalCollection[dataKey] = cachedCollection[dataKey]; - } - - PerformanceUtils.logSetStateCall(subscriber, prevState[subscriber.statePropertyName], finalCollection, 'keysChanged', collectionKey); - return { - [subscriber.statePropertyName]: finalCollection, - }; - }); - continue; - } - - // If a React component is only interested in a single key then we can set the cached value directly to the state name. - if (isSubscribedToCollectionMemberKey) { - // However, we only want to update this subscriber if the partial data contains a change. - // Otherwise, we would update them with a value they already have and trigger an unnecessary re-render. - const dataFromCollection = partialCollection[subscriber.key]; - if (_.isUndefined(dataFromCollection)) { - continue; - } - - // If the subscriber has a selector, then the component's state must only be updated with the data - // returned by the selector and the state should only change when the subset of data changes from what - // it was previously. - if (subscriber.selector) { - subscriber.withOnyxInstance.setStateProxy((prevState) => { - const prevData = prevState[subscriber.statePropertyName]; - const newData = getSubsetOfData(cachedCollection[subscriber.key], subscriber.selector, subscriber.withOnyxInstance.state); - if (!deepEqual(prevData, newData)) { - PerformanceUtils.logSetStateCall(subscriber, prevData, newData, 'keysChanged', collectionKey); - return { - [subscriber.statePropertyName]: newData, - }; - } - - return null; - }); - continue; - } - - subscriber.withOnyxInstance.setStateProxy((prevState) => { - const data = cachedCollection[subscriber.key]; - const previousData = prevState[subscriber.statePropertyName]; - - // Avoids triggering unnecessary re-renders when feeding empty objects - if (utils.isEmptyObject(data) && utils.isEmptyObject(previousData)) { - return null; - } - if (data === previousData) { - return null; - } - - PerformanceUtils.logSetStateCall(subscriber, previousData, data, 'keysChanged', collectionKey); - return { - [subscriber.statePropertyName]: data, - }; - }); - } - } - } -} - -/** - * When a key change happens, search for any callbacks matching the key or collection key and trigger those callbacks - * - * @example - * keyChanged(key, value, subscriber => subscriber.initWithStoredValues === false) - * - * @private - * @param {String} key - * @param {*} data - * @param {*} prevData - * @param {Function} [canUpdateSubscriber] only subscribers that pass this truth test will be updated - * @param {boolean} [notifyRegularSubscibers=true] - * @param {boolean} [notifyWithOnyxSubscibers=true] - */ -function keyChanged(key, data, prevData, canUpdateSubscriber = () => true, notifyRegularSubscibers = true, notifyWithOnyxSubscibers = true) { - // Add or remove this key from the recentlyAccessedKeys lists - if (!_.isNull(data)) { - addLastAccessedKey(key); - } else { - removeLastAccessedKey(key); - } - - // We are iterating over all subscribers to see if they are interested in the key that has just changed. If the subscriber's key is a collection key then we will - // notify them if the key that changed is a collection member. Or if it is a regular key notify them when there is an exact match. Depending on whether the subscriber - // was connected via withOnyx we will call setState() directly on the withOnyx instance. If it is a regular connection we will pass the data to the provided callback. - const stateMappingKeys = _.keys(callbackToStateMapping); - for (let i = 0; i < stateMappingKeys.length; i++) { - const subscriber = callbackToStateMapping[stateMappingKeys[i]]; - if (!subscriber || !isKeyMatch(subscriber.key, key) || !canUpdateSubscriber(subscriber)) { - continue; - } - - // Subscriber is a regular call to connect() and provided a callback - if (_.isFunction(subscriber.callback)) { - if (!notifyRegularSubscibers) { - continue; - } - if (isCollectionKey(subscriber.key) && subscriber.waitForCollectionCallback) { - const cachedCollection = getCachedCollection(subscriber.key); - cachedCollection[key] = data; - subscriber.callback(cachedCollection); - continue; - } - - subscriber.callback(data, key); - continue; - } - - // Subscriber connected via withOnyx() HOC - if (subscriber.withOnyxInstance) { - if (!notifyWithOnyxSubscibers) { - continue; - } - - // Check if we are subscribing to a collection key and overwrite the collection member key value in state - if (isCollectionKey(subscriber.key)) { - // If the subscriber has a selector, then the consumer of this data must only be given the data - // returned by the selector and only when the selected data has changed. - if (subscriber.selector) { - subscriber.withOnyxInstance.setStateProxy((prevState) => { - const prevWithOnyxData = prevState[subscriber.statePropertyName]; - const newWithOnyxData = { - [key]: getSubsetOfData(data, subscriber.selector, subscriber.withOnyxInstance.state), - }; - const prevDataWithNewData = { - ...prevWithOnyxData, - ...newWithOnyxData, - }; - if (!deepEqual(prevWithOnyxData, prevDataWithNewData)) { - PerformanceUtils.logSetStateCall(subscriber, prevWithOnyxData, newWithOnyxData, 'keyChanged', key); - return { - [subscriber.statePropertyName]: prevDataWithNewData, - }; - } - return null; - }); - continue; - } - - subscriber.withOnyxInstance.setStateProxy((prevState) => { - const collection = prevState[subscriber.statePropertyName] || {}; - const newCollection = { - ...collection, - [key]: data, - }; - PerformanceUtils.logSetStateCall(subscriber, collection, newCollection, 'keyChanged', key); - return { - [subscriber.statePropertyName]: newCollection, - }; - }); - continue; - } - - // If the subscriber has a selector, then the component's state must only be updated with the data - // returned by the selector and only if the selected data has changed. - if (subscriber.selector) { - subscriber.withOnyxInstance.setStateProxy(() => { - const previousValue = getSubsetOfData(prevData, subscriber.selector, subscriber.withOnyxInstance.state); - const newValue = getSubsetOfData(data, subscriber.selector, subscriber.withOnyxInstance.state); - - if (!deepEqual(previousValue, newValue)) { - return { - [subscriber.statePropertyName]: newValue, - }; - } - return null; - }); - continue; - } - - // If we did not match on a collection key then we just set the new data to the state property - subscriber.withOnyxInstance.setStateProxy((prevState) => { - const prevWithOnyxData = prevState[subscriber.statePropertyName]; - - // Avoids triggering unnecessary re-renders when feeding empty objects - if (utils.isEmptyObject(data) && utils.isEmptyObject(prevWithOnyxData)) { - return null; - } - if (prevWithOnyxData === data) { - return null; - } - - PerformanceUtils.logSetStateCall(subscriber, prevData, data, 'keyChanged', key); - return { - [subscriber.statePropertyName]: data, - }; - }); - continue; - } - - console.error('Warning: Found a matching subscriber to a key that changed, but no callback or withOnyxInstance could be found.'); - } -} - -/** - * Sends the data obtained from the keys to the connection. It either: - * - sets state on the withOnyxInstances - * - triggers the callback function - * - * @private - * @param {Object} mapping - * @param {Object} [mapping.withOnyxInstance] - * @param {String} [mapping.statePropertyName] - * @param {Function} [mapping.callback] - * @param {String} [mapping.selector] - * @param {*|null} val - * @param {String|undefined} matchedKey - * @param {Boolean} isBatched - */ -function sendDataToConnection(mapping, val, matchedKey, isBatched) { - // If the mapping no longer exists then we should not send any data. - // This means our subscriber disconnected or withOnyx wrapped component unmounted. - if (!callbackToStateMapping[mapping.connectionID]) { - return; - } - - if (mapping.withOnyxInstance) { - let newData = val; - - // If the mapping has a selector, then the component's state must only be updated with the data - // returned by the selector. - if (mapping.selector) { - if (isCollectionKey(mapping.key)) { - newData = reduceCollectionWithSelector(val, mapping.selector, mapping.withOnyxInstance.state); - } else { - newData = getSubsetOfData(val, mapping.selector, mapping.withOnyxInstance.state); - } - } - - PerformanceUtils.logSetStateCall(mapping, null, newData, 'sendDataToConnection'); - if (isBatched) { - batchUpdates(() => { - mapping.withOnyxInstance.setWithOnyxState(mapping.statePropertyName, newData); - }); - } else { - mapping.withOnyxInstance.setWithOnyxState(mapping.statePropertyName, newData); - } - return; - } - - if (_.isFunction(mapping.callback)) { - mapping.callback(val, matchedKey); - } -} - -/** - * We check to see if this key is flagged as safe for eviction and add it to the recentlyAccessedKeys list so that when we - * run out of storage the least recently accessed key can be removed. - * - * @private - * @param {Object} mapping - */ -function addKeyToRecentlyAccessedIfNeeded(mapping) { - if (!isSafeEvictionKey(mapping.key)) { - return; - } - - // Try to free some cache whenever we connect to a safe eviction key - cache.removeLeastRecentlyUsedKeys(); - - if (mapping.withOnyxInstance && !isCollectionKey(mapping.key)) { - // All React components subscribing to a key flagged as a safe eviction key must implement the canEvict property. - if (_.isUndefined(mapping.canEvict)) { - throw new Error(`Cannot subscribe to safe eviction key '${mapping.key}' without providing a canEvict value.`); - } - - addLastAccessedKey(mapping.key); - } -} - -/** - * Gets the data for a given an array of matching keys, combines them into an object, and sends the result back to the subscriber. - * - * @private - * @param {Array} matchingKeys - * @param {Object} mapping - */ -function getCollectionDataAndSendAsObject(matchingKeys, mapping) { - // Keys that are not in the cache - const missingKeys = []; - // Tasks that are pending - const pendingTasks = []; - // Keys for the tasks that are pending - const pendingKeys = []; - - // We are going to combine all the data from the matching keys into a single object - const data = {}; - - /** - * We are going to iterate over all the matching keys and check if we have the data in the cache. - * If we do then we add it to the data object. If we do not then we check if there is a pending task - * for the key. If there is then we add the promise to the pendingTasks array and the key to the pendingKeys - * array. If there is no pending task then we add the key to the missingKeys array. - * - * These missingKeys will be later to use to multiGet the data from the storage. - */ - matchingKeys.forEach((key) => { - const cacheValue = cache.getValue(key); - if (cacheValue) { - data[key] = cacheValue; - return; - } - - const pendingKey = `get:${key}`; - if (cache.hasPendingTask(pendingKey)) { - pendingTasks.push(cache.getTaskPromise(pendingKey)); - pendingKeys.push(key); - } else { - missingKeys.push(key); - } - }); - - Promise.all(pendingTasks) - // We are going to wait for all the pending tasks to resolve and then add the data to the data object. - .then((values) => { - values.forEach((value, index) => { - data[pendingKeys[index]] = value; - }); - - return Promise.resolve(); - }) - // We are going to get the missing keys using multiGet from the storage. - .then(() => { - if (missingKeys.length === 0) { - return Promise.resolve(); - } - return Storage.multiGet(missingKeys); - }) - // We are going to add the data from the missing keys to the data object and also merge it to the cache. - .then((values) => { - if (!values || values.length === 0) { - return Promise.resolve(); - } - - // temp object is used to merge the missing data into the cache - const temp = {}; - values.forEach((value) => { - data[value[0]] = value[1]; - temp[value[0]] = value[1]; - }); - cache.merge(temp); - return Promise.resolve(); - }) - // We are going to send the data to the subscriber. - .finally(() => { - sendDataToConnection(mapping, data, undefined, true); - }); -} - -/** - * Subscribes a react component's state directly to a store key - * - * @example - * const connectionID = Onyx.connect({ - * key: ONYXKEYS.SESSION, - * callback: onSessionChange, - * }); - * - * @param {Object} mapping the mapping information to connect Onyx to the components state - * @param {String} mapping.key ONYXKEY to subscribe to - * @param {String} [mapping.statePropertyName] the name of the property in the state to connect the data to - * @param {Object} [mapping.withOnyxInstance] whose setState() method will be called with any changed data - * This is used by React components to connect to Onyx - * @param {Function} [mapping.callback] a method that will be called with changed data - * This is used by any non-React code to connect to Onyx - * @param {Boolean} [mapping.initWithStoredValues] If set to false, then no data will be prefilled into the - * component - * @param {Boolean} [mapping.waitForCollectionCallback] If set to true, it will return the entire collection to the callback as a single object - * @param {Function} [mapping.selector] THIS PARAM IS ONLY USED WITH withOnyx(). If included, this will be used to subscribe to a subset of an Onyx key's data. - * The sourceData and withOnyx state are passed to the selector and should return the simplified data. Using this setting on `withOnyx` can have very positive - * performance benefits because the component will only re-render when the subset of data changes. Otherwise, any change of data on any property would normally - * cause the component to re-render (and that can be expensive from a performance standpoint). - * @param {String | Number | Boolean | Object} [mapping.initialValue] THIS PARAM IS ONLY USED WITH withOnyx(). - * If included, this will be passed to the component so that something can be rendered while data is being fetched from the DB. - * Note that it will not cause the component to have the loading prop set to true. | - * @returns {Number} an ID to use when calling disconnect - */ -function connect(mapping) { - const connectionID = lastConnectionID++; - callbackToStateMapping[connectionID] = mapping; - callbackToStateMapping[connectionID].connectionID = connectionID; - - if (mapping.initWithStoredValues === false) { - return connectionID; - } - - // Commit connection only after init passes - deferredInitTask.promise - .then(() => addKeyToRecentlyAccessedIfNeeded(mapping)) - .then(() => { - // Performance improvement - // If the mapping is connected to an onyx key that is not a collection - // we can skip the call to getAllKeys() and return an array with a single item - if (Boolean(mapping.key) && typeof mapping.key === 'string' && !mapping.key.endsWith('_') && cache.storageKeys.has(mapping.key)) { - return [mapping.key]; - } - return getAllKeys(); - }) - .then((keys) => { - // We search all the keys in storage to see if any are a "match" for the subscriber we are connecting so that we - // can send data back to the subscriber. Note that multiple keys can match as a subscriber could either be - // subscribed to a "collection key" or a single key. - const matchingKeys = _.filter(keys, (key) => isKeyMatch(mapping.key, key)); - - // If the key being connected to does not exist we initialize the value with null. For subscribers that connected - // directly via connect() they will simply get a null value sent to them without any information about which key matched - // since there are none matched. In withOnyx() we wait for all connected keys to return a value before rendering the child - // component. This null value will be filtered out so that the connected component can utilize defaultProps. - if (matchingKeys.length === 0) { - if (mapping.key && !isCollectionKey(mapping.key)) { - cache.set(mapping.key, null); - } - - // Here we cannot use batching because the null value is expected to be set immediately for default props - // or they will be undefined. - sendDataToConnection(mapping, null, undefined, false); - return; - } - - // When using a callback subscriber we will either trigger the provided callback for each key we find or combine all values - // into an object and just make a single call. The latter behavior is enabled by providing a waitForCollectionCallback key - // combined with a subscription to a collection key. - if (_.isFunction(mapping.callback)) { - if (isCollectionKey(mapping.key)) { - if (mapping.waitForCollectionCallback) { - getCollectionDataAndSendAsObject(matchingKeys, mapping); - return; - } - - // We did not opt into using waitForCollectionCallback mode so the callback is called for every matching key. - for (let i = 0; i < matchingKeys.length; i++) { - get(matchingKeys[i]).then((val) => sendDataToConnection(mapping, val, matchingKeys[i], true)); - } - return; - } - - // If we are not subscribed to a collection key then there's only a single key to send an update for. - get(mapping.key).then((val) => sendDataToConnection(mapping, val, mapping.key, true)); - return; - } - - // If we have a withOnyxInstance that means a React component has subscribed via the withOnyx() HOC and we need to - // group collection key member data into an object. - if (mapping.withOnyxInstance) { - if (isCollectionKey(mapping.key)) { - getCollectionDataAndSendAsObject(matchingKeys, mapping); - return; - } - - // If the subscriber is not using a collection key then we just send a single value back to the subscriber - get(mapping.key).then((val) => sendDataToConnection(mapping, val, mapping.key, true)); - return; - } - - console.error('Warning: Onyx.connect() was found without a callback or withOnyxInstance'); - }); - - // The connectionID is returned back to the caller so that it can be used to clean up the connection when it's no longer needed - // by calling Onyx.disconnect(connectionID). - return connectionID; -} - -/** - * Remove the listener for a react component - * @example - * Onyx.disconnect(connectionID); - * - * @param {Number} connectionID unique id returned by call to Onyx.connect() - * @param {String} [keyToRemoveFromEvictionBlocklist] - */ -function disconnect(connectionID, keyToRemoveFromEvictionBlocklist) { - if (!callbackToStateMapping[connectionID]) { - return; - } - - // Remove this key from the eviction block list as we are no longer - // subscribing to it and it should be safe to delete again - if (keyToRemoveFromEvictionBlocklist) { - removeFromEvictionBlockList(keyToRemoveFromEvictionBlocklist, connectionID); - } - - delete callbackToStateMapping[connectionID]; -} - -/** - * Schedules an update that will be appended to the macro task queue (so it doesn't update the subscribers immediately). - * - * @example - * scheduleSubscriberUpdate(key, value, subscriber => subscriber.initWithStoredValues === false) - * - * @param {String} key - * @param {*} value - * @param {*} prevValue - * @param {Function} [canUpdateSubscriber] only subscribers that pass this truth test will be updated - * @returns {Promise} - */ -function scheduleSubscriberUpdate(key, value, prevValue, canUpdateSubscriber = () => true) { - const promise = Promise.resolve().then(() => keyChanged(key, value, prevValue, canUpdateSubscriber, true, false)); - batchUpdates(() => keyChanged(key, value, prevValue, canUpdateSubscriber, false, true)); - return Promise.all([maybeFlushBatchUpdates(), promise]); -} - -/** - * This method is similar to notifySubscribersOnNextTick but it is built for working specifically with collections - * so that keysChanged() is triggered for the collection and not keyChanged(). If this was not done, then the - * subscriber callbacks receive the data in a different format than they normally expect and it breaks code. - * - * @param {String} key - * @param {*} value - * @returns {Promise} - */ -function scheduleNotifyCollectionSubscribers(key, value) { - const promise = Promise.resolve().then(() => keysChanged(key, value, true, false)); - batchUpdates(() => keysChanged(key, value, false, true)); - return Promise.all([maybeFlushBatchUpdates(), promise]); -} - -/** - * Remove a key from Onyx and update the subscribers - * - * @private - * @param {String} key - * @return {Promise} - */ -function remove(key) { - const prevValue = cache.getValue(key, false); - cache.drop(key); - scheduleSubscriberUpdate(key, null, prevValue); - return Storage.removeItem(key); -} - -/** - * @private - * @returns {Promise} - */ -function reportStorageQuota() { - return Storage.getDatabaseSize() - .then(({bytesUsed, bytesRemaining}) => { - Logger.logInfo(`Storage Quota Check -- bytesUsed: ${bytesUsed} bytesRemaining: ${bytesRemaining}`); - }) - .catch((dbSizeError) => { - Logger.logAlert(`Unable to get database size. Error: ${dbSizeError}`); - }); -} - -/** - * If we fail to set or merge we must handle this by - * evicting some data from Onyx and then retrying to do - * whatever it is we attempted to do. - * - * @private - * @param {Error} error - * @param {Function} onyxMethod - * @param {...any} args - * @return {Promise} - */ -function evictStorageAndRetry(error, onyxMethod, ...args) { - Logger.logInfo(`Failed to save to storage. Error: ${error}. onyxMethod: ${onyxMethod.name}`); - - if (error && Str.startsWith(error.message, "Failed to execute 'put' on 'IDBObjectStore'")) { - Logger.logAlert('Attempted to set invalid data set in Onyx. Please ensure all data is serializable.'); - throw error; - } - - // Find the first key that we can remove that has no subscribers in our blocklist - const keyForRemoval = _.find(recentlyAccessedKeys, (key) => !evictionBlocklist[key]); - if (!keyForRemoval) { - // If we have no acceptable keys to remove then we are possibly trying to save mission critical data. If this is the case, - // then we should stop retrying as there is not much the user can do to fix this. Instead of getting them stuck in an infinite loop we - // will allow this write to be skipped. - Logger.logAlert('Out of storage. But found no acceptable keys to remove.'); - return reportStorageQuota(); - } - - // Remove the least recently viewed key that is not currently being accessed and retry. - Logger.logInfo(`Out of storage. Evicting least recently accessed key (${keyForRemoval}) and retrying.`); - reportStorageQuota(); - return remove(keyForRemoval).then(() => onyxMethod(...args)); -} - -/** - * Notifys subscribers and writes current value to cache - * - * @param {String} key - * @param {*} value - * @param {String} method - * @param {Boolean} hasChanged - * @param {Boolean} wasRemoved - * @returns {Promise} - */ -function broadcastUpdate(key, value, method, hasChanged, wasRemoved = false) { - // Logging properties only since values could be sensitive things we don't want to log - Logger.logInfo(`${method}() called for key: ${key}${_.isObject(value) ? ` properties: ${_.keys(value).join(',')}` : ''}`); - const prevValue = cache.getValue(key, false); - - // Update subscribers if the cached value has changed, or when the subscriber specifically requires - // all updates regardless of value changes (indicated by initWithStoredValues set to false). - if (hasChanged && !wasRemoved) { - cache.set(key, value); - } else { - cache.addToAccessedKeys(key); - } - - return scheduleSubscriberUpdate(key, value, prevValue, (subscriber) => hasChanged || subscriber.initWithStoredValues === false); -} - -/** - * @param {String} key - * @returns {Boolean} - */ -function hasPendingMergeForKey(key) { - return Boolean(mergeQueue[key]); -} - -/** - * Removes a key from storage if the value is null. - * Otherwise removes all nested null values in objects and returns the object - * @param {String} key - * @param {Mixed} value - * @returns {Mixed} The value without null values and a boolean "wasRemoved", which indicates if the key got removed completely - */ -function removeNullValues(key, value) { - if (_.isNull(value)) { - remove(key); - return {value, wasRemoved: true}; - } - - // We can remove all null values in an object by merging it with itself - // utils.fastMerge recursively goes through the object and removes all null values - // Passing two identical objects as source and target to fastMerge will not change it, but only remove the null values - return {value: utils.removeNestedNullValues(value), wasRemoved: false}; -} - -/** - * Write a value to our store with the given key - * - * @param {String} key ONYXKEY to set - * @param {*} value value to store - * - * @returns {Promise} - */ -function set(key, value) { - // If the value is null, we remove the key from storage - const {value: valueAfterRemoving, wasRemoved} = removeNullValues(key, value); - - if (hasPendingMergeForKey(key)) { - delete mergeQueue[key]; - } - - const hasChanged = cache.hasValueChanged(key, valueAfterRemoving); - - // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. - const updatePromise = broadcastUpdate(key, valueAfterRemoving, 'set', hasChanged, wasRemoved); - - // If the value has not changed or the key got removed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead. - if (!hasChanged || wasRemoved) { - return updatePromise; - } - - return Storage.setItem(key, valueAfterRemoving) - .catch((error) => evictStorageAndRetry(error, set, key, valueAfterRemoving)) - .then(() => { - sendActionToDevTools(METHOD.SET, key, valueAfterRemoving); - return updatePromise; - }); -} - -/** - * Storage expects array like: [["@MyApp_user", value_1], ["@MyApp_key", value_2]] - * This method transforms an object like {'@MyApp_user': myUserValue, '@MyApp_key': myKeyValue} - * to an array of key-value pairs in the above format and removes key-value pairs that are being set to null - * @private - * @param {Record} data - * @return {Array} an array of key - value pairs <[key, value]> - */ -function prepareKeyValuePairsForStorage(data) { - const keyValuePairs = []; - - _.forEach(data, (value, key) => { - const {value: valueAfterRemoving, wasRemoved} = removeNullValues(key, value); - - if (wasRemoved) return; - - keyValuePairs.push([key, valueAfterRemoving]); - }); - - return keyValuePairs; -} - -/** - * Sets multiple keys and values - * - * @example Onyx.multiSet({'key1': 'a', 'key2': 'b'}); - * - * @param {Object} data object keyed by ONYXKEYS and the values to set - * @returns {Promise} - */ -function multiSet(data) { - const keyValuePairs = prepareKeyValuePairsForStorage(data); - - const updatePromises = _.map(keyValuePairs, ([key, value]) => { - const prevValue = cache.getValue(key, false); - - // Update cache and optimistically inform subscribers on the next tick - cache.set(key, value); - return scheduleSubscriberUpdate(key, value, prevValue); - }); - - return Storage.multiSet(keyValuePairs) - .catch((error) => evictStorageAndRetry(error, multiSet, data)) - .then(() => { - sendActionToDevTools(METHOD.MULTI_SET, undefined, data); - return Promise.all(updatePromises); - }); -} - -/** - * Merges an array of changes with an existing value - * - * @private - * @param {*} existingValue - * @param {Array<*>} changes Array of changes that should be applied to the existing value - * @param {Boolean} shouldRemoveNullObjectValues - * @returns {*} - */ -function applyMerge(existingValue, changes, shouldRemoveNullObjectValues) { - const lastChange = _.last(changes); - - if (_.isArray(lastChange)) { - return lastChange; - } - - if (_.some(changes, _.isObject)) { - // Object values are then merged one after the other - return _.reduce(changes, (modifiedData, change) => utils.fastMerge(modifiedData, change, shouldRemoveNullObjectValues), existingValue || {}); - } - - // If we have anything else we can't merge it so we'll - // simply return the last value that was queued - return lastChange; -} - -/** - * Merge a new value into an existing value at a key. - * - * The types of values that can be merged are `Object` and `Array`. To set another type of value use `Onyx.set()`. - * Values of type `Object` get merged with the old value, whilst for `Array`'s we simply replace the current value with the new one. - * - * Calls to `Onyx.merge()` are batched so that any calls performed in a single tick will stack in a queue and get - * applied in the order they were called. Note: `Onyx.set()` calls do not work this way so use caution when mixing - * `Onyx.merge()` and `Onyx.set()`. - * - * @example - * Onyx.merge(ONYXKEYS.EMPLOYEE_LIST, ['Joe']); // -> ['Joe'] - * Onyx.merge(ONYXKEYS.EMPLOYEE_LIST, ['Jack']); // -> ['Joe', 'Jack'] - * Onyx.merge(ONYXKEYS.POLICY, {id: 1}); // -> {id: 1} - * Onyx.merge(ONYXKEYS.POLICY, {name: 'My Workspace'}); // -> {id: 1, name: 'My Workspace'} - * - * @param {String} key ONYXKEYS key - * @param {(Object|Array)} changes Object or Array value to merge - * @returns {Promise} - */ -function merge(key, changes) { - // Top-level undefined values are ignored - // Therefore we need to prevent adding them to the merge queue - if (_.isUndefined(changes)) { - return mergeQueue[key] ? mergeQueuePromise[key] : Promise.resolve(); - } - - // Merge attempts are batched together. The delta should be applied after a single call to get() to prevent a race condition. - // Using the initial value from storage in subsequent merge attempts will lead to an incorrect final merged value. - if (mergeQueue[key]) { - mergeQueue[key].push(changes); - return mergeQueuePromise[key]; - } - mergeQueue[key] = [changes]; - - mergeQueuePromise[key] = get(key).then((existingValue) => { - // Calls to Onyx.set after a merge will terminate the current merge process and clear the merge queue - if (mergeQueue[key] == null) return; - - try { - // We first only merge the changes, so we can provide these to the native implementation (SQLite uses only delta changes in "JSON_PATCH" to merge) - // We don't want to remove null values from the "batchedChanges", because SQLite uses them to remove keys from storage natively. - let batchedChanges = applyMerge(undefined, mergeQueue[key], false); - - // The presence of a `null` in the merge queue instructs us to drop the existing value. - // In this case, we can't simply merge the batched changes with the existing value, because then the null in the merge queue would have no effect - const shouldOverwriteExistingValue = _.includes(mergeQueue[key], null); - - // Clean up the write queue, so we don't apply these changes again - delete mergeQueue[key]; - delete mergeQueuePromise[key]; - - // If the batched changes equal null, we want to remove the key from storage, to reduce storage size - const {wasRemoved} = removeNullValues(key, batchedChanges); - - // After that we merge the batched changes with the existing value - // We can remove null values from the "modifiedData", because "null" implicates that the user wants to remove a value from storage. - // The "modifiedData" will be directly "set" in storage instead of being merged - const modifiedData = shouldOverwriteExistingValue ? batchedChanges : applyMerge(existingValue, [batchedChanges], true); - - // On native platforms we use SQLite which utilises JSON_PATCH to merge changes. - // JSON_PATCH generally removes null values from the stored object. - // When there is no existing value though, SQLite will just insert the changes as a new value and thus the null values won't be removed. - // Therefore we need to remove null values from the `batchedChanges` which are sent to the SQLite, if no existing value is present. - if (!existingValue) { - batchedChanges = applyMerge(undefined, [batchedChanges], true); - } - - const hasChanged = cache.hasValueChanged(key, modifiedData); - - // This approach prioritizes fast UI changes without waiting for data to be stored in device storage. - const updatePromise = broadcastUpdate(key, modifiedData, 'merge', hasChanged, wasRemoved); - - // If the value has not changed, calling Storage.setItem() would be redundant and a waste of performance, so return early instead. - if (!hasChanged || wasRemoved) { - return updatePromise; - } - - return Storage.mergeItem(key, batchedChanges, modifiedData).then(() => { - sendActionToDevTools(METHOD.MERGE, key, changes, modifiedData); - return updatePromise; - }); - } catch (error) { - Logger.logAlert(`An error occurred while applying merge for key: ${key}, Error: ${error}`); - return Promise.resolve(); - } - }); - - return mergeQueuePromise[key]; -} - -/** - * Merge user provided default key value pairs. - * @private - * @returns {Promise} - */ -function initializeWithDefaultKeyStates() { - return Storage.multiGet(_.keys(defaultKeyStates)).then((pairs) => { - const existingDataAsObject = _.object(pairs); - - const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates); - cache.merge(merged); - - _.each(merged, (val, key) => keyChanged(key, val, existingDataAsObject)); - }); -} - -/** - * Clear out all the data in the store - * - * Note that calling Onyx.clear() and then Onyx.set() on a key with a default - * key state may store an unexpected value in Storage. - * - * E.g. - * Onyx.clear(); - * Onyx.set(ONYXKEYS.DEFAULT_KEY, 'default'); - * Storage.getItem(ONYXKEYS.DEFAULT_KEY) - * .then((storedValue) => console.log(storedValue)); - * null is logged instead of the expected 'default' - * - * Onyx.set() might call Storage.setItem() before Onyx.clear() calls - * Storage.setItem(). Use Onyx.merge() instead if possible. Onyx.merge() calls - * Onyx.get(key) before calling Storage.setItem() via Onyx.set(). - * Storage.setItem() from Onyx.clear() will have already finished and the merged - * value will be saved to storage after the default value. - * - * @param {Array} keysToPreserve is a list of ONYXKEYS that should not be cleared with the rest of the data - * @returns {Promise} - */ -function clear(keysToPreserve = []) { - return getAllKeys().then((keys) => { - const keysToBeClearedFromStorage = []; - const keyValuesToResetAsCollection = {}; - const keyValuesToResetIndividually = {}; - - // The only keys that should not be cleared are: - // 1. Anything specifically passed in keysToPreserve (because some keys like language preferences, offline - // status, or activeClients need to remain in Onyx even when signed out) - // 2. Any keys with a default state (because they need to remain in Onyx as their default, and setting them - // to null would cause unknown behavior) - _.each(keys, (key) => { - const isKeyToPreserve = _.contains(keysToPreserve, key); - const isDefaultKey = _.has(defaultKeyStates, key); - - // If the key is being removed or reset to default: - // 1. Update it in the cache - // 2. Figure out whether it is a collection key or not, - // since collection key subscribers need to be updated differently - if (!isKeyToPreserve) { - const oldValue = cache.getValue(key); - const newValue = _.get(defaultKeyStates, key, null); - if (newValue !== oldValue) { - cache.set(key, newValue); - const collectionKey = key.substring(0, key.indexOf('_') + 1); - if (collectionKey) { - if (!keyValuesToResetAsCollection[collectionKey]) { - keyValuesToResetAsCollection[collectionKey] = {}; - } - keyValuesToResetAsCollection[collectionKey][key] = newValue; - } else { - keyValuesToResetIndividually[key] = newValue; - } - } - } - - if (isKeyToPreserve || isDefaultKey) { - return; - } - - // If it isn't preserved and doesn't have a default, we'll remove it - keysToBeClearedFromStorage.push(key); - }); - - const updatePromises = []; - - // Notify the subscribers for each key/value group so they can receive the new values - _.each(keyValuesToResetIndividually, (value, key) => { - updatePromises.push(scheduleSubscriberUpdate(key, value, cache.getValue(key, false))); - }); - _.each(keyValuesToResetAsCollection, (value, key) => { - updatePromises.push(scheduleNotifyCollectionSubscribers(key, value)); - }); - - const defaultKeyValuePairs = _.pairs(_.omit(defaultKeyStates, keysToPreserve)); - - // Remove only the items that we want cleared from storage, and reset others to default - _.each(keysToBeClearedFromStorage, (key) => cache.drop(key)); - return Storage.removeItems(keysToBeClearedFromStorage) - .then(() => Storage.multiSet(defaultKeyValuePairs)) - .then(() => { - DevTools.clearState(keysToPreserve); - return Promise.all(updatePromises); - }); - }); -} - -/** - * Merges a collection based on their keys - * - * @example - * - * Onyx.mergeCollection(ONYXKEYS.COLLECTION.REPORT, { - * [`${ONYXKEYS.COLLECTION.REPORT}1`]: report1, - * [`${ONYXKEYS.COLLECTION.REPORT}2`]: report2, - * }); - * - * @param {String} collectionKey e.g. `ONYXKEYS.COLLECTION.REPORT` - * @param {Object} collection Object collection keyed by individual collection member keys and values - * @returns {Promise} - */ -function mergeCollection(collectionKey, collection) { - if (!_.isObject(collection) || _.isArray(collection) || _.isEmpty(collection)) { - Logger.logInfo('mergeCollection() called with invalid or empty value. Skipping this update.'); - return Promise.resolve(); - } - - // Confirm all the collection keys belong to the same parent - let hasCollectionKeyCheckFailed = false; - _.each(collection, (_data, dataKey) => { - if (isKeyMatch(collectionKey, dataKey)) { - return; - } - - if (process.env.NODE_ENV === 'development') { - throw new Error(`Provided collection doesn't have all its data belonging to the same parent. CollectionKey: ${collectionKey}, DataKey: ${dataKey}`); - } - - hasCollectionKeyCheckFailed = true; - Logger.logAlert(`Provided collection doesn't have all its data belonging to the same parent. CollectionKey: ${collectionKey}, DataKey: ${dataKey}`); - }); - - // Gracefully handle bad mergeCollection updates so it doesn't block the merge queue - if (hasCollectionKeyCheckFailed) { - return Promise.resolve(); - } - - return getAllKeys().then((persistedKeys) => { - // Split to keys that exist in storage and keys that don't - const [existingKeys, newKeys] = _.chain(collection) - .pick((value, key) => { - if (_.isNull(value)) { - remove(key); - return false; - } - return true; - }) - .keys() - .partition((key) => persistedKeys.includes(key)) - .value(); - - const existingKeyCollection = _.pick(collection, existingKeys); - const newCollection = _.pick(collection, newKeys); - const keyValuePairsForExistingCollection = prepareKeyValuePairsForStorage(existingKeyCollection); - const keyValuePairsForNewCollection = prepareKeyValuePairsForStorage(newCollection); - - const promises = []; - - // New keys will be added via multiSet while existing keys will be updated using multiMerge - // This is because setting a key that doesn't exist yet with multiMerge will throw errors - if (keyValuePairsForExistingCollection.length > 0) { - promises.push(Storage.multiMerge(keyValuePairsForExistingCollection)); - } - - if (keyValuePairsForNewCollection.length > 0) { - promises.push(Storage.multiSet(keyValuePairsForNewCollection)); - } - - // Prefill cache if necessary by calling get() on any existing keys and then merge original data to cache - // and update all subscribers - const promiseUpdate = Promise.all(_.map(existingKeys, get)).then(() => { - cache.merge(collection); - return scheduleNotifyCollectionSubscribers(collectionKey, collection); - }); - - return Promise.all(promises) - .catch((error) => evictStorageAndRetry(error, mergeCollection, collection)) - .then(() => { - sendActionToDevTools(METHOD.MERGE_COLLECTION, undefined, collection); - return promiseUpdate; - }); - }); -} - -/** - * Insert API responses and lifecycle data into Onyx - * - * @param {Array} data An array of objects with shape {onyxMethod: oneOf('set', 'merge', 'mergeCollection', 'multiSet', 'clear'), key: string, value: *} - * @returns {Promise} resolves when all operations are complete - */ -function update(data) { - // First, validate the Onyx object is in the format we expect - _.each(data, ({onyxMethod, key, value}) => { - if (!_.contains([METHOD.CLEAR, METHOD.SET, METHOD.MERGE, METHOD.MERGE_COLLECTION, METHOD.MULTI_SET], onyxMethod)) { - throw new Error(`Invalid onyxMethod ${onyxMethod} in Onyx update.`); - } - if (onyxMethod === METHOD.MULTI_SET) { - // For multiset, we just expect the value to be an object - if (!_.isObject(value) || _.isArray(value) || _.isFunction(value)) { - throw new Error('Invalid value provided in Onyx multiSet. Onyx multiSet value must be of type object.'); - } - } else if (onyxMethod !== METHOD.CLEAR && !_.isString(key)) { - throw new Error(`Invalid ${typeof key} key provided in Onyx update. Onyx key must be of type string.`); - } - }); - - const promises = []; - let clearPromise = Promise.resolve(); - - _.each(data, ({onyxMethod, key, value}) => { - switch (onyxMethod) { - case METHOD.SET: - promises.push(() => set(key, value)); - break; - case METHOD.MERGE: - promises.push(() => merge(key, value)); - break; - case METHOD.MERGE_COLLECTION: - promises.push(() => mergeCollection(key, value)); - break; - case METHOD.MULTI_SET: - promises.push(() => multiSet(value)); - break; - case METHOD.CLEAR: - clearPromise = clear(); - break; - default: - break; - } - }); - - return clearPromise.then(() => Promise.all(_.map(promises, (p) => p()))); -} - -/** - * When set these keys will not be persisted to storage - * @param {string[]} keyList - */ -function setMemoryOnlyKeys(keyList) { - Storage.setMemoryOnlyKeys(keyList); - - // When in memory only mode for certain keys we do not want to ever drop items from the cache as the user will have no way to recover them again via storage. - cache.setRecentKeysLimit(Infinity); -} - -/** - * Initialize the store with actions and listening for storage events - * - * @param {Object} [options={}] config object - * @param {Object} [options.keys={}] `ONYXKEYS` constants object - * @param {Object} [options.initialKeyStates={}] initial data to set when `init()` and `clear()` is called - * @param {String[]} [options.safeEvictionKeys=[]] This is an array of keys - * (individual or collection patterns) that when provided to Onyx are flagged - * as "safe" for removal. Any components subscribing to these keys must also - * implement a canEvict option. See the README for more info. - * @param {Number} [options.maxCachedKeysCount=55] Sets how many recent keys should we try to keep in cache - * Setting this to 0 would practically mean no cache - * We try to free cache when we connect to a safe eviction key - * @param {Boolean} [options.captureMetrics] Enables Onyx benchmarking and exposes the get/print/reset functions - * @param {Boolean} [options.shouldSyncMultipleInstances] Auto synchronize storage events between multiple instances - * of Onyx running in different tabs/windows. Defaults to true for platforms that support local storage (web/desktop) - * @param {Boolean} [options.debugSetState] Enables debugging setState() calls to connected components. - * @example - * Onyx.init({ - * keys: ONYXKEYS, - * initialKeyStates: { - * [ONYXKEYS.SESSION]: {loading: false}, - * }, - * }); - */ -function init({keys = {}, initialKeyStates = {}, safeEvictionKeys = [], maxCachedKeysCount = 1000, shouldSyncMultipleInstances = Boolean(global.localStorage), debugSetState = false} = {}) { - Storage.init(); - - if (shouldSyncMultipleInstances) { - Storage.keepInstancesSync((key, value) => { - const prevValue = cache.getValue(key, false); - cache.set(key, value); - keyChanged(key, value, prevValue); - }); - } - - if (debugSetState) { - PerformanceUtils.setShouldDebugSetState(true); - } - - if (maxCachedKeysCount > 0) { - cache.setRecentKeysLimit(maxCachedKeysCount); - } - - // We need the value of the collection keys later for checking if a - // key is a collection. We store it in a map for faster lookup. - const collectionValues = _.values(keys.COLLECTION); - onyxCollectionKeyMap = _.reduce( - collectionValues, - (acc, val) => { - acc.set(val, true); - return acc; - }, - new Map(), - ); - - // Set our default key states to use when initializing and clearing Onyx data - defaultKeyStates = initialKeyStates; - - DevTools.initState(initialKeyStates); - - // Let Onyx know about which keys are safe to evict - evictionAllowList = safeEvictionKeys; - - // Initialize all of our keys with data provided then give green light to any pending connections - Promise.all([addAllSafeEvictionKeysToRecentlyAccessedList(), initializeWithDefaultKeyStates()]).then(deferredInitTask.resolve); -} - -const Onyx = { - connect, - disconnect, - set, - multiSet, - merge, - mergeCollection, - update, - clear, - getAllKeys, - init, - registerLogger: Logger.registerLogger, - addToEvictionBlockList, - removeFromEvictionBlockList, - isSafeEvictionKey, - METHOD, - tryGetCachedValue, - hasPendingMergeForKey, -}; - -export default Onyx; diff --git a/lib/Onyx.ts b/lib/Onyx.ts index d1288d4b..12aad560 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -1578,6 +1578,12 @@ type InitOptions = { debugSetState?: boolean; }; +/** When set these keys will not be persisted to storage */ +function setMemoryOnlyKeys() { + // When in memory only mode for certain keys we do not want to ever drop items from the cache as the user will have no way to recover them again via storage. + cache.setRecentKeysLimit(Infinity); +} + /** Initialize the store with actions and listening for storage events */ function init({ keys = {}, @@ -1643,6 +1649,7 @@ const Onyx = { METHOD, tryGetCachedValue, hasPendingMergeForKey, + setMemoryOnlyKeys, } as const; export default Onyx; From 9f5d72a63cad060978200d46c83b75ce944f44b1 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 10:59:19 +0100 Subject: [PATCH 21/23] Add final assertion --- lib/Onyx.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index 12aad560..bdbc0b32 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -682,7 +682,7 @@ function keyChanged( * - sets state on the withOnyxInstances * - triggers the callback function */ -function sendDataToConnection(mapping: Mapping, val: OnyxValue, matchedKey: OnyxKey | undefined, isBatched: boolean) { +function sendDataToConnection(mapping: Mapping, val: OnyxValue | Record, matchedKey: OnyxKey | undefined, isBatched: boolean) { // If the mapping no longer exists then we should not send any data. // This means our subscriber disconnected or withOnyx wrapped component unmounted. if (!callbackToStateMapping[mapping.connectionID]) { @@ -696,7 +696,7 @@ function sendDataToConnection(mapping: Mapping, val: OnyxValue, matched // returned by the selector. if (mapping.selector) { if (isCollectionKey(mapping.key)) { - newData = reduceCollectionWithSelector(val, mapping.selector, mapping.withOnyxInstance.state); + newData = reduceCollectionWithSelector(val as Record, mapping.selector, mapping.withOnyxInstance.state); } else { newData = mapping.selector(val, mapping.withOnyxInstance.state); } From d34a077defea31dbbe8e6955a36777ad906865fd Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 11:32:09 +0100 Subject: [PATCH 22/23] Fix connect types --- lib/Onyx.ts | 25 ++++++++++++------------- lib/PerformanceUtils.ts | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index bdbc0b32..b02295bd 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -48,13 +48,10 @@ type WithOnyxInstance = Component = { - statePropertyName: string; + key: string; + selector?: Selector; withOnyxInstance?: WithOnyxInstance; initWithStoredValues?: boolean; - selector?: Selector; - connectionID: number; - key: string; - displayName: string; canEvict?: boolean; }; @@ -70,7 +67,7 @@ type BaseConnectOptions = { * If `waitForCollectionCallback` is `false` or not specified, the `key` can be any Onyx key and `callback` will be triggered with updates of each collection item * and will pass `value` as an `OnyxEntry`. */ -type Mapping = BaseConnectOptions & +type ConnectOptions = BaseConnectOptions & ( | { key: TKey extends CollectionKeyBase ? TKey : never; @@ -84,6 +81,8 @@ type Mapping = BaseConnectOptions & } ); +type Mapping = ConnectOptions & {connectionID: number; statePropertyName: string; displayName: string}; + // Method constants const METHOD = { SET: 'set', @@ -682,7 +681,7 @@ function keyChanged( * - sets state on the withOnyxInstances * - triggers the callback function */ -function sendDataToConnection(mapping: Mapping, val: OnyxValue | Record, matchedKey: OnyxKey | undefined, isBatched: boolean) { +function sendDataToConnection(mapping: Mapping, val: OnyxValue | Record, matchedKey: TKey | undefined, isBatched: boolean) { // If the mapping no longer exists then we should not send any data. // This means our subscriber disconnected or withOnyx wrapped component unmounted. if (!callbackToStateMapping[mapping.connectionID]) { @@ -714,7 +713,7 @@ function sendDataToConnection(mapping: Mapping, val: OnyxValue | Record } if (typeof mapping.callback === 'function') { - mapping.callback(val as Record, matchedKey as string); + mapping.callback(val as Record, matchedKey as TKey); } } @@ -722,7 +721,7 @@ function sendDataToConnection(mapping: Mapping, val: OnyxValue | Record * We check to see if this key is flagged as safe for eviction and add it to the recentlyAccessedKeys list so that when we * run out of storage the least recently accessed key can be removed. */ -function addKeyToRecentlyAccessedIfNeeded(mapping: Mapping) { +function addKeyToRecentlyAccessedIfNeeded(mapping: Mapping) { if (!isSafeEvictionKey(mapping.key)) { return; } @@ -743,7 +742,7 @@ function addKeyToRecentlyAccessedIfNeeded(mapping: Mapping) { /** * Gets the data for a given an array of matching keys, combines them into an object, and sends the result back to the subscriber. */ -function getCollectionDataAndSendAsObject(matchingKeys: CollectionKeyBase[], mapping: Mapping) { +function getCollectionDataAndSendAsObject(matchingKeys: CollectionKeyBase[], mapping: Mapping) { // Keys that are not in the cache const missingKeys: OnyxKey[] = []; // Tasks that are pending @@ -836,8 +835,8 @@ function getCollectionDataAndSendAsObject(matchingKeys: CollectionKeyBase[], map * @param [mapping.waitForCollectionCallback] If set to true, it will return the entire collection to the callback as a single object * @returns an ID to use when calling disconnect */ -function connect(mappingWithoutConnectionID: Omit, 'connectionID'>): number { - const mapping = mappingWithoutConnectionID as Mapping; +function connect(options: ConnectOptions): number { + const mapping = options as unknown as Mapping; const connectionID = lastConnectionID++; callbackToStateMapping[connectionID] = mapping; @@ -892,7 +891,7 @@ function connect(mappingWithoutConnectionID: Omit, 'connectionI // We did not opt into using waitForCollectionCallback mode so the callback is called for every matching key. for (let i = 0; i < matchingKeys.length; i++) { - get(matchingKeys[i]).then((val) => sendDataToConnection(mapping, val, matchingKeys[i], true)); + get(matchingKeys[i]).then((val) => sendDataToConnection(mapping, val, matchingKeys[i] as TKey, true)); } return; } diff --git a/lib/PerformanceUtils.ts b/lib/PerformanceUtils.ts index 8ac0b0aa..832aaa2a 100644 --- a/lib/PerformanceUtils.ts +++ b/lib/PerformanceUtils.ts @@ -41,7 +41,7 @@ function diffObject( /** * Provide insights into why a setState() call occurred by diffing the before and after values. */ -function logSetStateCall(mapping: Mapping, previousValue: unknown, newValue: unknown, caller: string, keyThatChanged?: string) { +function logSetStateCall(mapping: Mapping, previousValue: unknown, newValue: unknown, caller: string, keyThatChanged?: string) { if (!debugSetState) { return; } From 78ec402d5efe779492294b3c6a7b59cebbf44589 Mon Sep 17 00:00:00 2001 From: Blazej Kustra Date: Tue, 5 Mar 2024 12:23:30 +0100 Subject: [PATCH 23/23] Fix ConnectOptions type --- lib/Onyx.ts | 7 +++---- lib/index.ts | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/Onyx.ts b/lib/Onyx.ts index b02295bd..e840b735 100644 --- a/lib/Onyx.ts +++ b/lib/Onyx.ts @@ -13,7 +13,7 @@ import Storage from './storage'; import utils from './utils'; import unstable_batchedUpdates from './batch'; import DevTools from './DevTools'; -import type {CollectionKeyBase, DeepRecord, KeyValueMapping, NullishDeep, OnyxCollection, OnyxEntry, OnyxKey, OnyxValue, Selector, WithOnyxInstanceState} from './types'; +import type {CollectionKey, CollectionKeyBase, DeepRecord, KeyValueMapping, NullishDeep, OnyxCollection, OnyxEntry, OnyxKey, OnyxValue, Selector, WithOnyxInstanceState} from './types'; /** * Represents a mapping object where each `OnyxKey` maps to either a value of its corresponding type in `KeyValueMapping` or `null`. @@ -48,7 +48,6 @@ type WithOnyxInstance = Component = { - key: string; selector?: Selector; withOnyxInstance?: WithOnyxInstance; initWithStoredValues?: boolean; @@ -70,7 +69,7 @@ type BaseConnectOptions = { type ConnectOptions = BaseConnectOptions & ( | { - key: TKey extends CollectionKeyBase ? TKey : never; + key: TKey extends CollectionKey ? TKey : never; callback?: (value: OnyxCollection) => void; waitForCollectionCallback: true; } @@ -1652,4 +1651,4 @@ const Onyx = { } as const; export default Onyx; -export type {OnyxUpdate, Mapping}; +export type {OnyxUpdate, Mapping, ConnectOptions}; diff --git a/lib/index.ts b/lib/index.ts index a7b5cf63..7aa8e96a 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -1,8 +1,8 @@ import Onyx from './Onyx'; -import type {OnyxUpdate, Mapping} from './Onyx'; +import type {OnyxUpdate, ConnectOptions} from './Onyx'; import type {CustomTypeOptions, OnyxCollection, OnyxEntry, NullishDeep, KeyValueMapping, OnyxKey, Selector, WithOnyxInstanceState} from './types'; import withOnyx from './withOnyx'; export default Onyx; export {withOnyx}; -export type {CustomTypeOptions, OnyxCollection, OnyxEntry, OnyxUpdate, Mapping as ConnectOptions, NullishDeep, KeyValueMapping, OnyxKey, Selector, WithOnyxInstanceState}; +export type {CustomTypeOptions, OnyxCollection, OnyxEntry, OnyxUpdate, ConnectOptions, NullishDeep, KeyValueMapping, OnyxKey, Selector, WithOnyxInstanceState};