From 369593284de3b7126965e1fd8175ae8471e5678e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Wed, 18 Sep 2024 20:51:34 +0300 Subject: [PATCH] Memory Only Storage Option --- lib/countly-storage.js | 167 ++++++++++++++++++++++++++++------------- lib/countly.js | 41 +++++----- test/tests_storage.js | 47 ++++++------ 3 files changed, 161 insertions(+), 94 deletions(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 1ab308b..95ea68e 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -8,6 +8,121 @@ var defaultPath = "../data/"; // Default path var defaultBulkPath = "../bulk_data/"; // Default path var asyncWriteLock = false; var asyncWriteQueue = []; +let storageMethod = {}; + +// Memory-only storage methods +const memoryStorage = { + /** + * Save value in memory + * @param {String} key - key for value to store + * @param {varies} value - value to store + * @param {Function} callback - callback to call when done storing + */ + storeSet(key, value, callback) { + __data[key] = value; + if (typeof callback === "function") { + callback(null); // No errors for memory storage + } + }, + /** + * Get value from memory + * @param {String} key - key of value to get + * @param {varies} def - default value to use if not set + * @returns {varies} value for the key + */ + storeGet(key, def) { + cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`); + return typeof __data[key] !== "undefined" ? __data[key] : def; + }, + /** + * Remove value from memory + * @param {String} key - key of value to remove + */ + storeRemove(key) { + delete __data[key]; + }, +}; + +// File storage methods +const fileStorage = { + /** + * Save value in storage + * @param {String} key - key for value to store + * @param {varies} value - value to store + * @param {Function} callback - callback to call when done storing + */ + storeSet(key, value, callback) { + __data[key] = value; + if (!asyncWriteLock) { + asyncWriteLock = true; + writeFile(key, value, callback); + } + else { + asyncWriteQueue.push([key, value, callback]); + } + }, + /** + * Get value from storage + * @param {String} key - key of value to get + * @param {varies} def - default value to use if not set + * @returns {varies} value for the key + */ + storeGet(key, def) { + cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); + if (typeof __data[key] === "undefined") { + var ob = readFile(key); + var obLen; + // check if the 'read object' is empty or not + try { + obLen = Object.keys(ob).length; + } + catch (error) { + // if we can not even asses length set it to 0 so we can return the default value + obLen = 0; + } + + // if empty or falsy set default value + if (!ob || obLen === 0) { + __data[key] = def; + } + // else set the value read file has + else { + __data[key] = ob[key]; + } + } + return __data[key]; + }, + storeRemove(key) { + delete __data[key]; + var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); + fs.unlink(dir, (err) => { + if (err) { + cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err}].`); + } + }); + }, +}; + +/** + * Sets the storage method, by default sets file storage and storage path. + * @param {String} userPath - User provided storage path + * @param {Boolean} memoryOnly - Whether to use memory only storage or not + * @param {Boolean} isBulk - Whether the storage is for bulk data + * @param {Boolean} persistQueue - Whether to persist the queue until processed + * @returns {storageMethod} Prefered storage method + */ +var setStorage = function(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { + if (memoryOnly) { + cc.log(cc.logLevelEnums.DEBUG, "Using memory-only storage."); + storageMethod = memoryStorage; // Assign memory-only methods + } + else { + cc.log(cc.logLevelEnums.DEBUG, "Using persistent file storage."); + storageMethod = fileStorage; // Assign file storage methods + setStoragePath(userPath, isBulk, persistQueue); + } + return storageMethod; +}; /** * Sets the storage path, defaulting to a specified path if none is provided. @@ -139,59 +254,9 @@ var writeFile = function(key, value, callback) { }); }; -/** - * Save value in storage - * @param {String} key - key for value to store - * @param {varies} value - value to store - * @param {Function} callback - callback to call when done storing - */ -var storeSet = function(key, value, callback) { - __data[key] = value; - if (!asyncWriteLock) { - asyncWriteLock = true; - writeFile(key, value, callback); - } - else { - asyncWriteQueue.push([key, value, callback]); - } -}; - -/** - * Get value from storage - * @param {String} key - key of value to get - * @param {varies} def - default value to use if not set - * @returns {varies} value for the key - */ -var storeGet = function(key, def) { - cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); - if (typeof __data[key] === "undefined") { - var ob = readFile(key); - var obLen; - // check if the 'read object' is empty or not - try { - obLen = Object.keys(ob).length; - } - catch (error) { - // if we can not even asses length set it to 0 so we can return the default value - obLen = 0; - } - - // if empty or falsy set default value - if (!ob || obLen === 0) { - __data[key] = def; - } - // else set the value read file has - else { - __data[key] = ob[key]; - } - } - return __data[key]; -}; - module.exports = { + setStorage, writeFile, - storeGet, - storeSet, forceStore, getStoragePath, setStoragePath, diff --git a/lib/countly.js b/lib/countly.js index 9965868..5aa8e60 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -29,6 +29,7 @@ var Bulk = require("./countly-bulk"); var CountlyStorage = require("./countly-storage"); var Countly = {}; +var storageMethod; Countly.Bulk = Bulk; (function() { @@ -171,13 +172,13 @@ Countly.Bulk = Bulk; cc.debug = conf.debug; // Set the storage path - CountlyStorage.setStoragePath(conf.storage_path); + storageMethod = CountlyStorage.setStorage(conf.storage_path, true); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { cc.log(cc.logLevelEnums.WARNING, "init, clear_stored_device_id is true, erasing the stored ID."); - CountlyStorage.storeSet("cly_id", null); - CountlyStorage.storeSet("cly_id_type", null); + storageMethod.storeSet("cly_id", null); + storageMethod.storeSet("cly_id_type", null); } if (Countly.url === "") { @@ -223,8 +224,8 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { // fetch stored ID and ID type - var storedId = CountlyStorage.storeGet("cly_id", null); - var storedIdType = CountlyStorage.storeGet("cly_id_type", null); + var storedId = storageMethod.storeGet("cly_id", null); + var storedIdType = storageMethod.storeGet("cly_id_type", null); // if there was a stored ID if (storedId !== null) { Countly.device_id = storedId; @@ -253,12 +254,12 @@ Countly.Bulk = Bulk; deviceIdType = cc.deviceIdTypeEnums.SDK_GENERATED; } // save the ID and ID type - CountlyStorage.storeSet("cly_id", Countly.device_id); - CountlyStorage.storeSet("cly_id_type", deviceIdType); + storageMethod.storeSet("cly_id", Countly.device_id); + storageMethod.storeSet("cly_id_type", deviceIdType); // create queues - requestQueue = CountlyStorage.storeGet("cly_queue", []); - eventQueue = CountlyStorage.storeGet("cly_event", []); - remoteConfigs = CountlyStorage.storeGet("cly_remote_configs", {}); + requestQueue = storageMethod.storeGet("cly_queue", []); + eventQueue = storageMethod.storeGet("cly_event", []); + remoteConfigs = storageMethod.storeGet("cly_remote_configs", {}); heartBeat(); // listen to current workers if (cluster.workers) { @@ -635,7 +636,7 @@ Countly.Bulk = Bulk; if (eventQueue.length > 0) { toRequestQueue({ events: JSON.stringify(eventQueue) }); eventQueue = []; - CountlyStorage.storeSet("cly_event", eventQueue); + storageMethod.storeSet("cly_event", eventQueue); } // end current session Countly.end_session(); @@ -647,8 +648,8 @@ Countly.Bulk = Bulk; var oldId = Countly.device_id; Countly.device_id = newId; deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - CountlyStorage.storeSet("cly_id", Countly.device_id); - CountlyStorage.storeSet("cly_id_type", deviceIdType); + storageMethod.storeSet("cly_id", Countly.device_id); + storageMethod.storeSet("cly_id_type", deviceIdType); if (merge) { if (Countly.check_any_consent()) { toRequestQueue({ old_device_id: oldId }); @@ -664,7 +665,7 @@ Countly.Bulk = Bulk; if (Countly.remote_config) { remoteConfigs = {}; if (cluster.isMaster) { - CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); + storageMethod.storeSet("cly_remote_configs", remoteConfigs); } Countly.fetch_remote_config(Countly.remote_config); } @@ -744,7 +745,7 @@ Countly.Bulk = Bulk; e.dow = date.getDay(); cc.log(cc.logLevelEnums.DEBUG, "add_cly_events, Adding event: ", event); eventQueue.push(e); - CountlyStorage.storeSet("cly_event", eventQueue); + storageMethod.storeSet("cly_event", eventQueue); } else { process.send({ cly: { event } }); @@ -1155,7 +1156,7 @@ Countly.Bulk = Bulk; remoteConfigs = configs; } if (cluster.isMaster) { - CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); + storageMethod.storeSet("cly_remote_configs", remoteConfigs); cc.log(cc.logLevelEnums.INFO, `fetch_remote_config, Fetched remote config: [${remoteConfigs}].`); } } @@ -1339,7 +1340,7 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { cc.log(cc.logLevelEnums.INFO, "request, Adding the raw request to the queue."); requestQueue.push(request); - CountlyStorage.storeSet("cly_queue", requestQueue); + storageMethod.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "request, Sending message to the parent process. Adding the raw request to the queue."); @@ -1426,7 +1427,7 @@ Countly.Bulk = Bulk; cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Adding request to the queue."); requestQueue.push(request); - CountlyStorage.storeSet("cly_queue", requestQueue); + storageMethod.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Sending message to the parent process. Adding request to the queue."); @@ -1457,7 +1458,7 @@ Countly.Bulk = Bulk; var events = eventQueue.splice(0, maxEventBatch); toRequestQueue({ events: JSON.stringify(events) }); } - CountlyStorage.storeSet("cly_event", eventQueue); + storageMethod.storeSet("cly_event", eventQueue); } // process request queue with event queue @@ -1472,7 +1473,7 @@ Countly.Bulk = Bulk; failTimeout = cc.getTimestamp() + failTimeoutAmount; cc.log(cc.logLevelEnums.ERROR, `makeRequest, Encountered a problem while making the request: [${err}]`); } - CountlyStorage.storeSet("cly_queue", requestQueue); + storageMethod.storeSet("cly_queue", requestQueue); readyToProcess = true; }, "heartBeat", false); } diff --git a/test/tests_storage.js b/test/tests_storage.js index 6735b1f..6815b53 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -64,37 +64,38 @@ function validateDeviceId(deviceId, deviceIdType, expectedDeviceId, expectedDevi assert.equal(deviceIdType, expectedDeviceIdType); checkRequestsForT(rq, expectedDeviceIdType); } -function recordValuesToStorageAndValidate() { +function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { // Set values var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storage.storeSet("cly_id", "SpecialDeviceId"); - storage.storeSet("cly_id_type", deviceIdType); + var storageMethod = storage.setStorage(userPath, memoryOnly, isBulk, persistQueue); + storageMethod.storeSet("cly_id", "SpecialDeviceId"); + storageMethod.storeSet("cly_id_type", deviceIdType); // Set values with different data types - storage.storeSet("cly_count", 42); - storage.storeSet("cly_object", { key: "value" }); - storage.storeSet("cly_null", null); + storageMethod.storeSet("cly_count", 42); + storageMethod.storeSet("cly_object", { key: "value" }); + storageMethod.storeSet("cly_null", null); // Retrieve and assert values - assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storage.storeGet("cly_id_type"), deviceIdType); - assert.equal(storage.storeGet("cly_count"), 42); - assert.deepEqual(storage.storeGet("cly_object"), { key: "value" }); - assert.equal(storage.storeGet("cly_null"), null); + assert.equal(storageMethod.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storageMethod.storeGet("cly_id_type"), deviceIdType); + assert.equal(storageMethod.storeGet("cly_count"), 42); + assert.deepEqual(storageMethod.storeGet("cly_object"), { key: "value" }); + assert.equal(storageMethod.storeGet("cly_null"), null); // Remove specific items by overriding with null or empty array - storage.storeSet("cly_id", null); - storage.storeSet("cly_object", []); - assert.equal(storage.storeGet("cly_id"), null); - assert.deepEqual(storage.storeGet("cly_object"), []); + storageMethod.storeSet("cly_id", null); + storageMethod.storeSet("cly_object", []); + assert.equal(storageMethod.storeGet("cly_id"), null); + assert.deepEqual(storageMethod.storeGet("cly_object"), []); // Reset storage and check if it's empty again storage.resetStorage(); - assert.equal(storage.storeGet("cly_id"), undefined); - assert.equal(storage.storeGet("cly_id_type"), undefined); - assert.equal(storage.storeGet("cly_count"), undefined); - assert.equal(storage.storeGet("cly_object"), undefined); - assert.equal(storage.storeGet("cly_null"), undefined); + assert.equal(storageMethod.storeGet("cly_id"), undefined); + assert.equal(storageMethod.storeGet("cly_id_type"), undefined); + assert.equal(storageMethod.storeGet("cly_count"), undefined); + assert.equal(storageMethod.storeGet("cly_object"), undefined); + assert.equal(storageMethod.storeGet("cly_null"), undefined); } describe("Storage Tests", () => { @@ -247,7 +248,7 @@ describe("Storage Tests", () => { // will set to default storage path storage.setStoragePath("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - recordValuesToStorageAndValidate(); + recordValuesToStorageAndValidate("../test/customStorageDirectory/"); done(); }); @@ -257,7 +258,7 @@ describe("Storage Tests", () => { // To set the storage path to the default bulk storage path and persist the queue storage.setStoragePath(null, true, true); assert.equal(storage.getStoragePath(), "../bulk_data/"); - recordValuesToStorageAndValidate(); + recordValuesToStorageAndValidate(null, false, true, true); done(); }); @@ -266,7 +267,7 @@ describe("Storage Tests", () => { // will set to default storage path storage.setStoragePath("../test/customStorageDirectory/", true); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - recordValuesToStorageAndValidate(); + recordValuesToStorageAndValidate("../test/customStorageDirectory/", false, true); done(); }); });