From 369593284de3b7126965e1fd8175ae8471e5678e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Wed, 18 Sep 2024 20:51:34 +0300 Subject: [PATCH 01/13] Memory Only Storage Option --- lib/countly-storage.js | 167 ++++++++++++++++++++++++++++------------- lib/countly.js | 41 +++++----- test/tests_storage.js | 47 ++++++------ 3 files changed, 161 insertions(+), 94 deletions(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 1ab308b..95ea68e 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -8,6 +8,121 @@ var defaultPath = "../data/"; // Default path var defaultBulkPath = "../bulk_data/"; // Default path var asyncWriteLock = false; var asyncWriteQueue = []; +let storageMethod = {}; + +// Memory-only storage methods +const memoryStorage = { + /** + * Save value in memory + * @param {String} key - key for value to store + * @param {varies} value - value to store + * @param {Function} callback - callback to call when done storing + */ + storeSet(key, value, callback) { + __data[key] = value; + if (typeof callback === "function") { + callback(null); // No errors for memory storage + } + }, + /** + * Get value from memory + * @param {String} key - key of value to get + * @param {varies} def - default value to use if not set + * @returns {varies} value for the key + */ + storeGet(key, def) { + cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`); + return typeof __data[key] !== "undefined" ? __data[key] : def; + }, + /** + * Remove value from memory + * @param {String} key - key of value to remove + */ + storeRemove(key) { + delete __data[key]; + }, +}; + +// File storage methods +const fileStorage = { + /** + * Save value in storage + * @param {String} key - key for value to store + * @param {varies} value - value to store + * @param {Function} callback - callback to call when done storing + */ + storeSet(key, value, callback) { + __data[key] = value; + if (!asyncWriteLock) { + asyncWriteLock = true; + writeFile(key, value, callback); + } + else { + asyncWriteQueue.push([key, value, callback]); + } + }, + /** + * Get value from storage + * @param {String} key - key of value to get + * @param {varies} def - default value to use if not set + * @returns {varies} value for the key + */ + storeGet(key, def) { + cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); + if (typeof __data[key] === "undefined") { + var ob = readFile(key); + var obLen; + // check if the 'read object' is empty or not + try { + obLen = Object.keys(ob).length; + } + catch (error) { + // if we can not even asses length set it to 0 so we can return the default value + obLen = 0; + } + + // if empty or falsy set default value + if (!ob || obLen === 0) { + __data[key] = def; + } + // else set the value read file has + else { + __data[key] = ob[key]; + } + } + return __data[key]; + }, + storeRemove(key) { + delete __data[key]; + var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); + fs.unlink(dir, (err) => { + if (err) { + cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err}].`); + } + }); + }, +}; + +/** + * Sets the storage method, by default sets file storage and storage path. + * @param {String} userPath - User provided storage path + * @param {Boolean} memoryOnly - Whether to use memory only storage or not + * @param {Boolean} isBulk - Whether the storage is for bulk data + * @param {Boolean} persistQueue - Whether to persist the queue until processed + * @returns {storageMethod} Prefered storage method + */ +var setStorage = function(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { + if (memoryOnly) { + cc.log(cc.logLevelEnums.DEBUG, "Using memory-only storage."); + storageMethod = memoryStorage; // Assign memory-only methods + } + else { + cc.log(cc.logLevelEnums.DEBUG, "Using persistent file storage."); + storageMethod = fileStorage; // Assign file storage methods + setStoragePath(userPath, isBulk, persistQueue); + } + return storageMethod; +}; /** * Sets the storage path, defaulting to a specified path if none is provided. @@ -139,59 +254,9 @@ var writeFile = function(key, value, callback) { }); }; -/** - * Save value in storage - * @param {String} key - key for value to store - * @param {varies} value - value to store - * @param {Function} callback - callback to call when done storing - */ -var storeSet = function(key, value, callback) { - __data[key] = value; - if (!asyncWriteLock) { - asyncWriteLock = true; - writeFile(key, value, callback); - } - else { - asyncWriteQueue.push([key, value, callback]); - } -}; - -/** - * Get value from storage - * @param {String} key - key of value to get - * @param {varies} def - default value to use if not set - * @returns {varies} value for the key - */ -var storeGet = function(key, def) { - cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); - if (typeof __data[key] === "undefined") { - var ob = readFile(key); - var obLen; - // check if the 'read object' is empty or not - try { - obLen = Object.keys(ob).length; - } - catch (error) { - // if we can not even asses length set it to 0 so we can return the default value - obLen = 0; - } - - // if empty or falsy set default value - if (!ob || obLen === 0) { - __data[key] = def; - } - // else set the value read file has - else { - __data[key] = ob[key]; - } - } - return __data[key]; -}; - module.exports = { + setStorage, writeFile, - storeGet, - storeSet, forceStore, getStoragePath, setStoragePath, diff --git a/lib/countly.js b/lib/countly.js index 9965868..5aa8e60 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -29,6 +29,7 @@ var Bulk = require("./countly-bulk"); var CountlyStorage = require("./countly-storage"); var Countly = {}; +var storageMethod; Countly.Bulk = Bulk; (function() { @@ -171,13 +172,13 @@ Countly.Bulk = Bulk; cc.debug = conf.debug; // Set the storage path - CountlyStorage.setStoragePath(conf.storage_path); + storageMethod = CountlyStorage.setStorage(conf.storage_path, true); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { cc.log(cc.logLevelEnums.WARNING, "init, clear_stored_device_id is true, erasing the stored ID."); - CountlyStorage.storeSet("cly_id", null); - CountlyStorage.storeSet("cly_id_type", null); + storageMethod.storeSet("cly_id", null); + storageMethod.storeSet("cly_id_type", null); } if (Countly.url === "") { @@ -223,8 +224,8 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { // fetch stored ID and ID type - var storedId = CountlyStorage.storeGet("cly_id", null); - var storedIdType = CountlyStorage.storeGet("cly_id_type", null); + var storedId = storageMethod.storeGet("cly_id", null); + var storedIdType = storageMethod.storeGet("cly_id_type", null); // if there was a stored ID if (storedId !== null) { Countly.device_id = storedId; @@ -253,12 +254,12 @@ Countly.Bulk = Bulk; deviceIdType = cc.deviceIdTypeEnums.SDK_GENERATED; } // save the ID and ID type - CountlyStorage.storeSet("cly_id", Countly.device_id); - CountlyStorage.storeSet("cly_id_type", deviceIdType); + storageMethod.storeSet("cly_id", Countly.device_id); + storageMethod.storeSet("cly_id_type", deviceIdType); // create queues - requestQueue = CountlyStorage.storeGet("cly_queue", []); - eventQueue = CountlyStorage.storeGet("cly_event", []); - remoteConfigs = CountlyStorage.storeGet("cly_remote_configs", {}); + requestQueue = storageMethod.storeGet("cly_queue", []); + eventQueue = storageMethod.storeGet("cly_event", []); + remoteConfigs = storageMethod.storeGet("cly_remote_configs", {}); heartBeat(); // listen to current workers if (cluster.workers) { @@ -635,7 +636,7 @@ Countly.Bulk = Bulk; if (eventQueue.length > 0) { toRequestQueue({ events: JSON.stringify(eventQueue) }); eventQueue = []; - CountlyStorage.storeSet("cly_event", eventQueue); + storageMethod.storeSet("cly_event", eventQueue); } // end current session Countly.end_session(); @@ -647,8 +648,8 @@ Countly.Bulk = Bulk; var oldId = Countly.device_id; Countly.device_id = newId; deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - CountlyStorage.storeSet("cly_id", Countly.device_id); - CountlyStorage.storeSet("cly_id_type", deviceIdType); + storageMethod.storeSet("cly_id", Countly.device_id); + storageMethod.storeSet("cly_id_type", deviceIdType); if (merge) { if (Countly.check_any_consent()) { toRequestQueue({ old_device_id: oldId }); @@ -664,7 +665,7 @@ Countly.Bulk = Bulk; if (Countly.remote_config) { remoteConfigs = {}; if (cluster.isMaster) { - CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); + storageMethod.storeSet("cly_remote_configs", remoteConfigs); } Countly.fetch_remote_config(Countly.remote_config); } @@ -744,7 +745,7 @@ Countly.Bulk = Bulk; e.dow = date.getDay(); cc.log(cc.logLevelEnums.DEBUG, "add_cly_events, Adding event: ", event); eventQueue.push(e); - CountlyStorage.storeSet("cly_event", eventQueue); + storageMethod.storeSet("cly_event", eventQueue); } else { process.send({ cly: { event } }); @@ -1155,7 +1156,7 @@ Countly.Bulk = Bulk; remoteConfigs = configs; } if (cluster.isMaster) { - CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); + storageMethod.storeSet("cly_remote_configs", remoteConfigs); cc.log(cc.logLevelEnums.INFO, `fetch_remote_config, Fetched remote config: [${remoteConfigs}].`); } } @@ -1339,7 +1340,7 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { cc.log(cc.logLevelEnums.INFO, "request, Adding the raw request to the queue."); requestQueue.push(request); - CountlyStorage.storeSet("cly_queue", requestQueue); + storageMethod.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "request, Sending message to the parent process. Adding the raw request to the queue."); @@ -1426,7 +1427,7 @@ Countly.Bulk = Bulk; cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Adding request to the queue."); requestQueue.push(request); - CountlyStorage.storeSet("cly_queue", requestQueue); + storageMethod.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Sending message to the parent process. Adding request to the queue."); @@ -1457,7 +1458,7 @@ Countly.Bulk = Bulk; var events = eventQueue.splice(0, maxEventBatch); toRequestQueue({ events: JSON.stringify(events) }); } - CountlyStorage.storeSet("cly_event", eventQueue); + storageMethod.storeSet("cly_event", eventQueue); } // process request queue with event queue @@ -1472,7 +1473,7 @@ Countly.Bulk = Bulk; failTimeout = cc.getTimestamp() + failTimeoutAmount; cc.log(cc.logLevelEnums.ERROR, `makeRequest, Encountered a problem while making the request: [${err}]`); } - CountlyStorage.storeSet("cly_queue", requestQueue); + storageMethod.storeSet("cly_queue", requestQueue); readyToProcess = true; }, "heartBeat", false); } diff --git a/test/tests_storage.js b/test/tests_storage.js index 6735b1f..6815b53 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -64,37 +64,38 @@ function validateDeviceId(deviceId, deviceIdType, expectedDeviceId, expectedDevi assert.equal(deviceIdType, expectedDeviceIdType); checkRequestsForT(rq, expectedDeviceIdType); } -function recordValuesToStorageAndValidate() { +function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { // Set values var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storage.storeSet("cly_id", "SpecialDeviceId"); - storage.storeSet("cly_id_type", deviceIdType); + var storageMethod = storage.setStorage(userPath, memoryOnly, isBulk, persistQueue); + storageMethod.storeSet("cly_id", "SpecialDeviceId"); + storageMethod.storeSet("cly_id_type", deviceIdType); // Set values with different data types - storage.storeSet("cly_count", 42); - storage.storeSet("cly_object", { key: "value" }); - storage.storeSet("cly_null", null); + storageMethod.storeSet("cly_count", 42); + storageMethod.storeSet("cly_object", { key: "value" }); + storageMethod.storeSet("cly_null", null); // Retrieve and assert values - assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storage.storeGet("cly_id_type"), deviceIdType); - assert.equal(storage.storeGet("cly_count"), 42); - assert.deepEqual(storage.storeGet("cly_object"), { key: "value" }); - assert.equal(storage.storeGet("cly_null"), null); + assert.equal(storageMethod.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storageMethod.storeGet("cly_id_type"), deviceIdType); + assert.equal(storageMethod.storeGet("cly_count"), 42); + assert.deepEqual(storageMethod.storeGet("cly_object"), { key: "value" }); + assert.equal(storageMethod.storeGet("cly_null"), null); // Remove specific items by overriding with null or empty array - storage.storeSet("cly_id", null); - storage.storeSet("cly_object", []); - assert.equal(storage.storeGet("cly_id"), null); - assert.deepEqual(storage.storeGet("cly_object"), []); + storageMethod.storeSet("cly_id", null); + storageMethod.storeSet("cly_object", []); + assert.equal(storageMethod.storeGet("cly_id"), null); + assert.deepEqual(storageMethod.storeGet("cly_object"), []); // Reset storage and check if it's empty again storage.resetStorage(); - assert.equal(storage.storeGet("cly_id"), undefined); - assert.equal(storage.storeGet("cly_id_type"), undefined); - assert.equal(storage.storeGet("cly_count"), undefined); - assert.equal(storage.storeGet("cly_object"), undefined); - assert.equal(storage.storeGet("cly_null"), undefined); + assert.equal(storageMethod.storeGet("cly_id"), undefined); + assert.equal(storageMethod.storeGet("cly_id_type"), undefined); + assert.equal(storageMethod.storeGet("cly_count"), undefined); + assert.equal(storageMethod.storeGet("cly_object"), undefined); + assert.equal(storageMethod.storeGet("cly_null"), undefined); } describe("Storage Tests", () => { @@ -247,7 +248,7 @@ describe("Storage Tests", () => { // will set to default storage path storage.setStoragePath("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - recordValuesToStorageAndValidate(); + recordValuesToStorageAndValidate("../test/customStorageDirectory/"); done(); }); @@ -257,7 +258,7 @@ describe("Storage Tests", () => { // To set the storage path to the default bulk storage path and persist the queue storage.setStoragePath(null, true, true); assert.equal(storage.getStoragePath(), "../bulk_data/"); - recordValuesToStorageAndValidate(); + recordValuesToStorageAndValidate(null, false, true, true); done(); }); @@ -266,7 +267,7 @@ describe("Storage Tests", () => { // will set to default storage path storage.setStoragePath("../test/customStorageDirectory/", true); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - recordValuesToStorageAndValidate(); + recordValuesToStorageAndValidate("../test/customStorageDirectory/", false, true); done(); }); }); From c572bf1f9ff5badd6c4ed6ce6744c5e554e22196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 19 Sep 2024 13:49:22 +0300 Subject: [PATCH 02/13] Update countly.js --- lib/countly.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/countly.js b/lib/countly.js index 5aa8e60..1e642dc 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -172,7 +172,7 @@ Countly.Bulk = Bulk; cc.debug = conf.debug; // Set the storage path - storageMethod = CountlyStorage.setStorage(conf.storage_path, true); + storageMethod = CountlyStorage.setStorage(conf.storage_path); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { From b4a96bf56f49acc6d920c67cbbdfcb82d3551063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 19 Sep 2024 13:55:32 +0300 Subject: [PATCH 03/13] storage method in bulk --- lib/countly-bulk.js | 25 ++++++++++++++----------- lib/countly.js | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index ceb542e..1315973 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -25,6 +25,8 @@ var cc = require("./countly-common"); var BulkUser = require("./countly-bulk-user"); var CountlyStorage = require("./countly-storage"); +var storageMethod; + /** * @lends module:lib/countly-bulk * Initialize CountlyBulk server object @@ -102,7 +104,8 @@ function CountlyBulk(conf) { conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; - CountlyStorage.setStoragePath(conf.storage_path, true, conf.persist_queue); + // config time memory only option will be added + storageMethod = CountlyStorage.setStorage(conf.storage_path, false, true, conf.persist_queue); this.conf = conf; /** @@ -142,7 +145,7 @@ function CountlyBulk(conf) { requestQueue.push(query); cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_request, Adding request to the queue."); - CountlyStorage.storeSet("cly_req_queue", requestQueue); + storageMethod.storeSet("cly_req_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_request, Sending message to the parent process. Adding the raw request to the queue."); @@ -190,7 +193,7 @@ function CountlyBulk(conf) { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_bulk_request, adding the request into queue."); requestQueue.push(query); } - CountlyStorage.storeSet("cly_req_queue", requestQueue); + storageMethod.storeSet("cly_req_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_bulk_request, Sending message to the parent process. Adding the raw request to the queue."); @@ -245,7 +248,7 @@ function CountlyBulk(conf) { eventQueue[device_id] = []; } eventQueue[device_id].push(e); - CountlyStorage.storeSet("cly_bulk_event", eventQueue); + storageMethod.storeSet("cly_bulk_event", eventQueue); } else { cc.log(cc.logLevelEnums.INFO, `CountlyBulk add_event, Sending message to the parent process. Adding event: [${event.key}].`); @@ -343,7 +346,7 @@ function CountlyBulk(conf) { */ function toBulkRequestQueue(bulkRequest) { bulkQueue.push(bulkRequest); - CountlyStorage.storeSet("cly_bulk_queue", bulkQueue); + storageMethod.storeSet("cly_bulk_queue", bulkQueue); } var self = this; @@ -369,7 +372,7 @@ function CountlyBulk(conf) { } if (eventChanges) { isEmpty = false; - CountlyStorage.storeSet("cly_bulk_event", eventQueue); + storageMethod.storeSet("cly_bulk_event", eventQueue); } // process request queue into bulk requests @@ -383,7 +386,7 @@ function CountlyBulk(conf) { var requests = requestQueue.splice(0, conf.bulk_size); toBulkRequestQueue({ app_key: conf.app_key, requests: JSON.stringify(requests) }); } - CountlyStorage.storeSet("cly_req_queue", requestQueue); + storageMethod.storeSet("cly_req_queue", requestQueue); } // process bulk request queue @@ -398,7 +401,7 @@ function CountlyBulk(conf) { bulkQueue.unshift(res); failTimeout = cc.getTimestamp() + conf.fail_timeout; } - CountlyStorage.storeSet("cly_bulk_queue", bulkQueue); + storageMethod.storeSet("cly_bulk_queue", bulkQueue); readyToProcess = true; }, "heartBeat", false); } @@ -591,9 +594,9 @@ function CountlyBulk(conf) { worker.on("message", handleWorkerMessage); }); - var requestQueue = CountlyStorage.storeGet("cly_req_queue", []); - var eventQueue = CountlyStorage.storeGet("cly_bulk_event", {}); - var bulkQueue = CountlyStorage.storeGet("cly_bulk_queue", []); + var requestQueue = storageMethod.storeGet("cly_req_queue", []); + var eventQueue = storageMethod.storeGet("cly_bulk_event", {}); + var bulkQueue = storageMethod.storeGet("cly_bulk_queue", []); } module.exports = CountlyBulk; diff --git a/lib/countly.js b/lib/countly.js index 1e642dc..102377b 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -171,7 +171,7 @@ Countly.Bulk = Bulk; // Common module debug value is set to init time debug value cc.debug = conf.debug; - // Set the storage path + // Set the storage method and path storageMethod = CountlyStorage.setStorage(conf.storage_path); // clear stored device ID if flag is set From 9b9643ae773a0c8bf857e3276db0c0939a305244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 19 Sep 2024 14:55:23 +0300 Subject: [PATCH 04/13] Update tests_storage.js --- test/tests_storage.js | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/test/tests_storage.js b/test/tests_storage.js index 6815b53..024e532 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -270,4 +270,38 @@ describe("Storage Tests", () => { recordValuesToStorageAndValidate("../test/customStorageDirectory/", false, true); done(); }); -}); + + it("14- Setting storage path to default path via setStorage /no-init", (done) => { + storage.resetStorage(); + storage.setStorage(); + assert.equal(storage.getStoragePath(), "../data/"); + done(); + }); + + it("15- Setting bulk storage path to default path via setStorage /no-init", (done) => { + storage.resetStorage(); + storage.setStorage(null, false, true); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + done(); + }); + + it("16- Setting custom storage path via setStorage /no-init", (done) => { + storage.resetStorage(); + storage.setStorage("../test/customStorageDirectory/"); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + done(); + }); + + it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { + storage.resetStorage(); + storage.setStorage(null, true); + assert.equal(storage.getStoragePath(), undefined); + done(); + }); + + it("18- Recording to storage with memory only storage /no-init", (done) => { + storage.resetStorage(); + recordValuesToStorageAndValidate(null, true); + done(); + }); +}); \ No newline at end of file From ce59d66ffdad580e1de0589302b29d3d3374f9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 24 Sep 2024 16:06:01 +0300 Subject: [PATCH 05/13] Storing Methods Exported, Enums Added --- lib/countly-bulk.js | 30 ++++++++++++---------- lib/countly-common.js | 6 +++++ lib/countly-storage.js | 44 +++++++++++++++++++++------------ lib/countly.js | 47 +++++++++++++++++++---------------- test/tests_storage.js | 56 +++++++++++++++++++++--------------------- 5 files changed, 105 insertions(+), 78 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 1315973..ae58f20 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -25,7 +25,8 @@ var cc = require("./countly-common"); var BulkUser = require("./countly-bulk-user"); var CountlyStorage = require("./countly-storage"); -var storageMethod; +const StorageTypes = cc.storageTypeEnums; +var storageType = StorageTypes.FILE; /** * @lends module:lib/countly-bulk @@ -49,6 +50,7 @@ var storageMethod; * @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length + * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied * @example * var server = new CountlyBulk({ * app_key: "{YOUR-API-KEY}", @@ -104,8 +106,10 @@ function CountlyBulk(conf) { conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; - // config time memory only option will be added - storageMethod = CountlyStorage.setStorage(conf.storage_path, false, true, conf.persist_queue); + if (!conf.storage_type) { + storageType = conf.storage_type; + } + CountlyStorage.initStorage(conf.storage_path, storageType, true, conf.persist_queue); this.conf = conf; /** @@ -145,7 +149,7 @@ function CountlyBulk(conf) { requestQueue.push(query); cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_request, Adding request to the queue."); - storageMethod.storeSet("cly_req_queue", requestQueue); + CountlyStorage.storeSet("cly_req_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_request, Sending message to the parent process. Adding the raw request to the queue."); @@ -193,7 +197,7 @@ function CountlyBulk(conf) { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_bulk_request, adding the request into queue."); requestQueue.push(query); } - storageMethod.storeSet("cly_req_queue", requestQueue); + CountlyStorage.storeSet("cly_req_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_bulk_request, Sending message to the parent process. Adding the raw request to the queue."); @@ -248,7 +252,7 @@ function CountlyBulk(conf) { eventQueue[device_id] = []; } eventQueue[device_id].push(e); - storageMethod.storeSet("cly_bulk_event", eventQueue); + CountlyStorage.storeSet("cly_bulk_event", eventQueue); } else { cc.log(cc.logLevelEnums.INFO, `CountlyBulk add_event, Sending message to the parent process. Adding event: [${event.key}].`); @@ -346,7 +350,7 @@ function CountlyBulk(conf) { */ function toBulkRequestQueue(bulkRequest) { bulkQueue.push(bulkRequest); - storageMethod.storeSet("cly_bulk_queue", bulkQueue); + CountlyStorage.storeSet("cly_bulk_queue", bulkQueue); } var self = this; @@ -372,7 +376,7 @@ function CountlyBulk(conf) { } if (eventChanges) { isEmpty = false; - storageMethod.storeSet("cly_bulk_event", eventQueue); + CountlyStorage.storeSet("cly_bulk_event", eventQueue); } // process request queue into bulk requests @@ -386,7 +390,7 @@ function CountlyBulk(conf) { var requests = requestQueue.splice(0, conf.bulk_size); toBulkRequestQueue({ app_key: conf.app_key, requests: JSON.stringify(requests) }); } - storageMethod.storeSet("cly_req_queue", requestQueue); + CountlyStorage.storeSet("cly_req_queue", requestQueue); } // process bulk request queue @@ -401,7 +405,7 @@ function CountlyBulk(conf) { bulkQueue.unshift(res); failTimeout = cc.getTimestamp() + conf.fail_timeout; } - storageMethod.storeSet("cly_bulk_queue", bulkQueue); + CountlyStorage.storeSet("cly_bulk_queue", bulkQueue); readyToProcess = true; }, "heartBeat", false); } @@ -594,9 +598,9 @@ function CountlyBulk(conf) { worker.on("message", handleWorkerMessage); }); - var requestQueue = storageMethod.storeGet("cly_req_queue", []); - var eventQueue = storageMethod.storeGet("cly_bulk_event", {}); - var bulkQueue = storageMethod.storeGet("cly_bulk_queue", []); + var requestQueue = CountlyStorage.storeGet("cly_req_queue", []); + var eventQueue = CountlyStorage.storeGet("cly_bulk_event", {}); + var bulkQueue = CountlyStorage.storeGet("cly_bulk_queue", []); } module.exports = CountlyBulk; diff --git a/lib/countly-common.js b/lib/countly-common.js index d829250..7a961e5 100644 --- a/lib/countly-common.js +++ b/lib/countly-common.js @@ -51,6 +51,12 @@ var cc = { ACTION: '[CLY]_action', }, + storageTypeEnums: { + FILE: "file", + MEMORY: "memory", + CUSTOM: "custom", + }, + /** * Get current timestamp * @returns {number} unix timestamp in seconds diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 95ea68e..592fd79 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -9,6 +9,7 @@ var defaultBulkPath = "../bulk_data/"; // Default path var asyncWriteLock = false; var asyncWriteQueue = []; let storageMethod = {}; +const StorageTypes = cc.storageTypeEnums; // Memory-only storage methods const memoryStorage = { @@ -18,7 +19,7 @@ const memoryStorage = { * @param {varies} value - value to store * @param {Function} callback - callback to call when done storing */ - storeSet(key, value, callback) { + storeSet: function(key, value, callback) { __data[key] = value; if (typeof callback === "function") { callback(null); // No errors for memory storage @@ -30,7 +31,7 @@ const memoryStorage = { * @param {varies} def - default value to use if not set * @returns {varies} value for the key */ - storeGet(key, def) { + storeGet: function(key, def) { cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`); return typeof __data[key] !== "undefined" ? __data[key] : def; }, @@ -38,7 +39,7 @@ const memoryStorage = { * Remove value from memory * @param {String} key - key of value to remove */ - storeRemove(key) { + storeRemove: function(key) { delete __data[key]; }, }; @@ -51,7 +52,7 @@ const fileStorage = { * @param {varies} value - value to store * @param {Function} callback - callback to call when done storing */ - storeSet(key, value, callback) { + storeSet: function(key, value, callback) { __data[key] = value; if (!asyncWriteLock) { asyncWriteLock = true; @@ -67,7 +68,7 @@ const fileStorage = { * @param {varies} def - default value to use if not set * @returns {varies} value for the key */ - storeGet(key, def) { + storeGet: function(key, def) { cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); if (typeof __data[key] === "undefined") { var ob = readFile(key); @@ -92,7 +93,7 @@ const fileStorage = { } return __data[key]; }, - storeRemove(key) { + storeRemove: function(key) { delete __data[key]; var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); fs.unlink(dir, (err) => { @@ -106,22 +107,18 @@ const fileStorage = { /** * Sets the storage method, by default sets file storage and storage path. * @param {String} userPath - User provided storage path - * @param {Boolean} memoryOnly - Whether to use memory only storage or not + * @param {StorageTypes} storageType - Whether to use memory only storage or not * @param {Boolean} isBulk - Whether the storage is for bulk data * @param {Boolean} persistQueue - Whether to persist the queue until processed - * @returns {storageMethod} Prefered storage method */ -var setStorage = function(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { - if (memoryOnly) { - cc.log(cc.logLevelEnums.DEBUG, "Using memory-only storage."); - storageMethod = memoryStorage; // Assign memory-only methods +var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false) { + if (storageType === StorageTypes.MEMORY) { + storageMethod = memoryStorage; } else { - cc.log(cc.logLevelEnums.DEBUG, "Using persistent file storage."); - storageMethod = fileStorage; // Assign file storage methods + storageMethod = fileStorage; setStoragePath(userPath, isBulk, persistQueue); } - return storageMethod; }; /** @@ -254,8 +251,23 @@ var writeFile = function(key, value, callback) { }); }; +var storeSet = function(key, value, callback) { + storageMethod.storeSet(key, value, callback); +}; + +var storeGet = function(key, def) { + return storageMethod.storeGet(key, def); +}; + +var storeRemove = function(key) { + storageMethod.storeRemove(key); +}; + module.exports = { - setStorage, + initStorage, + storeSet, + storeGet, + storeRemove, writeFile, forceStore, getStoragePath, diff --git a/lib/countly.js b/lib/countly.js index 102377b..3fc1714 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -29,7 +29,8 @@ var Bulk = require("./countly-bulk"); var CountlyStorage = require("./countly-storage"); var Countly = {}; -var storageMethod; +const StorageTypes = cc.storageTypeEnums; +var storageType = StorageTypes.FILE; Countly.Bulk = Bulk; (function() { @@ -123,6 +124,7 @@ Countly.Bulk = Bulk; * @param {string} conf.metrics._density - screen density of the device * @param {string} conf.metrics._locale - locale or language of the device in ISO format * @param {string} conf.metrics._store - source from where the user/device/installation came from + * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied * @example * Countly.init({ * app_key: "{YOUR-APP-KEY}", @@ -172,13 +174,16 @@ Countly.Bulk = Bulk; cc.debug = conf.debug; // Set the storage method and path - storageMethod = CountlyStorage.setStorage(conf.storage_path); + if (!conf.storage_type) { + storageType = conf.storage_type; + } + CountlyStorage.initStorage(conf.storage_path, storageType); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { cc.log(cc.logLevelEnums.WARNING, "init, clear_stored_device_id is true, erasing the stored ID."); - storageMethod.storeSet("cly_id", null); - storageMethod.storeSet("cly_id_type", null); + CountlyStorage.storeSet("cly_id", null); + CountlyStorage.storeSet("cly_id_type", null); } if (Countly.url === "") { @@ -224,8 +229,8 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { // fetch stored ID and ID type - var storedId = storageMethod.storeGet("cly_id", null); - var storedIdType = storageMethod.storeGet("cly_id_type", null); + var storedId = CountlyStorage.storeGet("cly_id", null); + var storedIdType = CountlyStorage.storeGet("cly_id_type", null); // if there was a stored ID if (storedId !== null) { Countly.device_id = storedId; @@ -254,12 +259,12 @@ Countly.Bulk = Bulk; deviceIdType = cc.deviceIdTypeEnums.SDK_GENERATED; } // save the ID and ID type - storageMethod.storeSet("cly_id", Countly.device_id); - storageMethod.storeSet("cly_id_type", deviceIdType); + CountlyStorage.storeSet("cly_id", Countly.device_id); + CountlyStorage.storeSet("cly_id_type", deviceIdType); // create queues - requestQueue = storageMethod.storeGet("cly_queue", []); - eventQueue = storageMethod.storeGet("cly_event", []); - remoteConfigs = storageMethod.storeGet("cly_remote_configs", {}); + requestQueue = CountlyStorage.storeGet("cly_queue", []); + eventQueue = CountlyStorage.storeGet("cly_event", []); + remoteConfigs = CountlyStorage.storeGet("cly_remote_configs", {}); heartBeat(); // listen to current workers if (cluster.workers) { @@ -636,7 +641,7 @@ Countly.Bulk = Bulk; if (eventQueue.length > 0) { toRequestQueue({ events: JSON.stringify(eventQueue) }); eventQueue = []; - storageMethod.storeSet("cly_event", eventQueue); + CountlyStorage.storeSet("cly_event", eventQueue); } // end current session Countly.end_session(); @@ -648,8 +653,8 @@ Countly.Bulk = Bulk; var oldId = Countly.device_id; Countly.device_id = newId; deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storageMethod.storeSet("cly_id", Countly.device_id); - storageMethod.storeSet("cly_id_type", deviceIdType); + CountlyStorage.storeSet("cly_id", Countly.device_id); + CountlyStorage.storeSet("cly_id_type", deviceIdType); if (merge) { if (Countly.check_any_consent()) { toRequestQueue({ old_device_id: oldId }); @@ -665,7 +670,7 @@ Countly.Bulk = Bulk; if (Countly.remote_config) { remoteConfigs = {}; if (cluster.isMaster) { - storageMethod.storeSet("cly_remote_configs", remoteConfigs); + CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); } Countly.fetch_remote_config(Countly.remote_config); } @@ -745,7 +750,7 @@ Countly.Bulk = Bulk; e.dow = date.getDay(); cc.log(cc.logLevelEnums.DEBUG, "add_cly_events, Adding event: ", event); eventQueue.push(e); - storageMethod.storeSet("cly_event", eventQueue); + CountlyStorage.storeSet("cly_event", eventQueue); } else { process.send({ cly: { event } }); @@ -1156,7 +1161,7 @@ Countly.Bulk = Bulk; remoteConfigs = configs; } if (cluster.isMaster) { - storageMethod.storeSet("cly_remote_configs", remoteConfigs); + CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); cc.log(cc.logLevelEnums.INFO, `fetch_remote_config, Fetched remote config: [${remoteConfigs}].`); } } @@ -1340,7 +1345,7 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { cc.log(cc.logLevelEnums.INFO, "request, Adding the raw request to the queue."); requestQueue.push(request); - storageMethod.storeSet("cly_queue", requestQueue); + CountlyStorage.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "request, Sending message to the parent process. Adding the raw request to the queue."); @@ -1427,7 +1432,7 @@ Countly.Bulk = Bulk; cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Adding request to the queue."); requestQueue.push(request); - storageMethod.storeSet("cly_queue", requestQueue); + CountlyStorage.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Sending message to the parent process. Adding request to the queue."); @@ -1458,7 +1463,7 @@ Countly.Bulk = Bulk; var events = eventQueue.splice(0, maxEventBatch); toRequestQueue({ events: JSON.stringify(events) }); } - storageMethod.storeSet("cly_event", eventQueue); + CountlyStorage.storeSet("cly_event", eventQueue); } // process request queue with event queue @@ -1473,7 +1478,7 @@ Countly.Bulk = Bulk; failTimeout = cc.getTimestamp() + failTimeoutAmount; cc.log(cc.logLevelEnums.ERROR, `makeRequest, Encountered a problem while making the request: [${err}]`); } - storageMethod.storeSet("cly_queue", requestQueue); + CountlyStorage.storeSet("cly_queue", requestQueue); readyToProcess = true; }, "heartBeat", false); } diff --git a/test/tests_storage.js b/test/tests_storage.js index 024e532..338b66e 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -67,35 +67,35 @@ function validateDeviceId(deviceId, deviceIdType, expectedDeviceId, expectedDevi function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { // Set values var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - var storageMethod = storage.setStorage(userPath, memoryOnly, isBulk, persistQueue); - storageMethod.storeSet("cly_id", "SpecialDeviceId"); - storageMethod.storeSet("cly_id_type", deviceIdType); + storage.initStorage(userPath, memoryOnly, isBulk, persistQueue); + storage.storeSet("cly_id", "SpecialDeviceId"); + storage.storeSet("cly_id_type", deviceIdType); // Set values with different data types - storageMethod.storeSet("cly_count", 42); - storageMethod.storeSet("cly_object", { key: "value" }); - storageMethod.storeSet("cly_null", null); + storage.storeSet("cly_count", 42); + storage.storeSet("cly_object", { key: "value" }); + storage.storeSet("cly_null", null); // Retrieve and assert values - assert.equal(storageMethod.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storageMethod.storeGet("cly_id_type"), deviceIdType); - assert.equal(storageMethod.storeGet("cly_count"), 42); - assert.deepEqual(storageMethod.storeGet("cly_object"), { key: "value" }); - assert.equal(storageMethod.storeGet("cly_null"), null); + assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + assert.equal(storage.storeGet("cly_count"), 42); + assert.deepEqual(storage.storeGet("cly_object"), { key: "value" }); + assert.equal(storage.storeGet("cly_null"), null); // Remove specific items by overriding with null or empty array - storageMethod.storeSet("cly_id", null); - storageMethod.storeSet("cly_object", []); - assert.equal(storageMethod.storeGet("cly_id"), null); - assert.deepEqual(storageMethod.storeGet("cly_object"), []); + storage.storeSet("cly_id", null); + storage.storeSet("cly_object", []); + assert.equal(storage.storeGet("cly_id"), null); + assert.deepEqual(storage.storeGet("cly_object"), []); // Reset storage and check if it's empty again storage.resetStorage(); - assert.equal(storageMethod.storeGet("cly_id"), undefined); - assert.equal(storageMethod.storeGet("cly_id_type"), undefined); - assert.equal(storageMethod.storeGet("cly_count"), undefined); - assert.equal(storageMethod.storeGet("cly_object"), undefined); - assert.equal(storageMethod.storeGet("cly_null"), undefined); + assert.equal(storage.storeGet("cly_id"), undefined); + assert.equal(storage.storeGet("cly_id_type"), undefined); + assert.equal(storage.storeGet("cly_count"), undefined); + assert.equal(storage.storeGet("cly_object"), undefined); + assert.equal(storage.storeGet("cly_null"), undefined); } describe("Storage Tests", () => { @@ -271,37 +271,37 @@ describe("Storage Tests", () => { done(); }); - it("14- Setting storage path to default path via setStorage /no-init", (done) => { + it("14- Setting storage path to default path via initStorage /no-init", (done) => { storage.resetStorage(); - storage.setStorage(); + storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); done(); }); - it("15- Setting bulk storage path to default path via setStorage /no-init", (done) => { + it("15- Setting bulk storage path to default path via initStorage /no-init", (done) => { storage.resetStorage(); - storage.setStorage(null, false, true); + storage.initStorage(null, false, true); assert.equal(storage.getStoragePath(), "../bulk_data/"); done(); }); - it("16- Setting custom storage path via setStorage /no-init", (done) => { + it("16- Setting custom storage path via initStorage /no-init", (done) => { storage.resetStorage(); - storage.setStorage("../test/customStorageDirectory/"); + storage.initStorage("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); done(); }); it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { storage.resetStorage(); - storage.setStorage(null, true); + storage.initStorage(null, cc.storageTypeEnums.MEMORY); assert.equal(storage.getStoragePath(), undefined); done(); }); it("18- Recording to storage with memory only storage /no-init", (done) => { storage.resetStorage(); - recordValuesToStorageAndValidate(null, true); + recordValuesToStorageAndValidate(null, cc.storageTypeEnums.MEMORY); done(); }); }); \ No newline at end of file From f370c782d031c4158b4abcdaff05ca35ae653955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 1 Oct 2024 14:15:29 +0300 Subject: [PATCH 06/13] Updating Config Section --- lib/countly-bulk.js | 8 +++----- lib/countly.js | 11 +++-------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index ae58f20..9fc6b53 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -26,7 +26,6 @@ var BulkUser = require("./countly-bulk-user"); var CountlyStorage = require("./countly-storage"); const StorageTypes = cc.storageTypeEnums; -var storageType = StorageTypes.FILE; /** * @lends module:lib/countly-bulk @@ -76,6 +75,7 @@ function CountlyBulk(conf) { var maxBreadcrumbCount = 100; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; + var storageType = StorageTypes.FILE; cc.debugBulk = conf.debug || false; if (!conf.app_key) { @@ -105,11 +105,9 @@ function CountlyBulk(conf) { conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount; conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; + conf.storage_type = conf.storage_type || storageType; - if (!conf.storage_type) { - storageType = conf.storage_type; - } - CountlyStorage.initStorage(conf.storage_path, storageType, true, conf.persist_queue); + CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue); this.conf = conf; /** diff --git a/lib/countly.js b/lib/countly.js index 3fc1714..7a51f1e 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -30,7 +30,6 @@ var CountlyStorage = require("./countly-storage"); var Countly = {}; const StorageTypes = cc.storageTypeEnums; -var storageType = StorageTypes.FILE; Countly.Bulk = Bulk; (function() { @@ -73,7 +72,7 @@ Countly.Bulk = Bulk; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; var deviceIdType = null; - + var storageType = StorageTypes.FILE; /** * Array with list of available features that you can require consent for */ @@ -169,15 +168,11 @@ Countly.Bulk = Bulk; Countly.maxBreadcrumbCount = conf.max_breadcrumb_count || Countly.max_breadcrumb_count || conf.max_logs || Countly.max_logs || maxBreadcrumbCount; Countly.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || Countly.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; Countly.maxStackTraceLineLength = conf.max_stack_trace_line_length || Countly.max_stack_trace_line_length || maxStackTraceLineLength; - + conf.storage_type = conf.storage_type || storageType; // Common module debug value is set to init time debug value cc.debug = conf.debug; - // Set the storage method and path - if (!conf.storage_type) { - storageType = conf.storage_type; - } - CountlyStorage.initStorage(conf.storage_path, storageType); + CountlyStorage.initStorage(conf.storage_path, conf.storage_type); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { From 783749132953454ef9a305b354130f29b729e082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 1 Oct 2024 18:14:31 +0300 Subject: [PATCH 07/13] Updated Storage Tests --- lib/countly-storage.js | 24 +++++++-- test/tests_storage.js | 113 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 131 insertions(+), 6 deletions(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 592fd79..122bb94 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -20,9 +20,15 @@ const memoryStorage = { * @param {Function} callback - callback to call when done storing */ storeSet: function(key, value, callback) { - __data[key] = value; - if (typeof callback === "function") { - callback(null); // No errors for memory storage + if (key) { + cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting key: [${key}] & value: [${value}]!`); + __data[key] = value; + if (typeof callback === "function") { + callback(null); + } + } + else { + cc.log(cc.logLevelEnums.WARNING, `storeSet, Provioded key: [${key}] is null!`); } }, /** @@ -263,6 +269,17 @@ var storeRemove = function(key) { storageMethod.storeRemove(key); }; +/** + * Disclaimer: This method is mainly for testing purposes. + * @returns {StorageTypes} Returns the active storage type for the SDK + */ +var getStorageType = function() { + if (storageMethod === memoryStorage) { + return StorageTypes.MEMORY; + } + return StorageTypes.FILE; +}; + module.exports = { initStorage, storeSet, @@ -274,4 +291,5 @@ module.exports = { setStoragePath, resetStorage, readFile, + getStorageType, }; \ No newline at end of file diff --git a/test/tests_storage.js b/test/tests_storage.js index 338b66e..41124b7 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -4,6 +4,8 @@ var storage = require("../lib/countly-storage"); var cc = require("../lib/countly-common"); var hp = require("./helpers/helper_functions"); +const StorageTypes = cc.storageTypeEnums; + // example event object to use var eventObj = { key: "storage_check", @@ -294,14 +296,119 @@ describe("Storage Tests", () => { it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { storage.resetStorage(); - storage.initStorage(null, cc.storageTypeEnums.MEMORY); + storage.initStorage(null, StorageTypes.MEMORY); + assert.equal(storage.getStoragePath(), undefined); + done(); + }); + + it("18- Memory only storage Device-Id", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.MEMORY, + }); assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.storeGet("cly_id", null), "Test-Device-Id"); + assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); done(); }); - it("18- Recording to storage with memory only storage /no-init", (done) => { + it("19- Record event in memory only mode and validate the record", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.MEMORY, + }); + Countly.add_event(eventObj); + setTimeout(() => { + const storedData = storage.storeGet("cly_queue", null); + const eventArray = JSON.parse(storedData[0].events); + const eventFromQueue = eventArray[0]; + hp.eventValidator(eventObj, eventFromQueue); + done(); + }, hp.mWait); + }); + + it("20- Record and validate user details in memory only mode", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.MEMORY, + }); + Countly.user_details(userDetailObj); + const storedData = storage.storeGet("cly_queue", null); + const userDetailsReq = storedData[0]; + hp.userDetailRequestValidator(userDetailObj, userDetailsReq); + done(); + }); + + it("21- Memory only storage, change SDK Generated Device-Id", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + clear_stored_device_id: true, + storage_type: StorageTypes.MEMORY, + }); + assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.storeGet("cly_id", null), Countly.get_device_id()); + assert.equal(storage.storeGet("cly_id_type", null), Countly.get_device_id_type()); + + Countly.change_id("Test-Id-2"); + assert.equal(storage.storeGet("cly_id", null), "Test-Id-2"); + assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + done(); + }); + + it("22- Switch to file storage after init", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + clear_stored_device_id: true, + storage_type: StorageTypes.MEMORY, + }); + assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.getStorageType(), StorageTypes.MEMORY); + + storage.initStorage(); + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + done(); + }); + + it("23- storeRemove Memory Only /no-init", (done) => { storage.resetStorage(); - recordValuesToStorageAndValidate(null, cc.storageTypeEnums.MEMORY); + storage.initStorage(null, StorageTypes.MEMORY); + assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.getStorageType(), StorageTypes.MEMORY); + storage.storeSet("keyToStore", "valueToStore"); + assert.equal(storage.storeGet("keyToStore", null), "valueToStore"); + + storage.storeRemove("keyToStore"); + assert.equal(storage.storeGet("keyToStore", null), null); + done(); + }); + + it("24- storeRemove File Storage /no-init", (done) => { + storage.resetStorage(); + storage.initStorage(); + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + storage.storeSet("keyToStore", "valueToStore"); + assert.equal(storage.storeGet("keyToStore", null), "valueToStore"); + + storage.storeRemove("keyToStore"); + assert.equal(storage.storeGet("keyToStore", null), null); done(); }); }); \ No newline at end of file From ddf942b479f6da4f9fef1f34234a3e60aeb8a812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 00:22:04 +0300 Subject: [PATCH 08/13] Bug Fix Testing --- CHANGELOG.md | 3 ++- lib/countly-storage.js | 31 +++++++++++++++++++++-------- test/helpers/helper_functions.js | 34 +++++++++++++++++++++++--------- test/tests_storage.js | 19 ++++++++++++++++-- 4 files changed, 67 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c63f40f..a7cb28f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## xx.xx.xx -* Default max segmentation value count changed from 30 to 100 +- Default max segmentation value count changed from 30 to 100 +- Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file. ## 22.06.0 - Fixed a bug where remote config requests were rejected diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 122bb94..f5782fa 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -101,11 +101,20 @@ const fileStorage = { }, storeRemove: function(key) { delete __data[key]; - var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); - fs.unlink(dir, (err) => { - if (err) { - cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err}].`); + var filePath = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); + fs.access(filePath, fs.constants.F_OK, (accessErr) => { + if (accessErr) { + cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`); + return; } + fs.rmSync(filePath, { recursive: true, force: true }, (err) => { + if (err) { + cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`); + } + else { + cc.log(cc.logLevelEnums.INFO, `storeRemove, Successfully removed file with key: [${key}].`); + } + }); }); }, }; @@ -181,15 +190,21 @@ var resetStorage = function() { */ var readFile = function(key) { var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); - // try reading data file var data; try { - data = fs.readFileSync(dir); + data = fs.readFileSync(dir, 'utf8'); // read file as string } catch (ex) { // there was no file, probably new init - data = null; + cc.log(cc.logLevelEnums.WARN, `readFile, File not found for key: [${key}]. Returning null.`); + return null; // early exit if file doesn't exist + } + + // early exit if file is empty or whitespace + if (!data.trim()) { + cc.log(cc.logLevelEnums.WARN, `readFile, File is empty or contains only whitespace for key: [${key}]. Returning null.`); + return null; } try { @@ -202,7 +217,7 @@ var readFile = function(key) { // backup corrupted file data fs.writeFile(path.resolve(__dirname, `${getStoragePath()}__${key}.${cc.getTimestamp()}${Math.random()}.json`), data, () => { }); // start with new clean object - data = null; + return null; // return null in case of corrupted data } return data; }; diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index eaee194..1728687 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -3,10 +3,12 @@ var path = require("path"); var assert = require("assert"); var fs = require("fs"); var Countly = require("../../lib/countly"); +var CountlyStorage = require("../../lib/countly-storage"); // paths for convenience var dir = path.resolve(__dirname, "../../"); var idDir = (`${dir}/data/__cly_id.json`); +var idTypeDir = (`${dir}/data/__cly_id_type.json`); var eventDir = (`${dir}/data/__cly_event.json`); var reqDir = (`${dir}/data/__cly_queue.json`); var bulkEventDir = (`${dir}/bulk_data/__cly_bulk_event.json`); @@ -39,24 +41,37 @@ function readRequestQueue(givenPath = null, isBulk = false) { } return a; } -// queue files clearing logic +function doesFileStoragePathsExist(callback) { + fs.access(idDir, fs.constants.F_OK, (err1) => { + fs.access(idTypeDir, fs.constants.F_OK, (err2) => { + fs.access(eventDir, fs.constants.F_OK, (err3) => { + // If all err variables are null, all files exist + const allFilesExist = !err1 && !err2 && !err3; + callback(allFilesExist); + }); + }); + }); +} function clearStorage(keepID = false, isBulk = false, customDir = '') { // Resets Countly Countly.halt(true); // Determine the directory based on isBulk or customDir const eventDirectory = customDir || (isBulk ? bulkEventDir : eventDir); const reqDirectory = customDir || (isBulk ? bulkQueueDir : reqDir); - // Remove event directory if it exists - if (fs.existsSync(eventDirectory)) { - fs.rmSync(eventDirectory, { recursive: true, force: true }); + // Helper function to remove directory and files + function removeDir(directory) { + if (fs.existsSync(directory)) { + fs.rmSync(directory, { recursive: true, force: true }); + } } + // Remove event directory if it exists + removeDir(eventDirectory); // Remove request directory if it exists - if (fs.existsSync(reqDirectory)) { - fs.rmSync(reqDirectory, { recursive: true, force: true }); - } + removeDir(reqDirectory); // Optionally keep the ID directory - if (!keepID && fs.existsSync(idDir)) { - fs.rmSync(idDir, { recursive: true, force: true }); + if (!keepID) { + removeDir(idDir); + removeDir(idTypeDir); } } /** @@ -215,4 +230,5 @@ module.exports = { sessionRequestValidator, userDetailRequestValidator, viewEventValidator, + doesFileStoragePathsExist, }; \ No newline at end of file diff --git a/test/tests_storage.js b/test/tests_storage.js index 41124b7..bc7d5e1 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -310,6 +310,9 @@ describe("Storage Tests", () => { clear_stored_device_id: true, storage_type: StorageTypes.MEMORY, }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); assert.equal(storage.getStoragePath(), undefined); assert.equal(storage.storeGet("cly_id", null), "Test-Device-Id"); assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); @@ -325,6 +328,9 @@ describe("Storage Tests", () => { clear_stored_device_id: true, storage_type: StorageTypes.MEMORY, }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); Countly.add_event(eventObj); setTimeout(() => { const storedData = storage.storeGet("cly_queue", null); @@ -344,6 +350,9 @@ describe("Storage Tests", () => { clear_stored_device_id: true, storage_type: StorageTypes.MEMORY, }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); Countly.user_details(userDetailObj); const storedData = storage.storeGet("cly_queue", null); const userDetailsReq = storedData[0]; @@ -359,6 +368,9 @@ describe("Storage Tests", () => { clear_stored_device_id: true, storage_type: StorageTypes.MEMORY, }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); assert.equal(storage.getStoragePath(), undefined); assert.equal(storage.storeGet("cly_id", null), Countly.get_device_id()); assert.equal(storage.storeGet("cly_id_type", null), Countly.get_device_id_type()); @@ -379,6 +391,9 @@ describe("Storage Tests", () => { }); assert.equal(storage.getStoragePath(), undefined); assert.equal(storage.getStorageType(), StorageTypes.MEMORY); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); @@ -387,7 +402,7 @@ describe("Storage Tests", () => { }); it("23- storeRemove Memory Only /no-init", (done) => { - storage.resetStorage(); + hp.clearStorage(); storage.initStorage(null, StorageTypes.MEMORY); assert.equal(storage.getStoragePath(), undefined); assert.equal(storage.getStorageType(), StorageTypes.MEMORY); @@ -400,7 +415,7 @@ describe("Storage Tests", () => { }); it("24- storeRemove File Storage /no-init", (done) => { - storage.resetStorage(); + hp.clearStorage(); storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); assert.equal(storage.getStorageType(), StorageTypes.FILE); From 9cb8700f5a4cd0d119e1f294b01a2ff202b103ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 00:30:50 +0300 Subject: [PATCH 09/13] Update tests_internal_limits.js these were failing for a reason, updated the checks --- test/tests_internal_limits.js | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/test/tests_internal_limits.js b/test/tests_internal_limits.js index 5d38307..712da5d 100644 --- a/test/tests_internal_limits.js +++ b/test/tests_internal_limits.js @@ -79,9 +79,10 @@ describe("Testing internal limits", () => { assert.ok(event.segmentation["key of 3"]); assert.ok(!event.segmentation["key of 4"]); assert.equal(event.segmentation["key of 3"], "Value of"); - assert.ok(event.timestamp); - assert.ok(event.hour); - assert.ok(event.dow); + // common parameter validation + assert.ok(event.timestamp !== 'undefined'); + assert.ok(event.hour !== 'undefined'); + assert.ok(event.dow !== 'undefined'); done(); }, hp.sWait); }); @@ -101,9 +102,10 @@ describe("Testing internal limits", () => { assert.equal(event.segmentation.name, "a very l"); assert.equal(event.segmentation.visit, 1); assert.ok(event.segmentation.segment); - assert.ok(event.timestamp); - assert.ok(event.hour); - assert.ok(event.dow); + // common parameter validation + assert.ok(event.timestamp !== 'undefined'); + assert.ok(event.hour !== 'undefined'); + assert.ok(event.dow !== 'undefined'); done(); }, hp.sWait); }); @@ -134,9 +136,10 @@ describe("Testing internal limits", () => { assert.ok(req.device_id); assert.ok(req.sdk_name); assert.ok(req.sdk_version); - assert.ok(req.timestamp); - assert.ok(req.hour); - assert.ok(req.dow); + // common parameter validation + assert.ok(req.timestamp !== 'undefined'); + assert.ok(req.hour !== 'undefined'); + assert.ok(req.dow !== 'undefined'); var crash = JSON.parse(req.crash); assert.equal(crash._logs, "log5 too\nlog6\nlog7"); assert.ok(crash._os); @@ -184,9 +187,10 @@ describe("Testing internal limits", () => { assert.ok(req.device_id); assert.ok(req.sdk_name); assert.ok(req.sdk_version); - assert.ok(req.timestamp); - assert.ok(req.hour); - assert.ok(req.dow); + // common parameter validation + assert.ok(req.timestamp !== 'undefined'); + assert.ok(req.hour !== 'undefined'); + assert.ok(req.dow !== 'undefined'); var details = JSON.parse(req.user_details); assert.equal(details.name, 'Gottlob '); assert.equal(details.username, 'Grundges'); @@ -229,9 +233,10 @@ describe("Testing internal limits", () => { assert.ok(req.device_id); assert.ok(req.sdk_name); assert.ok(req.sdk_version); - assert.ok(req.timestamp); - assert.ok(req.hour); - assert.ok(req.dow); + // common parameter validation + assert.ok(req.timestamp !== 'undefined'); + assert.ok(req.hour !== 'undefined'); + assert.ok(req.dow !== 'undefined'); var details = JSON.parse(req.user_details).custom; // set assert.equal(details['name of '], 'Bertrand'); From 4dfbaa7357f309deee9117c0958bc574e1405e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 00:34:03 +0300 Subject: [PATCH 10/13] Changelog update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7cb28f..6311fef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## xx.xx.xx - Default max segmentation value count changed from 30 to 100 - Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file. +- Added a new init time config option (conf.storage_type) which enables Memory Only Storage. ## 22.06.0 - Fixed a bug where remote config requests were rejected From caf69b5c044922896473650839e06cf655ffff21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 09:20:08 +0300 Subject: [PATCH 11/13] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6311fef..bc0c778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ ## xx.xx.xx - Default max segmentation value count changed from 30 to 100 - Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file. -- Added a new init time config option (conf.storage_type) which enables Memory Only Storage. +- Added a new init time config option (conf.storage_type) which enables Memory Only Storage and File Storage. ## 22.06.0 - Fixed a bug where remote config requests were rejected From eb61043f0336549940658942c518b44364f8c623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 09:24:37 +0300 Subject: [PATCH 12/13] Update CHANGELOG.md --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc0c778..d78b2ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ ## xx.xx.xx - Default max segmentation value count changed from 30 to 100 - Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file. -- Added a new init time config option (conf.storage_type) which enables Memory Only Storage and File Storage. +- Added a new init time config option (conf.storage_type) which can make user set among these storage options: + - File Storage + - Memory Only Storage ## 22.06.0 - Fixed a bug where remote config requests were rejected From ae89395c9de7bd5e275e10916d43f088a03a8a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 09:49:59 +0300 Subject: [PATCH 13/13] Back to Unlink --- lib/countly-storage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index f5782fa..27e1a04 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -107,7 +107,7 @@ const fileStorage = { cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`); return; } - fs.rmSync(filePath, { recursive: true, force: true }, (err) => { + fs.unlink(filePath, (err) => { if (err) { cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`); }