Memory Only Storage Option

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## xx.xx.xx
-* Default max segmentation value count changed from 30 to 100
+- Default max segmentation value count changed from 30 to 100
+- Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file.
+- Added a new init time config option (conf.storage_type) which enables Memory Only Storage.
 
 ## 22.06.0
 - Fixed a bug where remote config requests were rejected

lib/countly-bulk.js

@@ -25,6 +25,8 @@ var cc = require("./countly-common");
 var BulkUser = require("./countly-bulk-user");
 var CountlyStorage = require("./countly-storage");
 
+const StorageTypes = cc.storageTypeEnums;
+
 /**
  * @lends module:lib/countly-bulk
  * Initialize CountlyBulk server object
@@ -47,6 +49,7 @@ var CountlyStorage = require("./countly-storage");
  * @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted
  * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread
  * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length
+ * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
  * @example
  * var server = new CountlyBulk({
  *   app_key: "{YOUR-API-KEY}",
@@ -72,6 +75,7 @@ function CountlyBulk(conf) {
     var maxBreadcrumbCount = 100;
     var maxStackTraceLinesPerThread = 30;
     var maxStackTraceLineLength = 200;
+    var storageType = StorageTypes.FILE;
 
     cc.debugBulk = conf.debug || false;
     if (!conf.app_key) {
@@ -101,8 +105,9 @@ function CountlyBulk(conf) {
     conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount;
     conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
     conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength;
+    conf.storage_type = conf.storage_type || storageType;
 
-    CountlyStorage.setStoragePath(conf.storage_path, true, conf.persist_queue);
+    CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue);
 
     this.conf = conf;
     /**

lib/countly-common.js

@@ -51,6 +51,12 @@ var cc = {
         ACTION: '[CLY]_action',
     },
 
+    storageTypeEnums: {
+        FILE: "file",
+        MEMORY: "memory",
+        CUSTOM: "custom",
+    },
+
     /**
      *  Get current timestamp
      *  @returns {number} unix timestamp in seconds

lib/countly-storage.js

@@ -8,6 +8,133 @@ var defaultPath = "../data/"; // Default path
 var defaultBulkPath = "../bulk_data/"; // Default path
 var asyncWriteLock = false;
 var asyncWriteQueue = [];
+let storageMethod = {};
+const StorageTypes = cc.storageTypeEnums;
+
+// Memory-only storage methods
+const memoryStorage = {
+    /**
+     *  Save value in memory
+     *  @param {String} key - key for value to store
+     *  @param {varies} value - value to store
+     *  @param {Function} callback - callback to call when done storing
+     */
+    storeSet: function(key, value, callback) {
+        if (key) {
+            cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting key: [${key}] & value: [${value}]!`);
+            __data[key] = value;
+            if (typeof callback === "function") {
+                callback(null);
+            }
+        }
+        else {
+            cc.log(cc.logLevelEnums.WARNING, `storeSet, Provioded key: [${key}] is null!`);
+        }
+    },
+    /**
+     *  Get value from memory
+     *  @param {String} key - key of value to get
+     *  @param {varies} def - default value to use if not set
+     *  @returns {varies} value for the key
+     */
+    storeGet: function(key, def) {
+        cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`);
+        return typeof __data[key] !== "undefined" ? __data[key] : def;
+    },
+    /**
+     *  Remove value from memory
+     *  @param {String} key - key of value to remove
+     */
+    storeRemove: function(key) {
+        delete __data[key];
+    },
+};
+
+// File storage methods
+const fileStorage = {
+    /**
+     *  Save value in storage
+     *  @param {String} key - key for value to store
+     *  @param {varies} value - value to store
+     *  @param {Function} callback - callback to call when done storing
+     */
+    storeSet: function(key, value, callback) {
+        __data[key] = value;
+        if (!asyncWriteLock) {
+            asyncWriteLock = true;
+            writeFile(key, value, callback);
+        }
+        else {
+            asyncWriteQueue.push([key, value, callback]);
+        }
+    },
+    /**
+     *  Get value from storage
+     *  @param {String} key - key of value to get
+     *  @param {varies} def - default value to use if not set
+     *  @returns {varies} value for the key
+     */
+    storeGet: function(key, def) {
+        cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`);
+        if (typeof __data[key] === "undefined") {
+            var ob = readFile(key);
+            var obLen;
+            // check if the 'read object' is empty or not
+            try {
+                obLen = Object.keys(ob).length;
+            }
+            catch (error) {
+                // if we can not even asses length set it to 0 so we can return the default value
+                obLen = 0;
+            }
+
+            // if empty or falsy set default value
+            if (!ob || obLen === 0) {
+                __data[key] = def;
+            }
+            // else set the value read file has
+            else {
+                __data[key] = ob[key];
+            }
+        }
+        return __data[key];
+    },
+    storeRemove: function(key) {
+        delete __data[key];
+        var filePath = path.resolve(__dirname, `${getStoragePath()}__${key}.json`);
+        fs.access(filePath, fs.constants.F_OK, (accessErr) => {
+            if (accessErr) {
+                cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`);
+                return;
+            }
+            fs.rmSync(filePath, { recursive: true, force: true }, (err) => {
+                if (err) {
+                    cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`);
+                }
+                else {
+                    cc.log(cc.logLevelEnums.INFO, `storeRemove, Successfully removed file with key: [${key}].`);
+                }
+            });
+        });
+    },
+};
+
+/**
+ * Sets the storage method, by default sets file storage and storage path.
+ * @param {String} userPath - User provided storage path
+ * @param {StorageTypes} storageType - Whether to use memory only storage or not
+ * @param {Boolean} isBulk - Whether the storage is for bulk data
+ * @param {Boolean} persistQueue - Whether to persist the queue until processed
+ */
+var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false) {
+    if (storageType === StorageTypes.MEMORY) {
+        storageMethod = memoryStorage;
+    }
+    else {
+        storageMethod = fileStorage;
+        setStoragePath(userPath, isBulk, persistQueue);
+    }
+};
 
 /**
  * Sets the storage path, defaulting to a specified path if none is provided.
@@ -63,15 +190,21 @@ var resetStorage = function() {
  */
 var readFile = function(key) {
     var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`);
-
     // try reading data file
     var data;
     try {
-        data = fs.readFileSync(dir);
+        data = fs.readFileSync(dir, 'utf8'); // read file as string
     }
     catch (ex) {
         // there was no file, probably new init
-        data = null;
+        cc.log(cc.logLevelEnums.WARN, `readFile, File not found for key: [${key}]. Returning null.`);
+        return null; // early exit if file doesn't exist
+    }
+
+    // early exit if file is empty or whitespace
+    if (!data.trim()) {
+        cc.log(cc.logLevelEnums.WARN, `readFile, File is empty or contains only whitespace for key: [${key}]. Returning null.`);
+        return null;
     }
 
     try {
@@ -84,7 +217,7 @@ var readFile = function(key) {
         // backup corrupted file data
         fs.writeFile(path.resolve(__dirname, `${getStoragePath()}__${key}.${cc.getTimestamp()}${Math.random()}.json`), data, () => { });
         // start with new clean object
-        data = null;
+        return null; // return null in case of corrupted data
     }
     return data;
 };
@@ -139,62 +272,39 @@ var writeFile = function(key, value, callback) {
     });
 };
 
-/**
- *  Save value in storage
- *  @param {String} key - key for value to store
- *  @param {varies} value - value to store
- *  @param {Function} callback - callback to call when done storing
- */
 var storeSet = function(key, value, callback) {
-    __data[key] = value;
-    if (!asyncWriteLock) {
-        asyncWriteLock = true;
-        writeFile(key, value, callback);
-    }
-    else {
-        asyncWriteQueue.push([key, value, callback]);
-    }
+    storageMethod.storeSet(key, value, callback);
 };
 
-/**
- *  Get value from storage
- *  @param {String} key - key of value to get
- *  @param {varies} def - default value to use if not set
- *  @returns {varies} value for the key
- */
 var storeGet = function(key, def) {
-    cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`);
-    if (typeof __data[key] === "undefined") {
-        var ob = readFile(key);
-        var obLen;
-        // check if the 'read object' is empty or not
-        try {
-            obLen = Object.keys(ob).length;
-        }
-        catch (error) {
-            // if we can not even asses length set it to 0 so we can return the default value
-            obLen = 0;
-        }
+    return storageMethod.storeGet(key, def);
+};
 
-        // if empty or falsy set default value
-        if (!ob || obLen === 0) {
-            __data[key] = def;
-        }
-        // else set the value read file has
-        else {
-            __data[key] = ob[key];
-        }
+var storeRemove = function(key) {
+    storageMethod.storeRemove(key);
+};
+
+/**
+ * Disclaimer: This method is mainly for testing purposes.
+ * @returns {StorageTypes} Returns the active storage type for the SDK
+ */
+var getStorageType = function() {
+    if (storageMethod === memoryStorage) {
+        return StorageTypes.MEMORY;
     }
-    return __data[key];
+    return StorageTypes.FILE;
 };
 
 module.exports = {
-    writeFile,
-    storeGet,
+    initStorage,
     storeSet,
+    storeGet,
+    storeRemove,
+    writeFile,
     forceStore,
     getStoragePath,
     setStoragePath,
     resetStorage,
     readFile,
+    getStorageType,
 };

lib/countly.js

@@ -29,6 +29,7 @@ var Bulk = require("./countly-bulk");
 var CountlyStorage = require("./countly-storage");
 
 var Countly = {};
+const StorageTypes = cc.storageTypeEnums;
 
 Countly.Bulk = Bulk;
 (function() {
@@ -71,7 +72,7 @@ Countly.Bulk = Bulk;
     var maxStackTraceLinesPerThread = 30;
     var maxStackTraceLineLength = 200;
     var deviceIdType = null;
-
+    var storageType = StorageTypes.FILE;
     /**
     * Array with list of available features that you can require consent for
     */
@@ -122,6 +123,7 @@ Countly.Bulk = Bulk;
  * @param {string} conf.metrics._density - screen density of the device
  * @param {string} conf.metrics._locale - locale or language of the device in ISO format
  * @param {string} conf.metrics._store - source from where the user/device/installation came from
+ * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied
  * @example
  * Countly.init({
  *   app_key: "{YOUR-APP-KEY}",
@@ -166,12 +168,11 @@ Countly.Bulk = Bulk;
             Countly.maxBreadcrumbCount = conf.max_breadcrumb_count || Countly.max_breadcrumb_count || conf.max_logs || Countly.max_logs || maxBreadcrumbCount;
             Countly.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || Countly.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread;
             Countly.maxStackTraceLineLength = conf.max_stack_trace_line_length || Countly.max_stack_trace_line_length || maxStackTraceLineLength;
-
+            conf.storage_type = conf.storage_type || storageType;
             // Common module debug value is set to init time debug value
             cc.debug = conf.debug;
 
-            // Set the storage path
-            CountlyStorage.setStoragePath(conf.storage_path);
+            CountlyStorage.initStorage(conf.storage_path, conf.storage_type);
 
             // clear stored device ID if flag is set
             if (conf.clear_stored_device_id) {