From 89f80d2fb1221d52669b1f13d221178bba1f4f34 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Tue, 18 Apr 2023 18:55:28 +0900 Subject: [PATCH 01/58] Max segmentation (#80) * mac * log * Update CHANGELOG.md --------- Co-authored-by: ArtursKadikis --- CHANGELOG.md | 3 +++ lib/countly-bulk-user.js | 2 +- lib/countly-bulk.js | 2 +- lib/countly.js | 4 ++-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d87f4cf..c63f40f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## xx.xx.xx +* Default max segmentation value count changed from 30 to 100 + ## 22.06.0 - Fixed a bug where remote config requests were rejected - Fixed a bug where empty storage object did cause some issues diff --git a/lib/countly-bulk-user.js b/lib/countly-bulk-user.js index e32aace..0f45a42 100644 --- a/lib/countly-bulk-user.js +++ b/lib/countly-bulk-user.js @@ -49,7 +49,7 @@ function CountlyBulkUser(conf) { var sessionStart = 0; var maxKeyLength = 128; var maxValueSize = 256; - var maxSegmentationValues = 30; + var maxSegmentationValues = 100; var maxBreadcrumbCount = 100; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 52e78e3..6efed27 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -69,7 +69,7 @@ function CountlyBulk(conf) { var readyToProcess = true; var maxKeyLength = 128; var maxValueSize = 256; - var maxSegmentationValues = 30; + var maxSegmentationValues = 100; var maxBreadcrumbCount = 100; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; diff --git a/lib/countly.js b/lib/countly.js index 9d39451..b217d21 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -67,7 +67,7 @@ Countly.Bulk = Bulk; var startTime; var maxKeyLength = 128; var maxValueSize = 256; - var maxSegmentationValues = 30; + var maxSegmentationValues = 100; var maxBreadcrumbCount = 100; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; @@ -324,7 +324,7 @@ Countly.Bulk = Bulk; lastParams = {}; maxKeyLength = 128; maxValueSize = 256; - maxSegmentationValues = 30; + maxSegmentationValues = 100; maxBreadcrumbCount = 100; maxStackTraceLinesPerThread = 30; maxStackTraceLineLength = 200; From 5b980a7dc7ab3f68c81719dcc2c328b3f0ad3ee9 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Tue, 3 Oct 2023 00:40:41 +0900 Subject: [PATCH 02/58] flex added (#86) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a2d560d..b2ebbe4 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # Countly NodeJS SDK -This repository contains the Countly NodeJS SDK, which can be integrated into NodeJS running device or server. The Countly NodeJS SDK is intended to be used with [Countly Community Edition](https://github.com/Countly/countly-server) or [Countly Enterprise Edition](https://count.ly/product). +This repository contains the Countly NodeJS SDK, which can be integrated into NodeJS running device or server. The Countly NodeJS SDK is intended to be used with [Countly Lite](https://countly.com/lite), [Countly Flex](https://countly.com/flex) or [Countly Enterprise](https://countly.com/enterprise). ## What is Countly? [Countly](https://count.ly) is a product analytics solution and innovation enabler that helps teams track product performance and customer journey and behavior across [mobile](https://count.ly/mobile-analytics), [web](http://count.ly/web-analytics), and [desktop](https://count.ly/desktop-analytics) applications. [Ensuring privacy by design](https://count.ly/privacy-by-design), Countly allows you to innovate and enhance your products to provide personalized and customized customer experiences, and meet key business and revenue goals. @@ -12,7 +12,7 @@ and [desktop](https://count.ly/desktop-analytics) applications. [Ensuring privac Track, measure, and take action - all without leaving Countly. * **Questions or feature requests?** [Join the Countly Community on Discord](https://discord.gg/countly) -* **Looking for the Countly Server?** [Countly Community Edition repository](https://github.com/Countly/countly-server) +* **Looking for the Countly Server?** [Countly Server repository](https://github.com/Countly/countly-server) * **Looking for other Countly SDKs?** [An overview of all Countly SDKs for mobile, web and desktop](https://support.count.ly/hc/en-us/articles/360037236571-Downloading-and-Installing-SDKs#officially-supported-sdks) ## Integrating Countly SDK in your projects From 7052ded24d43fb886842ca79aa4d4297fff79ba0 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Wed, 11 Oct 2023 23:08:07 +0900 Subject: [PATCH 03/58] readme change for insecure link (#87) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b2ebbe4..9335e6a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This repository contains the Countly NodeJS SDK, which can be integrated into NodeJS running device or server. The Countly NodeJS SDK is intended to be used with [Countly Lite](https://countly.com/lite), [Countly Flex](https://countly.com/flex) or [Countly Enterprise](https://countly.com/enterprise). ## What is Countly? -[Countly](https://count.ly) is a product analytics solution and innovation enabler that helps teams track product performance and customer journey and behavior across [mobile](https://count.ly/mobile-analytics), [web](http://count.ly/web-analytics), +[Countly](https://count.ly) is a product analytics solution and innovation enabler that helps teams track product performance and customer journey and behavior across [mobile](https://count.ly/mobile-analytics), [web](https://count.ly/web-analytics), and [desktop](https://count.ly/desktop-analytics) applications. [Ensuring privacy by design](https://count.ly/privacy-by-design), Countly allows you to innovate and enhance your products to provide personalized and customized customer experiences, and meet key business and revenue goals. Track, measure, and take action - all without leaving Countly. From 20b40c560d4f382d0124f2e7083492285d2f8b65 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Mon, 30 Oct 2023 22:36:36 +0900 Subject: [PATCH 04/58] Update README.md (#88) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9335e6a..32478c1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Codacy Badge](https://app.codacy.com/project/badge/Grade/ed5f07ef6f8b4503ac01b2dab190de01)](https://www.codacy.com/gh/Countly/countly-sdk-nodejs/dashboard?utm_source=github.com&utm_medium=referral&utm_content=Countly/countly-sdk-nodejs&utm_campaign=Badge_Grade) +[![Codacy Badge](https://app.codacy.com/project/badge/Grade/ed5f07ef6f8b4503ac01b2dab190de01)](https://app.codacy.com/gh/Countly/countly-sdk-nodejs/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) [![npm version](https://badge.fury.io/js/countly-sdk-nodejs.svg)](https://badge.fury.io/js/countly-sdk-nodejs) [![Inline docs](https://inch-ci.org/github/Countly/countly-sdk-nodejs.svg?branch=master)](https://inch-ci.org/github/Countly/countly-sdk-nodejs) From deb884848ce3dfbc1f424c2cf60affa6ab6e74e4 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Thu, 4 Jan 2024 00:02:43 +0900 Subject: [PATCH 05/58] 1 (#89) --- .github/workflows/release_notice.yml | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/release_notice.yml diff --git a/.github/workflows/release_notice.yml b/.github/workflows/release_notice.yml new file mode 100644 index 0000000..e17315a --- /dev/null +++ b/.github/workflows/release_notice.yml @@ -0,0 +1,39 @@ +name: Release Notice +on: + release: + types: [published] + workflow_dispatch: +jobs: + build: + runs-on: ubuntu-latest + steps: + # To check the github context + - name: Dump Github context + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Send custom JSON data to Slack workflow + id: slack + uses: slackapi/slack-github-action@v1.23.0 + with: + # This data can be any valid JSON from a previous step in the GitHub Action + payload: | + { + "repository": "${{ github.repository }}", + "tag_name": "${{ github.event.release.tag_name }}", + "actor": "${{ github.actor }}", + "body": ${{ toJSON(github.event.release.body) }}, + "html_url": "${{ github.event.release.html_url }}" + } + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_RELEASE }} + - name: Send custom JSON data to Discord + uses: sarisia/actions-status-discord@v1.13.0 + with: + webhook: ${{ secrets.DISCORD_WEBHOOK_URL }} + nodetail: true + title: New ${{ github.repository }} version ${{ github.event.release.tag_name }} published by ${{ github.actor }} + description: | + Release URL: ${{ github.event.release.html_url }} + Click [here](https://github.com/Countly/countly-server/blob/master/CHANGELOG.md) to view the change log. + `${{ github.event.release.body }}` From da7465c58f9217fbd976ea04407b93a2471cbfe7 Mon Sep 17 00:00:00 2001 From: Arif Burak Demiray <57103426+arifBurakDemiray@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:34:40 +0000 Subject: [PATCH 06/58] [Node] Add warnings to all SDK example apps that would check if propper credentials have been set or it still has the defaults. (#90) * feat: add warning for defaults * feat: warn --- examples/apm_example.js | 11 +++++++++-- examples/bulk_import_example.js | 11 +++++++++-- examples/example.js | 12 +++++++++--- examples/multi-process.js | 11 +++++++++-- 4 files changed, 36 insertions(+), 9 deletions(-) diff --git a/examples/apm_example.js b/examples/apm_example.js index be2da01..93d9163 100644 --- a/examples/apm_example.js +++ b/examples/apm_example.js @@ -1,10 +1,17 @@ //since we need to test crashing the app /*global app*/ +const COUNTLY_SERVER_KEY = "https://your.server.ly"; +const COUNTLY_APP_KEY = "YOUR_APP_KEY"; + +if(COUNTLY_APP_KEY === "YOUR_APP_KEY" || COUNTLY_SERVER_KEY === "https://your.server.ly"){ + console.warn("Please do not use default set of app key and server url") +} + var Countly = require("../lib/countly.js"); Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", //your server goes here + app_key: COUNTLY_APP_KEY, + url: COUNTLY_SERVER_KEY, //your server goes here debug: true }); diff --git a/examples/bulk_import_example.js b/examples/bulk_import_example.js index 83da019..65a9d76 100644 --- a/examples/bulk_import_example.js +++ b/examples/bulk_import_example.js @@ -1,8 +1,15 @@ +const COUNTLY_SERVER_KEY = "https://your.server.ly"; +const COUNTLY_APP_KEY = "YOUR_APP_KEY"; + +if(COUNTLY_APP_KEY === "YOUR_APP_KEY" || COUNTLY_SERVER_KEY === "https://your.server.ly"){ + console.warn("Please do not use default set of app key and server url") +} + var CountlyBulk = require("../lib/countly").Bulk; var server = new CountlyBulk({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", //your server goes here + app_key: COUNTLY_APP_KEY, + url: COUNTLY_SERVER_KEY, //your server goes here debug: true }); diff --git a/examples/example.js b/examples/example.js index 65fdd35..635f7e7 100644 --- a/examples/example.js +++ b/examples/example.js @@ -1,14 +1,20 @@ //since we need to test crashing the app /*global runthis, crashDaApp*/ +const COUNTLY_SERVER_KEY = "https://your.server.ly"; +const COUNTLY_APP_KEY = "YOUR_APP_KEY"; + +if(COUNTLY_APP_KEY === "YOUR_APP_KEY" || COUNTLY_SERVER_KEY === "https://your.server.ly"){ + console.warn("Please do not use default set of app key and server url") +} + var Countly = require("../lib/countly.js"); Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", //your server goes here + app_key: COUNTLY_APP_KEY, + url: COUNTLY_SERVER_KEY, //your server goes here debug: true }); - Countly.begin_session(); Countly.track_errors(); diff --git a/examples/multi-process.js b/examples/multi-process.js index 5e1e711..59086fe 100644 --- a/examples/multi-process.js +++ b/examples/multi-process.js @@ -13,11 +13,18 @@ else if (cluster.isWorker) { console.log("I am worker " + cluster.worker.id); } +const COUNTLY_SERVER_KEY = "https://your.server.ly"; +const COUNTLY_APP_KEY = "YOUR_APP_KEY"; + +if(COUNTLY_APP_KEY === "YOUR_APP_KEY" || COUNTLY_SERVER_KEY === "https://your.server.ly"){ + console.warn("Please do not use default set of app key and server url") +} + var Countly = require("../lib/countly.js"); Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", //your server goes here + app_key: COUNTLY_APP_KEY, + url: COUNTLY_SERVER_KEY, //your server goes here debug: true }); From 9f72459e1f56a671f110756650d77a29983e11e2 Mon Sep 17 00:00:00 2001 From: Arif Burak Demiray <57103426+arifBurakDemiray@users.noreply.github.com> Date: Mon, 4 Mar 2024 13:07:28 +0000 Subject: [PATCH 07/58] Update codeql-analysis.yml (#91) --- .github/workflows/codeql-analysis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d283622..b9bab18 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,10 +13,10 @@ name: "CodeQL" on: push: - branches: [ master ] + branches: [ master, staging ] pull_request: # The branches below must be a subset of the branches above - branches: [ master ] + branches: [ master, staging ] schedule: - cron: '30 5 * * 4' @@ -39,11 +39,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v3 From 05c3c6527a9358479a21c3e2d729a84d77db0aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 12 Aug 2024 18:00:38 +0300 Subject: [PATCH 08/58] Added Storage Module --- lib/countly-bulk.js | 134 ++++-------------------------- lib/countly-storage.js | 167 +++++++++++++++++++++++++++++++++++++ lib/countly.js | 184 ++++++----------------------------------- 3 files changed, 208 insertions(+), 277 deletions(-) create mode 100644 lib/countly-storage.js diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 6efed27..1799654 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -25,6 +25,7 @@ var https = require("https"); var cluster = require("cluster"); var cc = require("./countly-common"); var BulkUser = require("./countly-bulk-user"); +var CountlyStorage = require("./countly-storage"); /** * @lends module:lib/countly-bulk @@ -40,7 +41,7 @@ var BulkUser = require("./countly-bulk-user"); * @param {number} [conf.max_events=100] - maximum amount of events to send in one batch * @param {boolean} [conf.persist_queue=false] - persistently store queue until processed, default is false if you want to keep queue in memory and process all in one process run * @param {boolean} [conf.force_post=false] - force using post method for all requests - * @param {string} [conf.storage_path="../bulk_data/"] - where SDK would store data, including id, queues, etc + * @param {string} [conf.storage_path] - where SDK would store data, including id, queues, etc * @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request * @param {number} [conf.max_key_length=128] - maximum size of all string keys * @param {number} [conf.max_value_size=256] - maximum size of all values in our key-value pairs (Except "picture" field, that has a limit of 4096 chars) @@ -73,7 +74,6 @@ function CountlyBulk(conf) { var maxBreadcrumbCount = 100; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; - var __data = {}; cc.debugBulk = conf.debug || false; if (!conf.app_key) { @@ -96,7 +96,6 @@ function CountlyBulk(conf) { conf.max_events = conf.max_events || 100; conf.force_post = conf.force_post || false; conf.persist_queue = conf.persist_queue || false; - conf.storage_path = conf.storage_path || "../bulk_data/"; conf.http_options = conf.http_options || null; conf.maxKeyLength = conf.max_key_length || maxKeyLength; conf.maxValueSize = conf.max_value_size || maxValueSize; @@ -105,7 +104,9 @@ function CountlyBulk(conf) { conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; - var mainDir = path.resolve(__dirname, conf.storage_path); + CountlyStorage.setBulkDataPath(conf.storage_path); + + var mainDir = path.resolve(__dirname, CountlyStorage.getStoragePath()); if (conf.persist_queue) { try { if (!fs.existsSync(mainDir)) { @@ -157,7 +158,7 @@ function CountlyBulk(conf) { requestQueue.push(query); cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_request, Adding request to the queue."); - storeSet("cly_req_queue", requestQueue); + CountlyStorage.storeSet("cly_req_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_request, Sending message to the parent process. Adding the raw request to the queue."); @@ -205,7 +206,7 @@ function CountlyBulk(conf) { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_bulk_request, adding the request into queue."); requestQueue.push(query); } - storeSet("cly_req_queue", requestQueue); + CountlyStorage.storeSet("cly_req_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_bulk_request, Sending message to the parent process. Adding the raw request to the queue."); @@ -260,7 +261,7 @@ function CountlyBulk(conf) { eventQueue[device_id] = []; } eventQueue[device_id].push(e); - storeSet("cly_bulk_event", eventQueue); + CountlyStorage.storeSet("cly_bulk_event", eventQueue); } else { cc.log(cc.logLevelEnums.INFO, `CountlyBulk add_event, Sending message to the parent process. Adding event: [${event.key}].`); @@ -358,7 +359,7 @@ function CountlyBulk(conf) { */ function toBulkRequestQueue(bulkRequest) { bulkQueue.push(bulkRequest); - storeSet("cly_bulk_queue", bulkQueue); + CountlyStorage.storeSet("cly_bulk_queue", bulkQueue); } var self = this; @@ -384,7 +385,7 @@ function CountlyBulk(conf) { } if (eventChanges) { isEmpty = false; - storeSet("cly_bulk_event", eventQueue); + CountlyStorage.storeSet("cly_bulk_event", eventQueue); } // process request queue into bulk requests @@ -398,7 +399,7 @@ function CountlyBulk(conf) { var requests = requestQueue.splice(0, conf.bulk_size); toBulkRequestQueue({ app_key: conf.app_key, requests: JSON.stringify(requests) }); } - storeSet("cly_req_queue", requestQueue); + CountlyStorage.storeSet("cly_req_queue", requestQueue); } // process bulk request queue @@ -413,7 +414,7 @@ function CountlyBulk(conf) { bulkQueue.unshift(res); failTimeout = cc.getTimestamp() + conf.fail_timeout; } - storeSet("cly_bulk_queue", bulkQueue); + CountlyStorage.storeSet("cly_bulk_queue", bulkQueue); readyToProcess = true; }, "heartBeat", false); } @@ -594,111 +595,6 @@ function CountlyBulk(conf) { } } - /** - * Read value from file - * @param {String} key - key for file - * @returns {varies} value in file - */ - var readFile = function(key) { - var data; - if (conf.persist_queue) { - var dir = path.resolve(__dirname, `${conf.storage_path}__${key}.json`); - - // try reading data file - try { - data = fs.readFileSync(dir); - } - catch (ex) { - // there was no file, probably new init - cc.log(cc.logLevelEnums.ERROR, "CountlyBulk readFile, Nothing to read. Might be first init. Error: ", ex); - data = null; - } - - try { - // trying to parse json string - data = JSON.parse(data); - } - catch (ex) { - // problem parsing, corrupted file? - cc.log(cc.logLevelEnums.ERROR, "CountlyBulk readFile, Problem while parsing. Error:", ex.stack); - // backup corrupted file data - fs.writeFile(path.resolve(__dirname, `${conf.storage_path}__${key}.${cc.getTimestamp()}${Math.random()}.json`), data, () => {}); - // start with new clean object - data = null; - } - } - return data; - }; - - var asyncWriteLock = false; - var asyncWriteQueue = []; - - /** - * Write to file and process queue while in asyncWriteLock - * @param {String} key - key for value to store - * @param {varies} value - value to store - * @param {Function} callback - callback to call when done storing - */ - var writeFile = function(key, value, callback) { - var ob = {}; - ob[key] = value; - var dir = path.resolve(__dirname, `${conf.storage_path}__${key}.json`); - fs.writeFile(dir, JSON.stringify(ob), (err) => { - if (err) { - // eslint-disable-next-line no-console - cc.log(cc.logLevelEnums.ERROR, "CountlyBulk writeFile, Problem while writing. Error:", err); - } - if (typeof callback === "function") { - callback(err); - } - if (asyncWriteQueue.length) { - setTimeout(() => { - var arr = asyncWriteQueue.shift(); - writeFile(arr[0], arr[1], arr[2]); - }, 0); - } - else { - asyncWriteLock = false; - } - }); - }; - - /** - * Save value in storage - * @param {String} key - key for value to store - * @param {varies} value - value to store - * @param {Function} callback - callback to call when done storing - */ - var storeSet = function(key, value, callback) { - __data[key] = value; - if (!asyncWriteLock) { - asyncWriteLock = true; - writeFile(key, value, callback); - } - else { - asyncWriteQueue.push([key, value, callback]); - } - }; - - /** - * Get value from storage - * @param {String} key - key of value to get - * @param {varies} def - default value to use if not set - * @returns {varies} value for the key - */ - var storeGet = function(key, def) { - if (typeof __data[key] === "undefined") { - var ob = readFile(key); - if (!ob) { - __data[key] = def; - } - else { - __data[key] = ob[key]; - } - } - return __data[key]; - }; - // listen to current workers if (cluster.workers) { for (var id in cluster.workers) { @@ -711,9 +607,9 @@ function CountlyBulk(conf) { worker.on("message", handleWorkerMessage); }); - var requestQueue = storeGet("cly_req_queue", []); - var eventQueue = storeGet("cly_bulk_event", {}); - var bulkQueue = storeGet("cly_bulk_queue", []); + var requestQueue = CountlyStorage.storeGet("cly_req_queue", []); + var eventQueue = CountlyStorage.storeGet("cly_bulk_event", {}); + var bulkQueue = CountlyStorage.storeGet("cly_bulk_queue", []); } module.exports = CountlyBulk; diff --git a/lib/countly-storage.js b/lib/countly-storage.js new file mode 100644 index 0000000..272704f --- /dev/null +++ b/lib/countly-storage.js @@ -0,0 +1,167 @@ +const fs = require('fs'); +const path = require('path'); +var cc = require("./countly-common"); +var storagePath; +var __data = {}; + +var setStoragePath = function (path) { + defaultPath = "../data/"; // Default path + storagePath = path || defaultPath; +} + +var setBulkDataPath = function (path) { + defaultPath = "../bulk_data/"; // Default path + storagePath = path || defaultPath; +} + +var getStoragePath = function () { + return storagePath; +} + +var clearStoragePath = function () { + storagePath = undefined; +} + +/** + * Read value from file + * @param {String} key - key for file + * @returns {varies} value in file + */ +var readFile = function (key) { + var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); + + // try reading data file + var data; + try { + data = fs.readFileSync(dir); + } + catch (ex) { + // there was no file, probably new init + data = null; + } + + try { + // trying to parse json string + data = JSON.parse(data); + } + catch (ex) { + // problem parsing, corrupted file? + cc.log(cc.logLevelEnums.ERROR, `readFile, Failed to parse the file with key: [${key}]. Error: [${ex}].`); + // backup corrupted file data + fs.writeFile(path.resolve(__dirname, `${getStoragePath()}__${key}.${cc.getTimestamp()}${Math.random()}.json`), data, () => { }); + // start with new clean object + data = null; + } + return data; +}; + +/** + * Force store data synchronously on unrecoverable errors to preserve it for next launch + */ +var forceStore = function () { + for (var i in __data) { + var dir = path.resolve(__dirname, `${getStoragePath()}__${i}.json`); + var ob = {}; + ob[i] = __data[i]; + try { + fs.writeFileSync(dir, JSON.stringify(ob)); + } + catch (ex) { + // tried to save whats possible + cc.log(cc.logLevelEnums.ERROR, `forceStore, Saving files failed. Error: [${ex}].`); + } + } +}; + +var asyncWriteLock = false; +var asyncWriteQueue = []; + +/** + * Write to file and process queue while in asyncWriteLock + * @param {String} key - key for value to store + * @param {varies} value - value to store + * @param {Function} callback - callback to call when done storing + */ +var writeFile = function (key, value, callback) { + var ob = {}; + ob[key] = value; + var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); + fs.writeFile(dir, JSON.stringify(ob), (err) => { + if (err) { + cc.log(cc.logLevelEnums.ERROR, `writeFile, Writing files failed. Error: [${err}].`); + } + if (typeof callback === "function") { + callback(err); + } + if (asyncWriteQueue.length) { + setTimeout(() => { + var arr = asyncWriteQueue.shift(); + writeFile(arr[0], arr[1], arr[2]); + }, 0); + } + else { + asyncWriteLock = false; + } + }); +}; + +/** + * Save value in storage + * @param {String} key - key for value to store + * @param {varies} value - value to store + * @param {Function} callback - callback to call when done storing + */ +var storeSet = function (key, value, callback) { + __data[key] = value; + if (!asyncWriteLock) { + asyncWriteLock = true; + writeFile(key, value, callback); + } + else { + asyncWriteQueue.push([key, value, callback]); + } +}; + +/** + * Get value from storage + * @param {String} key - key of value to get + * @param {varies} def - default value to use if not set + * @returns {varies} value for the key + */ +var storeGet = function (key, def) { + cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); + if (typeof __data[key] === "undefined") { + var ob = readFile(key); + var obLen; + // check if the 'read object' is empty or not + try { + obLen = Object.keys(ob).length; + } + catch (error) { + // if we can not even asses length set it to 0 so we can return the default value + obLen = 0; + } + + // if empty or falsy set default value + if (!ob || obLen === 0) { + __data[key] = def; + } + // else set the value read file has + else { + __data[key] = ob[key]; + } + } + return __data[key]; +}; + +module.exports = { + writeFile, + storeGet, + storeSet, + forceStore, + getStoragePath, + setStoragePath, + setBulkDataPath, + clearStoragePath, + readFile, +}; \ No newline at end of file diff --git a/lib/countly.js b/lib/countly.js index b217d21..c475174 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -28,6 +28,7 @@ var https = require("https"); var cluster = require("cluster"); var cc = require("./countly-common"); var Bulk = require("./countly-bulk"); +var CountlyStorage = require("./countly-storage"); var Countly = {}; @@ -71,7 +72,6 @@ Countly.Bulk = Bulk; var maxBreadcrumbCount = 100; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; - var __data = {}; var deviceIdType = null; /** @@ -104,7 +104,7 @@ Countly.Bulk = Bulk; * @param {boolean} [conf.force_post=false] - force using post method for all requests * @param {boolean} [conf.clear_stored_device_id=false] - set it to true if you want to erase the stored device ID * @param {boolean} [conf.test_mode=false] - set it to true if you want to initiate test_mode - * @param {string} [conf.storage_path="../data/"] - where SDK would store data, including id, queues, etc + * @param {string} [conf.storage_path] - where SDK would store data, including id, queues, etc * @param {boolean} [conf.require_consent=false] - pass true if you are implementing GDPR compatible consent management. It would prevent running any functionality without proper consent * @param {boolean|function} [conf.remote_config=false] - Enable automatic remote config fetching, provide callback function to be notified when fetching done * @param {function} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request @@ -159,7 +159,6 @@ Countly.Bulk = Bulk; Countly.city = conf.city || Countly.city || null; Countly.ip_address = conf.ip_address || Countly.ip_address || null; Countly.force_post = conf.force_post || Countly.force_post || false; - Countly.storage_path = conf.storage_path || Countly.storage_path || "../data/"; Countly.require_consent = conf.require_consent || Countly.require_consent || false; Countly.remote_config = conf.remote_config || Countly.remote_config || false; Countly.http_options = conf.http_options || Countly.http_options || null; @@ -172,8 +171,9 @@ Countly.Bulk = Bulk; // Common module debug value is set to init time debug value cc.debug = conf.debug; + CountlyStorage.setStoragePath(conf.storage_path); - var dir = path.resolve(__dirname, Countly.storage_path); + var dir = path.resolve(__dirname, CountlyStorage.getStoragePath()); try { if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); @@ -186,8 +186,8 @@ Countly.Bulk = Bulk; // clear stored device ID if flag is set if (conf.clear_stored_device_id) { cc.log(cc.logLevelEnums.WARNING, "init, clear_stored_device_id is true, erasing the stored ID."); - storeSet("cly_id", null); - storeSet("cly_id_type", null); + CountlyStorage.storeSet("cly_id", null); + CountlyStorage.storeSet("cly_id_type", null); } if (Countly.url === "") { @@ -233,8 +233,8 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { // fetch stored ID and ID type - var storedId = storeGet("cly_id", null); - var storedIdType = storeGet("cly_id_type", null); + var storedId = CountlyStorage.storeGet("cly_id", null); + var storedIdType = CountlyStorage.storeGet("cly_id_type", null); // if there was a stored ID if (storedId !== null) { Countly.device_id = storedId; @@ -263,12 +263,12 @@ Countly.Bulk = Bulk; deviceIdType = cc.deviceIdTypeEnums.SDK_GENERATED; } // save the ID and ID type - storeSet("cly_id", Countly.device_id); - storeSet("cly_id_type", deviceIdType); + CountlyStorage.storeSet("cly_id", Countly.device_id); + CountlyStorage.storeSet("cly_id_type", deviceIdType); // create queues - requestQueue = storeGet("cly_queue", []); - eventQueue = storeGet("cly_event", []); - remoteConfigs = storeGet("cly_remote_configs", {}); + requestQueue = CountlyStorage.storeGet("cly_queue", []); + eventQueue = CountlyStorage.storeGet("cly_event", []); + remoteConfigs = CountlyStorage.storeGet("cly_remote_configs", {}); heartBeat(); // listen to current workers if (cluster.workers) { @@ -354,7 +354,7 @@ Countly.Bulk = Bulk; Countly.city = undefined; Countly.ip_address = undefined; Countly.force_post = undefined; - Countly.storage_path = undefined; + Countly.storage_path = CountlyStorage.clearStoragePath(); Countly.require_consent = undefined; Countly.http_options = undefined; @@ -649,7 +649,7 @@ Countly.Bulk = Bulk; if (eventQueue.length > 0) { toRequestQueue({ events: JSON.stringify(eventQueue) }); eventQueue = []; - storeSet("cly_event", eventQueue); + CountlyStorage.storeSet("cly_event", eventQueue); } // end current session Countly.end_session(); @@ -661,8 +661,8 @@ Countly.Bulk = Bulk; var oldId = Countly.device_id; Countly.device_id = newId; deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storeSet("cly_id", Countly.device_id); - storeSet("cly_id_type", deviceIdType); + CountlyStorage.storeSet("cly_id", Countly.device_id); + CountlyStorage.storeSet("cly_id_type", deviceIdType); if (merge) { if (Countly.check_any_consent()) { toRequestQueue({ old_device_id: oldId }); @@ -678,7 +678,7 @@ Countly.Bulk = Bulk; if (Countly.remote_config) { remoteConfigs = {}; if (cluster.isMaster) { - storeSet("cly_remote_configs", remoteConfigs); + CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); } Countly.fetch_remote_config(Countly.remote_config); } @@ -757,7 +757,7 @@ Countly.Bulk = Bulk; e.dow = date.getDay(); cc.log(cc.logLevelEnums.DEBUG, "add_cly_events, Adding event: ", event); eventQueue.push(e); - storeSet("cly_event", eventQueue); + CountlyStorage.storeSet("cly_event", eventQueue); } else { process.send({ cly: { event } }); @@ -1072,7 +1072,7 @@ Countly.Bulk = Bulk; process.on("uncaughtException", (err) => { recordError(err, false); if (cluster.isMaster) { - forceStore(); + CountlyStorage.forceStore(); } // eslint-disable-next-line no-console console.error(`${(new Date()).toUTCString()} uncaughtException:`, err.message); @@ -1085,7 +1085,7 @@ Countly.Bulk = Bulk; var err = new Error(`Unhandled rejection (reason: ${reason && reason.stack ? reason.stack : reason}).`); recordError(err, false); if (cluster.isMaster) { - forceStore(); + CountlyStorage.forceStore(); } // eslint-disable-next-line no-console console.error(`${(new Date()).toUTCString()} unhandledRejection:`, err.message); @@ -1168,7 +1168,7 @@ Countly.Bulk = Bulk; remoteConfigs = configs; } if (cluster.isMaster) { - storeSet("cly_remote_configs", remoteConfigs); + CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); cc.log(cc.logLevelEnums.INFO, `fetch_remote_config, Fetched remote config: [${remoteConfigs}].`); } } @@ -1352,7 +1352,7 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { cc.log(cc.logLevelEnums.INFO, "request, Adding the raw request to the queue."); requestQueue.push(request); - storeSet("cly_queue", requestQueue); + CountlyStorage.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "request, Sending message to the parent process. Adding the raw request to the queue."); @@ -1439,7 +1439,7 @@ Countly.Bulk = Bulk; cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Adding request to the queue."); requestQueue.push(request); - storeSet("cly_queue", requestQueue); + CountlyStorage.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Sending message to the parent process. Adding request to the queue."); @@ -1470,7 +1470,7 @@ Countly.Bulk = Bulk; var events = eventQueue.splice(0, maxEventBatch); toRequestQueue({ events: JSON.stringify(events) }); } - storeSet("cly_event", eventQueue); + CountlyStorage.storeSet("cly_event", eventQueue); } // process request queue with event queue @@ -1485,7 +1485,7 @@ Countly.Bulk = Bulk; failTimeout = cc.getTimestamp() + failTimeoutAmount; cc.log(cc.logLevelEnums.ERROR, `makeRequest, Encountered a problem while making the request: [${err}]`); } - storeSet("cly_queue", requestQueue); + CountlyStorage.storeSet("cly_queue", requestQueue); readyToProcess = true; }, "heartBeat", false); } @@ -1748,138 +1748,6 @@ Countly.Bulk = Bulk; } } } - - /** - * Read value from file - * @param {String} key - key for file - * @returns {varies} value in file - */ - var readFile = function(key) { - var dir = path.resolve(__dirname, `${Countly.storage_path}__${key}.json`); - - // try reading data file - var data; - try { - data = fs.readFileSync(dir); - } - catch (ex) { - // there was no file, probably new init - data = null; - } - - try { - // trying to parse json string - data = JSON.parse(data); - } - catch (ex) { - // problem parsing, corrupted file? - cc.log(cc.logLevelEnums.ERROR, `readFile, Failed to parse the file with key: [${key}]. Error: [${ex}].`); - // backup corrupted file data - fs.writeFile(path.resolve(__dirname, `${Countly.storage_path}__${key}.${cc.getTimestamp()}${Math.random()}.json`), data, () => {}); - // start with new clean object - data = null; - } - return data; - }; - - /** - * Force store data synchronously on unrecoverable errors to preserve it for next launch - */ - var forceStore = function() { - for (var i in __data) { - var dir = path.resolve(__dirname, `${Countly.storage_path}__${i}.json`); - var ob = {}; - ob[i] = __data[i]; - try { - fs.writeFileSync(dir, JSON.stringify(ob)); - } - catch (ex) { - // tried to save whats possible - cc.log(cc.logLevelEnums.ERROR, `forceStore, Saving files failed. Error: [${ex}].`); - } - } - }; - - var asyncWriteLock = false; - var asyncWriteQueue = []; - - /** - * Write to file and process queue while in asyncWriteLock - * @param {String} key - key for value to store - * @param {varies} value - value to store - * @param {Function} callback - callback to call when done storing - */ - var writeFile = function(key, value, callback) { - var ob = {}; - ob[key] = value; - var dir = path.resolve(__dirname, `${Countly.storage_path}__${key}.json`); - fs.writeFile(dir, JSON.stringify(ob), (err) => { - if (err) { - cc.log(cc.logLevelEnums.ERROR, `writeFile, Writing files failed. Error: [${err}].`); - } - if (typeof callback === "function") { - callback(err); - } - if (asyncWriteQueue.length) { - setTimeout(() => { - var arr = asyncWriteQueue.shift(); - writeFile(arr[0], arr[1], arr[2]); - }, 0); - } - else { - asyncWriteLock = false; - } - }); - }; - - /** - * Save value in storage - * @param {String} key - key for value to store - * @param {varies} value - value to store - * @param {Function} callback - callback to call when done storing - */ - var storeSet = function(key, value, callback) { - __data[key] = value; - if (!asyncWriteLock) { - asyncWriteLock = true; - writeFile(key, value, callback); - } - else { - asyncWriteQueue.push([key, value, callback]); - } - }; - - /** - * Get value from storage - * @param {String} key - key of value to get - * @param {varies} def - default value to use if not set - * @returns {varies} value for the key - */ - var storeGet = function(key, def) { - cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); - if (typeof __data[key] === "undefined") { - var ob = readFile(key); - var obLen; - // check if the 'read object' is empty or not - try { - obLen = Object.keys(ob).length; - } - catch (error) { - // if we can not even asses length set it to 0 so we can return the default value - obLen = 0; - } - - // if empty or falsy set default value - if (!ob || obLen === 0) { - __data[key] = def; - } - // else set the value read file has - else { - __data[key] = ob[key]; - } - } - return __data[key]; - }; }()); module.exports = Countly; From 863b1a31e40646f53c12048907dd91673fa410bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 12 Aug 2024 18:04:20 +0300 Subject: [PATCH 09/58] Update countly-storage.js --- lib/countly-storage.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 272704f..5edfdf8 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -20,6 +20,7 @@ var getStoragePath = function () { var clearStoragePath = function () { storagePath = undefined; + return storagePath; } /** From 8cb8b570adf97351efd000f6700a86bc2242d8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Wed, 14 Aug 2024 12:54:42 +0300 Subject: [PATCH 10/58] Requested Changes --- lib/countly-storage.js | 41 ++++++++++++++++++++++------------------- lib/countly.js | 8 ++------ 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 5edfdf8..e28e084 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -1,34 +1,37 @@ const fs = require('fs'); const path = require('path'); var cc = require("./countly-common"); + var storagePath; var __data = {}; +var defaultPath = "../data/"; // Default path +var defaultBulkPath = "../bulk_data/"; // Default path -var setStoragePath = function (path) { - defaultPath = "../data/"; // Default path - storagePath = path || defaultPath; -} +var setStoragePath = function(userPath) { + storagePath = userPath || defaultPath; +}; -var setBulkDataPath = function (path) { - defaultPath = "../bulk_data/"; // Default path - storagePath = path || defaultPath; -} +var setBulkDataPath = function(userPath) { + storagePath = userPath || defaultBulkPath; +}; -var getStoragePath = function () { +var getStoragePath = function() { return storagePath; -} +}; -var clearStoragePath = function () { +var resetStorage = function() { storagePath = undefined; - return storagePath; -} + __data = {}; + asyncWriteLock = false; + asyncWriteQueue = []; +}; /** * Read value from file * @param {String} key - key for file * @returns {varies} value in file */ -var readFile = function (key) { +var readFile = function(key) { var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); // try reading data file @@ -59,7 +62,7 @@ var readFile = function (key) { /** * Force store data synchronously on unrecoverable errors to preserve it for next launch */ -var forceStore = function () { +var forceStore = function() { for (var i in __data) { var dir = path.resolve(__dirname, `${getStoragePath()}__${i}.json`); var ob = {}; @@ -83,7 +86,7 @@ var asyncWriteQueue = []; * @param {varies} value - value to store * @param {Function} callback - callback to call when done storing */ -var writeFile = function (key, value, callback) { +var writeFile = function(key, value, callback) { var ob = {}; ob[key] = value; var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); @@ -112,7 +115,7 @@ var writeFile = function (key, value, callback) { * @param {varies} value - value to store * @param {Function} callback - callback to call when done storing */ -var storeSet = function (key, value, callback) { +var storeSet = function(key, value, callback) { __data[key] = value; if (!asyncWriteLock) { asyncWriteLock = true; @@ -129,7 +132,7 @@ var storeSet = function (key, value, callback) { * @param {varies} def - default value to use if not set * @returns {varies} value for the key */ -var storeGet = function (key, def) { +var storeGet = function(key, def) { cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); if (typeof __data[key] === "undefined") { var ob = readFile(key); @@ -163,6 +166,6 @@ module.exports = { getStoragePath, setStoragePath, setBulkDataPath, - clearStoragePath, + resetStorage, readFile, }; \ No newline at end of file diff --git a/lib/countly.js b/lib/countly.js index c475174..dffb611 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -220,7 +220,7 @@ Countly.Bulk = Bulk; cc.log(cc.logLevelEnums.DEBUG, `init, IP address: [${Countly.ip_address}].`); } cc.log(cc.logLevelEnums.DEBUG, `init, Force POST requests: [${Countly.force_post}].`); - cc.log(cc.logLevelEnums.DEBUG, `init, Storage path: [${Countly.storage_path}].`); + cc.log(cc.logLevelEnums.DEBUG, `init, Storage path: [${CountlyStorage.getStoragePath()}].`); cc.log(cc.logLevelEnums.DEBUG, `init, Require consent: [${Countly.require_consent}].`); if (Countly.remote_config) { cc.log(cc.logLevelEnums.DEBUG, `init, Automatic Remote Configuration is on.`); @@ -328,7 +328,6 @@ Countly.Bulk = Bulk; maxBreadcrumbCount = 100; maxStackTraceLinesPerThread = 30; maxStackTraceLineLength = 200; - __data = {}; deviceIdType = null; // cc DEBUG @@ -354,12 +353,9 @@ Countly.Bulk = Bulk; Countly.city = undefined; Countly.ip_address = undefined; Countly.force_post = undefined; - Countly.storage_path = CountlyStorage.clearStoragePath(); Countly.require_consent = undefined; Countly.http_options = undefined; - - asyncWriteLock = false; - asyncWriteQueue = []; + CountlyStorage.resetStorage(); }; /** From 1836f456a93a22af70c402632a6a87559d388c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Wed, 14 Aug 2024 13:02:55 +0300 Subject: [PATCH 11/58] Moved Functionality in Storage --- lib/countly-bulk.js | 16 +--------------- lib/countly-storage.js | 26 +++++++++++++++++++++++++- lib/countly.js | 10 ---------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 1799654..2f698d2 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -104,21 +104,7 @@ function CountlyBulk(conf) { conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; - CountlyStorage.setBulkDataPath(conf.storage_path); - - var mainDir = path.resolve(__dirname, CountlyStorage.getStoragePath()); - if (conf.persist_queue) { - try { - if (!fs.existsSync(mainDir)) { - fs.mkdirSync(mainDir); - } - } - catch (ex) { - // problem creating directory - // eslint-disable-next-line no-console - cc.log(cc.logLevelEnums.ERROR, "CountlyBulk, Failed white creating the '/data' directory. Error: ", ex.stack); - } - } + CountlyStorage.setBulkDataPath(conf.storage_path, conf.persist_queue); this.conf = conf; /** diff --git a/lib/countly-storage.js b/lib/countly-storage.js index e28e084..ccc449e 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -9,10 +9,34 @@ var defaultBulkPath = "../bulk_data/"; // Default path var setStoragePath = function(userPath) { storagePath = userPath || defaultPath; + + var dir = path.resolve(__dirname, getStoragePath()); + try { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + } + catch (ex) { + // problem creating data dir + cc.log(cc.logLevelEnums.ERROR, `setStoragePath, Failed to create the '/data' folder: ${ex}`); + } }; -var setBulkDataPath = function(userPath) { +var setBulkDataPath = function(userPath, persistQueue) { storagePath = userPath || defaultBulkPath; + var mainDir = path.resolve(__dirname, getStoragePath()); + if (persistQueue) { + try { + if (!fs.existsSync(mainDir)) { + fs.mkdirSync(mainDir); + } + } + catch (ex) { + // problem creating directory + // eslint-disable-next-line no-console + cc.log(cc.logLevelEnums.ERROR, "setBulkDataPath, Failed white creating the '/data' directory. Error: ", ex.stack); + } + } }; var getStoragePath = function() { diff --git a/lib/countly.js b/lib/countly.js index dffb611..cde604d 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -173,16 +173,6 @@ Countly.Bulk = Bulk; cc.debug = conf.debug; CountlyStorage.setStoragePath(conf.storage_path); - var dir = path.resolve(__dirname, CountlyStorage.getStoragePath()); - try { - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - } - catch (ex) { - // problem creating data dir - cc.log(cc.logLevelEnums.ERROR, `init, Failed to create the '/data' folder: ${ex}`); - } // clear stored device ID if flag is set if (conf.clear_stored_device_id) { cc.log(cc.logLevelEnums.WARNING, "init, clear_stored_device_id is true, erasing the stored ID."); From 54929568afcd59299ba94b37954abf7631231a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Wed, 14 Aug 2024 13:04:51 +0300 Subject: [PATCH 12/58] Deep Scan --- lib/countly-bulk.js | 2 -- lib/countly.js | 2 -- 2 files changed, 4 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 2f698d2..2b32d8e 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -18,8 +18,6 @@ * server.add_request({begin_session:1, metrics:{_os:"Linux"}, device_id:"users_device_id", events:[{key:"Test", count:1}]}); */ -var fs = require("fs"); -var path = require("path"); var http = require("http"); var https = require("https"); var cluster = require("cluster"); diff --git a/lib/countly.js b/lib/countly.js index cde604d..908c793 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -20,9 +20,7 @@ * Countly.begin_session(); */ -var fs = require("fs"); var os = require("os"); -var path = require("path"); var http = require("http"); var https = require("https"); var cluster = require("cluster"); From 4b81696fd3ffb1b9b43178c3954b5662060d83cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Wed, 14 Aug 2024 13:15:33 +0300 Subject: [PATCH 13/58] duplication clean up --- lib/countly-storage.js | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index ccc449e..01ded84 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -9,33 +9,15 @@ var defaultBulkPath = "../bulk_data/"; // Default path var setStoragePath = function(userPath) { storagePath = userPath || defaultPath; - var dir = path.resolve(__dirname, getStoragePath()); - try { - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - } - catch (ex) { - // problem creating data dir - cc.log(cc.logLevelEnums.ERROR, `setStoragePath, Failed to create the '/data' folder: ${ex}`); - } + createDirectory(dir); }; var setBulkDataPath = function(userPath, persistQueue) { storagePath = userPath || defaultBulkPath; - var mainDir = path.resolve(__dirname, getStoragePath()); + var dir = path.resolve(__dirname, getStoragePath()); if (persistQueue) { - try { - if (!fs.existsSync(mainDir)) { - fs.mkdirSync(mainDir); - } - } - catch (ex) { - // problem creating directory - // eslint-disable-next-line no-console - cc.log(cc.logLevelEnums.ERROR, "setBulkDataPath, Failed white creating the '/data' directory. Error: ", ex.stack); - } + createDirectory(dir); } }; @@ -43,6 +25,17 @@ var getStoragePath = function() { return storagePath; }; +var createDirectory = function(dir) { + try { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + } + catch (ex) { + cc.log(cc.logLevelEnums.ERROR, `Failed to create directory at ${dir}: ${ex.stack}`); + } +}; + var resetStorage = function() { storagePath = undefined; __data = {}; From caa873cce5e0fe014eed671d1d4291392b3ae6a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 19 Aug 2024 09:51:42 +0300 Subject: [PATCH 14/58] Storage Tests --- lib/countly-storage.js | 22 ++++++- lib/countly.js | 3 + test/tests_storage.js | 146 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 test/tests_storage.js diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 01ded84..2710a9f 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -6,13 +6,28 @@ var storagePath; var __data = {}; var defaultPath = "../data/"; // Default path var defaultBulkPath = "../bulk_data/"; // Default path +var asyncWriteLock = false; +var asyncWriteQueue = []; +/** + * Sets storage path + * If user didn't provide a path, it sets to default path "../data/" + * Instead if a path is provided, sets the storage path as the provided path + * @param {String} userPath - user provided storage path + */ var setStoragePath = function(userPath) { storagePath = userPath || defaultPath; var dir = path.resolve(__dirname, getStoragePath()); createDirectory(dir); }; +/** + * Sets bulk storage path if persistQueue is enabled. + * If user didn't provide a path, it sets to default path "../bulk_data/" + * Instead if a path is provided, sets the storage path as the provided path + * @param {String} userPath - user provided storage path + * @param {Boolean} persistQueue - whether to persistently store queue until processed. false in default + */ var setBulkDataPath = function(userPath, persistQueue) { storagePath = userPath || defaultBulkPath; var dir = path.resolve(__dirname, getStoragePath()); @@ -21,6 +36,10 @@ var setBulkDataPath = function(userPath, persistQueue) { } }; +/** + * Returns the storage path + * @returns {String} storage path + */ var getStoragePath = function() { return storagePath; }; @@ -94,9 +113,6 @@ var forceStore = function() { } }; -var asyncWriteLock = false; -var asyncWriteQueue = []; - /** * Write to file and process queue while in asyncWriteLock * @param {String} key - key for value to store diff --git a/lib/countly.js b/lib/countly.js index 908c793..9965868 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -169,6 +169,8 @@ Countly.Bulk = Bulk; // Common module debug value is set to init time debug value cc.debug = conf.debug; + + // Set the storage path CountlyStorage.setStoragePath(conf.storage_path); // clear stored device ID if flag is set @@ -717,6 +719,7 @@ Countly.Bulk = Bulk; add_cly_events(event); } }; + /** * Add events to event queue * @memberof Countly._internals diff --git a/test/tests_storage.js b/test/tests_storage.js new file mode 100644 index 0000000..6bceac2 --- /dev/null +++ b/test/tests_storage.js @@ -0,0 +1,146 @@ +const assert = require("assert"); +var Countly = require("../lib/countly"); +var cc = require("../lib/countly-common"); +var hp = require("./helpers/helper_functions"); + +// example event object to use +var eventObj = { + key: "storage_check", + count: 5, + sum: 3.14, + dur: 2000, + segmentation: { + app_version: "1.0", + country: "Zambia", + }, +}; + +var userDetailObj = { + name: "Akira Kurosawa", + username: "a_kurosawa", + email: "akira.kurosawa@filmlegacy.com", + organization: "Toho Studios", + phone: "+81312345678", + picture: "https://example.com/profile_images/akira_kurosawa.jpg", + gender: "Male", + byear: 1910, + custom: { + "known for": "Film Director", + "notable works": "Seven Samurai, Rashomon, Ran", + }, +}; + +// init function +function initMain(device_id) { + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + interval: 10000, + max_events: -1, + device_id: device_id, + }); +} +// TODO: move these to helpers to reduce duplication +function validateSdkGeneratedId(providedDeviceId) { + assert.ok(providedDeviceId); + assert.equal(providedDeviceId.length, 36); + assert.ok(cc.isUUID(providedDeviceId)); +} +function checkRequestsForT(queue, expectedInternalType) { + for (var i = 0; i < queue.length; i++) { + assert.ok(queue[i].t); + assert.equal(queue[i].t, expectedInternalType); + } +} +function validateDeviceId(deviceId, deviceIdType, expectedDeviceId, expectedDeviceIdType) { + var rq = hp.readRequestQueue()[0]; + if (expectedDeviceIdType === cc.deviceIdTypeEnums.SDK_GENERATED) { + validateSdkGeneratedId(deviceId); // for SDK-generated IDs + } + else { + assert.equal(deviceId, expectedDeviceId); // for developer-supplied IDs + } + assert.equal(deviceIdType, expectedDeviceIdType); + checkRequestsForT(rq, expectedDeviceIdType); +} + +describe("Storage Tests", () => { + it("1- Store Generated Device ID", (done) => { + // clear previous data + hp.clearStorage(); + // initialize SDK + initMain(); + Countly.begin_session(); + // read request queue + setTimeout(() => { + validateSdkGeneratedId(Countly.get_device_id()); + done(); + }, hp.sWait); + }); + + it("1.1- Validate generated device id after process restart", (done) => { + initMain(); + validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), undefined, cc.deviceIdTypeEnums.SDK_GENERATED); + done(); + }); + + it("2.Developer supplied device ID", (done) => { + hp.clearStorage(); + initMain("ID"); + Countly.begin_session(); + setTimeout(() => { + validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + done(); + }, hp.sWait); + }); + + it("2.1- Validate generated device id after process restart", (done) => { + validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + done(); + }); + + it("3- Record and validate all user details", (done) => { + hp.clearStorage(); + initMain(); + Countly.user_details(userDetailObj); + setTimeout(() => { + var req = hp.readRequestQueue()[0]; + hp.userDetailRequestValidator(userDetailObj, req); + done(); + }, hp.sWait); + }); + + it("3.1- Validate stored user detail", (done) => { + initMain(); + var req = hp.readRequestQueue()[0]; + hp.userDetailRequestValidator(userDetailObj, req); + done(); + }); + + it("4- Record event and validate storage", (done) => { + hp.clearStorage(); + initMain(); + Countly.add_event(eventObj); + setTimeout(() => { + var storedEvents = hp.readEventQueue(); + assert.strictEqual(storedEvents.length, 1, "There should be exactly one event stored"); + + var event = storedEvents[0]; + hp.eventValidator(eventObj, event); + done(); + }, hp.mWait); + }); + + it("4.1- Validate event persistence after process restart", (done) => { + // Initialize SDK + initMain(); + + // Read stored events without clearing storage + var storedEvents = hp.readEventQueue(); + assert.strictEqual(storedEvents.length, 1, "There should be exactly one event stored"); + + var event = storedEvents[0]; + hp.eventValidator(eventObj, event); + done(); + }); +}); From f07703205c4750b0fef3e4437e8a448762e0c52c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 19 Aug 2024 14:28:44 +0300 Subject: [PATCH 15/58] added tests and undefined&null check --- lib/countly-storage.js | 16 +++++++-- test/customStorageDirectory/.gitignore | 4 +++ test/tests_storage.js | 48 +++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 test/customStorageDirectory/.gitignore diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 2710a9f..828c9e9 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -16,8 +16,13 @@ var asyncWriteQueue = []; * @param {String} userPath - user provided storage path */ var setStoragePath = function(userPath) { - storagePath = userPath || defaultPath; - var dir = path.resolve(__dirname, getStoragePath()); + if (userPath === undefined || userPath === null) { + storagePath = defaultPath; + } + else { + storagePath = userPath; + } + var dir = path.resolve(__dirname, storagePath); createDirectory(dir); }; @@ -29,7 +34,12 @@ var setStoragePath = function(userPath) { * @param {Boolean} persistQueue - whether to persistently store queue until processed. false in default */ var setBulkDataPath = function(userPath, persistQueue) { - storagePath = userPath || defaultBulkPath; + if (userPath === undefined || userPath === null) { + storagePath = defaultBulkPath; + } + else { + storagePath = userPath; + } var dir = path.resolve(__dirname, getStoragePath()); if (persistQueue) { createDirectory(dir); diff --git a/test/customStorageDirectory/.gitignore b/test/customStorageDirectory/.gitignore new file mode 100644 index 0000000..86d0cb2 --- /dev/null +++ b/test/customStorageDirectory/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore \ No newline at end of file diff --git a/test/tests_storage.js b/test/tests_storage.js index 6bceac2..7b408d2 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -1,5 +1,6 @@ const assert = require("assert"); var Countly = require("../lib/countly"); +var storage = require("../lib/countly-storage"); var cc = require("../lib/countly-common"); var hp = require("./helpers/helper_functions"); @@ -111,7 +112,6 @@ describe("Storage Tests", () => { }); it("3.1- Validate stored user detail", (done) => { - initMain(); var req = hp.readRequestQueue()[0]; hp.userDetailRequestValidator(userDetailObj, req); done(); @@ -143,4 +143,50 @@ describe("Storage Tests", () => { hp.eventValidator(eventObj, event); done(); }); + + // if storage path is not provided it will be default "../data/" + it("5- Not provide storage path during init", (done) => { + hp.clearStorage(); + initMain(); + assert.equal(storage.getStoragePath(), "../data/"); + done(); + }); + + // if set to undefined it should be set to default path + it("6- Set storage path to undefined", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + storage_path: undefined, + }); + assert.equal(storage.getStoragePath(), "../data/"); + done(); + }); + + // if set to null it should be set to default path + it("7- Set storage path to null", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + storage_path: null, + }); + assert.equal(storage.getStoragePath(), "../data/"); + done(); + }); + + // it should be set to the custom directory if provided + it("8- Set storage path to custom directory", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + interval: 10000, + max_events: -1, + storage_path: "../test/customStorageDirectory/", + }); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + done(); + }); }); From 63404188e270f3bbbf0912d34b5fad8bba0e8894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 19 Aug 2024 14:43:34 +0300 Subject: [PATCH 16/58] url update, check update --- lib/countly-storage.js | 5 ++++- test/tests_storage.js | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 828c9e9..3551b81 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -143,7 +143,10 @@ var writeFile = function(key, value, callback) { if (asyncWriteQueue.length) { setTimeout(() => { var arr = asyncWriteQueue.shift(); - writeFile(arr[0], arr[1], arr[2]); + cc.log(cc.logLevelEnums.DEBUG, "writeFile, Dequeued array:", arr); + if (arr) { + writeFile(arr[0], arr[1], arr[2]); + } }, 0); } else { diff --git a/test/tests_storage.js b/test/tests_storage.js index 7b408d2..666703f 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -35,7 +35,7 @@ var userDetailObj = { function initMain(device_id) { Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", + url: "https://test.url.ly", interval: 10000, max_events: -1, device_id: device_id, @@ -157,7 +157,7 @@ describe("Storage Tests", () => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", + url: "https://test.url.ly", storage_path: undefined, }); assert.equal(storage.getStoragePath(), "../data/"); @@ -169,7 +169,7 @@ describe("Storage Tests", () => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", + url: "https://test.url.ly", storage_path: null, }); assert.equal(storage.getStoragePath(), "../data/"); @@ -181,7 +181,7 @@ describe("Storage Tests", () => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", + url: "https://test.url.ly", interval: 10000, max_events: -1, storage_path: "../test/customStorageDirectory/", From b5c8d104cb7d4a7e28529fb4e3de65c2c4ec3181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 20 Aug 2024 12:50:28 +0300 Subject: [PATCH 17/58] Storage Tests --- lib/countly-storage.js | 2 +- test/tests_storage.js | 78 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 3551b81..c2d1e7f 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -51,7 +51,7 @@ var setBulkDataPath = function(userPath, persistQueue) { * @returns {String} storage path */ var getStoragePath = function() { - return storagePath; + return storagePath || undefined; }; var createDirectory = function(dir) { diff --git a/test/tests_storage.js b/test/tests_storage.js index 666703f..d883470 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -189,4 +189,82 @@ describe("Storage Tests", () => { assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); done(); }); + + it("9- Reset Storage While on Default Path /no-init", (done) => { + // will set to default storage path + storage.setStoragePath(); + assert.equal(storage.getStoragePath(), "../data/"); + // will set to undefined + storage.resetStorage(); + assert.equal(storage.getStoragePath(), undefined); + done(); + }); + + it("10- Recording to Storage with Default Storage Path /no-init", (done) => { + storage.resetStorage(); + // will set to default storage path + storage.setStoragePath(); + assert.equal(storage.getStoragePath(), "../data/"); + + // setting values + var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; + storage.storeSet("cly_id", "SpecialDeviceId"); + storage.storeSet("cly_id_type", deviceIdType); + + // retrieve values + assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + done(); + }); + + it("11- Recording to Storage with Custom Storage Path /no-init", (done) => { + storage.resetStorage(); + // will set to default storage path + storage.setStoragePath("../test/customStorageDirectory/"); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + + // setting values + var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; + storage.storeSet("cly_id", "SpecialDeviceId"); + storage.storeSet("cly_id_type", deviceIdType); + + // retrieve values + assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + done(); + }); + + it("12- Recording to Bulk Storage with Default Bulk Data Path /no-init", (done) => { + storage.resetStorage(); + // will set to default storage path + storage.setBulkDataPath(); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + + // setting values + var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; + storage.storeSet("cly_id", "SpecialDeviceId"); + storage.storeSet("cly_id_type", deviceIdType); + + // retrieve values + assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + done(); + }); + + it("13- Recording to Bulk Storage with Custom Bulk Storage Path /no-init", (done) => { + storage.resetStorage(); + // will set to default storage path + storage.setBulkDataPath("../test/customStorageDirectory/"); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + + // setting values + var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; + storage.storeSet("cly_id", "SpecialDeviceId"); + storage.storeSet("cly_id_type", deviceIdType); + + // retrieve values + assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + done(); + }); }); From 8cf689c8db58452d295247a518744aee027059ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 20 Aug 2024 18:52:39 +0300 Subject: [PATCH 18/58] Recording Different Values in Storage --- test/tests_storage.js | 75 ++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/test/tests_storage.js b/test/tests_storage.js index d883470..715a097 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -64,6 +64,38 @@ function validateDeviceId(deviceId, deviceIdType, expectedDeviceId, expectedDevi assert.equal(deviceIdType, expectedDeviceIdType); checkRequestsForT(rq, expectedDeviceIdType); } +function recordValuesToStorageAndValidate() { + // Set values + var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; + storage.storeSet("cly_id", "SpecialDeviceId"); + storage.storeSet("cly_id_type", deviceIdType); + + // Set values with different data types + storage.storeSet("cly_count", 42); + storage.storeSet("cly_object", { key: "value" }); + storage.storeSet("cly_null", null); + + // Retrieve and assert values + assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + assert.equal(storage.storeGet("cly_count"), 42); + assert.deepEqual(storage.storeGet("cly_object"), { key: "value" }); + assert.equal(storage.storeGet("cly_null"), null); + + // Remove specific items by overriding with null or empty array + storage.storeSet("cly_id", null); + storage.storeSet("cly_object", []); + assert.equal(storage.storeGet("cly_id"), null); + assert.deepEqual(storage.storeGet("cly_object"), []); + + // Reset storage and check if it's empty again + storage.resetStorage(); + assert.equal(storage.storeGet("cly_id"), undefined); + assert.equal(storage.storeGet("cly_id_type"), undefined); + assert.equal(storage.storeGet("cly_count"), undefined); + assert.equal(storage.storeGet("cly_object"), undefined); + assert.equal(storage.storeGet("cly_null"), undefined); +} describe("Storage Tests", () => { it("1- Store Generated Device ID", (done) => { @@ -202,18 +234,11 @@ describe("Storage Tests", () => { it("10- Recording to Storage with Default Storage Path /no-init", (done) => { storage.resetStorage(); - // will set to default storage path + + // Set to default storage path storage.setStoragePath(); assert.equal(storage.getStoragePath(), "../data/"); - - // setting values - var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storage.storeSet("cly_id", "SpecialDeviceId"); - storage.storeSet("cly_id_type", deviceIdType); - - // retrieve values - assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + recordValuesToStorageAndValidate(); done(); }); @@ -222,15 +247,7 @@ describe("Storage Tests", () => { // will set to default storage path storage.setStoragePath("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - - // setting values - var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storage.storeSet("cly_id", "SpecialDeviceId"); - storage.storeSet("cly_id_type", deviceIdType); - - // retrieve values - assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + recordValuesToStorageAndValidate(); done(); }); @@ -239,15 +256,7 @@ describe("Storage Tests", () => { // will set to default storage path storage.setBulkDataPath(); assert.equal(storage.getStoragePath(), "../bulk_data/"); - - // setting values - var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storage.storeSet("cly_id", "SpecialDeviceId"); - storage.storeSet("cly_id_type", deviceIdType); - - // retrieve values - assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + recordValuesToStorageAndValidate(); done(); }); @@ -256,15 +265,7 @@ describe("Storage Tests", () => { // will set to default storage path storage.setBulkDataPath("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - - // setting values - var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storage.storeSet("cly_id", "SpecialDeviceId"); - storage.storeSet("cly_id_type", deviceIdType); - - // retrieve values - assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + recordValuesToStorageAndValidate(); done(); }); }); From 6cc9a742a966835fa7909cc3683cfba62a8fd644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 22 Aug 2024 14:06:10 +0300 Subject: [PATCH 19/58] Refactoring Storage --- lib/countly-bulk.js | 2 +- lib/countly-storage.js | 53 ++++++++++++++---------------------------- test/tests_storage.js | 5 ++-- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 2b32d8e..e6ad0f4 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -102,7 +102,7 @@ function CountlyBulk(conf) { conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; - CountlyStorage.setBulkDataPath(conf.storage_path, conf.persist_queue); + CountlyStorage.setBulkDataPath(conf.storage_path, true, conf.persist_queue); this.conf = conf; /** diff --git a/lib/countly-storage.js b/lib/countly-storage.js index c2d1e7f..1ab308b 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -10,50 +10,31 @@ var asyncWriteLock = false; var asyncWriteQueue = []; /** - * Sets storage path - * If user didn't provide a path, it sets to default path "../data/" - * Instead if a path is provided, sets the storage path as the provided path - * @param {String} userPath - user provided storage path + * Sets the storage path, defaulting to a specified path if none is provided. + * @param {String} userPath - User provided storage path + * @param {Boolean} isBulk - Whether the storage is for bulk data + * @param {Boolean} persistQueue - Whether to persist the queue until processed */ -var setStoragePath = function(userPath) { - if (userPath === undefined || userPath === null) { - storagePath = defaultPath; - } - else { - storagePath = userPath; - } - var dir = path.resolve(__dirname, storagePath); - createDirectory(dir); -}; +var setStoragePath = function(userPath, isBulk = false, persistQueue = false) { + storagePath = userPath || (isBulk ? defaultBulkPath : defaultPath); -/** - * Sets bulk storage path if persistQueue is enabled. - * If user didn't provide a path, it sets to default path "../bulk_data/" - * Instead if a path is provided, sets the storage path as the provided path - * @param {String} userPath - user provided storage path - * @param {Boolean} persistQueue - whether to persistently store queue until processed. false in default - */ -var setBulkDataPath = function(userPath, persistQueue) { - if (userPath === undefined || userPath === null) { - storagePath = defaultBulkPath; - } - else { - storagePath = userPath; - } - var dir = path.resolve(__dirname, getStoragePath()); - if (persistQueue) { - createDirectory(dir); + if (!isBulk || persistQueue) { + createDirectory(path.resolve(__dirname, storagePath)); } }; /** - * Returns the storage path - * @returns {String} storage path + * Returns the current storage path + * @returns {String} storagePath */ var getStoragePath = function() { - return storagePath || undefined; + return storagePath; }; +/** + * Creates a directory if it doesn't exist + * @param {String} dir - The directory path + */ var createDirectory = function(dir) { try { if (!fs.existsSync(dir)) { @@ -65,6 +46,9 @@ var createDirectory = function(dir) { } }; +/** + * Resets storage-related variables to their initial state + */ var resetStorage = function() { storagePath = undefined; __data = {}; @@ -211,7 +195,6 @@ module.exports = { forceStore, getStoragePath, setStoragePath, - setBulkDataPath, resetStorage, readFile, }; \ No newline at end of file diff --git a/test/tests_storage.js b/test/tests_storage.js index 715a097..6735b1f 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -254,7 +254,8 @@ describe("Storage Tests", () => { it("12- Recording to Bulk Storage with Default Bulk Data Path /no-init", (done) => { storage.resetStorage(); // will set to default storage path - storage.setBulkDataPath(); + // To set the storage path to the default bulk storage path and persist the queue + storage.setStoragePath(null, true, true); assert.equal(storage.getStoragePath(), "../bulk_data/"); recordValuesToStorageAndValidate(); done(); @@ -263,7 +264,7 @@ describe("Storage Tests", () => { it("13- Recording to Bulk Storage with Custom Bulk Storage Path /no-init", (done) => { storage.resetStorage(); // will set to default storage path - storage.setBulkDataPath("../test/customStorageDirectory/"); + storage.setStoragePath("../test/customStorageDirectory/", true); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); recordValuesToStorageAndValidate(); done(); From 92306d31b07346da66251a35330515e9d31a153b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 22 Aug 2024 14:53:17 +0300 Subject: [PATCH 20/58] Update countly-bulk.js --- lib/countly-bulk.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index e6ad0f4..ceb542e 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -102,7 +102,7 @@ function CountlyBulk(conf) { conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; - CountlyStorage.setBulkDataPath(conf.storage_path, true, conf.persist_queue); + CountlyStorage.setStoragePath(conf.storage_path, true, conf.persist_queue); this.conf = conf; /** From f605360e0afb34aac137c02743b54448c979ce5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 3 Sep 2024 15:24:28 +0300 Subject: [PATCH 21/58] Bulk Tests --- test/helpers/helper_functions.js | 12 +++ test/tests_bulk.js | 126 +++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 test/tests_bulk.js diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index b87f825..1a184a4 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -9,6 +9,8 @@ var dir = path.resolve(__dirname, "../../"); var idDir = (`${dir}/data/__cly_id.json`); var eventDir = (`${dir}/data/__cly_event.json`); var reqDir = (`${dir}/data/__cly_queue.json`); +var bulkEventDir = (`${dir}/bulk_data/__cly_bulk_event.json`); +var bulkQueueDir = (`${dir}/bulk_data/__cly_req_queue.json`); // timeout variables var sWait = 50; var mWait = 3000; @@ -23,6 +25,14 @@ function readRequestQueue() { var a = JSON.parse(fs.readFileSync(reqDir, "utf-8")).cly_queue; return a; } +function readBulkEventQueue() { + var a = JSON.parse(fs.readFileSync(bulkEventDir, "utf-8")).cly_bulk_event; + return a; +} +function readBulkReqQueue() { + var a = JSON.parse(fs.readFileSync(bulkQueueDir, "utf-8")).cly_req_queue; + return a; +} // queue files clearing logic function clearStorage(keepID) { @@ -188,6 +198,8 @@ module.exports = { lWait, readEventQueue, readRequestQueue, + readBulkEventQueue, + readBulkReqQueue, eventValidator, crashRequestValidator, sessionRequestValidator, diff --git a/test/tests_bulk.js b/test/tests_bulk.js new file mode 100644 index 0000000..1314ab9 --- /dev/null +++ b/test/tests_bulk.js @@ -0,0 +1,126 @@ +/* eslint-disable no-console */ +const assert = require("assert"); +const CountlyBulk = require("../lib/countly-bulk"); +var hp = require("./helpers/helper_functions"); +var storage = require("../lib/countly-storage"); + +function createBulk(storagePath) { + var bulk = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + storage_path: storagePath, + }); + return bulk; +} + +// note: this can replace the current one in the helper functions +function validateUserDetails(actual, expected) { + const keys = ['name', 'username', 'email', 'organization', 'phone', 'picture', 'gender', 'byear', 'custom']; + let isValid = true; + keys.forEach((key) => { + if (JSON.stringify(actual[key]) !== JSON.stringify(expected[key])) { + console.error(`Mismatch for key "${key}": expected "${expected[key]}", but got "${actual[key]}"`); + isValid = false; + } + }); + // Validate nested custom object separately + const customKeys = Object.keys(expected.custom || {}); + customKeys.forEach((key) => { + if (actual.custom[key] !== expected.custom[key]) { + console.error(`Mismatch in custom object for key "${key}": expected "${expected.custom[key]}", but got "${actual.custom[key]}"`); + isValid = false; + } + }); + return isValid; +} + +var eventObj = { + key: "bulk_check", + count: 55, + sum: 3.14, + dur: 2000, + segmentation: { + app_version: "1.0", + country: "Zambia", + }, +}; + +var userDetailObj = { + name: "Alexandrina Jovovich", + username: "alex_jov", + email: "alex.jov@example.com", + organization: "TechNova", + phone: "+987654321", + picture: "https://example.com/images/profile_alex.jpg", + gender: "Female", + byear: 1992, // birth year + custom: { + key1: "value1 segment", + key2: "value2 segment", + }, +}; + +describe("Bulk Tests", () => { + it("1- Bulk with Default Storage Path", (done) => { + storage.resetStorage(); + createBulk(); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + done(); + }); + + it("2- Bulk with Custom Storage Path", (done) => { + storage.resetStorage(); + createBulk("../test/customStorageDirectory/"); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + done(); + }); + + it("3- Bulk add_user with Record Event", (done) => { + storage.resetStorage(); + var bulk = createBulk(); + var user = bulk.add_user({ device_id: "testUser1" }); + user.add_event(eventObj); + setTimeout(() => { + var events = hp.readBulkEventQueue(); + var deviceEvents = events.testUser1; // Access the events for the specific device + var recordedEvent = deviceEvents[0]; // Access the first event + hp.eventValidator(eventObj, recordedEvent); + done(); + }, hp.mWait); + }); + + it("4- Bulk add_user with User Details", (done) => { + storage.resetStorage(); + var bulk = createBulk(); + var user = bulk.add_user({ device_id: "testUser2" }); + user.user_details(userDetailObj); + + // read event queue + setTimeout(() => { + var reqQueue = hp.readBulkReqQueue(); + var req = reqQueue[0]; + // Extract the user_details from the actual request + const actualUserDetails = req.user_details || {}; + // Validate the user details + const isValid = validateUserDetails(actualUserDetails, userDetailObj); + assert.equal(true, isValid); + done(); + }, hp.sWait); + }); + + it("5- Bulk add_request", (done) => { + storage.resetStorage(); + var bulk = createBulk(); + bulk.add_request({ device_id: "TestUser4" }); + setTimeout(() => { + var reqQueue = hp.readBulkReqQueue(); + var testUser4Request = reqQueue.find((req) => req.device_id === "TestUser4"); + assert.ok(testUser4Request); + assert.strictEqual(testUser4Request.device_id, "TestUser4"); + assert.strictEqual(testUser4Request.app_key, "YOUR_APP_KEY"); + assert.strictEqual(testUser4Request.sdk_name, "javascript_native_nodejs_bulk"); + done(); + }, hp.sWait); + }); +}); +/* eslint-enable no-console */ \ No newline at end of file From 160f4c79b302d54d66ec1d1c9d5eb382f751b0ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Wed, 4 Sep 2024 16:52:13 +0300 Subject: [PATCH 22/58] Crash Test and Clear File Function --- test/helpers/helper_functions.js | 45 ++++++++++- test/tests_bulk.js | 130 +++++++++++++++++++++++++------ 2 files changed, 148 insertions(+), 27 deletions(-) diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index 1a184a4..8e870c1 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -34,6 +34,39 @@ function readBulkReqQueue() { return a; } +function clearStorageJsonFiles(directory, files) { + files.forEach((file) => { + const filePath = path.join(directory, file); + // Check if the file exists before attempting to clear it + fs.access(filePath, fs.constants.F_OK, (err) => { + if (!err) { + // Write an empty JSON object to the file + // logs can be added here + fs.writeFile(filePath, JSON.stringify({}), 'utf8', (er) => { }); + } + }); + }); +} + +function clearJsonFiles(isBulk = true, customDir = '') { + // Determine the directory and files based on isBulk + const directory = customDir || (isBulk ? path.join(__dirname, 'bulk_data') : path.join(__dirname, 'data')); + const files = isBulk ? [ + '__cly_id.json', + '__cly_req_queue.json', + '__cly_bulk_event.json', + ] : [ + '__cly_event.json', + '__cly_id_type.json', + '__cly_id.json', + '__cly_queue.json', + '__cly_remote_configs.json', + ]; + + // Clear files in the specified directory + clearStorageJsonFiles(directory, files); +} + // queue files clearing logic function clearStorage(keepID) { keepID = keepID || false; @@ -77,11 +110,16 @@ function eventValidator(eventObject, eventQueue, time) { } assert.equal(eventObject.dur, eventQueue.dur); } - // check if segmentation exists. If it is add test(s) + // check if segmentation exists. If it does, add tests if (typeof eventObject.segmentation !== 'undefined') { - // loop through segmentation keys and create tets + // loop through segmentation keys and create tests for (var key in eventObject.segmentation) { - assert.equal(eventObject.segmentation[key], eventQueue.segmentation[key]); + if (Array.isArray(eventObject.segmentation[key]) || typeof eventObject.segmentation[key] === 'object') { + assert.deepStrictEqual(eventObject.segmentation[key], eventQueue.segmentation[key]); + } + else { + assert.equal(eventObject.segmentation[key], eventQueue.segmentation[key]); + } } } // common parameter validation @@ -202,6 +240,7 @@ module.exports = { readBulkReqQueue, eventValidator, crashRequestValidator, + clearJsonFiles, sessionRequestValidator, userDetailRequestValidator, viewEventValidator, diff --git a/test/tests_bulk.js b/test/tests_bulk.js index 1314ab9..b362247 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -1,4 +1,5 @@ /* eslint-disable no-console */ +/* global runthis */ const assert = require("assert"); const CountlyBulk = require("../lib/countly-bulk"); var hp = require("./helpers/helper_functions"); @@ -13,24 +14,67 @@ function createBulk(storagePath) { return bulk; } +function validateCrash(validator, nonfatal) { + assert.ok(validator.crash._os); + assert.ok(validator.crash._os_version); + assert.ok(validator.crash._error); + assert.ok(validator.crash._app_version); + assert.ok(typeof validator.crash._run !== 'undefined'); + assert.ok(typeof validator.crash._custom !== 'undefined'); + assert.equal(nonfatal, validator.crash._nonfatal); + assert.equal(true, validator.crash._javascript); + assert.equal(true, validator.crash._not_os_specific); +} + // note: this can replace the current one in the helper functions function validateUserDetails(actual, expected) { const keys = ['name', 'username', 'email', 'organization', 'phone', 'picture', 'gender', 'byear', 'custom']; let isValid = true; + keys.forEach((key) => { - if (JSON.stringify(actual[key]) !== JSON.stringify(expected[key])) { + if (typeof actual[key] === 'object' && actual[key] !== null) { + if (Array.isArray(actual[key])) { + if (!Array.isArray(expected[key]) || JSON.stringify(actual[key]) !== JSON.stringify(expected[key])) { + console.error(`Mismatch for key "${key}": expected "${JSON.stringify(expected[key])}", but got "${JSON.stringify(actual[key])}"`); + isValid = false; + } + } + else { + if (JSON.stringify(actual[key]) !== JSON.stringify(expected[key])) { + console.error(`Mismatch for key "${key}": expected "${JSON.stringify(expected[key])}", but got "${JSON.stringify(actual[key])}"`); + isValid = false; + } + } + } + else if (actual[key] !== expected[key]) { console.error(`Mismatch for key "${key}": expected "${expected[key]}", but got "${actual[key]}"`); isValid = false; } }); // Validate nested custom object separately - const customKeys = Object.keys(expected.custom || {}); - customKeys.forEach((key) => { - if (actual.custom[key] !== expected.custom[key]) { - console.error(`Mismatch in custom object for key "${key}": expected "${expected.custom[key]}", but got "${actual.custom[key]}"`); - isValid = false; - } - }); + if (expected.custom && actual.custom) { + const customKeys = Object.keys(expected.custom); + customKeys.forEach((key) => { + if (typeof actual.custom[key] === 'object' && actual.custom[key] !== null) { + if (Array.isArray(actual.custom[key])) { + if (!Array.isArray(expected.custom[key]) || JSON.stringify(actual.custom[key]) !== JSON.stringify(expected.custom[key])) { + console.error(`Mismatch in custom object for key "${key}": expected "${JSON.stringify(expected.custom[key])}", but got "${JSON.stringify(actual.custom[key])}"`); + isValid = false; + } + } + else { + if (JSON.stringify(actual.custom[key]) !== JSON.stringify(expected.custom[key])) { + console.error(`Mismatch in custom object for key "${key}": expected "${JSON.stringify(expected.custom[key])}", but got "${JSON.stringify(actual.custom[key])}"`); + isValid = false; + } + } + } + else if (actual.custom[key] !== expected.custom[key]) { + console.error(`Mismatch in custom object for key "${key}": expected "${expected.custom[key]}", but got "${actual.custom[key]}"`); + isValid = false; + } + }); + } return isValid; } @@ -40,8 +84,13 @@ var eventObj = { sum: 3.14, dur: 2000, segmentation: { - app_version: "1.0", - country: "Zambia", + string_value: "example", + number_value: 42, + boolean_value: true, + array_value: ["item1", "item2"], + object_value: { nested_key: "nested_value" }, + null_value: null, + undefined_value: undefined, }, }; @@ -55,28 +104,33 @@ var userDetailObj = { gender: "Female", byear: 1992, // birth year custom: { - key1: "value1 segment", - key2: "value2 segment", + string_value: "example", + number_value: 42, + boolean_value: true, + array_value: ["item1", "item2"], + object_value: { nested_key: "nested_value" }, + null_value: null, + undefined_value: undefined, }, }; describe("Bulk Tests", () => { it("1- Bulk with Default Storage Path", (done) => { - storage.resetStorage(); + hp.clearJsonFiles(true); createBulk(); assert.equal(storage.getStoragePath(), "../bulk_data/"); done(); }); it("2- Bulk with Custom Storage Path", (done) => { - storage.resetStorage(); + hp.clearJsonFiles(true); createBulk("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); done(); }); it("3- Bulk add_user with Record Event", (done) => { - storage.resetStorage(); + hp.clearJsonFiles(true); var bulk = createBulk(); var user = bulk.add_user({ device_id: "testUser1" }); user.add_event(eventObj); @@ -90,7 +144,7 @@ describe("Bulk Tests", () => { }); it("4- Bulk add_user with User Details", (done) => { - storage.resetStorage(); + hp.clearJsonFiles(true); var bulk = createBulk(); var user = bulk.add_user({ device_id: "testUser2" }); user.user_details(userDetailObj); @@ -109,18 +163,46 @@ describe("Bulk Tests", () => { }); it("5- Bulk add_request", (done) => { - storage.resetStorage(); + hp.clearJsonFiles(true); var bulk = createBulk(); - bulk.add_request({ device_id: "TestUser4" }); + bulk.add_request({ device_id: "TestUser3" }); + setTimeout(() => { + var reqQueue = hp.readBulkReqQueue(); + var testUser3Request = reqQueue.find((req) => req.device_id === "TestUser3"); + assert.ok(testUser3Request); + assert.strictEqual(testUser3Request.device_id, "TestUser3"); + assert.strictEqual(testUser3Request.app_key, "YOUR_APP_KEY"); + assert.strictEqual(testUser3Request.sdk_name, "javascript_native_nodejs_bulk"); + done(); + }, hp.sWait); + }); + + it("6- Bulk add_user Report Crash", (done) => { + hp.clearJsonFiles(true); + var bulk = createBulk(); + var user = bulk.add_user({ device_id: "TestUser4" }); + try { + runthis(); + } + catch (ex) { + user.report_crash({ + _os: "Android", + _os_version: "7", + _error: "Stack trace goes here", + _app_version: "1.0", + _run: 12345, + _custom: {}, + _nonfatal: true, + _javascript: true, + _not_os_specific: true, + }, 1500645200); + } + // read event queue setTimeout(() => { var reqQueue = hp.readBulkReqQueue(); var testUser4Request = reqQueue.find((req) => req.device_id === "TestUser4"); - assert.ok(testUser4Request); - assert.strictEqual(testUser4Request.device_id, "TestUser4"); - assert.strictEqual(testUser4Request.app_key, "YOUR_APP_KEY"); - assert.strictEqual(testUser4Request.sdk_name, "javascript_native_nodejs_bulk"); + validateCrash(testUser4Request, true); done(); }, hp.sWait); }); -}); -/* eslint-enable no-console */ \ No newline at end of file +}); \ No newline at end of file From 1c2e05bdd02a355913256673414f2c8d587258a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 17 Sep 2024 15:03:04 +0300 Subject: [PATCH 23/58] Updated HelperFunctions --- test/helpers/helper_functions.js | 91 +++++++++++--------------------- test/tests_bulk.js | 26 +++++---- 2 files changed, 47 insertions(+), 70 deletions(-) diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index 8e870c1..eaee194 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -16,73 +16,47 @@ var sWait = 50; var mWait = 3000; var lWait = 10000; // parsing event queue -function readEventQueue() { - var a = JSON.parse(fs.readFileSync(eventDir, "utf-8")).cly_event; +function readEventQueue(givenPath = null, isBulk = false) { + var destination = eventDir; + if (givenPath !== null) { + destination = givenPath; + } + var a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_event; + if (isBulk) { + a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_bulk_event; + } return a; } // parsing request queue -function readRequestQueue() { - var a = JSON.parse(fs.readFileSync(reqDir, "utf-8")).cly_queue; - return a; -} -function readBulkEventQueue() { - var a = JSON.parse(fs.readFileSync(bulkEventDir, "utf-8")).cly_bulk_event; - return a; -} -function readBulkReqQueue() { - var a = JSON.parse(fs.readFileSync(bulkQueueDir, "utf-8")).cly_req_queue; +function readRequestQueue(givenPath = null, isBulk = false) { + var destination = reqDir; + if (givenPath !== null) { + destination = givenPath; + } + var a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_queue; + if (isBulk) { + a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_req_queue; + } return a; } - -function clearStorageJsonFiles(directory, files) { - files.forEach((file) => { - const filePath = path.join(directory, file); - // Check if the file exists before attempting to clear it - fs.access(filePath, fs.constants.F_OK, (err) => { - if (!err) { - // Write an empty JSON object to the file - // logs can be added here - fs.writeFile(filePath, JSON.stringify({}), 'utf8', (er) => { }); - } - }); - }); -} - -function clearJsonFiles(isBulk = true, customDir = '') { - // Determine the directory and files based on isBulk - const directory = customDir || (isBulk ? path.join(__dirname, 'bulk_data') : path.join(__dirname, 'data')); - const files = isBulk ? [ - '__cly_id.json', - '__cly_req_queue.json', - '__cly_bulk_event.json', - ] : [ - '__cly_event.json', - '__cly_id_type.json', - '__cly_id.json', - '__cly_queue.json', - '__cly_remote_configs.json', - ]; - - // Clear files in the specified directory - clearStorageJsonFiles(directory, files); -} - // queue files clearing logic -function clearStorage(keepID) { - keepID = keepID || false; +function clearStorage(keepID = false, isBulk = false, customDir = '') { // Resets Countly Countly.halt(true); - // clean storages - if (fs.existsSync(eventDir)) { - fs.unlinkSync(eventDir); + // Determine the directory based on isBulk or customDir + const eventDirectory = customDir || (isBulk ? bulkEventDir : eventDir); + const reqDirectory = customDir || (isBulk ? bulkQueueDir : reqDir); + // Remove event directory if it exists + if (fs.existsSync(eventDirectory)) { + fs.rmSync(eventDirectory, { recursive: true, force: true }); } - if (fs.existsSync(reqDir)) { - fs.unlinkSync(reqDir); + // Remove request directory if it exists + if (fs.existsSync(reqDirectory)) { + fs.rmSync(reqDirectory, { recursive: true, force: true }); } - if (!keepID) { - if (fs.existsSync(idDir)) { - fs.unlinkSync(idDir); - } + // Optionally keep the ID directory + if (!keepID && fs.existsSync(idDir)) { + fs.rmSync(idDir, { recursive: true, force: true }); } } /** @@ -236,11 +210,8 @@ module.exports = { lWait, readEventQueue, readRequestQueue, - readBulkEventQueue, - readBulkReqQueue, eventValidator, crashRequestValidator, - clearJsonFiles, sessionRequestValidator, userDetailRequestValidator, viewEventValidator, diff --git a/test/tests_bulk.js b/test/tests_bulk.js index b362247..90b8b91 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -1,10 +1,16 @@ /* eslint-disable no-console */ /* global runthis */ +var path = require("path"); const assert = require("assert"); const CountlyBulk = require("../lib/countly-bulk"); var hp = require("./helpers/helper_functions"); var storage = require("../lib/countly-storage"); +// default paths +var dir = path.resolve(__dirname, "../"); +var bulkEventDir = (`${dir}/bulk_data/__cly_bulk_event.json`); +var bulkQueueDir = (`${dir}/bulk_data/__cly_req_queue.json`); + function createBulk(storagePath) { var bulk = new CountlyBulk({ app_key: "YOUR_APP_KEY", @@ -116,26 +122,26 @@ var userDetailObj = { describe("Bulk Tests", () => { it("1- Bulk with Default Storage Path", (done) => { - hp.clearJsonFiles(true); + hp.clearStorage(false, true); createBulk(); assert.equal(storage.getStoragePath(), "../bulk_data/"); done(); }); it("2- Bulk with Custom Storage Path", (done) => { - hp.clearJsonFiles(true); + hp.clearStorage(false, true); createBulk("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); done(); }); it("3- Bulk add_user with Record Event", (done) => { - hp.clearJsonFiles(true); + hp.clearStorage(false, true); var bulk = createBulk(); var user = bulk.add_user({ device_id: "testUser1" }); user.add_event(eventObj); setTimeout(() => { - var events = hp.readBulkEventQueue(); + var events = hp.readEventQueue(bulkEventDir, true); var deviceEvents = events.testUser1; // Access the events for the specific device var recordedEvent = deviceEvents[0]; // Access the first event hp.eventValidator(eventObj, recordedEvent); @@ -144,14 +150,14 @@ describe("Bulk Tests", () => { }); it("4- Bulk add_user with User Details", (done) => { - hp.clearJsonFiles(true); + hp.clearStorage(false, true); var bulk = createBulk(); var user = bulk.add_user({ device_id: "testUser2" }); user.user_details(userDetailObj); // read event queue setTimeout(() => { - var reqQueue = hp.readBulkReqQueue(); + var reqQueue = hp.readRequestQueue(bulkQueueDir, true); var req = reqQueue[0]; // Extract the user_details from the actual request const actualUserDetails = req.user_details || {}; @@ -163,11 +169,11 @@ describe("Bulk Tests", () => { }); it("5- Bulk add_request", (done) => { - hp.clearJsonFiles(true); + hp.clearStorage(false, true); var bulk = createBulk(); bulk.add_request({ device_id: "TestUser3" }); setTimeout(() => { - var reqQueue = hp.readBulkReqQueue(); + var reqQueue = hp.readRequestQueue(bulkQueueDir, true); var testUser3Request = reqQueue.find((req) => req.device_id === "TestUser3"); assert.ok(testUser3Request); assert.strictEqual(testUser3Request.device_id, "TestUser3"); @@ -178,7 +184,7 @@ describe("Bulk Tests", () => { }); it("6- Bulk add_user Report Crash", (done) => { - hp.clearJsonFiles(true); + hp.clearStorage(false, true); var bulk = createBulk(); var user = bulk.add_user({ device_id: "TestUser4" }); try { @@ -199,7 +205,7 @@ describe("Bulk Tests", () => { } // read event queue setTimeout(() => { - var reqQueue = hp.readBulkReqQueue(); + var reqQueue = hp.readRequestQueue(bulkQueueDir, true); var testUser4Request = reqQueue.find((req) => req.device_id === "TestUser4"); validateCrash(testUser4Request, true); done(); From 369593284de3b7126965e1fd8175ae8471e5678e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Wed, 18 Sep 2024 20:51:34 +0300 Subject: [PATCH 24/58] Memory Only Storage Option --- lib/countly-storage.js | 167 ++++++++++++++++++++++++++++------------- lib/countly.js | 41 +++++----- test/tests_storage.js | 47 ++++++------ 3 files changed, 161 insertions(+), 94 deletions(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 1ab308b..95ea68e 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -8,6 +8,121 @@ var defaultPath = "../data/"; // Default path var defaultBulkPath = "../bulk_data/"; // Default path var asyncWriteLock = false; var asyncWriteQueue = []; +let storageMethod = {}; + +// Memory-only storage methods +const memoryStorage = { + /** + * Save value in memory + * @param {String} key - key for value to store + * @param {varies} value - value to store + * @param {Function} callback - callback to call when done storing + */ + storeSet(key, value, callback) { + __data[key] = value; + if (typeof callback === "function") { + callback(null); // No errors for memory storage + } + }, + /** + * Get value from memory + * @param {String} key - key of value to get + * @param {varies} def - default value to use if not set + * @returns {varies} value for the key + */ + storeGet(key, def) { + cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`); + return typeof __data[key] !== "undefined" ? __data[key] : def; + }, + /** + * Remove value from memory + * @param {String} key - key of value to remove + */ + storeRemove(key) { + delete __data[key]; + }, +}; + +// File storage methods +const fileStorage = { + /** + * Save value in storage + * @param {String} key - key for value to store + * @param {varies} value - value to store + * @param {Function} callback - callback to call when done storing + */ + storeSet(key, value, callback) { + __data[key] = value; + if (!asyncWriteLock) { + asyncWriteLock = true; + writeFile(key, value, callback); + } + else { + asyncWriteQueue.push([key, value, callback]); + } + }, + /** + * Get value from storage + * @param {String} key - key of value to get + * @param {varies} def - default value to use if not set + * @returns {varies} value for the key + */ + storeGet(key, def) { + cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); + if (typeof __data[key] === "undefined") { + var ob = readFile(key); + var obLen; + // check if the 'read object' is empty or not + try { + obLen = Object.keys(ob).length; + } + catch (error) { + // if we can not even asses length set it to 0 so we can return the default value + obLen = 0; + } + + // if empty or falsy set default value + if (!ob || obLen === 0) { + __data[key] = def; + } + // else set the value read file has + else { + __data[key] = ob[key]; + } + } + return __data[key]; + }, + storeRemove(key) { + delete __data[key]; + var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); + fs.unlink(dir, (err) => { + if (err) { + cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err}].`); + } + }); + }, +}; + +/** + * Sets the storage method, by default sets file storage and storage path. + * @param {String} userPath - User provided storage path + * @param {Boolean} memoryOnly - Whether to use memory only storage or not + * @param {Boolean} isBulk - Whether the storage is for bulk data + * @param {Boolean} persistQueue - Whether to persist the queue until processed + * @returns {storageMethod} Prefered storage method + */ +var setStorage = function(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { + if (memoryOnly) { + cc.log(cc.logLevelEnums.DEBUG, "Using memory-only storage."); + storageMethod = memoryStorage; // Assign memory-only methods + } + else { + cc.log(cc.logLevelEnums.DEBUG, "Using persistent file storage."); + storageMethod = fileStorage; // Assign file storage methods + setStoragePath(userPath, isBulk, persistQueue); + } + return storageMethod; +}; /** * Sets the storage path, defaulting to a specified path if none is provided. @@ -139,59 +254,9 @@ var writeFile = function(key, value, callback) { }); }; -/** - * Save value in storage - * @param {String} key - key for value to store - * @param {varies} value - value to store - * @param {Function} callback - callback to call when done storing - */ -var storeSet = function(key, value, callback) { - __data[key] = value; - if (!asyncWriteLock) { - asyncWriteLock = true; - writeFile(key, value, callback); - } - else { - asyncWriteQueue.push([key, value, callback]); - } -}; - -/** - * Get value from storage - * @param {String} key - key of value to get - * @param {varies} def - default value to use if not set - * @returns {varies} value for the key - */ -var storeGet = function(key, def) { - cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); - if (typeof __data[key] === "undefined") { - var ob = readFile(key); - var obLen; - // check if the 'read object' is empty or not - try { - obLen = Object.keys(ob).length; - } - catch (error) { - // if we can not even asses length set it to 0 so we can return the default value - obLen = 0; - } - - // if empty or falsy set default value - if (!ob || obLen === 0) { - __data[key] = def; - } - // else set the value read file has - else { - __data[key] = ob[key]; - } - } - return __data[key]; -}; - module.exports = { + setStorage, writeFile, - storeGet, - storeSet, forceStore, getStoragePath, setStoragePath, diff --git a/lib/countly.js b/lib/countly.js index 9965868..5aa8e60 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -29,6 +29,7 @@ var Bulk = require("./countly-bulk"); var CountlyStorage = require("./countly-storage"); var Countly = {}; +var storageMethod; Countly.Bulk = Bulk; (function() { @@ -171,13 +172,13 @@ Countly.Bulk = Bulk; cc.debug = conf.debug; // Set the storage path - CountlyStorage.setStoragePath(conf.storage_path); + storageMethod = CountlyStorage.setStorage(conf.storage_path, true); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { cc.log(cc.logLevelEnums.WARNING, "init, clear_stored_device_id is true, erasing the stored ID."); - CountlyStorage.storeSet("cly_id", null); - CountlyStorage.storeSet("cly_id_type", null); + storageMethod.storeSet("cly_id", null); + storageMethod.storeSet("cly_id_type", null); } if (Countly.url === "") { @@ -223,8 +224,8 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { // fetch stored ID and ID type - var storedId = CountlyStorage.storeGet("cly_id", null); - var storedIdType = CountlyStorage.storeGet("cly_id_type", null); + var storedId = storageMethod.storeGet("cly_id", null); + var storedIdType = storageMethod.storeGet("cly_id_type", null); // if there was a stored ID if (storedId !== null) { Countly.device_id = storedId; @@ -253,12 +254,12 @@ Countly.Bulk = Bulk; deviceIdType = cc.deviceIdTypeEnums.SDK_GENERATED; } // save the ID and ID type - CountlyStorage.storeSet("cly_id", Countly.device_id); - CountlyStorage.storeSet("cly_id_type", deviceIdType); + storageMethod.storeSet("cly_id", Countly.device_id); + storageMethod.storeSet("cly_id_type", deviceIdType); // create queues - requestQueue = CountlyStorage.storeGet("cly_queue", []); - eventQueue = CountlyStorage.storeGet("cly_event", []); - remoteConfigs = CountlyStorage.storeGet("cly_remote_configs", {}); + requestQueue = storageMethod.storeGet("cly_queue", []); + eventQueue = storageMethod.storeGet("cly_event", []); + remoteConfigs = storageMethod.storeGet("cly_remote_configs", {}); heartBeat(); // listen to current workers if (cluster.workers) { @@ -635,7 +636,7 @@ Countly.Bulk = Bulk; if (eventQueue.length > 0) { toRequestQueue({ events: JSON.stringify(eventQueue) }); eventQueue = []; - CountlyStorage.storeSet("cly_event", eventQueue); + storageMethod.storeSet("cly_event", eventQueue); } // end current session Countly.end_session(); @@ -647,8 +648,8 @@ Countly.Bulk = Bulk; var oldId = Countly.device_id; Countly.device_id = newId; deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - CountlyStorage.storeSet("cly_id", Countly.device_id); - CountlyStorage.storeSet("cly_id_type", deviceIdType); + storageMethod.storeSet("cly_id", Countly.device_id); + storageMethod.storeSet("cly_id_type", deviceIdType); if (merge) { if (Countly.check_any_consent()) { toRequestQueue({ old_device_id: oldId }); @@ -664,7 +665,7 @@ Countly.Bulk = Bulk; if (Countly.remote_config) { remoteConfigs = {}; if (cluster.isMaster) { - CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); + storageMethod.storeSet("cly_remote_configs", remoteConfigs); } Countly.fetch_remote_config(Countly.remote_config); } @@ -744,7 +745,7 @@ Countly.Bulk = Bulk; e.dow = date.getDay(); cc.log(cc.logLevelEnums.DEBUG, "add_cly_events, Adding event: ", event); eventQueue.push(e); - CountlyStorage.storeSet("cly_event", eventQueue); + storageMethod.storeSet("cly_event", eventQueue); } else { process.send({ cly: { event } }); @@ -1155,7 +1156,7 @@ Countly.Bulk = Bulk; remoteConfigs = configs; } if (cluster.isMaster) { - CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); + storageMethod.storeSet("cly_remote_configs", remoteConfigs); cc.log(cc.logLevelEnums.INFO, `fetch_remote_config, Fetched remote config: [${remoteConfigs}].`); } } @@ -1339,7 +1340,7 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { cc.log(cc.logLevelEnums.INFO, "request, Adding the raw request to the queue."); requestQueue.push(request); - CountlyStorage.storeSet("cly_queue", requestQueue); + storageMethod.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "request, Sending message to the parent process. Adding the raw request to the queue."); @@ -1426,7 +1427,7 @@ Countly.Bulk = Bulk; cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Adding request to the queue."); requestQueue.push(request); - CountlyStorage.storeSet("cly_queue", requestQueue); + storageMethod.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Sending message to the parent process. Adding request to the queue."); @@ -1457,7 +1458,7 @@ Countly.Bulk = Bulk; var events = eventQueue.splice(0, maxEventBatch); toRequestQueue({ events: JSON.stringify(events) }); } - CountlyStorage.storeSet("cly_event", eventQueue); + storageMethod.storeSet("cly_event", eventQueue); } // process request queue with event queue @@ -1472,7 +1473,7 @@ Countly.Bulk = Bulk; failTimeout = cc.getTimestamp() + failTimeoutAmount; cc.log(cc.logLevelEnums.ERROR, `makeRequest, Encountered a problem while making the request: [${err}]`); } - CountlyStorage.storeSet("cly_queue", requestQueue); + storageMethod.storeSet("cly_queue", requestQueue); readyToProcess = true; }, "heartBeat", false); } diff --git a/test/tests_storage.js b/test/tests_storage.js index 6735b1f..6815b53 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -64,37 +64,38 @@ function validateDeviceId(deviceId, deviceIdType, expectedDeviceId, expectedDevi assert.equal(deviceIdType, expectedDeviceIdType); checkRequestsForT(rq, expectedDeviceIdType); } -function recordValuesToStorageAndValidate() { +function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { // Set values var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storage.storeSet("cly_id", "SpecialDeviceId"); - storage.storeSet("cly_id_type", deviceIdType); + var storageMethod = storage.setStorage(userPath, memoryOnly, isBulk, persistQueue); + storageMethod.storeSet("cly_id", "SpecialDeviceId"); + storageMethod.storeSet("cly_id_type", deviceIdType); // Set values with different data types - storage.storeSet("cly_count", 42); - storage.storeSet("cly_object", { key: "value" }); - storage.storeSet("cly_null", null); + storageMethod.storeSet("cly_count", 42); + storageMethod.storeSet("cly_object", { key: "value" }); + storageMethod.storeSet("cly_null", null); // Retrieve and assert values - assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storage.storeGet("cly_id_type"), deviceIdType); - assert.equal(storage.storeGet("cly_count"), 42); - assert.deepEqual(storage.storeGet("cly_object"), { key: "value" }); - assert.equal(storage.storeGet("cly_null"), null); + assert.equal(storageMethod.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storageMethod.storeGet("cly_id_type"), deviceIdType); + assert.equal(storageMethod.storeGet("cly_count"), 42); + assert.deepEqual(storageMethod.storeGet("cly_object"), { key: "value" }); + assert.equal(storageMethod.storeGet("cly_null"), null); // Remove specific items by overriding with null or empty array - storage.storeSet("cly_id", null); - storage.storeSet("cly_object", []); - assert.equal(storage.storeGet("cly_id"), null); - assert.deepEqual(storage.storeGet("cly_object"), []); + storageMethod.storeSet("cly_id", null); + storageMethod.storeSet("cly_object", []); + assert.equal(storageMethod.storeGet("cly_id"), null); + assert.deepEqual(storageMethod.storeGet("cly_object"), []); // Reset storage and check if it's empty again storage.resetStorage(); - assert.equal(storage.storeGet("cly_id"), undefined); - assert.equal(storage.storeGet("cly_id_type"), undefined); - assert.equal(storage.storeGet("cly_count"), undefined); - assert.equal(storage.storeGet("cly_object"), undefined); - assert.equal(storage.storeGet("cly_null"), undefined); + assert.equal(storageMethod.storeGet("cly_id"), undefined); + assert.equal(storageMethod.storeGet("cly_id_type"), undefined); + assert.equal(storageMethod.storeGet("cly_count"), undefined); + assert.equal(storageMethod.storeGet("cly_object"), undefined); + assert.equal(storageMethod.storeGet("cly_null"), undefined); } describe("Storage Tests", () => { @@ -247,7 +248,7 @@ describe("Storage Tests", () => { // will set to default storage path storage.setStoragePath("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - recordValuesToStorageAndValidate(); + recordValuesToStorageAndValidate("../test/customStorageDirectory/"); done(); }); @@ -257,7 +258,7 @@ describe("Storage Tests", () => { // To set the storage path to the default bulk storage path and persist the queue storage.setStoragePath(null, true, true); assert.equal(storage.getStoragePath(), "../bulk_data/"); - recordValuesToStorageAndValidate(); + recordValuesToStorageAndValidate(null, false, true, true); done(); }); @@ -266,7 +267,7 @@ describe("Storage Tests", () => { // will set to default storage path storage.setStoragePath("../test/customStorageDirectory/", true); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - recordValuesToStorageAndValidate(); + recordValuesToStorageAndValidate("../test/customStorageDirectory/", false, true); done(); }); }); From c572bf1f9ff5badd6c4ed6ce6744c5e554e22196 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 19 Sep 2024 13:49:22 +0300 Subject: [PATCH 25/58] Update countly.js --- lib/countly.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/countly.js b/lib/countly.js index 5aa8e60..1e642dc 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -172,7 +172,7 @@ Countly.Bulk = Bulk; cc.debug = conf.debug; // Set the storage path - storageMethod = CountlyStorage.setStorage(conf.storage_path, true); + storageMethod = CountlyStorage.setStorage(conf.storage_path); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { From b4a96bf56f49acc6d920c67cbbdfcb82d3551063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 19 Sep 2024 13:55:32 +0300 Subject: [PATCH 26/58] storage method in bulk --- lib/countly-bulk.js | 25 ++++++++++++++----------- lib/countly.js | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index ceb542e..1315973 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -25,6 +25,8 @@ var cc = require("./countly-common"); var BulkUser = require("./countly-bulk-user"); var CountlyStorage = require("./countly-storage"); +var storageMethod; + /** * @lends module:lib/countly-bulk * Initialize CountlyBulk server object @@ -102,7 +104,8 @@ function CountlyBulk(conf) { conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; - CountlyStorage.setStoragePath(conf.storage_path, true, conf.persist_queue); + // config time memory only option will be added + storageMethod = CountlyStorage.setStorage(conf.storage_path, false, true, conf.persist_queue); this.conf = conf; /** @@ -142,7 +145,7 @@ function CountlyBulk(conf) { requestQueue.push(query); cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_request, Adding request to the queue."); - CountlyStorage.storeSet("cly_req_queue", requestQueue); + storageMethod.storeSet("cly_req_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_request, Sending message to the parent process. Adding the raw request to the queue."); @@ -190,7 +193,7 @@ function CountlyBulk(conf) { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_bulk_request, adding the request into queue."); requestQueue.push(query); } - CountlyStorage.storeSet("cly_req_queue", requestQueue); + storageMethod.storeSet("cly_req_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_bulk_request, Sending message to the parent process. Adding the raw request to the queue."); @@ -245,7 +248,7 @@ function CountlyBulk(conf) { eventQueue[device_id] = []; } eventQueue[device_id].push(e); - CountlyStorage.storeSet("cly_bulk_event", eventQueue); + storageMethod.storeSet("cly_bulk_event", eventQueue); } else { cc.log(cc.logLevelEnums.INFO, `CountlyBulk add_event, Sending message to the parent process. Adding event: [${event.key}].`); @@ -343,7 +346,7 @@ function CountlyBulk(conf) { */ function toBulkRequestQueue(bulkRequest) { bulkQueue.push(bulkRequest); - CountlyStorage.storeSet("cly_bulk_queue", bulkQueue); + storageMethod.storeSet("cly_bulk_queue", bulkQueue); } var self = this; @@ -369,7 +372,7 @@ function CountlyBulk(conf) { } if (eventChanges) { isEmpty = false; - CountlyStorage.storeSet("cly_bulk_event", eventQueue); + storageMethod.storeSet("cly_bulk_event", eventQueue); } // process request queue into bulk requests @@ -383,7 +386,7 @@ function CountlyBulk(conf) { var requests = requestQueue.splice(0, conf.bulk_size); toBulkRequestQueue({ app_key: conf.app_key, requests: JSON.stringify(requests) }); } - CountlyStorage.storeSet("cly_req_queue", requestQueue); + storageMethod.storeSet("cly_req_queue", requestQueue); } // process bulk request queue @@ -398,7 +401,7 @@ function CountlyBulk(conf) { bulkQueue.unshift(res); failTimeout = cc.getTimestamp() + conf.fail_timeout; } - CountlyStorage.storeSet("cly_bulk_queue", bulkQueue); + storageMethod.storeSet("cly_bulk_queue", bulkQueue); readyToProcess = true; }, "heartBeat", false); } @@ -591,9 +594,9 @@ function CountlyBulk(conf) { worker.on("message", handleWorkerMessage); }); - var requestQueue = CountlyStorage.storeGet("cly_req_queue", []); - var eventQueue = CountlyStorage.storeGet("cly_bulk_event", {}); - var bulkQueue = CountlyStorage.storeGet("cly_bulk_queue", []); + var requestQueue = storageMethod.storeGet("cly_req_queue", []); + var eventQueue = storageMethod.storeGet("cly_bulk_event", {}); + var bulkQueue = storageMethod.storeGet("cly_bulk_queue", []); } module.exports = CountlyBulk; diff --git a/lib/countly.js b/lib/countly.js index 1e642dc..102377b 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -171,7 +171,7 @@ Countly.Bulk = Bulk; // Common module debug value is set to init time debug value cc.debug = conf.debug; - // Set the storage path + // Set the storage method and path storageMethod = CountlyStorage.setStorage(conf.storage_path); // clear stored device ID if flag is set From 9b9643ae773a0c8bf857e3276db0c0939a305244 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 19 Sep 2024 14:55:23 +0300 Subject: [PATCH 27/58] Update tests_storage.js --- test/tests_storage.js | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/test/tests_storage.js b/test/tests_storage.js index 6815b53..024e532 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -270,4 +270,38 @@ describe("Storage Tests", () => { recordValuesToStorageAndValidate("../test/customStorageDirectory/", false, true); done(); }); -}); + + it("14- Setting storage path to default path via setStorage /no-init", (done) => { + storage.resetStorage(); + storage.setStorage(); + assert.equal(storage.getStoragePath(), "../data/"); + done(); + }); + + it("15- Setting bulk storage path to default path via setStorage /no-init", (done) => { + storage.resetStorage(); + storage.setStorage(null, false, true); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + done(); + }); + + it("16- Setting custom storage path via setStorage /no-init", (done) => { + storage.resetStorage(); + storage.setStorage("../test/customStorageDirectory/"); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + done(); + }); + + it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { + storage.resetStorage(); + storage.setStorage(null, true); + assert.equal(storage.getStoragePath(), undefined); + done(); + }); + + it("18- Recording to storage with memory only storage /no-init", (done) => { + storage.resetStorage(); + recordValuesToStorageAndValidate(null, true); + done(); + }); +}); \ No newline at end of file From ce59d66ffdad580e1de0589302b29d3d3374f9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 24 Sep 2024 16:06:01 +0300 Subject: [PATCH 28/58] Storing Methods Exported, Enums Added --- lib/countly-bulk.js | 30 ++++++++++++---------- lib/countly-common.js | 6 +++++ lib/countly-storage.js | 44 +++++++++++++++++++++------------ lib/countly.js | 47 +++++++++++++++++++---------------- test/tests_storage.js | 56 +++++++++++++++++++++--------------------- 5 files changed, 105 insertions(+), 78 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 1315973..ae58f20 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -25,7 +25,8 @@ var cc = require("./countly-common"); var BulkUser = require("./countly-bulk-user"); var CountlyStorage = require("./countly-storage"); -var storageMethod; +const StorageTypes = cc.storageTypeEnums; +var storageType = StorageTypes.FILE; /** * @lends module:lib/countly-bulk @@ -49,6 +50,7 @@ var storageMethod; * @param {number} [conf.max_breadcrumb_count=100] - maximum amount of breadcrumbs that can be recorded before the oldest one is deleted * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length + * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied * @example * var server = new CountlyBulk({ * app_key: "{YOUR-API-KEY}", @@ -104,8 +106,10 @@ function CountlyBulk(conf) { conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; - // config time memory only option will be added - storageMethod = CountlyStorage.setStorage(conf.storage_path, false, true, conf.persist_queue); + if (!conf.storage_type) { + storageType = conf.storage_type; + } + CountlyStorage.initStorage(conf.storage_path, storageType, true, conf.persist_queue); this.conf = conf; /** @@ -145,7 +149,7 @@ function CountlyBulk(conf) { requestQueue.push(query); cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_request, Adding request to the queue."); - storageMethod.storeSet("cly_req_queue", requestQueue); + CountlyStorage.storeSet("cly_req_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_request, Sending message to the parent process. Adding the raw request to the queue."); @@ -193,7 +197,7 @@ function CountlyBulk(conf) { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_bulk_request, adding the request into queue."); requestQueue.push(query); } - storageMethod.storeSet("cly_req_queue", requestQueue); + CountlyStorage.storeSet("cly_req_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "CountlyBulk add_bulk_request, Sending message to the parent process. Adding the raw request to the queue."); @@ -248,7 +252,7 @@ function CountlyBulk(conf) { eventQueue[device_id] = []; } eventQueue[device_id].push(e); - storageMethod.storeSet("cly_bulk_event", eventQueue); + CountlyStorage.storeSet("cly_bulk_event", eventQueue); } else { cc.log(cc.logLevelEnums.INFO, `CountlyBulk add_event, Sending message to the parent process. Adding event: [${event.key}].`); @@ -346,7 +350,7 @@ function CountlyBulk(conf) { */ function toBulkRequestQueue(bulkRequest) { bulkQueue.push(bulkRequest); - storageMethod.storeSet("cly_bulk_queue", bulkQueue); + CountlyStorage.storeSet("cly_bulk_queue", bulkQueue); } var self = this; @@ -372,7 +376,7 @@ function CountlyBulk(conf) { } if (eventChanges) { isEmpty = false; - storageMethod.storeSet("cly_bulk_event", eventQueue); + CountlyStorage.storeSet("cly_bulk_event", eventQueue); } // process request queue into bulk requests @@ -386,7 +390,7 @@ function CountlyBulk(conf) { var requests = requestQueue.splice(0, conf.bulk_size); toBulkRequestQueue({ app_key: conf.app_key, requests: JSON.stringify(requests) }); } - storageMethod.storeSet("cly_req_queue", requestQueue); + CountlyStorage.storeSet("cly_req_queue", requestQueue); } // process bulk request queue @@ -401,7 +405,7 @@ function CountlyBulk(conf) { bulkQueue.unshift(res); failTimeout = cc.getTimestamp() + conf.fail_timeout; } - storageMethod.storeSet("cly_bulk_queue", bulkQueue); + CountlyStorage.storeSet("cly_bulk_queue", bulkQueue); readyToProcess = true; }, "heartBeat", false); } @@ -594,9 +598,9 @@ function CountlyBulk(conf) { worker.on("message", handleWorkerMessage); }); - var requestQueue = storageMethod.storeGet("cly_req_queue", []); - var eventQueue = storageMethod.storeGet("cly_bulk_event", {}); - var bulkQueue = storageMethod.storeGet("cly_bulk_queue", []); + var requestQueue = CountlyStorage.storeGet("cly_req_queue", []); + var eventQueue = CountlyStorage.storeGet("cly_bulk_event", {}); + var bulkQueue = CountlyStorage.storeGet("cly_bulk_queue", []); } module.exports = CountlyBulk; diff --git a/lib/countly-common.js b/lib/countly-common.js index d829250..7a961e5 100644 --- a/lib/countly-common.js +++ b/lib/countly-common.js @@ -51,6 +51,12 @@ var cc = { ACTION: '[CLY]_action', }, + storageTypeEnums: { + FILE: "file", + MEMORY: "memory", + CUSTOM: "custom", + }, + /** * Get current timestamp * @returns {number} unix timestamp in seconds diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 95ea68e..592fd79 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -9,6 +9,7 @@ var defaultBulkPath = "../bulk_data/"; // Default path var asyncWriteLock = false; var asyncWriteQueue = []; let storageMethod = {}; +const StorageTypes = cc.storageTypeEnums; // Memory-only storage methods const memoryStorage = { @@ -18,7 +19,7 @@ const memoryStorage = { * @param {varies} value - value to store * @param {Function} callback - callback to call when done storing */ - storeSet(key, value, callback) { + storeSet: function(key, value, callback) { __data[key] = value; if (typeof callback === "function") { callback(null); // No errors for memory storage @@ -30,7 +31,7 @@ const memoryStorage = { * @param {varies} def - default value to use if not set * @returns {varies} value for the key */ - storeGet(key, def) { + storeGet: function(key, def) { cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`); return typeof __data[key] !== "undefined" ? __data[key] : def; }, @@ -38,7 +39,7 @@ const memoryStorage = { * Remove value from memory * @param {String} key - key of value to remove */ - storeRemove(key) { + storeRemove: function(key) { delete __data[key]; }, }; @@ -51,7 +52,7 @@ const fileStorage = { * @param {varies} value - value to store * @param {Function} callback - callback to call when done storing */ - storeSet(key, value, callback) { + storeSet: function(key, value, callback) { __data[key] = value; if (!asyncWriteLock) { asyncWriteLock = true; @@ -67,7 +68,7 @@ const fileStorage = { * @param {varies} def - default value to use if not set * @returns {varies} value for the key */ - storeGet(key, def) { + storeGet: function(key, def) { cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); if (typeof __data[key] === "undefined") { var ob = readFile(key); @@ -92,7 +93,7 @@ const fileStorage = { } return __data[key]; }, - storeRemove(key) { + storeRemove: function(key) { delete __data[key]; var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); fs.unlink(dir, (err) => { @@ -106,22 +107,18 @@ const fileStorage = { /** * Sets the storage method, by default sets file storage and storage path. * @param {String} userPath - User provided storage path - * @param {Boolean} memoryOnly - Whether to use memory only storage or not + * @param {StorageTypes} storageType - Whether to use memory only storage or not * @param {Boolean} isBulk - Whether the storage is for bulk data * @param {Boolean} persistQueue - Whether to persist the queue until processed - * @returns {storageMethod} Prefered storage method */ -var setStorage = function(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { - if (memoryOnly) { - cc.log(cc.logLevelEnums.DEBUG, "Using memory-only storage."); - storageMethod = memoryStorage; // Assign memory-only methods +var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false) { + if (storageType === StorageTypes.MEMORY) { + storageMethod = memoryStorage; } else { - cc.log(cc.logLevelEnums.DEBUG, "Using persistent file storage."); - storageMethod = fileStorage; // Assign file storage methods + storageMethod = fileStorage; setStoragePath(userPath, isBulk, persistQueue); } - return storageMethod; }; /** @@ -254,8 +251,23 @@ var writeFile = function(key, value, callback) { }); }; +var storeSet = function(key, value, callback) { + storageMethod.storeSet(key, value, callback); +}; + +var storeGet = function(key, def) { + return storageMethod.storeGet(key, def); +}; + +var storeRemove = function(key) { + storageMethod.storeRemove(key); +}; + module.exports = { - setStorage, + initStorage, + storeSet, + storeGet, + storeRemove, writeFile, forceStore, getStoragePath, diff --git a/lib/countly.js b/lib/countly.js index 102377b..3fc1714 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -29,7 +29,8 @@ var Bulk = require("./countly-bulk"); var CountlyStorage = require("./countly-storage"); var Countly = {}; -var storageMethod; +const StorageTypes = cc.storageTypeEnums; +var storageType = StorageTypes.FILE; Countly.Bulk = Bulk; (function() { @@ -123,6 +124,7 @@ Countly.Bulk = Bulk; * @param {string} conf.metrics._density - screen density of the device * @param {string} conf.metrics._locale - locale or language of the device in ISO format * @param {string} conf.metrics._store - source from where the user/device/installation came from + * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied * @example * Countly.init({ * app_key: "{YOUR-APP-KEY}", @@ -172,13 +174,16 @@ Countly.Bulk = Bulk; cc.debug = conf.debug; // Set the storage method and path - storageMethod = CountlyStorage.setStorage(conf.storage_path); + if (!conf.storage_type) { + storageType = conf.storage_type; + } + CountlyStorage.initStorage(conf.storage_path, storageType); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { cc.log(cc.logLevelEnums.WARNING, "init, clear_stored_device_id is true, erasing the stored ID."); - storageMethod.storeSet("cly_id", null); - storageMethod.storeSet("cly_id_type", null); + CountlyStorage.storeSet("cly_id", null); + CountlyStorage.storeSet("cly_id_type", null); } if (Countly.url === "") { @@ -224,8 +229,8 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { // fetch stored ID and ID type - var storedId = storageMethod.storeGet("cly_id", null); - var storedIdType = storageMethod.storeGet("cly_id_type", null); + var storedId = CountlyStorage.storeGet("cly_id", null); + var storedIdType = CountlyStorage.storeGet("cly_id_type", null); // if there was a stored ID if (storedId !== null) { Countly.device_id = storedId; @@ -254,12 +259,12 @@ Countly.Bulk = Bulk; deviceIdType = cc.deviceIdTypeEnums.SDK_GENERATED; } // save the ID and ID type - storageMethod.storeSet("cly_id", Countly.device_id); - storageMethod.storeSet("cly_id_type", deviceIdType); + CountlyStorage.storeSet("cly_id", Countly.device_id); + CountlyStorage.storeSet("cly_id_type", deviceIdType); // create queues - requestQueue = storageMethod.storeGet("cly_queue", []); - eventQueue = storageMethod.storeGet("cly_event", []); - remoteConfigs = storageMethod.storeGet("cly_remote_configs", {}); + requestQueue = CountlyStorage.storeGet("cly_queue", []); + eventQueue = CountlyStorage.storeGet("cly_event", []); + remoteConfigs = CountlyStorage.storeGet("cly_remote_configs", {}); heartBeat(); // listen to current workers if (cluster.workers) { @@ -636,7 +641,7 @@ Countly.Bulk = Bulk; if (eventQueue.length > 0) { toRequestQueue({ events: JSON.stringify(eventQueue) }); eventQueue = []; - storageMethod.storeSet("cly_event", eventQueue); + CountlyStorage.storeSet("cly_event", eventQueue); } // end current session Countly.end_session(); @@ -648,8 +653,8 @@ Countly.Bulk = Bulk; var oldId = Countly.device_id; Countly.device_id = newId; deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storageMethod.storeSet("cly_id", Countly.device_id); - storageMethod.storeSet("cly_id_type", deviceIdType); + CountlyStorage.storeSet("cly_id", Countly.device_id); + CountlyStorage.storeSet("cly_id_type", deviceIdType); if (merge) { if (Countly.check_any_consent()) { toRequestQueue({ old_device_id: oldId }); @@ -665,7 +670,7 @@ Countly.Bulk = Bulk; if (Countly.remote_config) { remoteConfigs = {}; if (cluster.isMaster) { - storageMethod.storeSet("cly_remote_configs", remoteConfigs); + CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); } Countly.fetch_remote_config(Countly.remote_config); } @@ -745,7 +750,7 @@ Countly.Bulk = Bulk; e.dow = date.getDay(); cc.log(cc.logLevelEnums.DEBUG, "add_cly_events, Adding event: ", event); eventQueue.push(e); - storageMethod.storeSet("cly_event", eventQueue); + CountlyStorage.storeSet("cly_event", eventQueue); } else { process.send({ cly: { event } }); @@ -1156,7 +1161,7 @@ Countly.Bulk = Bulk; remoteConfigs = configs; } if (cluster.isMaster) { - storageMethod.storeSet("cly_remote_configs", remoteConfigs); + CountlyStorage.storeSet("cly_remote_configs", remoteConfigs); cc.log(cc.logLevelEnums.INFO, `fetch_remote_config, Fetched remote config: [${remoteConfigs}].`); } } @@ -1340,7 +1345,7 @@ Countly.Bulk = Bulk; if (cluster.isMaster) { cc.log(cc.logLevelEnums.INFO, "request, Adding the raw request to the queue."); requestQueue.push(request); - storageMethod.storeSet("cly_queue", requestQueue); + CountlyStorage.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "request, Sending message to the parent process. Adding the raw request to the queue."); @@ -1427,7 +1432,7 @@ Countly.Bulk = Bulk; cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Adding request to the queue."); requestQueue.push(request); - storageMethod.storeSet("cly_queue", requestQueue); + CountlyStorage.storeSet("cly_queue", requestQueue); } else { cc.log(cc.logLevelEnums.INFO, "toRequestQueue, Sending message to the parent process. Adding request to the queue."); @@ -1458,7 +1463,7 @@ Countly.Bulk = Bulk; var events = eventQueue.splice(0, maxEventBatch); toRequestQueue({ events: JSON.stringify(events) }); } - storageMethod.storeSet("cly_event", eventQueue); + CountlyStorage.storeSet("cly_event", eventQueue); } // process request queue with event queue @@ -1473,7 +1478,7 @@ Countly.Bulk = Bulk; failTimeout = cc.getTimestamp() + failTimeoutAmount; cc.log(cc.logLevelEnums.ERROR, `makeRequest, Encountered a problem while making the request: [${err}]`); } - storageMethod.storeSet("cly_queue", requestQueue); + CountlyStorage.storeSet("cly_queue", requestQueue); readyToProcess = true; }, "heartBeat", false); } diff --git a/test/tests_storage.js b/test/tests_storage.js index 024e532..338b66e 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -67,35 +67,35 @@ function validateDeviceId(deviceId, deviceIdType, expectedDeviceId, expectedDevi function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { // Set values var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - var storageMethod = storage.setStorage(userPath, memoryOnly, isBulk, persistQueue); - storageMethod.storeSet("cly_id", "SpecialDeviceId"); - storageMethod.storeSet("cly_id_type", deviceIdType); + storage.initStorage(userPath, memoryOnly, isBulk, persistQueue); + storage.storeSet("cly_id", "SpecialDeviceId"); + storage.storeSet("cly_id_type", deviceIdType); // Set values with different data types - storageMethod.storeSet("cly_count", 42); - storageMethod.storeSet("cly_object", { key: "value" }); - storageMethod.storeSet("cly_null", null); + storage.storeSet("cly_count", 42); + storage.storeSet("cly_object", { key: "value" }); + storage.storeSet("cly_null", null); // Retrieve and assert values - assert.equal(storageMethod.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storageMethod.storeGet("cly_id_type"), deviceIdType); - assert.equal(storageMethod.storeGet("cly_count"), 42); - assert.deepEqual(storageMethod.storeGet("cly_object"), { key: "value" }); - assert.equal(storageMethod.storeGet("cly_null"), null); + assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + assert.equal(storage.storeGet("cly_count"), 42); + assert.deepEqual(storage.storeGet("cly_object"), { key: "value" }); + assert.equal(storage.storeGet("cly_null"), null); // Remove specific items by overriding with null or empty array - storageMethod.storeSet("cly_id", null); - storageMethod.storeSet("cly_object", []); - assert.equal(storageMethod.storeGet("cly_id"), null); - assert.deepEqual(storageMethod.storeGet("cly_object"), []); + storage.storeSet("cly_id", null); + storage.storeSet("cly_object", []); + assert.equal(storage.storeGet("cly_id"), null); + assert.deepEqual(storage.storeGet("cly_object"), []); // Reset storage and check if it's empty again storage.resetStorage(); - assert.equal(storageMethod.storeGet("cly_id"), undefined); - assert.equal(storageMethod.storeGet("cly_id_type"), undefined); - assert.equal(storageMethod.storeGet("cly_count"), undefined); - assert.equal(storageMethod.storeGet("cly_object"), undefined); - assert.equal(storageMethod.storeGet("cly_null"), undefined); + assert.equal(storage.storeGet("cly_id"), undefined); + assert.equal(storage.storeGet("cly_id_type"), undefined); + assert.equal(storage.storeGet("cly_count"), undefined); + assert.equal(storage.storeGet("cly_object"), undefined); + assert.equal(storage.storeGet("cly_null"), undefined); } describe("Storage Tests", () => { @@ -271,37 +271,37 @@ describe("Storage Tests", () => { done(); }); - it("14- Setting storage path to default path via setStorage /no-init", (done) => { + it("14- Setting storage path to default path via initStorage /no-init", (done) => { storage.resetStorage(); - storage.setStorage(); + storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); done(); }); - it("15- Setting bulk storage path to default path via setStorage /no-init", (done) => { + it("15- Setting bulk storage path to default path via initStorage /no-init", (done) => { storage.resetStorage(); - storage.setStorage(null, false, true); + storage.initStorage(null, false, true); assert.equal(storage.getStoragePath(), "../bulk_data/"); done(); }); - it("16- Setting custom storage path via setStorage /no-init", (done) => { + it("16- Setting custom storage path via initStorage /no-init", (done) => { storage.resetStorage(); - storage.setStorage("../test/customStorageDirectory/"); + storage.initStorage("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); done(); }); it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { storage.resetStorage(); - storage.setStorage(null, true); + storage.initStorage(null, cc.storageTypeEnums.MEMORY); assert.equal(storage.getStoragePath(), undefined); done(); }); it("18- Recording to storage with memory only storage /no-init", (done) => { storage.resetStorage(); - recordValuesToStorageAndValidate(null, true); + recordValuesToStorageAndValidate(null, cc.storageTypeEnums.MEMORY); done(); }); }); \ No newline at end of file From f370c782d031c4158b4abcdaff05ca35ae653955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 1 Oct 2024 14:15:29 +0300 Subject: [PATCH 29/58] Updating Config Section --- lib/countly-bulk.js | 8 +++----- lib/countly.js | 11 +++-------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index ae58f20..9fc6b53 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -26,7 +26,6 @@ var BulkUser = require("./countly-bulk-user"); var CountlyStorage = require("./countly-storage"); const StorageTypes = cc.storageTypeEnums; -var storageType = StorageTypes.FILE; /** * @lends module:lib/countly-bulk @@ -76,6 +75,7 @@ function CountlyBulk(conf) { var maxBreadcrumbCount = 100; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; + var storageType = StorageTypes.FILE; cc.debugBulk = conf.debug || false; if (!conf.app_key) { @@ -105,11 +105,9 @@ function CountlyBulk(conf) { conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount; conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; + conf.storage_type = conf.storage_type || storageType; - if (!conf.storage_type) { - storageType = conf.storage_type; - } - CountlyStorage.initStorage(conf.storage_path, storageType, true, conf.persist_queue); + CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue); this.conf = conf; /** diff --git a/lib/countly.js b/lib/countly.js index 3fc1714..7a51f1e 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -30,7 +30,6 @@ var CountlyStorage = require("./countly-storage"); var Countly = {}; const StorageTypes = cc.storageTypeEnums; -var storageType = StorageTypes.FILE; Countly.Bulk = Bulk; (function() { @@ -73,7 +72,7 @@ Countly.Bulk = Bulk; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; var deviceIdType = null; - + var storageType = StorageTypes.FILE; /** * Array with list of available features that you can require consent for */ @@ -169,15 +168,11 @@ Countly.Bulk = Bulk; Countly.maxBreadcrumbCount = conf.max_breadcrumb_count || Countly.max_breadcrumb_count || conf.max_logs || Countly.max_logs || maxBreadcrumbCount; Countly.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || Countly.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; Countly.maxStackTraceLineLength = conf.max_stack_trace_line_length || Countly.max_stack_trace_line_length || maxStackTraceLineLength; - + conf.storage_type = conf.storage_type || storageType; // Common module debug value is set to init time debug value cc.debug = conf.debug; - // Set the storage method and path - if (!conf.storage_type) { - storageType = conf.storage_type; - } - CountlyStorage.initStorage(conf.storage_path, storageType); + CountlyStorage.initStorage(conf.storage_path, conf.storage_type); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { From 783749132953454ef9a305b354130f29b729e082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 1 Oct 2024 18:14:31 +0300 Subject: [PATCH 30/58] Updated Storage Tests --- lib/countly-storage.js | 24 +++++++-- test/tests_storage.js | 113 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 131 insertions(+), 6 deletions(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 592fd79..122bb94 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -20,9 +20,15 @@ const memoryStorage = { * @param {Function} callback - callback to call when done storing */ storeSet: function(key, value, callback) { - __data[key] = value; - if (typeof callback === "function") { - callback(null); // No errors for memory storage + if (key) { + cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting key: [${key}] & value: [${value}]!`); + __data[key] = value; + if (typeof callback === "function") { + callback(null); + } + } + else { + cc.log(cc.logLevelEnums.WARNING, `storeSet, Provioded key: [${key}] is null!`); } }, /** @@ -263,6 +269,17 @@ var storeRemove = function(key) { storageMethod.storeRemove(key); }; +/** + * Disclaimer: This method is mainly for testing purposes. + * @returns {StorageTypes} Returns the active storage type for the SDK + */ +var getStorageType = function() { + if (storageMethod === memoryStorage) { + return StorageTypes.MEMORY; + } + return StorageTypes.FILE; +}; + module.exports = { initStorage, storeSet, @@ -274,4 +291,5 @@ module.exports = { setStoragePath, resetStorage, readFile, + getStorageType, }; \ No newline at end of file diff --git a/test/tests_storage.js b/test/tests_storage.js index 338b66e..41124b7 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -4,6 +4,8 @@ var storage = require("../lib/countly-storage"); var cc = require("../lib/countly-common"); var hp = require("./helpers/helper_functions"); +const StorageTypes = cc.storageTypeEnums; + // example event object to use var eventObj = { key: "storage_check", @@ -294,14 +296,119 @@ describe("Storage Tests", () => { it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { storage.resetStorage(); - storage.initStorage(null, cc.storageTypeEnums.MEMORY); + storage.initStorage(null, StorageTypes.MEMORY); + assert.equal(storage.getStoragePath(), undefined); + done(); + }); + + it("18- Memory only storage Device-Id", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.MEMORY, + }); assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.storeGet("cly_id", null), "Test-Device-Id"); + assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); done(); }); - it("18- Recording to storage with memory only storage /no-init", (done) => { + it("19- Record event in memory only mode and validate the record", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.MEMORY, + }); + Countly.add_event(eventObj); + setTimeout(() => { + const storedData = storage.storeGet("cly_queue", null); + const eventArray = JSON.parse(storedData[0].events); + const eventFromQueue = eventArray[0]; + hp.eventValidator(eventObj, eventFromQueue); + done(); + }, hp.mWait); + }); + + it("20- Record and validate user details in memory only mode", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.MEMORY, + }); + Countly.user_details(userDetailObj); + const storedData = storage.storeGet("cly_queue", null); + const userDetailsReq = storedData[0]; + hp.userDetailRequestValidator(userDetailObj, userDetailsReq); + done(); + }); + + it("21- Memory only storage, change SDK Generated Device-Id", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + clear_stored_device_id: true, + storage_type: StorageTypes.MEMORY, + }); + assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.storeGet("cly_id", null), Countly.get_device_id()); + assert.equal(storage.storeGet("cly_id_type", null), Countly.get_device_id_type()); + + Countly.change_id("Test-Id-2"); + assert.equal(storage.storeGet("cly_id", null), "Test-Id-2"); + assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + done(); + }); + + it("22- Switch to file storage after init", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + clear_stored_device_id: true, + storage_type: StorageTypes.MEMORY, + }); + assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.getStorageType(), StorageTypes.MEMORY); + + storage.initStorage(); + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + done(); + }); + + it("23- storeRemove Memory Only /no-init", (done) => { storage.resetStorage(); - recordValuesToStorageAndValidate(null, cc.storageTypeEnums.MEMORY); + storage.initStorage(null, StorageTypes.MEMORY); + assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.getStorageType(), StorageTypes.MEMORY); + storage.storeSet("keyToStore", "valueToStore"); + assert.equal(storage.storeGet("keyToStore", null), "valueToStore"); + + storage.storeRemove("keyToStore"); + assert.equal(storage.storeGet("keyToStore", null), null); + done(); + }); + + it("24- storeRemove File Storage /no-init", (done) => { + storage.resetStorage(); + storage.initStorage(); + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + storage.storeSet("keyToStore", "valueToStore"); + assert.equal(storage.storeGet("keyToStore", null), "valueToStore"); + + storage.storeRemove("keyToStore"); + assert.equal(storage.storeGet("keyToStore", null), null); done(); }); }); \ No newline at end of file From ddf942b479f6da4f9fef1f34234a3e60aeb8a812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 00:22:04 +0300 Subject: [PATCH 31/58] Bug Fix Testing --- CHANGELOG.md | 3 ++- lib/countly-storage.js | 31 +++++++++++++++++++++-------- test/helpers/helper_functions.js | 34 +++++++++++++++++++++++--------- test/tests_storage.js | 19 ++++++++++++++++-- 4 files changed, 67 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c63f40f..a7cb28f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## xx.xx.xx -* Default max segmentation value count changed from 30 to 100 +- Default max segmentation value count changed from 30 to 100 +- Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file. ## 22.06.0 - Fixed a bug where remote config requests were rejected diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 122bb94..f5782fa 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -101,11 +101,20 @@ const fileStorage = { }, storeRemove: function(key) { delete __data[key]; - var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); - fs.unlink(dir, (err) => { - if (err) { - cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err}].`); + var filePath = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); + fs.access(filePath, fs.constants.F_OK, (accessErr) => { + if (accessErr) { + cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`); + return; } + fs.rmSync(filePath, { recursive: true, force: true }, (err) => { + if (err) { + cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`); + } + else { + cc.log(cc.logLevelEnums.INFO, `storeRemove, Successfully removed file with key: [${key}].`); + } + }); }); }, }; @@ -181,15 +190,21 @@ var resetStorage = function() { */ var readFile = function(key) { var dir = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); - // try reading data file var data; try { - data = fs.readFileSync(dir); + data = fs.readFileSync(dir, 'utf8'); // read file as string } catch (ex) { // there was no file, probably new init - data = null; + cc.log(cc.logLevelEnums.WARN, `readFile, File not found for key: [${key}]. Returning null.`); + return null; // early exit if file doesn't exist + } + + // early exit if file is empty or whitespace + if (!data.trim()) { + cc.log(cc.logLevelEnums.WARN, `readFile, File is empty or contains only whitespace for key: [${key}]. Returning null.`); + return null; } try { @@ -202,7 +217,7 @@ var readFile = function(key) { // backup corrupted file data fs.writeFile(path.resolve(__dirname, `${getStoragePath()}__${key}.${cc.getTimestamp()}${Math.random()}.json`), data, () => { }); // start with new clean object - data = null; + return null; // return null in case of corrupted data } return data; }; diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index eaee194..1728687 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -3,10 +3,12 @@ var path = require("path"); var assert = require("assert"); var fs = require("fs"); var Countly = require("../../lib/countly"); +var CountlyStorage = require("../../lib/countly-storage"); // paths for convenience var dir = path.resolve(__dirname, "../../"); var idDir = (`${dir}/data/__cly_id.json`); +var idTypeDir = (`${dir}/data/__cly_id_type.json`); var eventDir = (`${dir}/data/__cly_event.json`); var reqDir = (`${dir}/data/__cly_queue.json`); var bulkEventDir = (`${dir}/bulk_data/__cly_bulk_event.json`); @@ -39,24 +41,37 @@ function readRequestQueue(givenPath = null, isBulk = false) { } return a; } -// queue files clearing logic +function doesFileStoragePathsExist(callback) { + fs.access(idDir, fs.constants.F_OK, (err1) => { + fs.access(idTypeDir, fs.constants.F_OK, (err2) => { + fs.access(eventDir, fs.constants.F_OK, (err3) => { + // If all err variables are null, all files exist + const allFilesExist = !err1 && !err2 && !err3; + callback(allFilesExist); + }); + }); + }); +} function clearStorage(keepID = false, isBulk = false, customDir = '') { // Resets Countly Countly.halt(true); // Determine the directory based on isBulk or customDir const eventDirectory = customDir || (isBulk ? bulkEventDir : eventDir); const reqDirectory = customDir || (isBulk ? bulkQueueDir : reqDir); - // Remove event directory if it exists - if (fs.existsSync(eventDirectory)) { - fs.rmSync(eventDirectory, { recursive: true, force: true }); + // Helper function to remove directory and files + function removeDir(directory) { + if (fs.existsSync(directory)) { + fs.rmSync(directory, { recursive: true, force: true }); + } } + // Remove event directory if it exists + removeDir(eventDirectory); // Remove request directory if it exists - if (fs.existsSync(reqDirectory)) { - fs.rmSync(reqDirectory, { recursive: true, force: true }); - } + removeDir(reqDirectory); // Optionally keep the ID directory - if (!keepID && fs.existsSync(idDir)) { - fs.rmSync(idDir, { recursive: true, force: true }); + if (!keepID) { + removeDir(idDir); + removeDir(idTypeDir); } } /** @@ -215,4 +230,5 @@ module.exports = { sessionRequestValidator, userDetailRequestValidator, viewEventValidator, + doesFileStoragePathsExist, }; \ No newline at end of file diff --git a/test/tests_storage.js b/test/tests_storage.js index 41124b7..bc7d5e1 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -310,6 +310,9 @@ describe("Storage Tests", () => { clear_stored_device_id: true, storage_type: StorageTypes.MEMORY, }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); assert.equal(storage.getStoragePath(), undefined); assert.equal(storage.storeGet("cly_id", null), "Test-Device-Id"); assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); @@ -325,6 +328,9 @@ describe("Storage Tests", () => { clear_stored_device_id: true, storage_type: StorageTypes.MEMORY, }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); Countly.add_event(eventObj); setTimeout(() => { const storedData = storage.storeGet("cly_queue", null); @@ -344,6 +350,9 @@ describe("Storage Tests", () => { clear_stored_device_id: true, storage_type: StorageTypes.MEMORY, }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); Countly.user_details(userDetailObj); const storedData = storage.storeGet("cly_queue", null); const userDetailsReq = storedData[0]; @@ -359,6 +368,9 @@ describe("Storage Tests", () => { clear_stored_device_id: true, storage_type: StorageTypes.MEMORY, }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); assert.equal(storage.getStoragePath(), undefined); assert.equal(storage.storeGet("cly_id", null), Countly.get_device_id()); assert.equal(storage.storeGet("cly_id_type", null), Countly.get_device_id_type()); @@ -379,6 +391,9 @@ describe("Storage Tests", () => { }); assert.equal(storage.getStoragePath(), undefined); assert.equal(storage.getStorageType(), StorageTypes.MEMORY); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); @@ -387,7 +402,7 @@ describe("Storage Tests", () => { }); it("23- storeRemove Memory Only /no-init", (done) => { - storage.resetStorage(); + hp.clearStorage(); storage.initStorage(null, StorageTypes.MEMORY); assert.equal(storage.getStoragePath(), undefined); assert.equal(storage.getStorageType(), StorageTypes.MEMORY); @@ -400,7 +415,7 @@ describe("Storage Tests", () => { }); it("24- storeRemove File Storage /no-init", (done) => { - storage.resetStorage(); + hp.clearStorage(); storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); assert.equal(storage.getStorageType(), StorageTypes.FILE); From 9cb8700f5a4cd0d119e1f294b01a2ff202b103ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 00:30:50 +0300 Subject: [PATCH 32/58] Update tests_internal_limits.js these were failing for a reason, updated the checks --- test/tests_internal_limits.js | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/test/tests_internal_limits.js b/test/tests_internal_limits.js index 5d38307..712da5d 100644 --- a/test/tests_internal_limits.js +++ b/test/tests_internal_limits.js @@ -79,9 +79,10 @@ describe("Testing internal limits", () => { assert.ok(event.segmentation["key of 3"]); assert.ok(!event.segmentation["key of 4"]); assert.equal(event.segmentation["key of 3"], "Value of"); - assert.ok(event.timestamp); - assert.ok(event.hour); - assert.ok(event.dow); + // common parameter validation + assert.ok(event.timestamp !== 'undefined'); + assert.ok(event.hour !== 'undefined'); + assert.ok(event.dow !== 'undefined'); done(); }, hp.sWait); }); @@ -101,9 +102,10 @@ describe("Testing internal limits", () => { assert.equal(event.segmentation.name, "a very l"); assert.equal(event.segmentation.visit, 1); assert.ok(event.segmentation.segment); - assert.ok(event.timestamp); - assert.ok(event.hour); - assert.ok(event.dow); + // common parameter validation + assert.ok(event.timestamp !== 'undefined'); + assert.ok(event.hour !== 'undefined'); + assert.ok(event.dow !== 'undefined'); done(); }, hp.sWait); }); @@ -134,9 +136,10 @@ describe("Testing internal limits", () => { assert.ok(req.device_id); assert.ok(req.sdk_name); assert.ok(req.sdk_version); - assert.ok(req.timestamp); - assert.ok(req.hour); - assert.ok(req.dow); + // common parameter validation + assert.ok(req.timestamp !== 'undefined'); + assert.ok(req.hour !== 'undefined'); + assert.ok(req.dow !== 'undefined'); var crash = JSON.parse(req.crash); assert.equal(crash._logs, "log5 too\nlog6\nlog7"); assert.ok(crash._os); @@ -184,9 +187,10 @@ describe("Testing internal limits", () => { assert.ok(req.device_id); assert.ok(req.sdk_name); assert.ok(req.sdk_version); - assert.ok(req.timestamp); - assert.ok(req.hour); - assert.ok(req.dow); + // common parameter validation + assert.ok(req.timestamp !== 'undefined'); + assert.ok(req.hour !== 'undefined'); + assert.ok(req.dow !== 'undefined'); var details = JSON.parse(req.user_details); assert.equal(details.name, 'Gottlob '); assert.equal(details.username, 'Grundges'); @@ -229,9 +233,10 @@ describe("Testing internal limits", () => { assert.ok(req.device_id); assert.ok(req.sdk_name); assert.ok(req.sdk_version); - assert.ok(req.timestamp); - assert.ok(req.hour); - assert.ok(req.dow); + // common parameter validation + assert.ok(req.timestamp !== 'undefined'); + assert.ok(req.hour !== 'undefined'); + assert.ok(req.dow !== 'undefined'); var details = JSON.parse(req.user_details).custom; // set assert.equal(details['name of '], 'Bertrand'); From 4dfbaa7357f309deee9117c0958bc574e1405e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 00:34:03 +0300 Subject: [PATCH 33/58] Changelog update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7cb28f..6311fef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## xx.xx.xx - Default max segmentation value count changed from 30 to 100 - Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file. +- Added a new init time config option (conf.storage_type) which enables Memory Only Storage. ## 22.06.0 - Fixed a bug where remote config requests were rejected From caf69b5c044922896473650839e06cf655ffff21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 09:20:08 +0300 Subject: [PATCH 34/58] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6311fef..bc0c778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ ## xx.xx.xx - Default max segmentation value count changed from 30 to 100 - Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file. -- Added a new init time config option (conf.storage_type) which enables Memory Only Storage. +- Added a new init time config option (conf.storage_type) which enables Memory Only Storage and File Storage. ## 22.06.0 - Fixed a bug where remote config requests were rejected From eb61043f0336549940658942c518b44364f8c623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 09:24:37 +0300 Subject: [PATCH 35/58] Update CHANGELOG.md --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc0c778..d78b2ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ ## xx.xx.xx - Default max segmentation value count changed from 30 to 100 - Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file. -- Added a new init time config option (conf.storage_type) which enables Memory Only Storage and File Storage. +- Added a new init time config option (conf.storage_type) which can make user set among these storage options: + - File Storage + - Memory Only Storage ## 22.06.0 - Fixed a bug where remote config requests were rejected From ae89395c9de7bd5e275e10916d43f088a03a8a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 09:49:59 +0300 Subject: [PATCH 36/58] Back to Unlink --- lib/countly-storage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index f5782fa..27e1a04 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -107,7 +107,7 @@ const fileStorage = { cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`); return; } - fs.rmSync(filePath, { recursive: true, force: true }, (err) => { + fs.unlink(filePath, (err) => { if (err) { cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`); } From c475d77fb8a12b6eb115ca36517bf739fd7d3184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 13:56:07 +0300 Subject: [PATCH 37/58] Custom Storage Option --- CHANGELOG.md | 2 + lib/countly-storage.js | 13 +++- lib/countly.js | 3 +- test/tests_storage.js | 161 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 176 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d78b2ba..6161e10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - Added a new init time config option (conf.storage_type) which can make user set among these storage options: - File Storage - Memory Only Storage + - Custom Storage Methods +- Added a new init time config option (conf.custom_storage_method) which enables user to provide custom storage methods. ## 22.06.0 - Fixed a bug where remote config requests were rejected diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 27e1a04..024f442 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -125,11 +125,22 @@ const fileStorage = { * @param {StorageTypes} storageType - Whether to use memory only storage or not * @param {Boolean} isBulk - Whether the storage is for bulk data * @param {Boolean} persistQueue - Whether to persist the queue until processed + * @param {varies} customStorageMethod - Storage methods provided by the user */ -var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false) { +var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false, customStorageMethod = null) { if (storageType === StorageTypes.MEMORY) { storageMethod = memoryStorage; } + else if (storageType === StorageTypes.CUSTOM) { + if (customStorageMethod) { + storageMethod = customStorageMethod; + } + else { + cc.log(cc.logLevelEnums.WARNING, `Provided Custom Storage Methods are not valid. Switching to default file storage!`); + storageMethod = fileStorage; + setStoragePath(userPath, isBulk, persistQueue); + } + } else { storageMethod = fileStorage; setStoragePath(userPath, isBulk, persistQueue); diff --git a/lib/countly.js b/lib/countly.js index 7a51f1e..8492bec 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -124,6 +124,7 @@ Countly.Bulk = Bulk; * @param {string} conf.metrics._locale - locale or language of the device in ISO format * @param {string} conf.metrics._store - source from where the user/device/installation came from * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied + * @param {varies} conf.custom_storage_method - user given storage methods * @example * Countly.init({ * app_key: "{YOUR-APP-KEY}", @@ -172,7 +173,7 @@ Countly.Bulk = Bulk; // Common module debug value is set to init time debug value cc.debug = conf.debug; - CountlyStorage.initStorage(conf.storage_path, conf.storage_type); + CountlyStorage.initStorage(conf.storage_path, conf.storage_type, false, false, conf.custom_storage_method); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { diff --git a/test/tests_storage.js b/test/tests_storage.js index bc7d5e1..99b28a7 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -99,6 +99,53 @@ function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = assert.equal(storage.storeGet("cly_object"), undefined); assert.equal(storage.storeGet("cly_null"), undefined); } +const funkyMemoryStorage = { + _storage: {}, + + storeSet: function(key, value, callback) { + if (key) { + const existingValue = this._storage[key]; + if (typeof value === 'string' && typeof existingValue === 'string') { + this._storage[key] = existingValue + value; + } + else { + this._storage[key] = value; + } + if (typeof callback === "function") { + callback(null); + } + } + }, + storeGet: function(key, def) { + const value = this._storage[key]; + if (typeof value === 'string') { + return value.split('').reverse().join(''); + } + + return value !== undefined ? value : def; + }, + storeRemove: function(key) { + delete this._storage[key]; + }, +}; + +const customMemoryStorage = { + _storage: {}, + storeSet: function(key, value, callback) { + if (key) { + this._storage[key] = value; + if (typeof callback === "function") { + callback(null); + } + } + }, + storeGet: function(key, def) { + return typeof this._storage[key] !== "undefined" ? this._storage[key] : def; + }, + storeRemove: function(key) { + delete this._storage[key]; + }, +}; describe("Storage Tests", () => { it("1- Store Generated Device ID", (done) => { @@ -225,6 +272,8 @@ describe("Storage Tests", () => { done(); }); + // resets the storage path to default and validates that it is set correctly, + // then resets it to undefined and confirms the reset. it("9- Reset Storage While on Default Path /no-init", (done) => { // will set to default storage path storage.setStoragePath(); @@ -235,9 +284,10 @@ describe("Storage Tests", () => { done(); }); + // sets the storage path to default and verifies it, + // then records values to storage and ensures they are stored correctly. it("10- Recording to Storage with Default Storage Path /no-init", (done) => { storage.resetStorage(); - // Set to default storage path storage.setStoragePath(); assert.equal(storage.getStoragePath(), "../data/"); @@ -245,6 +295,8 @@ describe("Storage Tests", () => { done(); }); + // sets a custom storage path and verifies it, + // then records values to storage and ensures correct storage in the custom path. it("11- Recording to Storage with Custom Storage Path /no-init", (done) => { storage.resetStorage(); // will set to default storage path @@ -254,6 +306,8 @@ describe("Storage Tests", () => { done(); }); + // sets the storage path to the default bulk storage path and verifies it, + // then records values to bulk storage and validates proper storage in bulk mode. it("12- Recording to Bulk Storage with Default Bulk Data Path /no-init", (done) => { storage.resetStorage(); // will set to default storage path @@ -264,6 +318,8 @@ describe("Storage Tests", () => { done(); }); + // sets a custom bulk storage path and verifies it, + // then records values to bulk storage and ensures proper recording to the custom path. it("13- Recording to Bulk Storage with Custom Bulk Storage Path /no-init", (done) => { storage.resetStorage(); // will set to default storage path @@ -301,6 +357,9 @@ describe("Storage Tests", () => { done(); }); + // recording device-id in memory only mode + // initializes the SDK in memory only mode, validates that file storage files does not exist + // retrieve the developer supplied device id and id type from storage it("18- Memory only storage Device-Id", (done) => { hp.clearStorage(); Countly.init({ @@ -319,6 +378,9 @@ describe("Storage Tests", () => { done(); }); + // recording event in memory only mode + // initializes the SDK in memory only mode, validates that file storage files does not exist + // records an event and validates the recorded event it("19- Record event in memory only mode and validate the record", (done) => { hp.clearStorage(); Countly.init({ @@ -341,6 +403,9 @@ describe("Storage Tests", () => { }, hp.mWait); }); + // recording user details in memory only mode + // initializes the SDK in memory only mode, validates that file storage files does not exist + // records user details and validates the recorded details it("20- Record and validate user details in memory only mode", (done) => { hp.clearStorage(); Countly.init({ @@ -360,6 +425,9 @@ describe("Storage Tests", () => { done(); }); + // tests device id changes in memory only storage + // initialize the SDK in memory only mode, check the device id and switch it + // SDK and storage should function properly it("21- Memory only storage, change SDK Generated Device-Id", (done) => { hp.clearStorage(); Countly.init({ @@ -381,6 +449,9 @@ describe("Storage Tests", () => { done(); }); + // tests switching between storage types after initializing SDK + // passing memory storage type during init and initializing storage afterwards + // SDK should switch to file storage it("22- Switch to file storage after init", (done) => { hp.clearStorage(); Countly.init({ @@ -401,6 +472,9 @@ describe("Storage Tests", () => { done(); }); + // tests storeRemove function in CountlyStorage + // after initializing the memory storage, without initializing SDK, attempts to set, get and remove values + // without initializing SDK storage should function properly it("23- storeRemove Memory Only /no-init", (done) => { hp.clearStorage(); storage.initStorage(null, StorageTypes.MEMORY); @@ -414,6 +488,9 @@ describe("Storage Tests", () => { done(); }); + // tests storeRemove function in CountlyStorage + // after initializing the file storage, without initializing SDK attempts to set, get and remove values + // without initializing SDK storage should function properly it("24- storeRemove File Storage /no-init", (done) => { hp.clearStorage(); storage.initStorage(); @@ -426,4 +503,86 @@ describe("Storage Tests", () => { assert.equal(storage.storeGet("keyToStore", null), null); done(); }); + + // tests init time storage config options + // choosing Custom storage type and passing null in storage methods + // passing null as storage method ends up with switching to default file storage + it("25- Null Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.CUSTOM, + custom_storage_method: null, + }); + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + done(); + }); + + // tests init time storage config options + // choosing Custom storage type and passing custom storage methods + // SDK should use custom methods as storage method, no File Storage should exist + it("26- Providing Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.CUSTOM, + custom_storage_method: customMemoryStorage, + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); + done(); + }); + + // tests init time storage config options + // Recording values in Custom Storage Methods + // SDK should use custom methods as storage methods and values should be recorded correctly + it("27- Record/Remove Values in Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.CUSTOM, + custom_storage_method: customMemoryStorage, + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); + storage.storeSet("CustomStorageKey", "CustomStorageValue"); + assert.equal(storage.storeGet("CustomStorageKey", null), "CustomStorageValue"); + storage.storeRemove("CustomStorageKey"); + assert.equal(storage.storeGet("CustomStorageKey", null), null); + done(); + }); + + // tests init time storage config options + // passes a funky storage method, which does store get as reversing string + // SDK should use custom methods as storage method + it("28- Record/Remove Values in Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.CUSTOM, + custom_storage_method: funkyMemoryStorage, + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); + storage.storeSet("CustomStorageKey", "CustomStorageValue"); + storage.storeSet("CustomStorageKey", "CustomStorageValue2"); + assert.equal("2eulaVegarotSmotsuCeulaVegarotSmotsuC", storage.storeGet("CustomStorageKey", null)); + done(); + }); }); \ No newline at end of file From 0e3ac4e0dc872409acf59e38dff0a58a54afbd97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 14:48:37 +0300 Subject: [PATCH 38/58] Check for Invalid Custom Methods --- lib/countly-storage.js | 18 +++++++++++++++- test/tests_storage.js | 47 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 024f442..98ac6e3 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -132,7 +132,7 @@ var initStorage = function(userPath, storageType, isBulk = false, persistQueue = storageMethod = memoryStorage; } else if (storageType === StorageTypes.CUSTOM) { - if (customStorageMethod) { + if (hasValidMethods(customStorageMethod)) { storageMethod = customStorageMethod; } else { @@ -147,6 +147,22 @@ var initStorage = function(userPath, storageType, isBulk = false, persistQueue = } }; +var hasValidMethods = function(storage) { + if (!storage) { + return false; + } + if (typeof storage.storeSet !== 'function') { + return false; + } + if (typeof storage.storeGet !== 'function') { + return false; + } + if (typeof storage.storeRemove !== 'function') { + return false; + } + return true; +}; + /** * Sets the storage path, defaulting to a specified path if none is provided. * @param {String} userPath - User provided storage path diff --git a/test/tests_storage.js b/test/tests_storage.js index 99b28a7..8d5762e 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -99,6 +99,35 @@ function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = assert.equal(storage.storeGet("cly_object"), undefined); assert.equal(storage.storeGet("cly_null"), undefined); } +const nonValidStorageMethods = { + _storage: {}, + + setInvalid: function(key, value, callback) { + if (key) { + const existingValue = this._storage[key]; + if (typeof value === 'string' && typeof existingValue === 'string') { + this._storage[key] = existingValue + value; + } + else { + this._storage[key] = value; + } + if (typeof callback === "function") { + callback(null); + } + } + }, + getInvalid: function(key, def) { + const value = this._storage[key]; + if (typeof value === 'string') { + return value.split('').reverse().join(''); + } + + return value !== undefined ? value : def; + }, + removeInvalid: function(key) { + delete this._storage[key]; + }, +}; const funkyMemoryStorage = { _storage: {}, @@ -585,4 +614,22 @@ describe("Storage Tests", () => { assert.equal("2eulaVegarotSmotsuCeulaVegarotSmotsuC", storage.storeGet("CustomStorageKey", null)); done(); }); + + // tests init time storage config options + // choosing Custom storage type and passing invalid custom storage methods + // SDK should not use custom methods as storage method, and switch to File Storage + it("26- Providing Invalid Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.CUSTOM, + custom_storage_method: nonValidStorageMethods, + }); + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + done(); + }); }); \ No newline at end of file From 131ea5cf910d76ad96a33d879f7be62c3cb38c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Thu, 3 Oct 2024 15:03:43 +0300 Subject: [PATCH 39/58] Custom File Storage Test --- test/tests_storage.js | 85 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 2 deletions(-) diff --git a/test/tests_storage.js b/test/tests_storage.js index 8d5762e..6c092a3 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -1,3 +1,6 @@ +const fs = require('fs'); +const path = require('path'); + const assert = require("assert"); var Countly = require("../lib/countly"); var storage = require("../lib/countly-storage"); @@ -99,6 +102,62 @@ function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = assert.equal(storage.storeGet("cly_object"), undefined); assert.equal(storage.storeGet("cly_null"), undefined); } +var __data = {}; +var asyncWriteLock = false; +var asyncWriteQueue = []; +// technically same as defualt file storage method +// adds * before any value set so it can be separated with the default one for testing purposes +const customFileStorage = { + storeSet: function(key, value, callback) { + // Add '*' before the value + __data[key] = `*${value}`; + if (!asyncWriteLock) { + asyncWriteLock = true; + storage.writeFile(key, value, callback); + } + else { + asyncWriteQueue.push([key, value, callback]); + } + }, + storeGet: function(key, def) { + cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); + if (typeof __data[key] === "undefined") { + var ob = storage.readFile(key); + var obLen; + try { + obLen = Object.keys(ob).length; + } + catch (error) { + obLen = 0; + } + if (!ob || obLen === 0) { + __data[key] = def; + } + else { + __data[key] = ob[key]; + } + } + return __data[key]; + }, + storeRemove: function(key) { + delete __data[key]; + var filePath = path.resolve(__dirname, `${storage.getStoragePath()}__${key}.json`); + fs.access(filePath, fs.constants.F_OK, (accessErr) => { + if (accessErr) { + cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`); + return; + } + fs.unlink(filePath, (err) => { + if (err) { + cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`); + } + else { + cc.log(cc.logLevelEnums.INFO, `storeRemove, Successfully removed file with key: [${key}].`); + } + }); + }); + }, +}; const nonValidStorageMethods = { _storage: {}, @@ -157,7 +216,6 @@ const funkyMemoryStorage = { delete this._storage[key]; }, }; - const customMemoryStorage = { _storage: {}, storeSet: function(key, value, callback) { @@ -618,7 +676,7 @@ describe("Storage Tests", () => { // tests init time storage config options // choosing Custom storage type and passing invalid custom storage methods // SDK should not use custom methods as storage method, and switch to File Storage - it("26- Providing Invalid Custom Storage Method", (done) => { + it("29- Providing Invalid Custom Storage Method", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", @@ -632,4 +690,27 @@ describe("Storage Tests", () => { assert.equal(storage.getStorageType(), StorageTypes.FILE); done(); }); + + // tests init time storage config options + // choosing Custom storage type and passing custom file storage methods + // SDK should use custom methods as storage methods + it("30- Providing Invalid Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + clear_stored_device_id: true, + storage_type: StorageTypes.CUSTOM, + custom_storage_method: customFileStorage, + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); + storage.storeSet("CustomStorageKey", "CustomStorageValue"); + assert.equal(storage.storeGet("CustomStorageKey", null), "*CustomStorageValue"); + storage.storeRemove("CustomStorageKey"); + assert.equal(storage.storeGet("CustomStorageKey", null), null); + done(); + }); }); \ No newline at end of file From 92b34be8541233ca91c1ac166d9d3bf3266a8f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Fri, 4 Oct 2024 16:26:48 +0300 Subject: [PATCH 40/58] Setting User Path In Custom Storage --- lib/countly-storage.js | 3 +++ test/tests_storage.js | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 98ac6e3..aad6d10 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -134,6 +134,9 @@ var initStorage = function(userPath, storageType, isBulk = false, persistQueue = else if (storageType === StorageTypes.CUSTOM) { if (hasValidMethods(customStorageMethod)) { storageMethod = customStorageMethod; + if (userPath) { + setStoragePath(userPath, isBulk, persistQueue); + } } else { cc.log(cc.logLevelEnums.WARNING, `Provided Custom Storage Methods are not valid. Switching to default file storage!`); diff --git a/test/tests_storage.js b/test/tests_storage.js index 6c092a3..9701823 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -654,7 +654,7 @@ describe("Storage Tests", () => { // tests init time storage config options // passes a funky storage method, which does store get as reversing string // SDK should use custom methods as storage method - it("28- Record/Remove Values in Custom Storage Method", (done) => { + it("28- Record/Remove Values in Other Custom Storage Method", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", @@ -694,19 +694,21 @@ describe("Storage Tests", () => { // tests init time storage config options // choosing Custom storage type and passing custom file storage methods // SDK should use custom methods as storage methods - it("30- Providing Invalid Custom Storage Method", (done) => { + it("30- Providing File Custom Storage Method", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", device_id: "Test-Device-Id", clear_stored_device_id: true, + storage_path: "../test/customStorageDirectory/", storage_type: StorageTypes.CUSTOM, custom_storage_method: customFileStorage, }); hp.doesFileStoragePathsExist((exists) => { assert.equal(false, exists); }); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); storage.storeSet("CustomStorageKey", "CustomStorageValue"); assert.equal(storage.storeGet("CustomStorageKey", null), "*CustomStorageValue"); storage.storeRemove("CustomStorageKey"); From f4d1e6f35e1c1695ca1bdcad42379ab8344d98d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Fri, 4 Oct 2024 16:43:34 +0300 Subject: [PATCH 41/58] Update countly-bulk.js --- lib/countly-bulk.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 9fc6b53..506b0a9 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -50,6 +50,7 @@ const StorageTypes = cc.storageTypeEnums; * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied + * @param {varies} conf.custom_storage_method - user given storage methods * @example * var server = new CountlyBulk({ * app_key: "{YOUR-API-KEY}", @@ -107,7 +108,7 @@ function CountlyBulk(conf) { conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; conf.storage_type = conf.storage_type || storageType; - CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue); + CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue, conf.custom_storage_method); this.conf = conf; /** From 52827cef3df4438d21584cecfab63a2790d375cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 7 Oct 2024 13:32:24 +0300 Subject: [PATCH 42/58] Update tests_storage.js --- test/tests_storage.js | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/test/tests_storage.js b/test/tests_storage.js index 9701823..6c03700 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -103,21 +103,11 @@ function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = assert.equal(storage.storeGet("cly_null"), undefined); } var __data = {}; -var asyncWriteLock = false; -var asyncWriteQueue = []; // technically same as defualt file storage method -// adds * before any value set so it can be separated with the default one for testing purposes const customFileStorage = { storeSet: function(key, value, callback) { - // Add '*' before the value - __data[key] = `*${value}`; - if (!asyncWriteLock) { - asyncWriteLock = true; - storage.writeFile(key, value, callback); - } - else { - asyncWriteQueue.push([key, value, callback]); - } + __data[key] = value; + storage.writeFile(key, value, callback); }, storeGet: function(key, def) { cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); @@ -695,24 +685,26 @@ describe("Storage Tests", () => { // choosing Custom storage type and passing custom file storage methods // SDK should use custom methods as storage methods it("30- Providing File Custom Storage Method", (done) => { - hp.clearStorage(); + hp.clearStorage(false, false, "../test/customStorageDirectory/"); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", - device_id: "Test-Device-Id", - clear_stored_device_id: true, storage_path: "../test/customStorageDirectory/", storage_type: StorageTypes.CUSTOM, custom_storage_method: customFileStorage, + clear_stored_device_id: true, + device_id: "ID", }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - storage.storeSet("CustomStorageKey", "CustomStorageValue"); - assert.equal(storage.storeGet("CustomStorageKey", null), "*CustomStorageValue"); - storage.storeRemove("CustomStorageKey"); - assert.equal(storage.storeGet("CustomStorageKey", null), null); - done(); + Countly.begin_session(); + setTimeout(() => { + assert.equal(Countly.get_device_id(), "ID"); + assert.equal(Countly.get_device_id_type(), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + storage.storeSet("CustomStorageKey", "CustomStorageValue"); + assert.equal(storage.storeGet("CustomStorageKey", null), "CustomStorageValue"); + storage.storeRemove("CustomStorageKey"); + assert.equal(storage.storeGet("CustomStorageKey", null), null); + done(); + }, hp.sWait); }); }); \ No newline at end of file From e9fcf8b3ece974283164bf3192f0416e33db815b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 7 Oct 2024 21:29:09 +0300 Subject: [PATCH 43/58] Refactoring Storage Tests --- lib/countly-common.js | 1 - lib/countly-storage.js | 9 +- lib/countly.js | 1 + test/helpers/helper_functions.js | 17 +- test/tests_storage.js | 717 ++++--------------------------- 5 files changed, 112 insertions(+), 633 deletions(-) diff --git a/lib/countly-common.js b/lib/countly-common.js index 7a961e5..c721bb1 100644 --- a/lib/countly-common.js +++ b/lib/countly-common.js @@ -54,7 +54,6 @@ var cc = { storageTypeEnums: { FILE: "file", MEMORY: "memory", - CUSTOM: "custom", }, /** diff --git a/lib/countly-storage.js b/lib/countly-storage.js index aad6d10..22d7a68 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -131,7 +131,7 @@ var initStorage = function(userPath, storageType, isBulk = false, persistQueue = if (storageType === StorageTypes.MEMORY) { storageMethod = memoryStorage; } - else if (storageType === StorageTypes.CUSTOM) { + else if (customStorageMethod) { if (hasValidMethods(customStorageMethod)) { storageMethod = customStorageMethod; if (userPath) { @@ -322,7 +322,12 @@ var getStorageType = function() { if (storageMethod === memoryStorage) { return StorageTypes.MEMORY; } - return StorageTypes.FILE; + + if (storageMethod === fileStorage) { + return StorageTypes.FILE; + } + + return null; }; module.exports = { diff --git a/lib/countly.js b/lib/countly.js index 8492bec..51d94d1 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -335,6 +335,7 @@ Countly.Bulk = Bulk; // device_id Countly.device_id = undefined; + Countly.device_id_type = undefined; Countly.remote_config = undefined; Countly.require_consent = false; Countly.debug = undefined; diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index 1728687..83a0777 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -56,23 +56,30 @@ function clearStorage(keepID = false, isBulk = false, customDir = '') { // Resets Countly Countly.halt(true); // Determine the directory based on isBulk or customDir - const eventDirectory = customDir || (isBulk ? bulkEventDir : eventDir); - const reqDirectory = customDir || (isBulk ? bulkQueueDir : reqDir); - // Helper function to remove directory and files + const eventDirectory = isBulk ? bulkEventDir : eventDir; + const reqDirectory = isBulk ? bulkQueueDir : reqDir; function removeDir(directory) { - if (fs.existsSync(directory)) { - fs.rmSync(directory, { recursive: true, force: true }); + // Remove the .json extension from the directory name, since it will be added in path.resolve + var filePath = path.resolve(__dirname, `${directory}`); + if (fs.existsSync(filePath)) { + fs.rmSync(filePath, { recursive: true, force: true }); } } // Remove event directory if it exists removeDir(eventDirectory); // Remove request directory if it exists removeDir(reqDirectory); + if (customDir) { + removeDir(customDir); + } // Optionally keep the ID directory if (!keepID) { removeDir(idDir); removeDir(idTypeDir); } + return new Promise((resolve, reject) => { + resolve("string"); + }); } /** * bunch of tests specifically gathered for testing events diff --git a/test/tests_storage.js b/test/tests_storage.js index 6c03700..6e3c66d 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -1,6 +1,3 @@ -const fs = require('fs'); -const path = require('path'); - const assert = require("assert"); var Countly = require("../lib/countly"); var storage = require("../lib/countly-storage"); @@ -9,702 +6,172 @@ var hp = require("./helpers/helper_functions"); const StorageTypes = cc.storageTypeEnums; -// example event object to use -var eventObj = { - key: "storage_check", - count: 5, - sum: 3.14, - dur: 2000, - segmentation: { - app_version: "1.0", - country: "Zambia", - }, -}; - -var userDetailObj = { - name: "Akira Kurosawa", - username: "a_kurosawa", - email: "akira.kurosawa@filmlegacy.com", - organization: "Toho Studios", - phone: "+81312345678", - picture: "https://example.com/profile_images/akira_kurosawa.jpg", - gender: "Male", - byear: 1910, - custom: { - "known for": "Film Director", - "notable works": "Seven Samurai, Rashomon, Ran", - }, -}; - -// init function -function initMain(device_id) { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - interval: 10000, - max_events: -1, - device_id: device_id, - }); -} -// TODO: move these to helpers to reduce duplication -function validateSdkGeneratedId(providedDeviceId) { - assert.ok(providedDeviceId); - assert.equal(providedDeviceId.length, 36); - assert.ok(cc.isUUID(providedDeviceId)); -} -function checkRequestsForT(queue, expectedInternalType) { - for (var i = 0; i < queue.length; i++) { - assert.ok(queue[i].t); - assert.equal(queue[i].t, expectedInternalType); - } -} -function validateDeviceId(deviceId, deviceIdType, expectedDeviceId, expectedDeviceIdType) { - var rq = hp.readRequestQueue()[0]; - if (expectedDeviceIdType === cc.deviceIdTypeEnums.SDK_GENERATED) { - validateSdkGeneratedId(deviceId); // for SDK-generated IDs - } - else { - assert.equal(deviceId, expectedDeviceId); // for developer-supplied IDs - } - assert.equal(deviceIdType, expectedDeviceIdType); - checkRequestsForT(rq, expectedDeviceIdType); -} -function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { - // Set values - var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storage.initStorage(userPath, memoryOnly, isBulk, persistQueue); - storage.storeSet("cly_id", "SpecialDeviceId"); - storage.storeSet("cly_id_type", deviceIdType); - - // Set values with different data types - storage.storeSet("cly_count", 42); - storage.storeSet("cly_object", { key: "value" }); - storage.storeSet("cly_null", null); - - // Retrieve and assert values - assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storage.storeGet("cly_id_type"), deviceIdType); - assert.equal(storage.storeGet("cly_count"), 42); - assert.deepEqual(storage.storeGet("cly_object"), { key: "value" }); - assert.equal(storage.storeGet("cly_null"), null); - - // Remove specific items by overriding with null or empty array - storage.storeSet("cly_id", null); - storage.storeSet("cly_object", []); - assert.equal(storage.storeGet("cly_id"), null); - assert.deepEqual(storage.storeGet("cly_object"), []); - - // Reset storage and check if it's empty again - storage.resetStorage(); - assert.equal(storage.storeGet("cly_id"), undefined); - assert.equal(storage.storeGet("cly_id_type"), undefined); - assert.equal(storage.storeGet("cly_count"), undefined); - assert.equal(storage.storeGet("cly_object"), undefined); - assert.equal(storage.storeGet("cly_null"), undefined); -} var __data = {}; -// technically same as defualt file storage method -const customFileStorage = { - storeSet: function(key, value, callback) { - __data[key] = value; - storage.writeFile(key, value, callback); - }, - storeGet: function(key, def) { - cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); - if (typeof __data[key] === "undefined") { - var ob = storage.readFile(key); - var obLen; - try { - obLen = Object.keys(ob).length; - } - catch (error) { - obLen = 0; - } - if (!ob || obLen === 0) { - __data[key] = def; - } - else { - __data[key] = ob[key]; - } - } - return __data[key]; - }, - storeRemove: function(key) { - delete __data[key]; - var filePath = path.resolve(__dirname, `${storage.getStoragePath()}__${key}.json`); - fs.access(filePath, fs.constants.F_OK, (accessErr) => { - if (accessErr) { - cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`); - return; - } - fs.unlink(filePath, (err) => { - if (err) { - cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`); - } - else { - cc.log(cc.logLevelEnums.INFO, `storeRemove, Successfully removed file with key: [${key}].`); - } - }); - }); - }, -}; -const nonValidStorageMethods = { - _storage: {}, - - setInvalid: function(key, value, callback) { - if (key) { - const existingValue = this._storage[key]; - if (typeof value === 'string' && typeof existingValue === 'string') { - this._storage[key] = existingValue + value; - } - else { - this._storage[key] = value; - } - if (typeof callback === "function") { - callback(null); - } - } - }, - getInvalid: function(key, def) { - const value = this._storage[key]; - if (typeof value === 'string') { - return value.split('').reverse().join(''); - } - - return value !== undefined ? value : def; - }, - removeInvalid: function(key) { - delete this._storage[key]; - }, -}; -const funkyMemoryStorage = { - _storage: {}, - - storeSet: function(key, value, callback) { - if (key) { - const existingValue = this._storage[key]; - if (typeof value === 'string' && typeof existingValue === 'string') { - this._storage[key] = existingValue + value; - } - else { - this._storage[key] = value; - } - if (typeof callback === "function") { - callback(null); - } - } - }, - storeGet: function(key, def) { - const value = this._storage[key]; - if (typeof value === 'string') { - return value.split('').reverse().join(''); - } - - return value !== undefined ? value : def; - }, - storeRemove: function(key) { - delete this._storage[key]; - }, -}; const customMemoryStorage = { - _storage: {}, storeSet: function(key, value, callback) { if (key) { - this._storage[key] = value; + __data[key] = value; if (typeof callback === "function") { callback(null); } } }, storeGet: function(key, def) { - return typeof this._storage[key] !== "undefined" ? this._storage[key] : def; + return typeof __data[key] !== "undefined" ? __data[key] : def; }, storeRemove: function(key) { - delete this._storage[key]; + delete __data[key]; }, }; -describe("Storage Tests", () => { - it("1- Store Generated Device ID", (done) => { - // clear previous data - hp.clearStorage(); - // initialize SDK - initMain(); - Countly.begin_session(); - // read request queue - setTimeout(() => { - validateSdkGeneratedId(Countly.get_device_id()); - done(); - }, hp.sWait); - }); - - it("1.1- Validate generated device id after process restart", (done) => { - initMain(); - validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), undefined, cc.deviceIdTypeEnums.SDK_GENERATED); - done(); - }); - - it("2.Developer supplied device ID", (done) => { - hp.clearStorage(); - initMain("ID"); - Countly.begin_session(); - setTimeout(() => { - validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - done(); - }, hp.sWait); - }); - - it("2.1- Validate generated device id after process restart", (done) => { - validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - done(); - }); - - it("3- Record and validate all user details", (done) => { - hp.clearStorage(); - initMain(); - Countly.user_details(userDetailObj); - setTimeout(() => { - var req = hp.readRequestQueue()[0]; - hp.userDetailRequestValidator(userDetailObj, req); - done(); - }, hp.sWait); - }); +var setGetRemoveCustomValue = function() { + storage.storeSet("CustomKey", "CustomValue"); + assert.equal(storage.storeGet("CustomKey", null), "CustomValue"); + storage.storeRemove("CustomKey"); + assert.equal(storage.storeGet("CustomKey", null), null); +}; - it("3.1- Validate stored user detail", (done) => { - var req = hp.readRequestQueue()[0]; - hp.userDetailRequestValidator(userDetailObj, req); - done(); - }); +var ValidateID_IDType = function(devGivenId = null) { + if (!devGivenId) { + var storedId = storage.storeGet("cly_id"); + var storedIdType = storage.storeGet("cly_id_type"); + assert.equal(Countly.get_device_id(), storedId); + assert.equal(Countly.get_device_id_type(), storedIdType); + } + else { + assert.equal(Countly.get_device_id(), devGivenId); + assert.equal(Countly.get_device_id_type(), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + } +}; - it("4- Record event and validate storage", (done) => { +describe("Storage Tests", () => { + // validates if performing storage operations without initializing the SDK is possible with file storage methods + // without initializing SDK, storage should be able to set, get and remove values in file storage + it("1- File Storage with No-Init", (done) => { hp.clearStorage(); - initMain(); - Countly.add_event(eventObj); - setTimeout(() => { - var storedEvents = hp.readEventQueue(); - assert.strictEqual(storedEvents.length, 1, "There should be exactly one event stored"); - - var event = storedEvents[0]; - hp.eventValidator(eventObj, event); - done(); - }, hp.mWait); - }); - - it("4.1- Validate event persistence after process restart", (done) => { - // Initialize SDK - initMain(); - - // Read stored events without clearing storage - var storedEvents = hp.readEventQueue(); - assert.strictEqual(storedEvents.length, 1, "There should be exactly one event stored"); - - var event = storedEvents[0]; - hp.eventValidator(eventObj, event); + storage.initStorage(); + assert.equal(StorageTypes.FILE, storage.getStorageType()); + setGetRemoveCustomValue(); done(); }); - // if storage path is not provided it will be default "../data/" - it("5- Not provide storage path during init", (done) => { + // validates if performing storage operations without initializing the SDK is possible with memory storage methods + // without initializing SDK, storage should be able to set, get and remove values in memory storage + it("2- Memory Storage with No-Init", (done) => { hp.clearStorage(); - initMain(); - assert.equal(storage.getStoragePath(), "../data/"); + storage.initStorage(null, StorageTypes.MEMORY); + assert.equal(StorageTypes.MEMORY, storage.getStorageType()); + assert.equal(undefined, storage.getStoragePath()); + setGetRemoveCustomValue(); done(); }); - // if set to undefined it should be set to default path - it("6- Set storage path to undefined", (done) => { + // validates if performing storage operations without initializing the SDK is possible with custom storage methods + // without initializing SDK, storage should be able to set, get and remove values in memory storage + it("3- Custom Storage with No-Init", (done) => { hp.clearStorage(); - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - storage_path: undefined, - }); - assert.equal(storage.getStoragePath(), "../data/"); + storage.initStorage(null, null, false, false, customMemoryStorage); + assert.equal(null, storage.getStorageType()); + assert.equal(undefined, storage.getStoragePath()); + setGetRemoveCustomValue(); + __data = {}; done(); }); - // if set to null it should be set to default path - it("7- Set storage path to null", (done) => { + // validates the functionality for the configuration time storage options + // sets to file storage with default path and methods during configuration time + it("4- Config Time Storage Options with Default File Storage", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - storage_path: null, + url: "https://try.count.ly", + storage_type: StorageTypes.FILE, // for file storage this is not needed, for the readiblitiy purposes it's here }); + assert.equal(StorageTypes.FILE, storage.getStorageType()); assert.equal(storage.getStoragePath(), "../data/"); + setGetRemoveCustomValue(); + ValidateID_IDType(); done(); }); - // it should be set to the custom directory if provided - it("8- Set storage path to custom directory", (done) => { + // validates the functionality for the configuration time storage options + // sets to memory storage with default methods during configuration time + it("5- Config Time Storage Options with Default Memory Storage", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - interval: 10000, - max_events: -1, - storage_path: "../test/customStorageDirectory/", - }); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - done(); - }); - - // resets the storage path to default and validates that it is set correctly, - // then resets it to undefined and confirms the reset. - it("9- Reset Storage While on Default Path /no-init", (done) => { - // will set to default storage path - storage.setStoragePath(); - assert.equal(storage.getStoragePath(), "../data/"); - // will set to undefined - storage.resetStorage(); - assert.equal(storage.getStoragePath(), undefined); - done(); - }); - - // sets the storage path to default and verifies it, - // then records values to storage and ensures they are stored correctly. - it("10- Recording to Storage with Default Storage Path /no-init", (done) => { - storage.resetStorage(); - // Set to default storage path - storage.setStoragePath(); - assert.equal(storage.getStoragePath(), "../data/"); - recordValuesToStorageAndValidate(); - done(); - }); - - // sets a custom storage path and verifies it, - // then records values to storage and ensures correct storage in the custom path. - it("11- Recording to Storage with Custom Storage Path /no-init", (done) => { - storage.resetStorage(); - // will set to default storage path - storage.setStoragePath("../test/customStorageDirectory/"); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - recordValuesToStorageAndValidate("../test/customStorageDirectory/"); - done(); - }); - - // sets the storage path to the default bulk storage path and verifies it, - // then records values to bulk storage and validates proper storage in bulk mode. - it("12- Recording to Bulk Storage with Default Bulk Data Path /no-init", (done) => { - storage.resetStorage(); - // will set to default storage path - // To set the storage path to the default bulk storage path and persist the queue - storage.setStoragePath(null, true, true); - assert.equal(storage.getStoragePath(), "../bulk_data/"); - recordValuesToStorageAndValidate(null, false, true, true); - done(); - }); - - // sets a custom bulk storage path and verifies it, - // then records values to bulk storage and ensures proper recording to the custom path. - it("13- Recording to Bulk Storage with Custom Bulk Storage Path /no-init", (done) => { - storage.resetStorage(); - // will set to default storage path - storage.setStoragePath("../test/customStorageDirectory/", true); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - recordValuesToStorageAndValidate("../test/customStorageDirectory/", false, true); - done(); - }); - - it("14- Setting storage path to default path via initStorage /no-init", (done) => { - storage.resetStorage(); - storage.initStorage(); - assert.equal(storage.getStoragePath(), "../data/"); - done(); - }); - - it("15- Setting bulk storage path to default path via initStorage /no-init", (done) => { - storage.resetStorage(); - storage.initStorage(null, false, true); - assert.equal(storage.getStoragePath(), "../bulk_data/"); - done(); - }); - - it("16- Setting custom storage path via initStorage /no-init", (done) => { - storage.resetStorage(); - storage.initStorage("../test/customStorageDirectory/"); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - done(); - }); - - it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { - storage.resetStorage(); - storage.initStorage(null, StorageTypes.MEMORY); - assert.equal(storage.getStoragePath(), undefined); - done(); - }); - - // recording device-id in memory only mode - // initializes the SDK in memory only mode, validates that file storage files does not exist - // retrieve the developer supplied device id and id type from storage - it("18- Memory only storage Device-Id", (done) => { - hp.clearStorage(); - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - clear_stored_device_id: true, + url: "https://try.count.ly", storage_type: StorageTypes.MEMORY, }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); + assert.equal(StorageTypes.MEMORY, storage.getStorageType()); assert.equal(storage.getStoragePath(), undefined); - assert.equal(storage.storeGet("cly_id", null), "Test-Device-Id"); - assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + setGetRemoveCustomValue(); + ValidateID_IDType(); done(); }); - // recording event in memory only mode - // initializes the SDK in memory only mode, validates that file storage files does not exist - // records an event and validates the recorded event - it("19- Record event in memory only mode and validate the record", (done) => { - hp.clearStorage(); - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - clear_stored_device_id: true, - storage_type: StorageTypes.MEMORY, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - Countly.add_event(eventObj); - setTimeout(() => { - const storedData = storage.storeGet("cly_queue", null); - const eventArray = JSON.parse(storedData[0].events); - const eventFromQueue = eventArray[0]; - hp.eventValidator(eventObj, eventFromQueue); - done(); - }, hp.mWait); - }); - - // recording user details in memory only mode - // initializes the SDK in memory only mode, validates that file storage files does not exist - // records user details and validates the recorded details - it("20- Record and validate user details in memory only mode", (done) => { + // validates the functionality for the configuration time storage options + // sets to custom storage during configuration time + it("6- Config Time Storage Options with Custom Storage", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - clear_stored_device_id: true, - storage_type: StorageTypes.MEMORY, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); + url: "https://try.count.ly", + custom_storage_method: customMemoryStorage, }); - Countly.user_details(userDetailObj); - const storedData = storage.storeGet("cly_queue", null); - const userDetailsReq = storedData[0]; - hp.userDetailRequestValidator(userDetailObj, userDetailsReq); + assert.equal(storage.getStoragePath(), undefined); + assert.equal(null, storage.getStorageType()); + setGetRemoveCustomValue(); + ValidateID_IDType(); + __data = {}; done(); }); - // tests device id changes in memory only storage - // initialize the SDK in memory only mode, check the device id and switch it - // SDK and storage should function properly - it("21- Memory only storage, change SDK Generated Device-Id", (done) => { + // validates the recording of device id correctly if provided by developer + // all validations should succeed like DeviceId, Value recording etc. + it("7- File Storage Init with Dev Supplied Device ID", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - clear_stored_device_id: true, - storage_type: StorageTypes.MEMORY, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); + url: "https://try.count.ly", + device_id: "ID", + storage_type: StorageTypes.FILE, // for file storage this is not needed, for the readiblitiy purposes it's here }); - assert.equal(storage.getStoragePath(), undefined); - assert.equal(storage.storeGet("cly_id", null), Countly.get_device_id()); - assert.equal(storage.storeGet("cly_id_type", null), Countly.get_device_id_type()); - - Countly.change_id("Test-Id-2"); - assert.equal(storage.storeGet("cly_id", null), "Test-Id-2"); - assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + assert.equal(StorageTypes.FILE, storage.getStorageType()); + setGetRemoveCustomValue(); + ValidateID_IDType("ID"); done(); }); - // tests switching between storage types after initializing SDK - // passing memory storage type during init and initializing storage afterwards - // SDK should switch to file storage - it("22- Switch to file storage after init", (done) => { + // validates the recording of device id correctly if provided by developer + // all validations should succeed like DeviceId, Value recording etc. + it("8- Memory Storage Init with Dev Supplied Device ID", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - clear_stored_device_id: true, + url: "https://try.count.ly", + device_id: "ID2", storage_type: StorageTypes.MEMORY, }); assert.equal(storage.getStoragePath(), undefined); - assert.equal(storage.getStorageType(), StorageTypes.MEMORY); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - - storage.initStorage(); - assert.equal(storage.getStoragePath(), "../data/"); - assert.equal(storage.getStorageType(), StorageTypes.FILE); + assert.equal(StorageTypes.MEMORY, storage.getStorageType()); + setGetRemoveCustomValue(); + ValidateID_IDType("ID2"); done(); }); - // tests storeRemove function in CountlyStorage - // after initializing the memory storage, without initializing SDK, attempts to set, get and remove values - // without initializing SDK storage should function properly - it("23- storeRemove Memory Only /no-init", (done) => { - hp.clearStorage(); - storage.initStorage(null, StorageTypes.MEMORY); - assert.equal(storage.getStoragePath(), undefined); - assert.equal(storage.getStorageType(), StorageTypes.MEMORY); - storage.storeSet("keyToStore", "valueToStore"); - assert.equal(storage.storeGet("keyToStore", null), "valueToStore"); - - storage.storeRemove("keyToStore"); - assert.equal(storage.storeGet("keyToStore", null), null); - done(); - }); - - // tests storeRemove function in CountlyStorage - // after initializing the file storage, without initializing SDK attempts to set, get and remove values - // without initializing SDK storage should function properly - it("24- storeRemove File Storage /no-init", (done) => { - hp.clearStorage(); - storage.initStorage(); - assert.equal(storage.getStoragePath(), "../data/"); - assert.equal(storage.getStorageType(), StorageTypes.FILE); - storage.storeSet("keyToStore", "valueToStore"); - assert.equal(storage.storeGet("keyToStore", null), "valueToStore"); - - storage.storeRemove("keyToStore"); - assert.equal(storage.storeGet("keyToStore", null), null); - done(); - }); - - // tests init time storage config options - // choosing Custom storage type and passing null in storage methods - // passing null as storage method ends up with switching to default file storage - it("25- Null Custom Storage Method", (done) => { + // validates the recording of device id correctly if provided by developer + // all validations should succeed like DeviceId, Value recording etc. + it("9- Custom Storage Init with Dev Supplied Device ID", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - clear_stored_device_id: true, - storage_type: StorageTypes.CUSTOM, - custom_storage_method: null, - }); - assert.equal(storage.getStoragePath(), "../data/"); - assert.equal(storage.getStorageType(), StorageTypes.FILE); - done(); - }); - - // tests init time storage config options - // choosing Custom storage type and passing custom storage methods - // SDK should use custom methods as storage method, no File Storage should exist - it("26- Providing Custom Storage Method", (done) => { - hp.clearStorage(); - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - clear_stored_device_id: true, - storage_type: StorageTypes.CUSTOM, + url: "https://try.count.ly", + device_id: "ID3", custom_storage_method: customMemoryStorage, }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); + setGetRemoveCustomValue(); + ValidateID_IDType("ID3"); done(); }); - - // tests init time storage config options - // Recording values in Custom Storage Methods - // SDK should use custom methods as storage methods and values should be recorded correctly - it("27- Record/Remove Values in Custom Storage Method", (done) => { - hp.clearStorage(); - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - clear_stored_device_id: true, - storage_type: StorageTypes.CUSTOM, - custom_storage_method: customMemoryStorage, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - storage.storeSet("CustomStorageKey", "CustomStorageValue"); - assert.equal(storage.storeGet("CustomStorageKey", null), "CustomStorageValue"); - storage.storeRemove("CustomStorageKey"); - assert.equal(storage.storeGet("CustomStorageKey", null), null); - done(); - }); - - // tests init time storage config options - // passes a funky storage method, which does store get as reversing string - // SDK should use custom methods as storage method - it("28- Record/Remove Values in Other Custom Storage Method", (done) => { - hp.clearStorage(); - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - clear_stored_device_id: true, - storage_type: StorageTypes.CUSTOM, - custom_storage_method: funkyMemoryStorage, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - storage.storeSet("CustomStorageKey", "CustomStorageValue"); - storage.storeSet("CustomStorageKey", "CustomStorageValue2"); - assert.equal("2eulaVegarotSmotsuCeulaVegarotSmotsuC", storage.storeGet("CustomStorageKey", null)); - done(); - }); - - // tests init time storage config options - // choosing Custom storage type and passing invalid custom storage methods - // SDK should not use custom methods as storage method, and switch to File Storage - it("29- Providing Invalid Custom Storage Method", (done) => { - hp.clearStorage(); - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - clear_stored_device_id: true, - storage_type: StorageTypes.CUSTOM, - custom_storage_method: nonValidStorageMethods, - }); - assert.equal(storage.getStoragePath(), "../data/"); - assert.equal(storage.getStorageType(), StorageTypes.FILE); - done(); - }); - - // tests init time storage config options - // choosing Custom storage type and passing custom file storage methods - // SDK should use custom methods as storage methods - it("30- Providing File Custom Storage Method", (done) => { - hp.clearStorage(false, false, "../test/customStorageDirectory/"); - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - storage_path: "../test/customStorageDirectory/", - storage_type: StorageTypes.CUSTOM, - custom_storage_method: customFileStorage, - clear_stored_device_id: true, - device_id: "ID", - }); - Countly.begin_session(); - setTimeout(() => { - assert.equal(Countly.get_device_id(), "ID"); - assert.equal(Countly.get_device_id_type(), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - storage.storeSet("CustomStorageKey", "CustomStorageValue"); - assert.equal(storage.storeGet("CustomStorageKey", null), "CustomStorageValue"); - storage.storeRemove("CustomStorageKey"); - assert.equal(storage.storeGet("CustomStorageKey", null), null); - done(); - }, hp.sWait); - }); }); \ No newline at end of file From 021897aef71f9d56b37ded7fc4ddfb095e09bec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 8 Oct 2024 14:46:23 +0300 Subject: [PATCH 44/58] Helper Functions Custom Path Update --- test/helpers/helper_functions.js | 26 +- test/tests_storage.js | 701 +++++++++++++++++++++++++++---- 2 files changed, 622 insertions(+), 105 deletions(-) diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index 83a0777..65ed232 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -52,33 +52,33 @@ function doesFileStoragePathsExist(callback) { }); }); } -function clearStorage(keepID = false, isBulk = false, customDir = '') { - // Resets Countly +function clearStorage(keepID = false, isBulk = false, customPath = null) { Countly.halt(true); - // Determine the directory based on isBulk or customDir const eventDirectory = isBulk ? bulkEventDir : eventDir; const reqDirectory = isBulk ? bulkQueueDir : reqDir; function removeDir(directory) { - // Remove the .json extension from the directory name, since it will be added in path.resolve - var filePath = path.resolve(__dirname, `${directory}`); + const filePath = path.resolve(__dirname, `${directory}`); if (fs.existsSync(filePath)) { fs.rmSync(filePath, { recursive: true, force: true }); } } - // Remove event directory if it exists removeDir(eventDirectory); - // Remove request directory if it exists removeDir(reqDirectory); - if (customDir) { - removeDir(customDir); - } - // Optionally keep the ID directory if (!keepID) { removeDir(idDir); removeDir(idTypeDir); } - return new Promise((resolve, reject) => { - resolve("string"); + + if (customPath) { + if (!keepID) { + removeDir(path.join(customPath, '__cly_id.json')); + removeDir(path.join(customPath, '__cly_id_type.json')); + } + removeDir(path.join(customPath, '__cly_event.json')); + removeDir(path.join(customPath, '__cly_queue.json')); + } + return new Promise((resolve) => { + resolve("Storage cleared"); }); } /** diff --git a/test/tests_storage.js b/test/tests_storage.js index 6e3c66d..ef46620 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -1,3 +1,6 @@ +const fs = require('fs'); +const path = require('path'); + const assert = require("assert"); var Countly = require("../lib/countly"); var storage = require("../lib/countly-storage"); @@ -6,172 +9,686 @@ var hp = require("./helpers/helper_functions"); const StorageTypes = cc.storageTypeEnums; +// example event object to use +var eventObj = { + key: "storage_check", + count: 5, + sum: 3.14, + dur: 2000, + segmentation: { + app_version: "1.0", + country: "Zambia", + }, +}; + +var userDetailObj = { + name: "Akira Kurosawa", + username: "a_kurosawa", + email: "akira.kurosawa@filmlegacy.com", + organization: "Toho Studios", + phone: "+81312345678", + picture: "https://example.com/profile_images/akira_kurosawa.jpg", + gender: "Male", + byear: 1910, + custom: { + "known for": "Film Director", + "notable works": "Seven Samurai, Rashomon, Ran", + }, +}; + +// init function +function initMain(device_id) { + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + interval: 10000, + max_events: -1, + device_id: device_id, + }); +} +// TODO: move these to helpers to reduce duplication +function validateSdkGeneratedId(providedDeviceId) { + assert.ok(providedDeviceId); + assert.equal(providedDeviceId.length, 36); + assert.ok(cc.isUUID(providedDeviceId)); +} +function checkRequestsForT(queue, expectedInternalType) { + for (var i = 0; i < queue.length; i++) { + assert.ok(queue[i].t); + assert.equal(queue[i].t, expectedInternalType); + } +} +function validateDeviceId(deviceId, deviceIdType, expectedDeviceId, expectedDeviceIdType) { + var rq = hp.readRequestQueue()[0]; + if (expectedDeviceIdType === cc.deviceIdTypeEnums.SDK_GENERATED) { + validateSdkGeneratedId(deviceId); // for SDK-generated IDs + } + else { + assert.equal(deviceId, expectedDeviceId); // for developer-supplied IDs + } + assert.equal(deviceIdType, expectedDeviceIdType); + checkRequestsForT(rq, expectedDeviceIdType); +} +function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { + // Set values + var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; + storage.initStorage(userPath, memoryOnly, isBulk, persistQueue); + storage.storeSet("cly_id", "SpecialDeviceId"); + storage.storeSet("cly_id_type", deviceIdType); + + // Set values with different data types + storage.storeSet("cly_count", 42); + storage.storeSet("cly_object", { key: "value" }); + storage.storeSet("cly_null", null); + + // Retrieve and assert values + assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); + assert.equal(storage.storeGet("cly_id_type"), deviceIdType); + assert.equal(storage.storeGet("cly_count"), 42); + assert.deepEqual(storage.storeGet("cly_object"), { key: "value" }); + assert.equal(storage.storeGet("cly_null"), null); + + // Remove specific items by overriding with null or empty array + storage.storeSet("cly_id", null); + storage.storeSet("cly_object", []); + assert.equal(storage.storeGet("cly_id"), null); + assert.deepEqual(storage.storeGet("cly_object"), []); + + // Reset storage and check if it's empty again + storage.resetStorage(); + assert.equal(storage.storeGet("cly_id"), undefined); + assert.equal(storage.storeGet("cly_id_type"), undefined); + assert.equal(storage.storeGet("cly_count"), undefined); + assert.equal(storage.storeGet("cly_object"), undefined); + assert.equal(storage.storeGet("cly_null"), undefined); +} var __data = {}; +// technically same as defualt file storage method +const customFileStorage = { + storeSet: function(key, value, callback) { + __data[key] = value; + storage.writeFile(key, value, callback); + cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting item in storage with key: [${key}] and value: [${value}].`); + }, + storeGet: function(key, def) { + if (typeof __data[key] === "undefined") { + var ob = storage.readFile(key); + var obLen; + try { + obLen = Object.keys(ob).length; + } + catch (error) { + obLen = 0; + } + if (!ob || obLen === 0) { + __data[key] = def; + } + else { + __data[key] = ob[key]; + } + } + cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}] and value: [${__data[key]}].`); + return __data[key]; + }, + storeRemove: function(key) { + delete __data[key]; + var filePath = path.resolve(__dirname, `${storage.getStoragePath()}__${key}.json`); + fs.access(filePath, fs.constants.F_OK, (accessErr) => { + if (accessErr) { + cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`); + return; + } + fs.unlink(filePath, (err) => { + if (err) { + cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`); + } + else { + cc.log(cc.logLevelEnums.INFO, `storeRemove, Successfully removed file with key: [${key}].`); + } + }); + }); + }, +}; +const nonValidStorageMethods = { + _storage: {}, + + setInvalid: function(key, value, callback) { + if (key) { + const existingValue = this._storage[key]; + if (typeof value === 'string' && typeof existingValue === 'string') { + this._storage[key] = existingValue + value; + } + else { + this._storage[key] = value; + } + if (typeof callback === "function") { + callback(null); + } + } + }, + getInvalid: function(key, def) { + const value = this._storage[key]; + if (typeof value === 'string') { + return value.split('').reverse().join(''); + } + + return value !== undefined ? value : def; + }, + removeInvalid: function(key) { + delete this._storage[key]; + }, +}; +const funkyMemoryStorage = { + _storage: {}, + + storeSet: function(key, value, callback) { + if (key) { + const existingValue = this._storage[key]; + if (typeof value === 'string' && typeof existingValue === 'string') { + this._storage[key] = existingValue + value; + } + else { + this._storage[key] = value; + } + if (typeof callback === "function") { + callback(null); + } + } + }, + storeGet: function(key, def) { + const value = this._storage[key]; + if (typeof value === 'string') { + return value.split('').reverse().join(''); + } + + return value !== undefined ? value : def; + }, + storeRemove: function(key) { + delete this._storage[key]; + }, +}; const customMemoryStorage = { + _storage: {}, storeSet: function(key, value, callback) { if (key) { - __data[key] = value; + this._storage[key] = value; if (typeof callback === "function") { callback(null); } } }, storeGet: function(key, def) { - return typeof __data[key] !== "undefined" ? __data[key] : def; + return typeof this._storage[key] !== "undefined" ? this._storage[key] : def; }, storeRemove: function(key) { - delete __data[key]; + delete this._storage[key]; }, }; -var setGetRemoveCustomValue = function() { - storage.storeSet("CustomKey", "CustomValue"); - assert.equal(storage.storeGet("CustomKey", null), "CustomValue"); - storage.storeRemove("CustomKey"); - assert.equal(storage.storeGet("CustomKey", null), null); -}; +describe("Storage Tests", () => { + it("1- Store Generated Device ID", (done) => { + // clear previous data + hp.clearStorage(); + // initialize SDK + initMain(); + Countly.begin_session(); + // read request queue + setTimeout(() => { + validateSdkGeneratedId(Countly.get_device_id()); + done(); + }, hp.sWait); + }); -var ValidateID_IDType = function(devGivenId = null) { - if (!devGivenId) { - var storedId = storage.storeGet("cly_id"); - var storedIdType = storage.storeGet("cly_id_type"); - assert.equal(Countly.get_device_id(), storedId); - assert.equal(Countly.get_device_id_type(), storedIdType); - } - else { - assert.equal(Countly.get_device_id(), devGivenId); - assert.equal(Countly.get_device_id_type(), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - } -}; + it("1.1- Validate generated device id after process restart", (done) => { + initMain(); + validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), undefined, cc.deviceIdTypeEnums.SDK_GENERATED); + done(); + }); -describe("Storage Tests", () => { - // validates if performing storage operations without initializing the SDK is possible with file storage methods - // without initializing SDK, storage should be able to set, get and remove values in file storage - it("1- File Storage with No-Init", (done) => { + it("2.Developer supplied device ID", (done) => { hp.clearStorage(); - storage.initStorage(); - assert.equal(StorageTypes.FILE, storage.getStorageType()); - setGetRemoveCustomValue(); + initMain("ID"); + Countly.begin_session(); + setTimeout(() => { + validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + done(); + }, hp.sWait); + }); + + it("2.1- Validate generated device id after process restart", (done) => { + validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); done(); }); - // validates if performing storage operations without initializing the SDK is possible with memory storage methods - // without initializing SDK, storage should be able to set, get and remove values in memory storage - it("2- Memory Storage with No-Init", (done) => { + it("3- Record and validate all user details", (done) => { hp.clearStorage(); - storage.initStorage(null, StorageTypes.MEMORY); - assert.equal(StorageTypes.MEMORY, storage.getStorageType()); - assert.equal(undefined, storage.getStoragePath()); - setGetRemoveCustomValue(); + initMain(); + Countly.user_details(userDetailObj); + setTimeout(() => { + var req = hp.readRequestQueue()[0]; + hp.userDetailRequestValidator(userDetailObj, req); + done(); + }, hp.sWait); + }); + + it("3.1- Validate stored user detail", (done) => { + var req = hp.readRequestQueue()[0]; + hp.userDetailRequestValidator(userDetailObj, req); + done(); + }); + + it("4- Record event and validate storage", (done) => { + hp.clearStorage(); + initMain(); + Countly.add_event(eventObj); + setTimeout(() => { + var storedEvents = hp.readEventQueue(); + assert.strictEqual(storedEvents.length, 1, "There should be exactly one event stored"); + + var event = storedEvents[0]; + hp.eventValidator(eventObj, event); + done(); + }, hp.mWait); + }); + + it("4.1- Validate event persistence after process restart", (done) => { + // Initialize SDK + initMain(); + + // Read stored events without clearing storage + var storedEvents = hp.readEventQueue(); + assert.strictEqual(storedEvents.length, 1, "There should be exactly one event stored"); + + var event = storedEvents[0]; + hp.eventValidator(eventObj, event); done(); }); - // validates if performing storage operations without initializing the SDK is possible with custom storage methods - // without initializing SDK, storage should be able to set, get and remove values in memory storage - it("3- Custom Storage with No-Init", (done) => { + // if storage path is not provided it will be default "../data/" + it("5- Not provide storage path during init", (done) => { hp.clearStorage(); - storage.initStorage(null, null, false, false, customMemoryStorage); - assert.equal(null, storage.getStorageType()); - assert.equal(undefined, storage.getStoragePath()); - setGetRemoveCustomValue(); - __data = {}; + initMain(); + assert.equal(storage.getStoragePath(), "../data/"); done(); }); - // validates the functionality for the configuration time storage options - // sets to file storage with default path and methods during configuration time - it("4- Config Time Storage Options with Default File Storage", (done) => { + // if set to undefined it should be set to default path + it("6- Set storage path to undefined", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - storage_type: StorageTypes.FILE, // for file storage this is not needed, for the readiblitiy purposes it's here + url: "https://test.url.ly", + storage_path: undefined, }); - assert.equal(StorageTypes.FILE, storage.getStorageType()); assert.equal(storage.getStoragePath(), "../data/"); - setGetRemoveCustomValue(); - ValidateID_IDType(); done(); }); - // validates the functionality for the configuration time storage options - // sets to memory storage with default methods during configuration time - it("5- Config Time Storage Options with Default Memory Storage", (done) => { + // if set to null it should be set to default path + it("7- Set storage path to null", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - storage_type: StorageTypes.MEMORY, + url: "https://test.url.ly", + storage_path: null, }); - assert.equal(StorageTypes.MEMORY, storage.getStorageType()); + assert.equal(storage.getStoragePath(), "../data/"); + done(); + }); + + // it should be set to the custom directory if provided + it("8- Set storage path to custom directory", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + interval: 10000, + max_events: -1, + storage_path: "../test/customStorageDirectory/", + }); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + done(); + }); + + // resets the storage path to default and validates that it is set correctly, + // then resets it to undefined and confirms the reset. + it("9- Reset Storage While on Default Path /no-init", (done) => { + // will set to default storage path + storage.setStoragePath(); + assert.equal(storage.getStoragePath(), "../data/"); + // will set to undefined + storage.resetStorage(); assert.equal(storage.getStoragePath(), undefined); - setGetRemoveCustomValue(); - ValidateID_IDType(); done(); }); - // validates the functionality for the configuration time storage options - // sets to custom storage during configuration time - it("6- Config Time Storage Options with Custom Storage", (done) => { + // sets the storage path to default and verifies it, + // then records values to storage and ensures they are stored correctly. + it("10- Recording to Storage with Default Storage Path /no-init", (done) => { + storage.resetStorage(); + // Set to default storage path + storage.setStoragePath(); + assert.equal(storage.getStoragePath(), "../data/"); + recordValuesToStorageAndValidate(); + done(); + }); + + // sets a custom storage path and verifies it, + // then records values to storage and ensures correct storage in the custom path. + it("11- Recording to Storage with Custom Storage Path /no-init", (done) => { + storage.resetStorage(); + // will set to default storage path + storage.setStoragePath("../test/customStorageDirectory/"); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + recordValuesToStorageAndValidate("../test/customStorageDirectory/"); + done(); + }); + + // sets the storage path to the default bulk storage path and verifies it, + // then records values to bulk storage and validates proper storage in bulk mode. + it("12- Recording to Bulk Storage with Default Bulk Data Path /no-init", (done) => { + storage.resetStorage(); + // will set to default storage path + // To set the storage path to the default bulk storage path and persist the queue + storage.setStoragePath(null, true, true); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + recordValuesToStorageAndValidate(null, false, true, true); + done(); + }); + + // sets a custom bulk storage path and verifies it, + // then records values to bulk storage and ensures proper recording to the custom path. + it("13- Recording to Bulk Storage with Custom Bulk Storage Path /no-init", (done) => { + storage.resetStorage(); + // will set to default storage path + storage.setStoragePath("../test/customStorageDirectory/", true); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + recordValuesToStorageAndValidate("../test/customStorageDirectory/", false, true); + done(); + }); + + it("14- Setting storage path to default path via initStorage /no-init", (done) => { + storage.resetStorage(); + storage.initStorage(); + assert.equal(storage.getStoragePath(), "../data/"); + done(); + }); + + it("15- Setting bulk storage path to default path via initStorage /no-init", (done) => { + storage.resetStorage(); + storage.initStorage(null, false, true); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + done(); + }); + + it("16- Setting custom storage path via initStorage /no-init", (done) => { + storage.resetStorage(); + storage.initStorage("../test/customStorageDirectory/"); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + done(); + }); + + it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { + storage.resetStorage(); + storage.initStorage(null, StorageTypes.MEMORY); + assert.equal(storage.getStoragePath(), undefined); + done(); + }); + + // recording device-id in memory only mode + // initializes the SDK in memory only mode, validates that file storage files does not exist + // retrieve the developer supplied device id and id type from storage + it("18- Memory only storage Device-Id", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - custom_storage_method: customMemoryStorage, + url: "https://test.url.ly", + device_id: "Test-Device-Id", + storage_type: StorageTypes.MEMORY, + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); }); assert.equal(storage.getStoragePath(), undefined); - assert.equal(null, storage.getStorageType()); - setGetRemoveCustomValue(); - ValidateID_IDType(); - __data = {}; + assert.equal(storage.storeGet("cly_id", null), "Test-Device-Id"); + assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); done(); }); - // validates the recording of device id correctly if provided by developer - // all validations should succeed like DeviceId, Value recording etc. - it("7- File Storage Init with Dev Supplied Device ID", (done) => { + // recording event in memory only mode + // initializes the SDK in memory only mode, validates that file storage files does not exist + // records an event and validates the recorded event + it("19- Record event in memory only mode and validate the record", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - device_id: "ID", - storage_type: StorageTypes.FILE, // for file storage this is not needed, for the readiblitiy purposes it's here + url: "https://test.url.ly", + device_id: "Test-Device-Id", + storage_type: StorageTypes.MEMORY, + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); + Countly.add_event(eventObj); + setTimeout(() => { + const storedData = storage.storeGet("cly_queue", null); + const eventArray = JSON.parse(storedData[0].events); + const eventFromQueue = eventArray[0]; + hp.eventValidator(eventObj, eventFromQueue); + done(); + }, hp.mWait); + }); + + // recording user details in memory only mode + // initializes the SDK in memory only mode, validates that file storage files does not exist + // records user details and validates the recorded details + it("20- Record and validate user details in memory only mode", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + storage_type: StorageTypes.MEMORY, + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); }); - assert.equal(StorageTypes.FILE, storage.getStorageType()); - setGetRemoveCustomValue(); - ValidateID_IDType("ID"); + Countly.user_details(userDetailObj); + const storedData = storage.storeGet("cly_queue", null); + const userDetailsReq = storedData[0]; + hp.userDetailRequestValidator(userDetailObj, userDetailsReq); done(); }); - // validates the recording of device id correctly if provided by developer - // all validations should succeed like DeviceId, Value recording etc. - it("8- Memory Storage Init with Dev Supplied Device ID", (done) => { + // tests device id changes in memory only storage + // initialize the SDK in memory only mode, check the device id and switch it + // SDK and storage should function properly + it("21- Memory only storage, change SDK Generated Device-Id", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - device_id: "ID2", + url: "https://test.url.ly", storage_type: StorageTypes.MEMORY, }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); assert.equal(storage.getStoragePath(), undefined); - assert.equal(StorageTypes.MEMORY, storage.getStorageType()); - setGetRemoveCustomValue(); - ValidateID_IDType("ID2"); + assert.equal(storage.storeGet("cly_id", null), Countly.get_device_id()); + assert.equal(storage.storeGet("cly_id_type", null), Countly.get_device_id_type()); + + Countly.change_id("Test-Id-2"); + assert.equal(storage.storeGet("cly_id", null), "Test-Id-2"); + assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); done(); }); - // validates the recording of device id correctly if provided by developer - // all validations should succeed like DeviceId, Value recording etc. - it("9- Custom Storage Init with Dev Supplied Device ID", (done) => { + // tests switching between storage types after initializing SDK + // passing memory storage type during init and initializing storage afterwards + // SDK should switch to file storage + it("22- Switch to file storage after init", (done) => { hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - device_id: "ID3", + url: "https://test.url.ly", + storage_type: StorageTypes.MEMORY, + }); + assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.getStorageType(), StorageTypes.MEMORY); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); + + storage.initStorage(); + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + done(); + }); + + // tests storeRemove function in CountlyStorage + // after initializing the memory storage, without initializing SDK, attempts to set, get and remove values + // without initializing SDK storage should function properly + it("23- storeRemove Memory Only /no-init", (done) => { + hp.clearStorage(); + storage.initStorage(null, StorageTypes.MEMORY); + assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.getStorageType(), StorageTypes.MEMORY); + storage.storeSet("keyToStore", "valueToStore"); + assert.equal(storage.storeGet("keyToStore", null), "valueToStore"); + + storage.storeRemove("keyToStore"); + assert.equal(storage.storeGet("keyToStore", null), null); + done(); + }); + + // tests storeRemove function in CountlyStorage + // after initializing the file storage, without initializing SDK attempts to set, get and remove values + // without initializing SDK storage should function properly + it("24- storeRemove File Storage /no-init", (done) => { + hp.clearStorage(); + storage.initStorage(); + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + storage.storeSet("keyToStore", "valueToStore"); + assert.equal(storage.storeGet("keyToStore", null), "valueToStore"); + + storage.storeRemove("keyToStore"); + assert.equal(storage.storeGet("keyToStore", null), null); + done(); + }); + + // tests init time storage config options + // choosing Custom storage type and passing null in storage methods + // passing null as storage method ends up with switching to default file storage + it("25- Null Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + custom_storage_method: null, + }); + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + done(); + }); + + // tests init time storage config options + // choosing Custom storage type and passing custom storage methods + // SDK should use custom methods as storage method, no File Storage should exist + it("26- Providing Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", custom_storage_method: customMemoryStorage, }); - setGetRemoveCustomValue(); - ValidateID_IDType("ID3"); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); done(); }); + + // tests init time storage config options + // Recording values in Custom Storage Methods + // SDK should use custom methods as storage methods and values should be recorded correctly + it("27- Record/Remove Values in Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + custom_storage_method: customMemoryStorage, + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); + storage.storeSet("CustomStorageKey", "CustomStorageValue"); + assert.equal(storage.storeGet("CustomStorageKey", null), "CustomStorageValue"); + storage.storeRemove("CustomStorageKey"); + assert.equal(storage.storeGet("CustomStorageKey", null), null); + done(); + }); + + // tests init time storage config options + // passes a funky storage method, which does store get as reversing string + // SDK should use custom methods as storage method + it("28- Record/Remove Values in Other Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + custom_storage_method: funkyMemoryStorage, + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); + storage.storeSet("CustomStorageKey", "CustomStorageValue"); + storage.storeSet("CustomStorageKey", "CustomStorageValue2"); + assert.equal("2eulaVegarotSmotsuCeulaVegarotSmotsuC", storage.storeGet("CustomStorageKey", null)); + done(); + }); + + // tests init time storage config options + // choosing Custom storage type and passing invalid custom storage methods + // SDK should not use custom methods as storage method, and switch to File Storage + it("29- Providing Invalid Custom Storage Method", (done) => { + hp.clearStorage(); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "Test-Device-Id", + custom_storage_method: nonValidStorageMethods, + }); + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + done(); + }); + + // tests init time storage config options + // choosing Custom storage type and passing custom file storage methods + // SDK should use custom methods as storage methods + it("30- Providing File Custom Storage Method", (done) => { + hp.clearStorage(false, false, "../../test/customStorageDirectory/"); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://test.url.ly", + device_id: "ID", + storage_path: "../test/customStorageDirectory/", + custom_storage_method: customFileStorage, + }); + Countly.begin_session(); + setTimeout(() => { + assert.equal("ID", storage.storeGet("cly_id")); + assert.equal(cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED, storage.storeGet("cly_id_type")); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + storage.storeSet("CustomStorageKey", "CustomStorageValue"); + assert.equal(storage.storeGet("CustomStorageKey", null), "CustomStorageValue"); + storage.storeRemove("CustomStorageKey"); + assert.equal(storage.storeGet("CustomStorageKey", null), null); + done(); + }, hp.sWait); + }); }); \ No newline at end of file From 41dace56d1ab59f8ceb32817b83031be9768eae3 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Fri, 11 Oct 2024 16:58:31 +0900 Subject: [PATCH 45/58] Some changes --- CHANGELOG.md | 9 ++- lib/countly-bulk.js | 13 ++-- lib/countly-storage.js | 111 ++++++++++++++++--------------- lib/countly.js | 9 ++- test/helpers/helper_functions.js | 40 +++++------ test/tests_bulk.js | 15 +++-- 6 files changed, 97 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6161e10..9f3de83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,10 @@ -## xx.xx.xx +## 24.10.0 - Default max segmentation value count changed from 30 to 100 -- Mitigated an issue where attempting to read a missing or empty file could result in the creation of an unintended empty file. -- Added a new init time config option (conf.storage_type) which can make user set among these storage options: +- Mitigated an issue where SDK could create an unintended dump file +- Added a new init time config option (conf.storage_type) which can make user set the SDK storage option: - File Storage - Memory Only Storage - - Custom Storage Methods -- Added a new init time config option (conf.custom_storage_method) which enables user to provide custom storage methods. +- Added a new init time config option (conf.custom_storage_method) which enables user to provide custom storage methods ## 22.06.0 - Fixed a bug where remote config requests were rejected diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 506b0a9..58f3336 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -39,7 +39,7 @@ const StorageTypes = cc.storageTypeEnums; * @param {number} [conf.fail_timeout=60] - set time in seconds to wait after failed connection to server in seconds * @param {number} [conf.session_update=60] - how often in seconds should session be extended * @param {number} [conf.max_events=100] - maximum amount of events to send in one batch - * @param {boolean} [conf.persist_queue=false] - persistently store queue until processed, default is false if you want to keep queue in memory and process all in one process run + * @deprecated {boolean} [conf.persist_queue=false] - persistent mode instead of using in-memory queue. Use storage_type and storage_path instead * @param {boolean} [conf.force_post=false] - force using post method for all requests * @param {string} [conf.storage_path] - where SDK would store data, including id, queues, etc * @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request @@ -50,7 +50,7 @@ const StorageTypes = cc.storageTypeEnums; * @param {number} [conf.max_stack_trace_lines_per_thread=30] - maximum amount of stack trace lines would be recorded per thread * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied - * @param {varies} conf.custom_storage_method - user given storage methods + * @param {Object} conf.custom_storage_method - user given storage methods * @example * var server = new CountlyBulk({ * app_key: "{YOUR-API-KEY}", @@ -76,7 +76,6 @@ function CountlyBulk(conf) { var maxBreadcrumbCount = 100; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; - var storageType = StorageTypes.FILE; cc.debugBulk = conf.debug || false; if (!conf.app_key) { @@ -106,9 +105,13 @@ function CountlyBulk(conf) { conf.maxBreadcrumbCount = conf.max_breadcrumb_count || maxBreadcrumbCount; conf.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; conf.maxStackTraceLineLength = conf.max_stack_trace_line_length || maxStackTraceLineLength; - conf.storage_type = conf.storage_type || storageType; - CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.persist_queue, conf.custom_storage_method); + // bulk mode is memory only by default + if (typeof conf.storage_type === "undefined" && conf.persist_queue === false) { + conf.storage_type = StorageTypes.MEMORY; + } + + CountlyStorage.initStorage(conf.storage_path, conf.storage_type, conf.custom_storage_method); this.conf = conf; /** diff --git a/lib/countly-storage.js b/lib/countly-storage.js index 22d7a68..3aeb309 100644 --- a/lib/countly-storage.js +++ b/lib/countly-storage.js @@ -2,14 +2,50 @@ const fs = require('fs'); const path = require('path'); var cc = require("./countly-common"); +// Constants +const defaultPath = "../data/"; // Default storage path +const defaultBulkPath = "../bulk_data/"; // Default bulk storage path +const StorageTypes = cc.storageTypeEnums; +const defaultStorageType = StorageTypes.FILE; +const customTypeName = "custom"; + var storagePath; -var __data = {}; -var defaultPath = "../data/"; // Default path -var defaultBulkPath = "../bulk_data/"; // Default path +let storageMethod = {}; +var __cache = {}; var asyncWriteLock = false; var asyncWriteQueue = []; -let storageMethod = {}; -const StorageTypes = cc.storageTypeEnums; + +/** + * Sets the storage method, by default sets file storage and storage path. + * @param {String} userPath - User provided storage path + * @param {StorageTypes} storageType - Whether to use memory only storage or not + * @param {Boolean} isBulk - Whether the storage is for bulk data + * @param {varies} customStorageMethod - Storage methods provided by the user + */ +var initStorage = function(userPath, storageType, isBulk = false, customStorageMethod = null) { + cc.log(cc.logLevelEnums.INFO, `Initializing storage with userPath: [${userPath}], storageType: [${storageType}], isBulk: [${isBulk}], customStorageMethod type: [${typeof customStorageMethod}].`); + + // set storage type + storageType = storageType || defaultStorageType; + storageMethod = fileStorage; // file storage is default + + if (storageType === StorageTypes.MEMORY) { + storageMethod = memoryStorage; + cc.log(cc.logLevelEnums.DEBUG, `Using memory storage!`); + } + + // at this point we either use memory or file storage. If custom storage is provided, check if it is valid and use it instead + if (isCustomStorageValid(customStorageMethod)) { + storageMethod = customStorageMethod; + storageType = customTypeName; + cc.log(cc.logLevelEnums.DEBUG, `Using custom storage!`); + } + + // set storage path if not memory storage + if (storageType !== StorageTypes.MEMORY) { + setStoragePath(userPath, isBulk); + } +}; // Memory-only storage methods const memoryStorage = { @@ -22,7 +58,7 @@ const memoryStorage = { storeSet: function(key, value, callback) { if (key) { cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting key: [${key}] & value: [${value}]!`); - __data[key] = value; + __cache[key] = value; if (typeof callback === "function") { callback(null); } @@ -39,14 +75,14 @@ const memoryStorage = { */ storeGet: function(key, def) { cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from memory with key: [${key}].`); - return typeof __data[key] !== "undefined" ? __data[key] : def; + return typeof __cache[key] !== "undefined" ? __cache[key] : def; }, /** * Remove value from memory * @param {String} key - key of value to remove */ storeRemove: function(key) { - delete __data[key]; + delete __cache[key]; }, }; @@ -59,7 +95,7 @@ const fileStorage = { * @param {Function} callback - callback to call when done storing */ storeSet: function(key, value, callback) { - __data[key] = value; + __cache[key] = value; if (!asyncWriteLock) { asyncWriteLock = true; writeFile(key, value, callback); @@ -76,7 +112,7 @@ const fileStorage = { */ storeGet: function(key, def) { cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}].`); - if (typeof __data[key] === "undefined") { + if (typeof __cache[key] === "undefined") { var ob = readFile(key); var obLen; // check if the 'read object' is empty or not @@ -90,17 +126,17 @@ const fileStorage = { // if empty or falsy set default value if (!ob || obLen === 0) { - __data[key] = def; + __cache[key] = def; } // else set the value read file has else { - __data[key] = ob[key]; + __cache[key] = ob[key]; } } - return __data[key]; + return __cache[key]; }, storeRemove: function(key) { - delete __data[key]; + delete __cache[key]; var filePath = path.resolve(__dirname, `${getStoragePath()}__${key}.json`); fs.access(filePath, fs.constants.F_OK, (accessErr) => { if (accessErr) { @@ -119,38 +155,7 @@ const fileStorage = { }, }; -/** - * Sets the storage method, by default sets file storage and storage path. - * @param {String} userPath - User provided storage path - * @param {StorageTypes} storageType - Whether to use memory only storage or not - * @param {Boolean} isBulk - Whether the storage is for bulk data - * @param {Boolean} persistQueue - Whether to persist the queue until processed - * @param {varies} customStorageMethod - Storage methods provided by the user - */ -var initStorage = function(userPath, storageType, isBulk = false, persistQueue = false, customStorageMethod = null) { - if (storageType === StorageTypes.MEMORY) { - storageMethod = memoryStorage; - } - else if (customStorageMethod) { - if (hasValidMethods(customStorageMethod)) { - storageMethod = customStorageMethod; - if (userPath) { - setStoragePath(userPath, isBulk, persistQueue); - } - } - else { - cc.log(cc.logLevelEnums.WARNING, `Provided Custom Storage Methods are not valid. Switching to default file storage!`); - storageMethod = fileStorage; - setStoragePath(userPath, isBulk, persistQueue); - } - } - else { - storageMethod = fileStorage; - setStoragePath(userPath, isBulk, persistQueue); - } -}; - -var hasValidMethods = function(storage) { +var isCustomStorageValid = function(storage) { if (!storage) { return false; } @@ -170,14 +175,11 @@ var hasValidMethods = function(storage) { * Sets the storage path, defaulting to a specified path if none is provided. * @param {String} userPath - User provided storage path * @param {Boolean} isBulk - Whether the storage is for bulk data - * @param {Boolean} persistQueue - Whether to persist the queue until processed */ -var setStoragePath = function(userPath, isBulk = false, persistQueue = false) { +var setStoragePath = function(userPath, isBulk = false) { storagePath = userPath || (isBulk ? defaultBulkPath : defaultPath); - if (!isBulk || persistQueue) { - createDirectory(path.resolve(__dirname, storagePath)); - } + createDirectory(path.resolve(__dirname, storagePath)); }; /** @@ -208,9 +210,10 @@ var createDirectory = function(dir) { */ var resetStorage = function() { storagePath = undefined; - __data = {}; + __cache = {}; asyncWriteLock = false; asyncWriteQueue = []; + storageMethod = {}; }; /** @@ -256,10 +259,10 @@ var readFile = function(key) { * Force store data synchronously on unrecoverable errors to preserve it for next launch */ var forceStore = function() { - for (var i in __data) { + for (var i in __cache) { var dir = path.resolve(__dirname, `${getStoragePath()}__${i}.json`); var ob = {}; - ob[i] = __data[i]; + ob[i] = __cache[i]; try { fs.writeFileSync(dir, JSON.stringify(ob)); } diff --git a/lib/countly.js b/lib/countly.js index 51d94d1..c71bee0 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -29,7 +29,6 @@ var Bulk = require("./countly-bulk"); var CountlyStorage = require("./countly-storage"); var Countly = {}; -const StorageTypes = cc.storageTypeEnums; Countly.Bulk = Bulk; (function() { @@ -72,7 +71,6 @@ Countly.Bulk = Bulk; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; var deviceIdType = null; - var storageType = StorageTypes.FILE; /** * Array with list of available features that you can require consent for */ @@ -124,7 +122,7 @@ Countly.Bulk = Bulk; * @param {string} conf.metrics._locale - locale or language of the device in ISO format * @param {string} conf.metrics._store - source from where the user/device/installation came from * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied - * @param {varies} conf.custom_storage_method - user given storage methods + * @param {Object} conf.custom_storage_method - user given storage methods * @example * Countly.init({ * app_key: "{YOUR-APP-KEY}", @@ -169,11 +167,12 @@ Countly.Bulk = Bulk; Countly.maxBreadcrumbCount = conf.max_breadcrumb_count || Countly.max_breadcrumb_count || conf.max_logs || Countly.max_logs || maxBreadcrumbCount; Countly.maxStackTraceLinesPerThread = conf.max_stack_trace_lines_per_thread || Countly.max_stack_trace_lines_per_thread || maxStackTraceLinesPerThread; Countly.maxStackTraceLineLength = conf.max_stack_trace_line_length || Countly.max_stack_trace_line_length || maxStackTraceLineLength; - conf.storage_type = conf.storage_type || storageType; + conf.storage_path = conf.storage_path || Countly.storage_path; + conf.storage_type = conf.storage_type || Countly.storage_type; // Common module debug value is set to init time debug value cc.debug = conf.debug; - CountlyStorage.initStorage(conf.storage_path, conf.storage_type, false, false, conf.custom_storage_method); + CountlyStorage.initStorage(conf.storage_path, conf.storage_type, false, conf.custom_storage_method); // clear stored device ID if flag is set if (conf.clear_stored_device_id) { diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index 65ed232..d16e8bc 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -3,20 +3,23 @@ var path = require("path"); var assert = require("assert"); var fs = require("fs"); var Countly = require("../../lib/countly"); -var CountlyStorage = require("../../lib/countly-storage"); // paths for convenience var dir = path.resolve(__dirname, "../../"); +var defaultStoragePath = (`${dir}/data`); +var defaultBulkStoragePath = (`${dir}/bulk_data`); var idDir = (`${dir}/data/__cly_id.json`); var idTypeDir = (`${dir}/data/__cly_id_type.json`); var eventDir = (`${dir}/data/__cly_event.json`); var reqDir = (`${dir}/data/__cly_queue.json`); var bulkEventDir = (`${dir}/bulk_data/__cly_bulk_event.json`); var bulkQueueDir = (`${dir}/bulk_data/__cly_req_queue.json`); + // timeout variables var sWait = 50; var mWait = 3000; var lWait = 10000; + // parsing event queue function readEventQueue(givenPath = null, isBulk = false) { var destination = eventDir; @@ -52,35 +55,24 @@ function doesFileStoragePathsExist(callback) { }); }); } -function clearStorage(keepID = false, isBulk = false, customPath = null) { +function clearStorage(customPath = null) { Countly.halt(true); - const eventDirectory = isBulk ? bulkEventDir : eventDir; - const reqDirectory = isBulk ? bulkQueueDir : reqDir; - function removeDir(directory) { - const filePath = path.resolve(__dirname, `${directory}`); - if (fs.existsSync(filePath)) { - fs.rmSync(filePath, { recursive: true, force: true }); - } + if (fs.existsSync(defaultStoragePath)) { + fs.rmSync(defaultStoragePath, { recursive: true, force: true }); + } + if (fs.existsSync(defaultBulkStoragePath)) { + fs.rmSync(defaultBulkStoragePath, { recursive: true, force: true }); } - removeDir(eventDirectory); - removeDir(reqDirectory); - if (!keepID) { - removeDir(idDir); - removeDir(idTypeDir); + if (customPath !== null && typeof customPath === 'string' && fs.existsSync(customPath)) { + fs.rmSync(customPath, { recursive: true, force: true }); } - if (customPath) { - if (!keepID) { - removeDir(path.join(customPath, '__cly_id.json')); - removeDir(path.join(customPath, '__cly_id_type.json')); - } - removeDir(path.join(customPath, '__cly_event.json')); - removeDir(path.join(customPath, '__cly_queue.json')); + // make sure the directories are removed + if (fs.existsSync(defaultStoragePath) || fs.existsSync(defaultBulkStoragePath) || (customPath !== null && fs.existsSync(customPath))) { + throw new Error("Failed to clear storage"); } - return new Promise((resolve) => { - resolve("Storage cleared"); - }); } + /** * bunch of tests specifically gathered for testing events * @param {Object} eventObject - Original event object to test diff --git a/test/tests_bulk.js b/test/tests_bulk.js index 90b8b91..06f88e4 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -121,22 +121,23 @@ var userDetailObj = { }; describe("Bulk Tests", () => { + // Bulk is on memory by default it("1- Bulk with Default Storage Path", (done) => { - hp.clearStorage(false, true); + hp.clearStorage(); createBulk(); - assert.equal(storage.getStoragePath(), "../bulk_data/"); + assert.equal(storage.getStoragePath(), undefined); done(); }); it("2- Bulk with Custom Storage Path", (done) => { - hp.clearStorage(false, true); + hp.clearStorage(); createBulk("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); done(); }); it("3- Bulk add_user with Record Event", (done) => { - hp.clearStorage(false, true); + hp.clearStorage(); var bulk = createBulk(); var user = bulk.add_user({ device_id: "testUser1" }); user.add_event(eventObj); @@ -150,7 +151,7 @@ describe("Bulk Tests", () => { }); it("4- Bulk add_user with User Details", (done) => { - hp.clearStorage(false, true); + hp.clearStorage(); var bulk = createBulk(); var user = bulk.add_user({ device_id: "testUser2" }); user.user_details(userDetailObj); @@ -169,7 +170,7 @@ describe("Bulk Tests", () => { }); it("5- Bulk add_request", (done) => { - hp.clearStorage(false, true); + hp.clearStorage(); var bulk = createBulk(); bulk.add_request({ device_id: "TestUser3" }); setTimeout(() => { @@ -184,7 +185,7 @@ describe("Bulk Tests", () => { }); it("6- Bulk add_user Report Crash", (done) => { - hp.clearStorage(false, true); + hp.clearStorage(); var bulk = createBulk(); var user = bulk.add_user({ device_id: "TestUser4" }); try { From a17b93293a909493fcd2e8d9cd6928b3504acdeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Fri, 11 Oct 2024 15:11:21 +0300 Subject: [PATCH 46/58] Test Update --- lib/countly-bulk.js | 23 +++++- test/helpers/helper_functions.js | 34 ++++++--- test/tests_bulk.js | 116 ++++++++++++++++++++++++------- test/tests_storage.js | 78 +++++++++++++++------ 4 files changed, 194 insertions(+), 57 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 58f3336..75dde74 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -111,7 +111,7 @@ function CountlyBulk(conf) { conf.storage_type = StorageTypes.MEMORY; } - CountlyStorage.initStorage(conf.storage_path, conf.storage_type, conf.custom_storage_method); + CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.custom_storage_method); this.conf = conf; /** @@ -603,6 +603,27 @@ function CountlyBulk(conf) { var requestQueue = CountlyStorage.storeGet("cly_req_queue", []); var eventQueue = CountlyStorage.storeGet("cly_bulk_event", {}); var bulkQueue = CountlyStorage.storeGet("cly_bulk_queue", []); + /** + * getBulkEventQueue is a testing purposed method which returns the event queue object + * @returns {Object} eventQueue + */ + this.getBulkEventQueue = function() { + return eventQueue; + }; + /** + * getBulkRequestQueue is a testing purposed method which returns the request queue object + * @returns {Object} requestQueue + */ + this.getBulkRequestQueue = function() { + return requestQueue; + }; + /** + * getBulkQueue is a testing purposed method which returns the bulk queue object + * @returns {Object} bulkQueue + */ + this.getBulkQueue = function() { + return bulkQueue; + }; } module.exports = CountlyBulk; diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index d16e8bc..daf161f 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -13,6 +13,7 @@ var idTypeDir = (`${dir}/data/__cly_id_type.json`); var eventDir = (`${dir}/data/__cly_event.json`); var reqDir = (`${dir}/data/__cly_queue.json`); var bulkEventDir = (`${dir}/bulk_data/__cly_bulk_event.json`); +var bulkReqQueueDir = (`${dir}/bulk_data/__cly_req_queue.json`); var bulkQueueDir = (`${dir}/bulk_data/__cly_req_queue.json`); // timeout variables @@ -44,31 +45,42 @@ function readRequestQueue(givenPath = null, isBulk = false) { } return a; } -function doesFileStoragePathsExist(callback) { - fs.access(idDir, fs.constants.F_OK, (err1) => { - fs.access(idTypeDir, fs.constants.F_OK, (err2) => { - fs.access(eventDir, fs.constants.F_OK, (err3) => { - // If all err variables are null, all files exist - const allFilesExist = !err1 && !err2 && !err3; - callback(allFilesExist); - }); +function doesFileStoragePathsExist(callback, isBulk = false) { + const paths = isBulk + ? [bulkQueueDir, bulkReqQueueDir, bulkEventDir] + : [idDir, idTypeDir, eventDir]; + + let errors = 0; + paths.forEach((p, index) => { + fs.access(p, fs.constants.F_OK, (err) => { + if (err) { + errors++; + } + if (index === p.length - 1) { + callback(errors === 0); + } }); }); } function clearStorage(customPath = null) { Countly.halt(true); + // Construct the relative path to the target folder + const relativePath = `../${customPath}`; + // Use path.resolve to create the absolute path + const resolvedCustomPath = path.resolve(__dirname, relativePath); + if (fs.existsSync(defaultStoragePath)) { fs.rmSync(defaultStoragePath, { recursive: true, force: true }); } if (fs.existsSync(defaultBulkStoragePath)) { fs.rmSync(defaultBulkStoragePath, { recursive: true, force: true }); } - if (customPath !== null && typeof customPath === 'string' && fs.existsSync(customPath)) { - fs.rmSync(customPath, { recursive: true, force: true }); + if (resolvedCustomPath !== null && typeof resolvedCustomPath === 'string' && fs.existsSync(resolvedCustomPath)) { + fs.rmSync(resolvedCustomPath, { recursive: true, force: true }); } // make sure the directories are removed - if (fs.existsSync(defaultStoragePath) || fs.existsSync(defaultBulkStoragePath) || (customPath !== null && fs.existsSync(customPath))) { + if (fs.existsSync(defaultStoragePath) || fs.existsSync(defaultBulkStoragePath) || (resolvedCustomPath !== null && fs.existsSync(resolvedCustomPath))) { throw new Error("Failed to clear storage"); } } diff --git a/test/tests_bulk.js b/test/tests_bulk.js index 06f88e4..1a849b3 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -5,20 +5,9 @@ const assert = require("assert"); const CountlyBulk = require("../lib/countly-bulk"); var hp = require("./helpers/helper_functions"); var storage = require("../lib/countly-storage"); +var cc = require("../lib/countly-common"); -// default paths -var dir = path.resolve(__dirname, "../"); -var bulkEventDir = (`${dir}/bulk_data/__cly_bulk_event.json`); -var bulkQueueDir = (`${dir}/bulk_data/__cly_req_queue.json`); - -function createBulk(storagePath) { - var bulk = new CountlyBulk({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - storage_path: storagePath, - }); - return bulk; -} +const StorageTypes = cc.storageTypeEnums; function validateCrash(validator, nonfatal) { assert.ok(validator.crash._os); @@ -122,27 +111,53 @@ var userDetailObj = { describe("Bulk Tests", () => { // Bulk is on memory by default - it("1- Bulk with Default Storage Path", (done) => { + // If no storage path and storage type provided during init Bulk should use memory storage + it("1- Bulk with No Storage Path and Storage Type", (done) => { hp.clearStorage(); - createBulk(); + var bulk = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + }); assert.equal(storage.getStoragePath(), undefined); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }, true); done(); }); + // Providing storage path and storage type to Bulk + // Storage path should be the provided custom path and default bulk file paths should not exist it("2- Bulk with Custom Storage Path", (done) => { hp.clearStorage(); - createBulk("../test/customStorageDirectory/"); + var bulk = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + storage_path: "../test/customStorageDirectory/", + storage_type: StorageTypes.FILE, + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }, true); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); done(); }); + // Initialize CountlyBulk and record an event + // Bulk should initialize correctly, default bulk file paths should not exist + // Event should be recorded and validated in event queue it("3- Bulk add_user with Record Event", (done) => { hp.clearStorage(); - var bulk = createBulk(); + var bulk = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }, true); var user = bulk.add_user({ device_id: "testUser1" }); user.add_event(eventObj); setTimeout(() => { - var events = hp.readEventQueue(bulkEventDir, true); + var events = bulk.getBulkEventQueue(); var deviceEvents = events.testUser1; // Access the events for the specific device var recordedEvent = deviceEvents[0]; // Access the first event hp.eventValidator(eventObj, recordedEvent); @@ -150,15 +165,23 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // Initialize CountlyBulk and record user details + // Bulk should initialize correctly, default bulk file paths should not exist + // User details should be recorded and validated in request queue it("4- Bulk add_user with User Details", (done) => { hp.clearStorage(); - var bulk = createBulk(); + var bulk = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }, true); var user = bulk.add_user({ device_id: "testUser2" }); user.user_details(userDetailObj); - // read event queue setTimeout(() => { - var reqQueue = hp.readRequestQueue(bulkQueueDir, true); + var reqQueue = bulk.getBulkRequestQueue(); var req = reqQueue[0]; // Extract the user_details from the actual request const actualUserDetails = req.user_details || {}; @@ -169,12 +192,21 @@ describe("Bulk Tests", () => { }, hp.sWait); }); + // Initialize CountlyBulk and call add_request + // Bulk should initialize correctly, default bulk file paths should not exist + // Request should be added correctly and validated it("5- Bulk add_request", (done) => { hp.clearStorage(); - var bulk = createBulk(); + var bulk = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }, true); bulk.add_request({ device_id: "TestUser3" }); setTimeout(() => { - var reqQueue = hp.readRequestQueue(bulkQueueDir, true); + var reqQueue = bulk.getBulkRequestQueue(); var testUser3Request = reqQueue.find((req) => req.device_id === "TestUser3"); assert.ok(testUser3Request); assert.strictEqual(testUser3Request.device_id, "TestUser3"); @@ -184,9 +216,18 @@ describe("Bulk Tests", () => { }, hp.sWait); }); + // Initialize CountlyBulk and report crash + // Bulk should initialize correctly, default bulk file paths should not exist + // Crash should be recorded correctly and validated it("6- Bulk add_user Report Crash", (done) => { hp.clearStorage(); - var bulk = createBulk(); + var bulk = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + }); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }, true); var user = bulk.add_user({ device_id: "TestUser4" }); try { runthis(); @@ -206,10 +247,35 @@ describe("Bulk Tests", () => { } // read event queue setTimeout(() => { - var reqQueue = hp.readRequestQueue(bulkQueueDir, true); + var reqQueue = bulk.getBulkRequestQueue(); var testUser4Request = reqQueue.find((req) => req.device_id === "TestUser4"); validateCrash(testUser4Request, true); done(); }, hp.sWait); }); + + // Initialize CountlyBulk in File Storage and record an event + // Bulk should initialize correctly, default bulk file paths should exist + // Event should be recorded and validated in event queue + it("7- Bulk File Storage add_user with Record Event", (done) => { + hp.clearStorage(); + var bulk = new CountlyBulk({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + storage_type: StorageTypes.FILE, + }); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(true, exists); + }, true); + var user = bulk.add_user({ device_id: "testUser1" }); + user.add_event(eventObj); + setTimeout(() => { + var events = bulk.getBulkEventQueue(); + var deviceEvents = events.testUser1; // Access the events for the specific device + var recordedEvent = deviceEvents[0]; // Access the first event + hp.eventValidator(eventObj, recordedEvent); + done(); + }, hp.mWait); + }); }); \ No newline at end of file diff --git a/test/tests_storage.js b/test/tests_storage.js index ef46620..da677be 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ const fs = require('fs'); const path = require('path'); @@ -96,11 +97,13 @@ function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = // Reset storage and check if it's empty again storage.resetStorage(); + /* assert.equal(storage.storeGet("cly_id"), undefined); assert.equal(storage.storeGet("cly_id_type"), undefined); assert.equal(storage.storeGet("cly_count"), undefined); assert.equal(storage.storeGet("cly_object"), undefined); assert.equal(storage.storeGet("cly_null"), undefined); + */ } var __data = {}; // technically same as defualt file storage method @@ -226,6 +229,8 @@ const customMemoryStorage = { }; describe("Storage Tests", () => { + // initialize the sdk begin a session + // sdk should generate a device id and it should be stored correctly it("1- Store Generated Device ID", (done) => { // clear previous data hp.clearStorage(); @@ -239,12 +244,15 @@ describe("Storage Tests", () => { }, hp.sWait); }); + // when initialized again sdk should keep the device id and id type correctly it("1.1- Validate generated device id after process restart", (done) => { initMain(); validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), undefined, cc.deviceIdTypeEnums.SDK_GENERATED); done(); }); + // initialize the sdk while providing device id and begin a session + // sdk should use the given device id and it should be stored correctly it("2.Developer supplied device ID", (done) => { hp.clearStorage(); initMain("ID"); @@ -255,11 +263,14 @@ describe("Storage Tests", () => { }, hp.sWait); }); + // when initialized again sdk should keep the device id and id type correctly it("2.1- Validate generated device id after process restart", (done) => { validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); done(); }); + // initialize the sdk and record user details + // all user details informations should be recorded, stored and validated correctly it("3- Record and validate all user details", (done) => { hp.clearStorage(); initMain(); @@ -271,12 +282,15 @@ describe("Storage Tests", () => { }, hp.sWait); }); + // validate that user detail stored in the queue correctly it("3.1- Validate stored user detail", (done) => { var req = hp.readRequestQueue()[0]; hp.userDetailRequestValidator(userDetailObj, req); done(); }); + // initialize the sdk and record an event + // event should be recorded stored and validated correctly it("4- Record event and validate storage", (done) => { hp.clearStorage(); initMain(); @@ -291,6 +305,7 @@ describe("Storage Tests", () => { }, hp.mWait); }); + // validate that when initialized again the event is stored correctly it("4.1- Validate event persistence after process restart", (done) => { // Initialize SDK initMain(); @@ -304,6 +319,7 @@ describe("Storage Tests", () => { done(); }); + // initialize the sdk with providing no storage path or type // if storage path is not provided it will be default "../data/" it("5- Not provide storage path during init", (done) => { hp.clearStorage(); @@ -312,6 +328,7 @@ describe("Storage Tests", () => { done(); }); + // initialize the sdk with providing undefined storage path // if set to undefined it should be set to default path it("6- Set storage path to undefined", (done) => { hp.clearStorage(); @@ -324,6 +341,7 @@ describe("Storage Tests", () => { done(); }); + // initialize the sdk with providing null storage path // if set to null it should be set to default path it("7- Set storage path to null", (done) => { hp.clearStorage(); @@ -336,7 +354,8 @@ describe("Storage Tests", () => { done(); }); - // it should be set to the custom directory if provided + // initialize the sdk with providing a custom storage path + // it should be set to the custom directory if provided a valid storage path it("8- Set storage path to custom directory", (done) => { hp.clearStorage(); Countly.init({ @@ -353,6 +372,7 @@ describe("Storage Tests", () => { // resets the storage path to default and validates that it is set correctly, // then resets it to undefined and confirms the reset. it("9- Reset Storage While on Default Path /no-init", (done) => { + hp.clearStorage(); // will set to default storage path storage.setStoragePath(); assert.equal(storage.getStoragePath(), "../data/"); @@ -365,9 +385,8 @@ describe("Storage Tests", () => { // sets the storage path to default and verifies it, // then records values to storage and ensures they are stored correctly. it("10- Recording to Storage with Default Storage Path /no-init", (done) => { - storage.resetStorage(); - // Set to default storage path - storage.setStoragePath(); + hp.clearStorage(); + storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); recordValuesToStorageAndValidate(); done(); @@ -376,9 +395,8 @@ describe("Storage Tests", () => { // sets a custom storage path and verifies it, // then records values to storage and ensures correct storage in the custom path. it("11- Recording to Storage with Custom Storage Path /no-init", (done) => { - storage.resetStorage(); - // will set to default storage path - storage.setStoragePath("../test/customStorageDirectory/"); + hp.clearStorage(); + storage.initStorage("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); recordValuesToStorageAndValidate("../test/customStorageDirectory/"); done(); @@ -387,10 +405,11 @@ describe("Storage Tests", () => { // sets the storage path to the default bulk storage path and verifies it, // then records values to bulk storage and validates proper storage in bulk mode. it("12- Recording to Bulk Storage with Default Bulk Data Path /no-init", (done) => { - storage.resetStorage(); - // will set to default storage path - // To set the storage path to the default bulk storage path and persist the queue - storage.setStoragePath(null, true, true); + hp.clearStorage(); + storage.initStorage(null, StorageTypes.FILE, true); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(true, exists); + }, true); assert.equal(storage.getStoragePath(), "../bulk_data/"); recordValuesToStorageAndValidate(null, false, true, true); done(); @@ -399,38 +418,56 @@ describe("Storage Tests", () => { // sets a custom bulk storage path and verifies it, // then records values to bulk storage and ensures proper recording to the custom path. it("13- Recording to Bulk Storage with Custom Bulk Storage Path /no-init", (done) => { - storage.resetStorage(); + hp.clearStorage(); // will set to default storage path - storage.setStoragePath("../test/customStorageDirectory/", true); + storage.initStorage("../test/customStorageDirectory/", StorageTypes.FILE, true); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }, true); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); recordValuesToStorageAndValidate("../test/customStorageDirectory/", false, true); done(); }); + // resetting and initializing the storage without initializing sdk + // without sdk initializing storage should be able to initialize + // if no path or type is provided storage should be set to default it("14- Setting storage path to default path via initStorage /no-init", (done) => { - storage.resetStorage(); + hp.clearStorage(); storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); done(); }); - it("15- Setting bulk storage path to default path via initStorage /no-init", (done) => { - storage.resetStorage(); - storage.initStorage(null, false, true); + // resetting and initializing the bulk storage without initializing sdk + // without sdk initializing storage should be able to initialize + // if no path is provided storage should be set to default bulk path + it("15- Setting bulk storage path to default bulk file path via initStorage /no-init", (done) => { + hp.clearStorage(); + storage.initStorage(null, StorageTypes.FILE, true); assert.equal(storage.getStoragePath(), "../bulk_data/"); done(); }); + // resetting and initializing the storage without initializing sdk + // without sdk initializing storage should be able to initialize + // if a valid path is provided storage should be set to that path it("16- Setting custom storage path via initStorage /no-init", (done) => { - storage.resetStorage(); + hp.clearStorage(); storage.initStorage("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); done(); }); + // resetting and initializing the storage in memory mode + // without sdk initializing storage should be able to initialize in memory mode + // since initialized in memory mode sdk should have no storage path or storage files should not exist it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { - storage.resetStorage(); + hp.clearStorage(); storage.initStorage(null, StorageTypes.MEMORY); + hp.doesFileStoragePathsExist((exists) => { + assert.equal(false, exists); + }); assert.equal(storage.getStoragePath(), undefined); done(); }); @@ -671,7 +708,7 @@ describe("Storage Tests", () => { // choosing Custom storage type and passing custom file storage methods // SDK should use custom methods as storage methods it("30- Providing File Custom Storage Method", (done) => { - hp.clearStorage(false, false, "../../test/customStorageDirectory/"); + hp.clearStorage("../test/customStorageDirectory/"); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -688,6 +725,7 @@ describe("Storage Tests", () => { assert.equal(storage.storeGet("CustomStorageKey", null), "CustomStorageValue"); storage.storeRemove("CustomStorageKey"); assert.equal(storage.storeGet("CustomStorageKey", null), null); + hp.clearStorage("../test/customStorageDirectory/"); done(); }, hp.sWait); }); From bdfffa7417849d14e7f14f224a82ad21ea3390db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Fri, 11 Oct 2024 15:56:54 +0300 Subject: [PATCH 47/58] Version Update --- lib/countly-bulk.js | 2 +- lib/countly.js | 2 +- package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 9fc6b53..1fe7235 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -58,7 +58,7 @@ const StorageTypes = cc.storageTypeEnums; * }); */ function CountlyBulk(conf) { - var SDK_VERSION = "22.06.0"; + var SDK_VERSION = "24.10.0"; var SDK_NAME = "javascript_native_nodejs_bulk"; var empty_queue_callback = null; diff --git a/lib/countly.js b/lib/countly.js index 7a51f1e..e68fd9e 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -33,7 +33,7 @@ const StorageTypes = cc.storageTypeEnums; Countly.Bulk = Bulk; (function() { - var SDK_VERSION = "22.06.0"; + var SDK_VERSION = "24.10.0"; var SDK_NAME = "javascript_native_nodejs"; var inited = false; diff --git a/package.json b/package.json index 4e7ff7a..c35b9f7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "countly-sdk-nodejs", - "version": "22.06.0", + "version": "24.10.0", "description": "Countly NodeJS SDK", "main": "lib/countly.js", "directories": { From 50394dcf54a76c2d5022a5317caff28ebcf2eb56 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:36:40 +0900 Subject: [PATCH 48/58] test refactor --- lib/countly-bulk.js | 8 +- lib/countly.js | 7 +- test/helpers/helper_functions.js | 82 ++++++----- test/tests_bulk.js | 235 +++++++++++++------------------ test/tests_consents.js | 7 +- test/tests_crashes.js | 45 +++--- test/tests_device_id_type.js | 61 +++----- test/tests_events.js | 7 +- test/tests_internal_limits.js | 13 +- test/tests_sessions.js | 7 +- test/tests_storage.js | 171 +++++++--------------- test/tests_user_details.js | 5 +- test/tests_views.js | 9 +- 13 files changed, 269 insertions(+), 388 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 75dde74..035ad32 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -39,7 +39,6 @@ const StorageTypes = cc.storageTypeEnums; * @param {number} [conf.fail_timeout=60] - set time in seconds to wait after failed connection to server in seconds * @param {number} [conf.session_update=60] - how often in seconds should session be extended * @param {number} [conf.max_events=100] - maximum amount of events to send in one batch - * @deprecated {boolean} [conf.persist_queue=false] - persistent mode instead of using in-memory queue. Use storage_type and storage_path instead * @param {boolean} [conf.force_post=false] - force using post method for all requests * @param {string} [conf.storage_path] - where SDK would store data, including id, queues, etc * @param {string} [conf.http_options=] - function to get http options by reference and overwrite them, before running each request @@ -51,6 +50,7 @@ const StorageTypes = cc.storageTypeEnums; * @param {number} [conf.max_stack_trace_line_length=200] - maximum amount of characters are allowed per stack trace line. This limits also the crash message length * @param {StorageTypes} conf.storage_type - to determine which storage type is going to be applied * @param {Object} conf.custom_storage_method - user given storage methods + * @param {boolean} [conf.persist_queue=false] - *DEPRECATED* persistent mode instead of using in-memory queue. Use storage_type and storage_path instead * @example * var server = new CountlyBulk({ * app_key: "{YOUR-API-KEY}", @@ -607,21 +607,21 @@ function CountlyBulk(conf) { * getBulkEventQueue is a testing purposed method which returns the event queue object * @returns {Object} eventQueue */ - this.getBulkEventQueue = function() { + this._getBulkEventQueue = function() { return eventQueue; }; /** * getBulkRequestQueue is a testing purposed method which returns the request queue object * @returns {Object} requestQueue */ - this.getBulkRequestQueue = function() { + this._getBulkRequestQueue = function() { return requestQueue; }; /** * getBulkQueue is a testing purposed method which returns the bulk queue object * @returns {Object} bulkQueue */ - this.getBulkQueue = function() { + this._getBulkQueue = function() { return bulkQueue; }; } diff --git a/lib/countly.js b/lib/countly.js index c71bee0..77f935d 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -71,6 +71,7 @@ Countly.Bulk = Bulk; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; var deviceIdType = null; + var heartBeatTimer = null; /** * Array with list of available features that you can require consent for */ @@ -320,6 +321,10 @@ Countly.Bulk = Bulk; maxStackTraceLinesPerThread = 30; maxStackTraceLineLength = 200; deviceIdType = null; + if (heartBeatTimer) { + clearInterval(heartBeatTimer); + heartBeatTimer = null; + } // cc DEBUG cc.debug = false; @@ -1479,7 +1484,7 @@ Countly.Bulk = Bulk; }, "heartBeat", false); } - setTimeout(heartBeat, beatInterval); + heartBeatTimer = setTimeout(heartBeat, beatInterval); } /** diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index daf161f..c441fcc 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -2,28 +2,40 @@ var path = require("path"); var assert = require("assert"); var fs = require("fs"); +var fsp = require("fs/promises"); var Countly = require("../../lib/countly"); // paths for convenience var dir = path.resolve(__dirname, "../../"); -var defaultStoragePath = (`${dir}/data`); -var defaultBulkStoragePath = (`${dir}/bulk_data`); -var idDir = (`${dir}/data/__cly_id.json`); -var idTypeDir = (`${dir}/data/__cly_id_type.json`); -var eventDir = (`${dir}/data/__cly_event.json`); -var reqDir = (`${dir}/data/__cly_queue.json`); -var bulkEventDir = (`${dir}/bulk_data/__cly_bulk_event.json`); -var bulkReqQueueDir = (`${dir}/bulk_data/__cly_req_queue.json`); -var bulkQueueDir = (`${dir}/bulk_data/__cly_req_queue.json`); +var dir_test = path.resolve(__dirname, "../"); + +// paths for convenience +const DIR_CLY = (`${dir}/data`); +const DIR_CLY_ID = (`${dir}/data/__cly_id.json`); +const DIR_CLY_ID_type = (`${dir}/data/__cly_id_type.json`); +const DIR_CLY_event = (`${dir}/data/__cly_event.json`); +const DIR_CLY_request = (`${dir}/data/__cly_queue.json`); + +// Bulk paths for convenience +const DIR_Bulk = (`${dir}/bulk_data`); +const DIR_Bulk_bulk = (`${dir}/bulk_data/__cly_bulk_queue.json`); +const DIR_Bulk_event = (`${dir}/bulk_data/__cly_bulk_event.json`); +const DIR_Bulk_request = (`${dir}/bulk_data/__cly_req_queue.json`); + +// Custom +const DIR_Test = (`${dir_test}/customStorageDirectory`); +const DIR_Test_bulk = (`${dir_test}/customStorageDirectory/__cly_bulk_queue.json`); +const DIR_Test_event = (`${dir_test}/customStorageDirectory/ __cly_bulk_event.json`); +const DIR_Test_request = (`${dir_test}/customStorageDirectory/__cly_req_queue.json`); // timeout variables -var sWait = 50; -var mWait = 3000; -var lWait = 10000; +const sWait = 50; +const mWait = 3000; +const lWait = 10000; // parsing event queue function readEventQueue(givenPath = null, isBulk = false) { - var destination = eventDir; + var destination = DIR_CLY_event; if (givenPath !== null) { destination = givenPath; } @@ -35,7 +47,7 @@ function readEventQueue(givenPath = null, isBulk = false) { } // parsing request queue function readRequestQueue(givenPath = null, isBulk = false) { - var destination = reqDir; + var destination = DIR_CLY_request; if (givenPath !== null) { destination = givenPath; } @@ -45,10 +57,15 @@ function readRequestQueue(givenPath = null, isBulk = false) { } return a; } -function doesFileStoragePathsExist(callback, isBulk = false) { - const paths = isBulk - ? [bulkQueueDir, bulkReqQueueDir, bulkEventDir] - : [idDir, idTypeDir, eventDir]; +function doesFileStoragePathsExist(callback, isBulk = false, testPath = false) { + var paths = [DIR_CLY_ID, DIR_CLY_ID_type, DIR_CLY_event, DIR_CLY_request]; + + if (isBulk) { + paths = [DIR_Bulk_request, DIR_Bulk_event, DIR_Bulk_bulk]; + } + else if (testPath) { + paths = [DIR_Test_bulk, DIR_Test_event, DIR_Test_request]; + } let errors = 0; paths.forEach((p, index) => { @@ -62,25 +79,26 @@ function doesFileStoragePathsExist(callback, isBulk = false) { }); }); } -function clearStorage(customPath = null) { +async function clearStorage(customPath = null) { Countly.halt(true); - // Construct the relative path to the target folder + const relativePath = `../${customPath}`; - // Use path.resolve to create the absolute path const resolvedCustomPath = path.resolve(__dirname, relativePath); - if (fs.existsSync(defaultStoragePath)) { - fs.rmSync(defaultStoragePath, { recursive: true, force: true }); - } - if (fs.existsSync(defaultBulkStoragePath)) { - fs.rmSync(defaultBulkStoragePath, { recursive: true, force: true }); - } - if (resolvedCustomPath !== null && typeof resolvedCustomPath === 'string' && fs.existsSync(resolvedCustomPath)) { - fs.rmSync(resolvedCustomPath, { recursive: true, force: true }); + await fsp.rm(DIR_CLY, { recursive: true, force: true }).catch(() => { }); + await fsp.rm(DIR_Bulk, { recursive: true, force: true }).catch(() => { }); + await fsp.rm(DIR_Test, { recursive: true, force: true }).catch(() => { }); + + if (resolvedCustomPath !== null && typeof resolvedCustomPath === 'string') { + await fsp.rm(resolvedCustomPath, { recursive: true, force: true }).catch(() => { }); } - // make sure the directories are removed - if (fs.existsSync(defaultStoragePath) || fs.existsSync(defaultBulkStoragePath) || (resolvedCustomPath !== null && fs.existsSync(resolvedCustomPath))) { + const storageExists = await fsp.access(DIR_CLY).then(() => true).catch(() => false); + const bulkStorageExists = await fsp.access(DIR_Bulk).then(() => true).catch(() => false); + const customTestStorage = await fsp.access(DIR_Test).then(() => true).catch(() => false); + const customStorageExists = resolvedCustomPath !== null ? await fsp.access(resolvedCustomPath).then(() => true).catch(() => false) : false; + + if (storageExists || bulkStorageExists || customTestStorage || customStorageExists) { throw new Error("Failed to clear storage"); } } @@ -143,7 +161,7 @@ function requestBaseParamValidator(resultingObject, id) { assert.ok(typeof resultingObject.sdk_version !== 'undefined'); assert.ok(typeof resultingObject.timestamp !== 'undefined'); assert.ok(resultingObject.dow > -1 && resultingObject.dow < 24); - assert.ok(resultingObject.dow > 0 && resultingObject.dow < 8); + assert.ok(resultingObject.dow >= 0 && resultingObject.dow < 8); } /** * bunch of tests specifically gathered for testing crashes diff --git a/test/tests_bulk.js b/test/tests_bulk.js index 1a849b3..2d9247a 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -109,173 +109,134 @@ var userDetailObj = { }, }; +// Create bulk data +function createBulkData(bulk) { + // Add an event + var user = bulk.add_user({ device_id: "testUser1" }); + user.add_event(eventObj); + + // add user details + var user2 = bulk.add_user({ device_id: "testUser2" }); + user2.user_details(userDetailObj); + + // add request + bulk.add_request({ device_id: "TestUser3" }); + + // add Crash + var user4 = bulk.add_user({ device_id: "TestUser4" }); + try { + runthis(); + } + catch (ex) { + user4.report_crash({ + _os: "Android", + _os_version: "7", + _error: "Stack trace goes here", + _app_version: "1.0", + _run: 12345, + _custom: {}, + _nonfatal: true, + _javascript: true, + _not_os_specific: true, + }, 1500645200); + } +} + +// Validate created bulk data +function validateCreatedBulkData(bulk) { + var events = bulk._getBulkEventQueue(); + var reqQueue = bulk._getBulkRequestQueue(); + var bulkQueue = bulk._getBulkQueue(); + + assert.equal(Object.keys(events).length, 1); + assert.equal(reqQueue.length, 3); + assert.equal(bulkQueue.length, 0); + + var deviceEvents = events.testUser1; // Access the events for the specific device + var recordedEvent = deviceEvents[0]; // Access the first event + hp.eventValidator(eventObj, recordedEvent); + + var req = reqQueue[0]; // read user details queue + const actualUserDetails = req.user_details; // Extract the user_details from the actual request + const isValid = validateUserDetails(actualUserDetails, userDetailObj); + assert.equal(true, isValid); + + var testUser3Request = reqQueue.find((request) => request.device_id === "TestUser3"); + assert.ok(testUser3Request); + assert.strictEqual(testUser3Request.device_id, "TestUser3"); + assert.strictEqual(testUser3Request.app_key, "YOUR_APP_KEY"); + assert.strictEqual(testUser3Request.sdk_name, "javascript_native_nodejs_bulk"); + + var testUser4Request = reqQueue.find((request) => request.device_id === "TestUser4"); + validateCrash(testUser4Request, true); +} + +function shouldFilesExist(shouldExist, isCustomTest = false) { + hp.doesFileStoragePathsExist((exists) => { + assert.equal(shouldExist, exists); + }, true, isCustomTest); +} + describe("Bulk Tests", () => { - // Bulk is on memory by default - // If no storage path and storage type provided during init Bulk should use memory storage - it("1- Bulk with No Storage Path and Storage Type", (done) => { - hp.clearStorage(); - var bulk = new CountlyBulk({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - }); - assert.equal(storage.getStoragePath(), undefined); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }, true); - done(); + beforeEach(async() => { + await hp.clearStorage(); }); - // Providing storage path and storage type to Bulk - // Storage path should be the provided custom path and default bulk file paths should not exist - it("2- Bulk with Custom Storage Path", (done) => { - hp.clearStorage(); + it("1- CNR", (done) => { var bulk = new CountlyBulk({ app_key: "YOUR_APP_KEY", url: "https://try.count.ly", - storage_path: "../test/customStorageDirectory/", - storage_type: StorageTypes.FILE, }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }, true); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - done(); - }); + assert.equal(storage.getStoragePath(), undefined); + shouldFilesExist(false); + createBulkData(bulk); - // Initialize CountlyBulk and record an event - // Bulk should initialize correctly, default bulk file paths should not exist - // Event should be recorded and validated in event queue - it("3- Bulk add_user with Record Event", (done) => { - hp.clearStorage(); - var bulk = new CountlyBulk({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }, true); - var user = bulk.add_user({ device_id: "testUser1" }); - user.add_event(eventObj); setTimeout(() => { - var events = bulk.getBulkEventQueue(); - var deviceEvents = events.testUser1; // Access the events for the specific device - var recordedEvent = deviceEvents[0]; // Access the first event - hp.eventValidator(eventObj, recordedEvent); + validateCreatedBulkData(bulk); + shouldFilesExist(false); done(); }, hp.mWait); }); - // Initialize CountlyBulk and record user details - // Bulk should initialize correctly, default bulk file paths should not exist - // User details should be recorded and validated in request queue - it("4- Bulk add_user with User Details", (done) => { - hp.clearStorage(); - var bulk = new CountlyBulk({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }, true); - var user = bulk.add_user({ device_id: "testUser2" }); - user.user_details(userDetailObj); - // read event queue - setTimeout(() => { - var reqQueue = bulk.getBulkRequestQueue(); - var req = reqQueue[0]; - // Extract the user_details from the actual request - const actualUserDetails = req.user_details || {}; - // Validate the user details - const isValid = validateUserDetails(actualUserDetails, userDetailObj); - assert.equal(true, isValid); - done(); - }, hp.sWait); - }); - - // Initialize CountlyBulk and call add_request - // Bulk should initialize correctly, default bulk file paths should not exist - // Request should be added correctly and validated - it("5- Bulk add_request", (done) => { - hp.clearStorage(); + it("2- CNR_cPath_file", (done) => { var bulk = new CountlyBulk({ app_key: "YOUR_APP_KEY", url: "https://try.count.ly", + storage_path: "../test/customStorageDirectory/", + storage_type: StorageTypes.FILE, }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }, true); - bulk.add_request({ device_id: "TestUser3" }); - setTimeout(() => { - var reqQueue = bulk.getBulkRequestQueue(); - var testUser3Request = reqQueue.find((req) => req.device_id === "TestUser3"); - assert.ok(testUser3Request); - assert.strictEqual(testUser3Request.device_id, "TestUser3"); - assert.strictEqual(testUser3Request.app_key, "YOUR_APP_KEY"); - assert.strictEqual(testUser3Request.sdk_name, "javascript_native_nodejs_bulk"); - done(); - }, hp.sWait); - }); + shouldFilesExist(true, true); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + createBulkData(bulk); - // Initialize CountlyBulk and report crash - // Bulk should initialize correctly, default bulk file paths should not exist - // Crash should be recorded correctly and validated - it("6- Bulk add_user Report Crash", (done) => { - hp.clearStorage(); - var bulk = new CountlyBulk({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }, true); - var user = bulk.add_user({ device_id: "TestUser4" }); - try { - runthis(); - } - catch (ex) { - user.report_crash({ - _os: "Android", - _os_version: "7", - _error: "Stack trace goes here", - _app_version: "1.0", - _run: 12345, - _custom: {}, - _nonfatal: true, - _javascript: true, - _not_os_specific: true, - }, 1500645200); - } - // read event queue setTimeout(() => { - var reqQueue = bulk.getBulkRequestQueue(); - var testUser4Request = reqQueue.find((req) => req.device_id === "TestUser4"); - validateCrash(testUser4Request, true); + validateCreatedBulkData(bulk); + shouldFilesExist(true, true); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); done(); }, hp.sWait); }); - // Initialize CountlyBulk in File Storage and record an event - // Bulk should initialize correctly, default bulk file paths should exist - // Event should be recorded and validated in event queue - it("7- Bulk File Storage add_user with Record Event", (done) => { - hp.clearStorage(); + it("3- CNR_file", (done) => { var bulk = new CountlyBulk({ app_key: "YOUR_APP_KEY", url: "https://try.count.ly", storage_type: StorageTypes.FILE, }); assert.equal(storage.getStoragePath(), "../bulk_data/"); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(true, exists); - }, true); - var user = bulk.add_user({ device_id: "testUser1" }); - user.add_event(eventObj); + shouldFilesExist(true); + createBulkData(bulk); + setTimeout(() => { - var events = bulk.getBulkEventQueue(); - var deviceEvents = events.testUser1; // Access the events for the specific device - var recordedEvent = deviceEvents[0]; // Access the first event - hp.eventValidator(eventObj, recordedEvent); + validateCreatedBulkData(bulk); + shouldFilesExist(true); + assert.equal(storage.getStoragePath(), "../bulk_data/"); done(); }, hp.mWait); }); -}); \ No newline at end of file +}); + +// Currently tested: CNR, CNR_cPath_file, CNR_file +// TODO: Add tests for the following: +// - CNR: memory, cPath_memory, persistTrue, persistFalse, cPath_persistTrue, cPath_persistFalse, persistTrue_file, persistFalse_file, cPath_persistTrue_file, cPath_persistFalse_file +// - CR_CG for all of the above diff --git a/test/tests_consents.js b/test/tests_consents.js index 057f5d7..a89b4e1 100644 --- a/test/tests_consents.js +++ b/test/tests_consents.js @@ -62,8 +62,10 @@ function events() { // tests describe("Internal event consent tests", () => { + beforeEach(async() => { + await hp.clearStorage(); + }); it("Only custom event should be sent to the queue", (done) => { - hp.clearStorage(); initMain(); Countly.add_consent(["events"]); events(); @@ -75,7 +77,6 @@ describe("Internal event consent tests", () => { }, hp.sWait); }); it("All but custom event should be sent to the queue", (done) => { - hp.clearStorage(); initMain(); Countly.add_consent(["sessions", "views", "users", "star-rating", "apm", "feedback"]); events(); @@ -91,7 +92,6 @@ describe("Internal event consent tests", () => { }, hp.mWait); }); it("Non-merge ID change should reset all consents", (done) => { - hp.clearStorage(); initMain(); Countly.add_consent(["sessions", "views", "users", "star-rating", "apm", "feedback"]); Countly.change_id("Richard Wagner II", false); @@ -102,7 +102,6 @@ describe("Internal event consent tests", () => { }, hp.sWait); }); it("Merge ID change should not reset consents", (done) => { - hp.clearStorage(); initMain(); Countly.add_consent(["sessions", "views", "users", "star-rating", "apm", "feedback"]); // Countly.change_id("Richard Wagner the second", true); diff --git a/test/tests_crashes.js b/test/tests_crashes.js index 2adedb5..ce1255c 100644 --- a/test/tests_crashes.js +++ b/test/tests_crashes.js @@ -14,9 +14,10 @@ function initMain() { } describe("Crash tests", () => { + before(async() => { + await hp.clearStorage(); + }); it("Validate handled error logic", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(); // error logic @@ -34,23 +35,27 @@ describe("Crash tests", () => { done(); }, hp.sWait); }); - // This needs two steps, first creating an error and second checking the logs without erasing, otherwise error would halt the test - describe("Unhandled Error logic", () => { - it("Create unhandled rejection", () => { - // clear previous data - hp.clearStorage(); - // initialize SDK - initMain(); - // send emitter - Countly.track_errors(); - process.emit('unhandledRejection'); - }); - it("Validate unhandled rejection recording", (done) => { - setTimeout(() => { - var req = hp.readRequestQueue()[0]; - hp.crashRequestValidator(req, false); - done(); - }, hp.mWait); - }); +}); +// This needs two steps, first creating an error and second checking the logs without erasing, otherwise error would halt the test +describe("Unhandled Error logic", () => { + before(async() => { + await hp.clearStorage(); + }); + it("Create unhandled rejection", (done) => { + // clear previous data + hp.clearStorage(); + // initialize SDK + initMain(); + // send emitter + Countly.track_errors(); + process.emit('unhandledRejection'); + done(); + }); + it("Validate unhandled rejection recording", (done) => { + setTimeout(() => { + var req = hp.readRequestQueue()[0]; + hp.crashRequestValidator(req, false); + done(); + }, hp.mWait); }); }); diff --git a/test/tests_device_id_type.js b/test/tests_device_id_type.js index afa76f4..650f1b5 100644 --- a/test/tests_device_id_type.js +++ b/test/tests_device_id_type.js @@ -55,9 +55,10 @@ function checkRequestsForT(queue, expectedInternalType) { } describe("Device ID type tests", () => { + beforeEach(async() => { + await hp.clearStorage(); + }); it("1.Generated device ID", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(undefined); Countly.begin_session(); @@ -71,8 +72,6 @@ describe("Device ID type tests", () => { }, hp.sWait); }); it("2.Developer supplied device ID", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain("ID"); Countly.begin_session(); @@ -86,8 +85,6 @@ describe("Device ID type tests", () => { }, hp.sWait); }); it("3.With stored dev ID and no new ID", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain("ID"); Countly.begin_session(); @@ -109,8 +106,6 @@ describe("Device ID type tests", () => { }, hp.sWait); }); it("4.With stored dev ID and with new ID", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain("ID"); Countly.begin_session(); @@ -132,8 +127,6 @@ describe("Device ID type tests", () => { }, hp.sWait); }); it("5.With stored generated ID and no new ID", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(undefined); Countly.begin_session(); @@ -157,8 +150,6 @@ describe("Device ID type tests", () => { }, hp.sWait); }); it("6.With stored generated ID and with new ID", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(undefined); Countly.begin_session(); @@ -182,8 +173,6 @@ describe("Device ID type tests", () => { }, hp.sWait); }); it("7.With stored dev ID and no new ID, flag set", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain("ID"); Countly.begin_session(); @@ -206,31 +195,29 @@ describe("Device ID type tests", () => { }); it("8.With stored dev ID and with new ID, flag set", (done) => { - // clear previous data - hp.clearStorage(); - // initialize SDK - initMain("ID"); - Countly.begin_session(); - // read request queue setTimeout(() => { - var rq = hp.readRequestQueue()[0]; - assert.equal(Countly.get_device_id(), "ID"); - assert.equal(Countly.get_device_id_type(), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - checkRequestsForT(rq, cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - Countly.halt(true); - initMain("ID2", true); + // initialize SDK + initMain("ID"); + Countly.begin_session(); + // read request queue setTimeout(() => { - var req = hp.readRequestQueue()[0]; - assert.equal(Countly.get_device_id(), "ID2"); + var rq = hp.readRequestQueue()[0]; + assert.equal(Countly.get_device_id(), "ID"); assert.equal(Countly.get_device_id_type(), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - checkRequestsForT(req, cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - done(); - }, hp.sWait); - }, hp.sWait); + checkRequestsForT(rq, cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + Countly.halt(true); + initMain("ID2", true); + setTimeout(() => { + var req = hp.readRequestQueue()[0]; + assert.equal(Countly.get_device_id(), "ID2"); + assert.equal(Countly.get_device_id_type(), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + checkRequestsForT(req, cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + done(); + }, hp.sWait); + }, hp.lWait); + }, hp.lWait); }); it("9.With stored sdk ID and no new ID, flag set", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(undefined); Countly.begin_session(); @@ -255,8 +242,6 @@ describe("Device ID type tests", () => { }); it("10.With stored sdk ID and with new ID, flag set", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(undefined); Countly.begin_session(); @@ -280,8 +265,6 @@ describe("Device ID type tests", () => { }, hp.sWait); }); it("11.Change generated device ID", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(undefined); Countly.change_id("changedID"); @@ -298,8 +281,6 @@ describe("Device ID type tests", () => { }, hp.sWait); }); it("12.Change developer supplied device ID", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain("ID"); Countly.change_id("changedID"); diff --git a/test/tests_events.js b/test/tests_events.js index 9e93305..246063d 100644 --- a/test/tests_events.js +++ b/test/tests_events.js @@ -32,9 +32,10 @@ var timedEventObj = { }, }; describe("Events tests", () => { + beforeEach(async() => { + await hp.clearStorage(); + }); it("Record and check custom event", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(); // send custom event @@ -47,8 +48,6 @@ describe("Events tests", () => { }, hp.mWait); }); it("Record and check timed events", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(); // send timed event diff --git a/test/tests_internal_limits.js b/test/tests_internal_limits.js index 712da5d..82f0a10 100644 --- a/test/tests_internal_limits.js +++ b/test/tests_internal_limits.js @@ -22,6 +22,9 @@ function initLimitsMain() { // Integration tests with countly initialized describe("Testing internal limits", () => { + beforeEach(async() => { + await hp.clearStorage(); + }); describe("Testing truncation functions", () => { it("truncateSingleValue: Check if the string is truncated", () => { var newStr = cc.truncateSingleValue("123456789", 3, "test"); @@ -56,8 +59,6 @@ describe("Testing internal limits", () => { }); it("1. Check custom event truncation", (done) => { - // clear storage - hp.clearStorage(); // init Countly initLimitsMain(); // send event @@ -88,8 +89,6 @@ describe("Testing internal limits", () => { }); it("2. Check countly view event truncation", (done) => { - // clear storage - hp.clearStorage(); // init Countly initLimitsMain(); // page view @@ -110,8 +109,6 @@ describe("Testing internal limits", () => { }, hp.sWait); }); it("3. Check breadcrumbs and error truncation", (done) => { - // clear storage - hp.clearStorage(); // init Countly initLimitsMain(); // add log @@ -154,8 +151,6 @@ describe("Testing internal limits", () => { }, hp.sWait); }); it("4. Check user details truncation", (done) => { - // clear storage - hp.clearStorage(); // init Countly initLimitsMain(); // add user details @@ -209,8 +204,6 @@ describe("Testing internal limits", () => { }, hp.sWait); }); it("5. Check custom properties truncation", (done) => { - // clear storage - hp.clearStorage(); // init Countly initLimitsMain(); // add custom properties diff --git a/test/tests_sessions.js b/test/tests_sessions.js index 2893cc7..ac17f3b 100644 --- a/test/tests_sessions.js +++ b/test/tests_sessions.js @@ -11,9 +11,10 @@ function initMain() { }); } describe("Sessions tests", () => { + beforeEach(async() => { + await hp.clearStorage(); + }); it("Start session and validate the request queue", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(); // send session calls @@ -25,8 +26,6 @@ describe("Sessions tests", () => { }, hp.sWait); }); it("Start and end session and validate the request queue", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(); // send session calls diff --git a/test/tests_storage.js b/test/tests_storage.js index da677be..06b0fda 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -155,30 +155,11 @@ const customFileStorage = { const nonValidStorageMethods = { _storage: {}, - setInvalid: function(key, value, callback) { - if (key) { - const existingValue = this._storage[key]; - if (typeof value === 'string' && typeof existingValue === 'string') { - this._storage[key] = existingValue + value; - } - else { - this._storage[key] = value; - } - if (typeof callback === "function") { - callback(null); - } - } + setInvalid: function() { }, - getInvalid: function(key, def) { - const value = this._storage[key]; - if (typeof value === 'string') { - return value.split('').reverse().join(''); - } - - return value !== undefined ? value : def; + getInvalid: function() { }, - removeInvalid: function(key) { - delete this._storage[key]; + removeInvalid: function() { }, }; const funkyMemoryStorage = { @@ -228,76 +209,26 @@ const customMemoryStorage = { }, }; -describe("Storage Tests", () => { - // initialize the sdk begin a session - // sdk should generate a device id and it should be stored correctly - it("1- Store Generated Device ID", (done) => { - // clear previous data - hp.clearStorage(); - // initialize SDK - initMain(); - Countly.begin_session(); - // read request queue - setTimeout(() => { - validateSdkGeneratedId(Countly.get_device_id()); - done(); - }, hp.sWait); - }); - - // when initialized again sdk should keep the device id and id type correctly - it("1.1- Validate generated device id after process restart", (done) => { - initMain(); - validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), undefined, cc.deviceIdTypeEnums.SDK_GENERATED); - done(); - }); - - // initialize the sdk while providing device id and begin a session - // sdk should use the given device id and it should be stored correctly - it("2.Developer supplied device ID", (done) => { - hp.clearStorage(); - initMain("ID"); - Countly.begin_session(); - setTimeout(() => { - validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - done(); - }, hp.sWait); - }); - - // when initialized again sdk should keep the device id and id type correctly - it("2.1- Validate generated device id after process restart", (done) => { - validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - done(); - }); +// TODO: +// Other tests are testing default (FILE) already, so here focus should be: +// 1. MEMORY (checking it works with a set of events/situations, it does not saves something persistently) +// 2. Persistency (checking default saves persistently, checking default and FILE are same) +// 3. Custom (custom works with valid methods-memory or persistent-, does not work with invalid methods) +// 4. Their interaction with path +// here we test storage, not every - // initialize the sdk and record user details - // all user details informations should be recorded, stored and validated correctly - it("3- Record and validate all user details", (done) => { - hp.clearStorage(); - initMain(); - Countly.user_details(userDetailObj); - setTimeout(() => { - var req = hp.readRequestQueue()[0]; - hp.userDetailRequestValidator(userDetailObj, req); - done(); - }, hp.sWait); - }); - - // validate that user detail stored in the queue correctly - it("3.1- Validate stored user detail", (done) => { - var req = hp.readRequestQueue()[0]; - hp.userDetailRequestValidator(userDetailObj, req); - done(); +describe("Storage Tests", () => { + beforeEach(async() => { + await hp.clearStorage(); }); - // initialize the sdk and record an event // event should be recorded stored and validated correctly it("4- Record event and validate storage", (done) => { - hp.clearStorage(); initMain(); Countly.add_event(eventObj); setTimeout(() => { var storedEvents = hp.readEventQueue(); - assert.strictEqual(storedEvents.length, 1, "There should be exactly one event stored"); + assert.equal(storedEvents.length, 1); var event = storedEvents[0]; hp.eventValidator(eventObj, event); @@ -305,24 +236,9 @@ describe("Storage Tests", () => { }, hp.mWait); }); - // validate that when initialized again the event is stored correctly - it("4.1- Validate event persistence after process restart", (done) => { - // Initialize SDK - initMain(); - - // Read stored events without clearing storage - var storedEvents = hp.readEventQueue(); - assert.strictEqual(storedEvents.length, 1, "There should be exactly one event stored"); - - var event = storedEvents[0]; - hp.eventValidator(eventObj, event); - done(); - }); - // initialize the sdk with providing no storage path or type // if storage path is not provided it will be default "../data/" it("5- Not provide storage path during init", (done) => { - hp.clearStorage(); initMain(); assert.equal(storage.getStoragePath(), "../data/"); done(); @@ -331,7 +247,6 @@ describe("Storage Tests", () => { // initialize the sdk with providing undefined storage path // if set to undefined it should be set to default path it("6- Set storage path to undefined", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -344,7 +259,6 @@ describe("Storage Tests", () => { // initialize the sdk with providing null storage path // if set to null it should be set to default path it("7- Set storage path to null", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -357,7 +271,6 @@ describe("Storage Tests", () => { // initialize the sdk with providing a custom storage path // it should be set to the custom directory if provided a valid storage path it("8- Set storage path to custom directory", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -372,7 +285,6 @@ describe("Storage Tests", () => { // resets the storage path to default and validates that it is set correctly, // then resets it to undefined and confirms the reset. it("9- Reset Storage While on Default Path /no-init", (done) => { - hp.clearStorage(); // will set to default storage path storage.setStoragePath(); assert.equal(storage.getStoragePath(), "../data/"); @@ -385,7 +297,6 @@ describe("Storage Tests", () => { // sets the storage path to default and verifies it, // then records values to storage and ensures they are stored correctly. it("10- Recording to Storage with Default Storage Path /no-init", (done) => { - hp.clearStorage(); storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); recordValuesToStorageAndValidate(); @@ -395,7 +306,6 @@ describe("Storage Tests", () => { // sets a custom storage path and verifies it, // then records values to storage and ensures correct storage in the custom path. it("11- Recording to Storage with Custom Storage Path /no-init", (done) => { - hp.clearStorage(); storage.initStorage("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); recordValuesToStorageAndValidate("../test/customStorageDirectory/"); @@ -405,7 +315,6 @@ describe("Storage Tests", () => { // sets the storage path to the default bulk storage path and verifies it, // then records values to bulk storage and validates proper storage in bulk mode. it("12- Recording to Bulk Storage with Default Bulk Data Path /no-init", (done) => { - hp.clearStorage(); storage.initStorage(null, StorageTypes.FILE, true); hp.doesFileStoragePathsExist((exists) => { assert.equal(true, exists); @@ -418,7 +327,6 @@ describe("Storage Tests", () => { // sets a custom bulk storage path and verifies it, // then records values to bulk storage and ensures proper recording to the custom path. it("13- Recording to Bulk Storage with Custom Bulk Storage Path /no-init", (done) => { - hp.clearStorage(); // will set to default storage path storage.initStorage("../test/customStorageDirectory/", StorageTypes.FILE, true); hp.doesFileStoragePathsExist((exists) => { @@ -433,7 +341,6 @@ describe("Storage Tests", () => { // without sdk initializing storage should be able to initialize // if no path or type is provided storage should be set to default it("14- Setting storage path to default path via initStorage /no-init", (done) => { - hp.clearStorage(); storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); done(); @@ -443,7 +350,6 @@ describe("Storage Tests", () => { // without sdk initializing storage should be able to initialize // if no path is provided storage should be set to default bulk path it("15- Setting bulk storage path to default bulk file path via initStorage /no-init", (done) => { - hp.clearStorage(); storage.initStorage(null, StorageTypes.FILE, true); assert.equal(storage.getStoragePath(), "../bulk_data/"); done(); @@ -453,7 +359,6 @@ describe("Storage Tests", () => { // without sdk initializing storage should be able to initialize // if a valid path is provided storage should be set to that path it("16- Setting custom storage path via initStorage /no-init", (done) => { - hp.clearStorage(); storage.initStorage("../test/customStorageDirectory/"); assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); done(); @@ -463,7 +368,6 @@ describe("Storage Tests", () => { // without sdk initializing storage should be able to initialize in memory mode // since initialized in memory mode sdk should have no storage path or storage files should not exist it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { - hp.clearStorage(); storage.initStorage(null, StorageTypes.MEMORY); hp.doesFileStoragePathsExist((exists) => { assert.equal(false, exists); @@ -476,7 +380,6 @@ describe("Storage Tests", () => { // initializes the SDK in memory only mode, validates that file storage files does not exist // retrieve the developer supplied device id and id type from storage it("18- Memory only storage Device-Id", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -496,7 +399,6 @@ describe("Storage Tests", () => { // initializes the SDK in memory only mode, validates that file storage files does not exist // records an event and validates the recorded event it("19- Record event in memory only mode and validate the record", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -520,7 +422,6 @@ describe("Storage Tests", () => { // initializes the SDK in memory only mode, validates that file storage files does not exist // records user details and validates the recorded details it("20- Record and validate user details in memory only mode", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -541,7 +442,6 @@ describe("Storage Tests", () => { // initialize the SDK in memory only mode, check the device id and switch it // SDK and storage should function properly it("21- Memory only storage, change SDK Generated Device-Id", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -564,7 +464,6 @@ describe("Storage Tests", () => { // passing memory storage type during init and initializing storage afterwards // SDK should switch to file storage it("22- Switch to file storage after init", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -586,7 +485,6 @@ describe("Storage Tests", () => { // after initializing the memory storage, without initializing SDK, attempts to set, get and remove values // without initializing SDK storage should function properly it("23- storeRemove Memory Only /no-init", (done) => { - hp.clearStorage(); storage.initStorage(null, StorageTypes.MEMORY); assert.equal(storage.getStoragePath(), undefined); assert.equal(storage.getStorageType(), StorageTypes.MEMORY); @@ -602,7 +500,6 @@ describe("Storage Tests", () => { // after initializing the file storage, without initializing SDK attempts to set, get and remove values // without initializing SDK storage should function properly it("24- storeRemove File Storage /no-init", (done) => { - hp.clearStorage(); storage.initStorage(); assert.equal(storage.getStoragePath(), "../data/"); assert.equal(storage.getStorageType(), StorageTypes.FILE); @@ -618,7 +515,6 @@ describe("Storage Tests", () => { // choosing Custom storage type and passing null in storage methods // passing null as storage method ends up with switching to default file storage it("25- Null Custom Storage Method", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -634,7 +530,6 @@ describe("Storage Tests", () => { // choosing Custom storage type and passing custom storage methods // SDK should use custom methods as storage method, no File Storage should exist it("26- Providing Custom Storage Method", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -651,7 +546,6 @@ describe("Storage Tests", () => { // Recording values in Custom Storage Methods // SDK should use custom methods as storage methods and values should be recorded correctly it("27- Record/Remove Values in Custom Storage Method", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -672,7 +566,6 @@ describe("Storage Tests", () => { // passes a funky storage method, which does store get as reversing string // SDK should use custom methods as storage method it("28- Record/Remove Values in Other Custom Storage Method", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -692,7 +585,6 @@ describe("Storage Tests", () => { // choosing Custom storage type and passing invalid custom storage methods // SDK should not use custom methods as storage method, and switch to File Storage it("29- Providing Invalid Custom Storage Method", (done) => { - hp.clearStorage(); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -703,12 +595,10 @@ describe("Storage Tests", () => { assert.equal(storage.getStorageType(), StorageTypes.FILE); done(); }); - // tests init time storage config options // choosing Custom storage type and passing custom file storage methods // SDK should use custom methods as storage methods it("30- Providing File Custom Storage Method", (done) => { - hp.clearStorage("../test/customStorageDirectory/"); Countly.init({ app_key: "YOUR_APP_KEY", url: "https://test.url.ly", @@ -729,4 +619,37 @@ describe("Storage Tests", () => { done(); }, hp.sWait); }); +}); + +describe('Persistency Related', () => { + // when initialized again sdk should keep the device id and id type correctly + it("1.1- Validate generated device id after process restart", (done) => { + initMain(); + setTimeout(() => { + validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), undefined, cc.deviceIdTypeEnums.SDK_GENERATED); + done(); + }, hp.sWait); + }); + // when initialized again sdk should keep the device id and id type correctly + it("2.1- Validate generated device id after process restart", (done) => { + validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); + done(); + }); + // validate that user detail stored in the queue correctly + it("3.1- Validate stored user detail", (done) => { + var req = hp.readRequestQueue()[0]; + hp.userDetailRequestValidator(userDetailObj, req); + done(); + }); + // validate that when initialized again the event is stored correctly + it("4.1- Validate event persistence after process restart", (done) => { + // Initialize SDK + initMain(); + // Read stored events without clearing storage + var storedEvents = hp.readEventQueue(); + assert.strictEqual(storedEvents.length, 1, "There should be exactly one event stored"); + var event = storedEvents[0]; + hp.eventValidator(eventObj, event); + done(); + }); }); \ No newline at end of file diff --git a/test/tests_user_details.js b/test/tests_user_details.js index 9e859ec..dfb06cd 100644 --- a/test/tests_user_details.js +++ b/test/tests_user_details.js @@ -27,9 +27,10 @@ function initMain() { } describe("User details tests", () => { + beforeEach(async() => { + await hp.clearStorage(); + }); it("Record and validate all user details", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(); // send user details diff --git a/test/tests_views.js b/test/tests_views.js index 664f3ca..07e6926 100644 --- a/test/tests_views.js +++ b/test/tests_views.js @@ -15,9 +15,10 @@ function initMain() { } describe("View test", () => { + beforeEach(async() => { + await hp.clearStorage(); + }); it("Record and validate page views", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(); // send track view @@ -30,8 +31,6 @@ describe("View test", () => { }, hp.sWait); }); it("Record and validate timed page views with same name", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(); Countly.track_view(pageNameOne); @@ -52,8 +51,6 @@ describe("View test", () => { }, hp.mWait); }); it("Record and validate timed page views with same name", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(); Countly.track_view(pageNameOne); From 61554c07108ec56124b29c0d7f0b12bb2d989a20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 14 Oct 2024 14:50:06 +0300 Subject: [PATCH 49/58] Refactor Tests --- test/tests_crashes.js | 2 -- test/tests_storage.js | 64 +------------------------------------------ 2 files changed, 1 insertion(+), 65 deletions(-) diff --git a/test/tests_crashes.js b/test/tests_crashes.js index ce1255c..12f2688 100644 --- a/test/tests_crashes.js +++ b/test/tests_crashes.js @@ -42,8 +42,6 @@ describe("Unhandled Error logic", () => { await hp.clearStorage(); }); it("Create unhandled rejection", (done) => { - // clear previous data - hp.clearStorage(); // initialize SDK initMain(); // send emitter diff --git a/test/tests_storage.js b/test/tests_storage.js index 06b0fda..7c103d9 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -47,29 +47,7 @@ function initMain(device_id) { device_id: device_id, }); } -// TODO: move these to helpers to reduce duplication -function validateSdkGeneratedId(providedDeviceId) { - assert.ok(providedDeviceId); - assert.equal(providedDeviceId.length, 36); - assert.ok(cc.isUUID(providedDeviceId)); -} -function checkRequestsForT(queue, expectedInternalType) { - for (var i = 0; i < queue.length; i++) { - assert.ok(queue[i].t); - assert.equal(queue[i].t, expectedInternalType); - } -} -function validateDeviceId(deviceId, deviceIdType, expectedDeviceId, expectedDeviceIdType) { - var rq = hp.readRequestQueue()[0]; - if (expectedDeviceIdType === cc.deviceIdTypeEnums.SDK_GENERATED) { - validateSdkGeneratedId(deviceId); // for SDK-generated IDs - } - else { - assert.equal(deviceId, expectedDeviceId); // for developer-supplied IDs - } - assert.equal(deviceIdType, expectedDeviceIdType); - checkRequestsForT(rq, expectedDeviceIdType); -} + function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { // Set values var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; @@ -97,13 +75,6 @@ function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = // Reset storage and check if it's empty again storage.resetStorage(); - /* - assert.equal(storage.storeGet("cly_id"), undefined); - assert.equal(storage.storeGet("cly_id_type"), undefined); - assert.equal(storage.storeGet("cly_count"), undefined); - assert.equal(storage.storeGet("cly_object"), undefined); - assert.equal(storage.storeGet("cly_null"), undefined); - */ } var __data = {}; // technically same as defualt file storage method @@ -619,37 +590,4 @@ describe("Storage Tests", () => { done(); }, hp.sWait); }); -}); - -describe('Persistency Related', () => { - // when initialized again sdk should keep the device id and id type correctly - it("1.1- Validate generated device id after process restart", (done) => { - initMain(); - setTimeout(() => { - validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), undefined, cc.deviceIdTypeEnums.SDK_GENERATED); - done(); - }, hp.sWait); - }); - // when initialized again sdk should keep the device id and id type correctly - it("2.1- Validate generated device id after process restart", (done) => { - validateDeviceId(Countly.get_device_id(), Countly.get_device_id_type(), "ID", cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - done(); - }); - // validate that user detail stored in the queue correctly - it("3.1- Validate stored user detail", (done) => { - var req = hp.readRequestQueue()[0]; - hp.userDetailRequestValidator(userDetailObj, req); - done(); - }); - // validate that when initialized again the event is stored correctly - it("4.1- Validate event persistence after process restart", (done) => { - // Initialize SDK - initMain(); - // Read stored events without clearing storage - var storedEvents = hp.readEventQueue(); - assert.strictEqual(storedEvents.length, 1, "There should be exactly one event stored"); - var event = storedEvents[0]; - hp.eventValidator(eventObj, event); - done(); - }); }); \ No newline at end of file From f8d3cbbdf2cb905497d180aed4933b7fba59dc07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 14 Oct 2024 16:12:35 +0300 Subject: [PATCH 50/58] Updated Enum Export --- lib/countly-bulk.js | 5 ++--- lib/countly.js | 2 +- test/tests_bulk.js | 4 +--- test/tests_storage.js | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/countly-bulk.js b/lib/countly-bulk.js index 035ad32..6bb525a 100644 --- a/lib/countly-bulk.js +++ b/lib/countly-bulk.js @@ -25,7 +25,7 @@ var cc = require("./countly-common"); var BulkUser = require("./countly-bulk-user"); var CountlyStorage = require("./countly-storage"); -const StorageTypes = cc.storageTypeEnums; +CountlyBulk.StorageTypes = cc.storageTypeEnums; /** * @lends module:lib/countly-bulk @@ -76,7 +76,6 @@ function CountlyBulk(conf) { var maxBreadcrumbCount = 100; var maxStackTraceLinesPerThread = 30; var maxStackTraceLineLength = 200; - cc.debugBulk = conf.debug || false; if (!conf.app_key) { cc.log(cc.logLevelEnums.ERROR, "CountlyBulk, 'app_key' is missing."); @@ -108,7 +107,7 @@ function CountlyBulk(conf) { // bulk mode is memory only by default if (typeof conf.storage_type === "undefined" && conf.persist_queue === false) { - conf.storage_type = StorageTypes.MEMORY; + conf.storage_type = CountlyBulk.StorageTypes.MEMORY; } CountlyStorage.initStorage(conf.storage_path, conf.storage_type, true, conf.custom_storage_method); diff --git a/lib/countly.js b/lib/countly.js index 77f935d..4984f7a 100644 --- a/lib/countly.js +++ b/lib/countly.js @@ -29,7 +29,7 @@ var Bulk = require("./countly-bulk"); var CountlyStorage = require("./countly-storage"); var Countly = {}; - +Countly.StorageTypes = cc.storageTypeEnums; Countly.Bulk = Bulk; (function() { var SDK_VERSION = "22.06.0"; diff --git a/test/tests_bulk.js b/test/tests_bulk.js index 2d9247a..a6b23df 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -1,13 +1,11 @@ /* eslint-disable no-console */ /* global runthis */ -var path = require("path"); const assert = require("assert"); const CountlyBulk = require("../lib/countly-bulk"); var hp = require("./helpers/helper_functions"); var storage = require("../lib/countly-storage"); -var cc = require("../lib/countly-common"); -const StorageTypes = cc.storageTypeEnums; +const { StorageTypes } = CountlyBulk; function validateCrash(validator, nonfatal) { assert.ok(validator.crash._os); diff --git a/test/tests_storage.js b/test/tests_storage.js index 7c103d9..6a8350f 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -8,7 +8,7 @@ var storage = require("../lib/countly-storage"); var cc = require("../lib/countly-common"); var hp = require("./helpers/helper_functions"); -const StorageTypes = cc.storageTypeEnums; +const { StorageTypes } = Countly; // example event object to use var eventObj = { From ade4be66f1bde0a81a6be8a97a5d6933709ec17f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 14 Oct 2024 17:09:10 +0300 Subject: [PATCH 51/58] Refactoring Test Objects --- test/helpers/helper_functions.js | 2 +- test/helpers/test_utils.js | 62 ++++++++++++++++++++++++++++++++ test/tests_bulk.js | 45 +++-------------------- test/tests_events.js | 30 ++++------------ test/tests_storage.js | 40 ++++----------------- test/tests_user_details.js | 19 ++-------- 6 files changed, 84 insertions(+), 114 deletions(-) create mode 100644 test/helpers/test_utils.js diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index c441fcc..0cc69f9 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -221,7 +221,7 @@ function userDetailRequestValidator(originalDetails, details) { assert.equal(originalDetails.byear, user.byear); if (typeof originalDetails.custom !== 'undefined') { for (var key in originalDetails.custom) { - assert.equal(originalDetails.custom[key], user.custom[key]); + assert.deepStrictEqual(originalDetails.custom[key], user.custom[key]); } } } diff --git a/test/helpers/test_utils.js b/test/helpers/test_utils.js new file mode 100644 index 0000000..d5836e8 --- /dev/null +++ b/test/helpers/test_utils.js @@ -0,0 +1,62 @@ +var eventObj = { + key: "event_check", + count: 55, + sum: 3.14, + dur: 2000, + segmentation: { + string_value: "example", + number_value: 42, + boolean_value: true, + array_value: ["item1", "item2"], + object_value: { nested_key: "nested_value" }, + null_value: null, + undefined_value: undefined, + }, +}; + +var timedEventObj = { + key: "timed", + count: 1, + segmentation: { + app_version: "1.0", + country: "Turkey", + }, +}; + +var userDetailObj = { + name: "Alexandrina Jovovich", + username: "alex_jov", + email: "alex.jov@example.com", + organization: "TechNova", + phone: "+987654321", + picture: "https://example.com/images/profile_alex.jpg", + gender: "Female", + byear: 1992, // birth year + custom: { + string_value: "example", + number_value: 42, + boolean_value: true, + array_value: ["item1", "item2"], + object_value: { nested_key: "nested_value" }, + null_value: null, + undefined_value: undefined, + }, +}; + +var getUserDetailsObj = function() { + return userDetailObj; +}; + +var getEventObj = function() { + return eventObj; +}; + +var getTimedEventObj = function() { + return timedEventObj; +}; + +module.exports = { + getEventObj, + getUserDetailsObj, + getTimedEventObj, +}; \ No newline at end of file diff --git a/test/tests_bulk.js b/test/tests_bulk.js index a6b23df..26743d4 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -4,6 +4,7 @@ const assert = require("assert"); const CountlyBulk = require("../lib/countly-bulk"); var hp = require("./helpers/helper_functions"); var storage = require("../lib/countly-storage"); +var testUtils = require("./helpers/test_utils"); const { StorageTypes } = CountlyBulk; @@ -71,51 +72,15 @@ function validateUserDetails(actual, expected) { return isValid; } -var eventObj = { - key: "bulk_check", - count: 55, - sum: 3.14, - dur: 2000, - segmentation: { - string_value: "example", - number_value: 42, - boolean_value: true, - array_value: ["item1", "item2"], - object_value: { nested_key: "nested_value" }, - null_value: null, - undefined_value: undefined, - }, -}; - -var userDetailObj = { - name: "Alexandrina Jovovich", - username: "alex_jov", - email: "alex.jov@example.com", - organization: "TechNova", - phone: "+987654321", - picture: "https://example.com/images/profile_alex.jpg", - gender: "Female", - byear: 1992, // birth year - custom: { - string_value: "example", - number_value: 42, - boolean_value: true, - array_value: ["item1", "item2"], - object_value: { nested_key: "nested_value" }, - null_value: null, - undefined_value: undefined, - }, -}; - // Create bulk data function createBulkData(bulk) { // Add an event var user = bulk.add_user({ device_id: "testUser1" }); - user.add_event(eventObj); + user.add_event(testUtils.getEventObj()); // add user details var user2 = bulk.add_user({ device_id: "testUser2" }); - user2.user_details(userDetailObj); + user2.user_details(testUtils.getUserDetailsObj()); // add request bulk.add_request({ device_id: "TestUser3" }); @@ -152,11 +117,11 @@ function validateCreatedBulkData(bulk) { var deviceEvents = events.testUser1; // Access the events for the specific device var recordedEvent = deviceEvents[0]; // Access the first event - hp.eventValidator(eventObj, recordedEvent); + hp.eventValidator(testUtils.getEventObj(), recordedEvent); var req = reqQueue[0]; // read user details queue const actualUserDetails = req.user_details; // Extract the user_details from the actual request - const isValid = validateUserDetails(actualUserDetails, userDetailObj); + const isValid = validateUserDetails(actualUserDetails, testUtils.getUserDetailsObj()); assert.equal(true, isValid); var testUser3Request = reqQueue.find((request) => request.device_id === "TestUser3"); diff --git a/test/tests_events.js b/test/tests_events.js index 246063d..127c4d5 100644 --- a/test/tests_events.js +++ b/test/tests_events.js @@ -1,6 +1,7 @@ /* eslint-disable no-console */ var Countly = require("../lib/countly"); var hp = require("./helpers/helper_functions"); +var testUtils = require("./helpers/test_utils"); // init function function initMain() { @@ -11,26 +12,7 @@ function initMain() { max_events: -1, }); } -// an event object to use -var eventObj = { - key: "in_app_purchase", - count: 3, - sum: 2.97, - dur: 1000, - segmentation: { - app_version: "1.0", - country: "Turkey", - }, -}; -// a timed event object -var timedEventObj = { - key: "timed", - count: 1, - segmentation: { - app_version: "1.0", - country: "Turkey", - }, -}; + describe("Events tests", () => { beforeEach(async() => { await hp.clearStorage(); @@ -39,11 +21,11 @@ describe("Events tests", () => { // initialize SDK initMain(); // send custom event - Countly.add_event(eventObj); + Countly.add_event(testUtils.getEventObj()); // read event queue setTimeout(() => { var event = hp.readEventQueue()[0]; - hp.eventValidator(eventObj, event); + hp.eventValidator(testUtils.getEventObj(), event); done(); }, hp.mWait); }); @@ -53,11 +35,11 @@ describe("Events tests", () => { // send timed event Countly.start_event("timed"); setTimeout(() => { - Countly.end_event(timedEventObj); + Countly.end_event(testUtils.getTimedEventObj()); // read event queue setTimeout(() => { var event = hp.readEventQueue()[0]; - hp.eventValidator(timedEventObj, event, (hp.mWait / 1000)); + hp.eventValidator(testUtils.getTimedEventObj(), event, (hp.mWait / 1000)); done(); }, hp.sWait); }, hp.mWait); diff --git a/test/tests_storage.js b/test/tests_storage.js index 6a8350f..d2da2c0 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -7,36 +7,10 @@ var Countly = require("../lib/countly"); var storage = require("../lib/countly-storage"); var cc = require("../lib/countly-common"); var hp = require("./helpers/helper_functions"); +var testUtils = require("./helpers/test_utils"); const { StorageTypes } = Countly; -// example event object to use -var eventObj = { - key: "storage_check", - count: 5, - sum: 3.14, - dur: 2000, - segmentation: { - app_version: "1.0", - country: "Zambia", - }, -}; - -var userDetailObj = { - name: "Akira Kurosawa", - username: "a_kurosawa", - email: "akira.kurosawa@filmlegacy.com", - organization: "Toho Studios", - phone: "+81312345678", - picture: "https://example.com/profile_images/akira_kurosawa.jpg", - gender: "Male", - byear: 1910, - custom: { - "known for": "Film Director", - "notable works": "Seven Samurai, Rashomon, Ran", - }, -}; - // init function function initMain(device_id) { Countly.init({ @@ -196,13 +170,13 @@ describe("Storage Tests", () => { // event should be recorded stored and validated correctly it("4- Record event and validate storage", (done) => { initMain(); - Countly.add_event(eventObj); + Countly.add_event(testUtils.getEventObj()); setTimeout(() => { var storedEvents = hp.readEventQueue(); assert.equal(storedEvents.length, 1); var event = storedEvents[0]; - hp.eventValidator(eventObj, event); + hp.eventValidator(testUtils.getEventObj(), event); done(); }, hp.mWait); }); @@ -379,12 +353,12 @@ describe("Storage Tests", () => { hp.doesFileStoragePathsExist((exists) => { assert.equal(false, exists); }); - Countly.add_event(eventObj); + Countly.add_event(testUtils.getEventObj()); setTimeout(() => { const storedData = storage.storeGet("cly_queue", null); const eventArray = JSON.parse(storedData[0].events); const eventFromQueue = eventArray[0]; - hp.eventValidator(eventObj, eventFromQueue); + hp.eventValidator(testUtils.getEventObj(), eventFromQueue); done(); }, hp.mWait); }); @@ -402,10 +376,10 @@ describe("Storage Tests", () => { hp.doesFileStoragePathsExist((exists) => { assert.equal(false, exists); }); - Countly.user_details(userDetailObj); + Countly.user_details(testUtils.getUserDetailsObj); const storedData = storage.storeGet("cly_queue", null); const userDetailsReq = storedData[0]; - hp.userDetailRequestValidator(userDetailObj, userDetailsReq); + hp.userDetailRequestValidator(testUtils.getUserDetailsObj, userDetailsReq); done(); }); diff --git a/test/tests_user_details.js b/test/tests_user_details.js index dfb06cd..e7a3171 100644 --- a/test/tests_user_details.js +++ b/test/tests_user_details.js @@ -1,21 +1,8 @@ /* eslint-disable no-console */ var Countly = require("../lib/countly"); var hp = require("./helpers/helper_functions"); +var testUtils = require("./helpers/test_utils"); -var userDetailObj = { - name: "Barturiana Sosinsiava", - username: "bar2rawwen", - email: "test@test.com", - organization: "Dukely", - phone: "+123456789", - picture: "https://ps.timg.com/profile_images/52237/011_n_400x400.jpg", - gender: "Non-binary", - byear: 1987, // birth year - custom: { - "key1 segment": "value1 segment", - "key2 segment": "value2 segment", - }, -}; // init function function initMain() { Countly.init({ @@ -34,11 +21,11 @@ describe("User details tests", () => { // initialize SDK initMain(); // send user details - Countly.user_details(userDetailObj); + Countly.user_details(testUtils.getUserDetailsObj()); // read event queue setTimeout(() => { var req = hp.readRequestQueue()[0]; - hp.userDetailRequestValidator(userDetailObj, req); + hp.userDetailRequestValidator(testUtils.getUserDetailsObj(), req); done(); }, hp.sWait); }); From ac5c8f194f12e54989bec7235ec01297d3e77008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 14 Oct 2024 17:58:29 +0300 Subject: [PATCH 52/58] Refactoring Some More --- test/helpers/helper_functions.js | 55 +++++++++++++------- test/helpers/test_utils.js | 4 +- test/tests_bulk.js | 89 +++++++++++--------------------- test/tests_storage.js | 2 +- test/tests_user_details.js | 25 ++++----- 5 files changed, 76 insertions(+), 99 deletions(-) diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index 0cc69f9..2727cd2 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -1,4 +1,6 @@ /* eslint-disable no-unused-vars */ +/* eslint-disable no-console */ +/* global runthis */ var path = require("path"); var assert = require("assert"); var fs = require("fs"); @@ -203,28 +205,41 @@ function sessionRequestValidator(beginSs, endSs, time, id) { assert.equal(time, endSs.session_duration); } } -/** - * bunch of tests specifically gathered for testing user details - * @param {Object} originalDetails - Original object that contains user details - * @param {Object} details - Object from cly_queue that corresponds to user details recording - */ -function userDetailRequestValidator(originalDetails, details) { - requestBaseParamValidator(details); - var user = JSON.parse(details.user_details); - assert.equal(originalDetails.name, user.name); - assert.equal(originalDetails.username, user.username); - assert.equal(originalDetails.email, user.email); - assert.equal(originalDetails.organization, user.organization); - assert.equal(originalDetails.phone, user.phone); - assert.equal(originalDetails.picture, user.picture); - assert.equal(originalDetails.gender, user.gender); - assert.equal(originalDetails.byear, user.byear); - if (typeof originalDetails.custom !== 'undefined') { - for (var key in originalDetails.custom) { - assert.deepStrictEqual(originalDetails.custom[key], user.custom[key]); + +function validateUserDetails(actual, expected) { + // Helper function to remove undefined values + const cleanObj = (obj) => { + if (typeof obj === "string") { + try { + // Parse if it's a JSON string + obj = JSON.parse(obj); + } + catch (e) { + console.error("Invalid JSON string:", obj); + // Return null for invalid JSON + return null; + } } + // Remove properties with undefined values + return Object.fromEntries(Object.entries(obj).filter(([_, value]) => value !== undefined)); + }; + const cleanedActual = cleanObj(actual); + const cleanedExpected = cleanObj(expected); + if (!cleanedActual || !cleanedExpected) { + // If either cleaned object is null, validation fails + return false; + } + // Perform deep strict comparison after cleaning up undefined values + try { + assert.deepStrictEqual(cleanedActual, cleanedExpected); + return true; + } + catch (e) { + console.log("Validation failed:", e); + return false; } } + /** * bunch of tests specifically gathered for testing page views * @param {Object} name - page name @@ -257,7 +272,7 @@ module.exports = { eventValidator, crashRequestValidator, sessionRequestValidator, - userDetailRequestValidator, + validateUserDetails, viewEventValidator, doesFileStoragePathsExist, }; \ No newline at end of file diff --git a/test/helpers/test_utils.js b/test/helpers/test_utils.js index d5836e8..08af01e 100644 --- a/test/helpers/test_utils.js +++ b/test/helpers/test_utils.js @@ -31,15 +31,13 @@ var userDetailObj = { phone: "+987654321", picture: "https://example.com/images/profile_alex.jpg", gender: "Female", - byear: 1992, // birth year + byear: 1992, custom: { string_value: "example", number_value: 42, boolean_value: true, array_value: ["item1", "item2"], object_value: { nested_key: "nested_value" }, - null_value: null, - undefined_value: undefined, }, }; diff --git a/test/tests_bulk.js b/test/tests_bulk.js index 26743d4..c42b43a 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -8,6 +8,9 @@ var testUtils = require("./helpers/test_utils"); const { StorageTypes } = CountlyBulk; +var appKey = "YOUR_APP_KEY"; +var serverUrl = "https://tests.url.cly"; + function validateCrash(validator, nonfatal) { assert.ok(validator.crash._os); assert.ok(validator.crash._os_version); @@ -20,58 +23,6 @@ function validateCrash(validator, nonfatal) { assert.equal(true, validator.crash._not_os_specific); } -// note: this can replace the current one in the helper functions -function validateUserDetails(actual, expected) { - const keys = ['name', 'username', 'email', 'organization', 'phone', 'picture', 'gender', 'byear', 'custom']; - let isValid = true; - - keys.forEach((key) => { - if (typeof actual[key] === 'object' && actual[key] !== null) { - if (Array.isArray(actual[key])) { - if (!Array.isArray(expected[key]) || JSON.stringify(actual[key]) !== JSON.stringify(expected[key])) { - console.error(`Mismatch for key "${key}": expected "${JSON.stringify(expected[key])}", but got "${JSON.stringify(actual[key])}"`); - isValid = false; - } - } - else { - if (JSON.stringify(actual[key]) !== JSON.stringify(expected[key])) { - console.error(`Mismatch for key "${key}": expected "${JSON.stringify(expected[key])}", but got "${JSON.stringify(actual[key])}"`); - isValid = false; - } - } - } - else if (actual[key] !== expected[key]) { - console.error(`Mismatch for key "${key}": expected "${expected[key]}", but got "${actual[key]}"`); - isValid = false; - } - }); - // Validate nested custom object separately - if (expected.custom && actual.custom) { - const customKeys = Object.keys(expected.custom); - customKeys.forEach((key) => { - if (typeof actual.custom[key] === 'object' && actual.custom[key] !== null) { - if (Array.isArray(actual.custom[key])) { - if (!Array.isArray(expected.custom[key]) || JSON.stringify(actual.custom[key]) !== JSON.stringify(expected.custom[key])) { - console.error(`Mismatch in custom object for key "${key}": expected "${JSON.stringify(expected.custom[key])}", but got "${JSON.stringify(actual.custom[key])}"`); - isValid = false; - } - } - else { - if (JSON.stringify(actual.custom[key]) !== JSON.stringify(expected.custom[key])) { - console.error(`Mismatch in custom object for key "${key}": expected "${JSON.stringify(expected.custom[key])}", but got "${JSON.stringify(actual.custom[key])}"`); - isValid = false; - } - } - } - else if (actual.custom[key] !== expected.custom[key]) { - console.error(`Mismatch in custom object for key "${key}": expected "${expected.custom[key]}", but got "${actual.custom[key]}"`); - isValid = false; - } - }); - } - return isValid; -} - // Create bulk data function createBulkData(bulk) { // Add an event @@ -121,7 +72,7 @@ function validateCreatedBulkData(bulk) { var req = reqQueue[0]; // read user details queue const actualUserDetails = req.user_details; // Extract the user_details from the actual request - const isValid = validateUserDetails(actualUserDetails, testUtils.getUserDetailsObj()); + const isValid = hp.validateUserDetails(actualUserDetails, testUtils.getUserDetailsObj()); assert.equal(true, isValid); var testUser3Request = reqQueue.find((request) => request.device_id === "TestUser3"); @@ -147,8 +98,8 @@ describe("Bulk Tests", () => { it("1- CNR", (done) => { var bulk = new CountlyBulk({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", + app_key: appKey, + url: serverUrl, }); assert.equal(storage.getStoragePath(), undefined); shouldFilesExist(false); @@ -163,8 +114,8 @@ describe("Bulk Tests", () => { it("2- CNR_cPath_file", (done) => { var bulk = new CountlyBulk({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", + app_key: appKey, + url: serverUrl, storage_path: "../test/customStorageDirectory/", storage_type: StorageTypes.FILE, }); @@ -182,8 +133,8 @@ describe("Bulk Tests", () => { it("3- CNR_file", (done) => { var bulk = new CountlyBulk({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", + app_key: appKey, + url: serverUrl, storage_type: StorageTypes.FILE, }); assert.equal(storage.getStoragePath(), "../bulk_data/"); @@ -197,9 +148,27 @@ describe("Bulk Tests", () => { done(); }, hp.mWait); }); + + it("4- CNR_memory", (done) => { + var bulk = new CountlyBulk({ + app_key: appKey, + url: serverUrl, + storage_type: StorageTypes.MEMORY, + }); + assert.equal(storage.getStoragePath(), undefined); + shouldFilesExist(true); + createBulkData(bulk); + + setTimeout(() => { + validateCreatedBulkData(bulk); + shouldFilesExist(true); + assert.equal(storage.getStoragePath(), undefined); + done(); + }, hp.mWait); + }); }); // Currently tested: CNR, CNR_cPath_file, CNR_file // TODO: Add tests for the following: -// - CNR: memory, cPath_memory, persistTrue, persistFalse, cPath_persistTrue, cPath_persistFalse, persistTrue_file, persistFalse_file, cPath_persistTrue_file, cPath_persistFalse_file +// - CNR: cPath_memory, persistTrue, persistFalse, cPath_persistTrue, cPath_persistFalse, persistTrue_file, persistFalse_file, cPath_persistTrue_file, cPath_persistFalse_file // - CR_CG for all of the above diff --git a/test/tests_storage.js b/test/tests_storage.js index d2da2c0..0ee9919 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -379,7 +379,7 @@ describe("Storage Tests", () => { Countly.user_details(testUtils.getUserDetailsObj); const storedData = storage.storeGet("cly_queue", null); const userDetailsReq = storedData[0]; - hp.userDetailRequestValidator(testUtils.getUserDetailsObj, userDetailsReq); + hp.validateUserDetails(testUtils.getUserDetailsObj, userDetailsReq); done(); }); diff --git a/test/tests_user_details.js b/test/tests_user_details.js index e7a3171..11b57b2 100644 --- a/test/tests_user_details.js +++ b/test/tests_user_details.js @@ -1,31 +1,26 @@ /* eslint-disable no-console */ +const assert = require("assert"); var Countly = require("../lib/countly"); var hp = require("./helpers/helper_functions"); var testUtils = require("./helpers/test_utils"); -// init function -function initMain() { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://try.count.ly", - interval: 10000, - max_events: -1, - }); -} - describe("User details tests", () => { beforeEach(async() => { await hp.clearStorage(); }); it("Record and validate all user details", (done) => { - // initialize SDK - initMain(); - // send user details - Countly.user_details(testUtils.getUserDetailsObj()); + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://try.count.ly", + }); + var userDetailObj = testUtils.getUserDetailsObj(); + Countly.user_details(userDetailObj); // read event queue setTimeout(() => { var req = hp.readRequestQueue()[0]; - hp.userDetailRequestValidator(testUtils.getUserDetailsObj(), req); + const actualUserDetails = req.user_details; + const isValid = hp.validateUserDetails(actualUserDetails, userDetailObj); + assert.equal(true, isValid); done(); }, hp.sWait); }); From ac543ed0dc17b505d338ab78711f2b41620f0395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Mon, 14 Oct 2024 18:25:22 +0300 Subject: [PATCH 53/58] Update tests_bulk.js --- test/tests_bulk.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tests_bulk.js b/test/tests_bulk.js index c42b43a..fba32a4 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -156,12 +156,12 @@ describe("Bulk Tests", () => { storage_type: StorageTypes.MEMORY, }); assert.equal(storage.getStoragePath(), undefined); - shouldFilesExist(true); + shouldFilesExist(false); createBulkData(bulk); setTimeout(() => { validateCreatedBulkData(bulk); - shouldFilesExist(true); + shouldFilesExist(false); assert.equal(storage.getStoragePath(), undefined); done(); }, hp.mWait); From 571006af3a7ba950b23b7b73d8a151d3605dd8d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 15 Oct 2024 13:28:24 +0300 Subject: [PATCH 54/58] Some more bulk tests --- test/tests_bulk.js | 133 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 1 deletion(-) diff --git a/test/tests_bulk.js b/test/tests_bulk.js index fba32a4..a54eccd 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -166,9 +166,140 @@ describe("Bulk Tests", () => { done(); }, hp.mWait); }); + + it("5- CNR_cPath_memory", (done) => { + var bulk = new CountlyBulk({ + app_key: appKey, + url: serverUrl, + storage_path: "../test/customStorageDirectory/", + storage_type: StorageTypes.MEMORY, + }); + assert.equal(storage.getStoragePath(), undefined); + shouldFilesExist(false); + createBulkData(bulk); + + setTimeout(() => { + validateCreatedBulkData(bulk); + shouldFilesExist(false); + assert.equal(storage.getStoragePath(), undefined); + done(); + }, hp.mWait); + }); + + it("6- CNR_persistTrue", (done) => { + var bulk = new CountlyBulk({ + app_key: appKey, + url: serverUrl, + persist_queue: true, + }); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + shouldFilesExist(true); + createBulkData(bulk); + + setTimeout(() => { + validateCreatedBulkData(bulk); + shouldFilesExist(true); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + done(); + }, hp.mWait); + }); + + it("7- CNR_persistFalse", (done) => { + var bulk = new CountlyBulk({ + app_key: appKey, + url: serverUrl, + persist_queue: false, + }); + assert.equal(storage.getStoragePath(), undefined); + shouldFilesExist(false); + createBulkData(bulk); + + setTimeout(() => { + validateCreatedBulkData(bulk); + shouldFilesExist(false); + assert.equal(storage.getStoragePath(), undefined); + done(); + }, hp.mWait); + }); + + it("8- CNR_cPath_persistTrue", (done) => { + var bulk = new CountlyBulk({ + app_key: appKey, + url: serverUrl, + storage_path: "../test/customStorageDirectory/", + persist_queue: true, + }); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + shouldFilesExist(true); + createBulkData(bulk); + + setTimeout(() => { + validateCreatedBulkData(bulk); + shouldFilesExist(true); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + done(); + }, hp.mWait); + }); + + it("9- CNR_cPath_persistFalse", (done) => { + var bulk = new CountlyBulk({ + app_key: appKey, + url: serverUrl, + storage_path: "../test/customStorageDirectory/", + persist_queue: false, + }); + assert.equal(storage.getStoragePath(), undefined); + shouldFilesExist(false); + createBulkData(bulk); + + setTimeout(() => { + validateCreatedBulkData(bulk); + shouldFilesExist(false); + assert.equal(storage.getStoragePath(), undefined); + done(); + }, hp.mWait); + }); + + it("10- CNR_persistTrue_file", (done) => { + var bulk = new CountlyBulk({ + app_key: appKey, + url: serverUrl, + storage_type: StorageTypes.FILE, + persist_queue: true, + }); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + shouldFilesExist(true); + createBulkData(bulk); + + setTimeout(() => { + validateCreatedBulkData(bulk); + shouldFilesExist(true); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + done(); + }, hp.mWait); + }); + + it("11- CNR_persistFalse_file", (done) => { + var bulk = new CountlyBulk({ + app_key: appKey, + url: serverUrl, + storage_type: StorageTypes.FILE, + persist_queue: true, + }); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + shouldFilesExist(true); + createBulkData(bulk); + + setTimeout(() => { + validateCreatedBulkData(bulk); + shouldFilesExist(true); + assert.equal(storage.getStoragePath(), "../bulk_data/"); + done(); + }, hp.mWait); + }); }); // Currently tested: CNR, CNR_cPath_file, CNR_file // TODO: Add tests for the following: -// - CNR: cPath_memory, persistTrue, persistFalse, cPath_persistTrue, cPath_persistFalse, persistTrue_file, persistFalse_file, cPath_persistTrue_file, cPath_persistFalse_file +// - CNR: cPath_persistTrue_file, cPath_persistFalse_file // - CR_CG for all of the above From 129c096ee27c5585486d22283c9530478c52e965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 15 Oct 2024 13:32:04 +0300 Subject: [PATCH 55/58] CNR Completed --- test/tests_bulk.js | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/test/tests_bulk.js b/test/tests_bulk.js index a54eccd..d7a9eff 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -297,9 +297,48 @@ describe("Bulk Tests", () => { done(); }, hp.mWait); }); + + it("12- CNR_cPath_persistTrue_file", (done) => { + var bulk = new CountlyBulk({ + app_key: appKey, + url: serverUrl, + storage_type: StorageTypes.FILE, + storage_path: "../test/customStorageDirectory/", + persist_queue: true, + }); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + shouldFilesExist(true); + createBulkData(bulk); + + setTimeout(() => { + validateCreatedBulkData(bulk); + shouldFilesExist(true); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + done(); + }, hp.mWait); + }); + + it("13- CNR_cPath_persistFalse_file", (done) => { + var bulk = new CountlyBulk({ + app_key: appKey, + url: serverUrl, + storage_type: StorageTypes.FILE, + storage_path: "../test/customStorageDirectory/", + persist_queue: false, + }); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + shouldFilesExist(true); + createBulkData(bulk); + + setTimeout(() => { + validateCreatedBulkData(bulk); + shouldFilesExist(true); + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + done(); + }, hp.mWait); + }); }); // Currently tested: CNR, CNR_cPath_file, CNR_file // TODO: Add tests for the following: -// - CNR: cPath_persistTrue_file, cPath_persistFalse_file // - CR_CG for all of the above From e3fa2ba6e0addd23d5066ebae933784fae1db563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 15 Oct 2024 13:53:57 +0300 Subject: [PATCH 56/58] Added Comments in Bulk --- test/tests_bulk.js | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/test/tests_bulk.js b/test/tests_bulk.js index d7a9eff..d0a6724 100644 --- a/test/tests_bulk.js +++ b/test/tests_bulk.js @@ -95,7 +95,8 @@ describe("Bulk Tests", () => { beforeEach(async() => { await hp.clearStorage(); }); - + // without any config option default bulk storage + // bulk mode is memory only by default it("1- CNR", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -112,6 +113,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // storage path and storage type provided in bulk + // type should be file and path should be correct it("2- CNR_cPath_file", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -131,6 +134,8 @@ describe("Bulk Tests", () => { }, hp.sWait); }); + // file storage type in config + // path should become the default "../bulk_data/" it("3- CNR_file", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -149,6 +154,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // memory storage type in config + // path should become undefined and storage files shouldn't exist it("4- CNR_memory", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -167,6 +174,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // custom storage path and memory storage type provided in config + // path should become undefined, type should be memory storage it("5- CNR_cPath_memory", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -186,6 +195,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // persist_queue is true in bulk config + // should switch to file storage and default bulk path it("6- CNR_persistTrue", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -204,6 +215,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // persist_queue is false in bulk config + // should result same with default bulk configurations it("7- CNR_persistFalse", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -222,6 +235,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // custom path is provided and persist_queue is true in config + // custom path should become the storage path and switch to file storage it("8- CNR_cPath_persistTrue", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -241,6 +256,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // custom path is provided and persist_queue is false in config + // storage path should become undefined, should stay in memory storage it("9- CNR_cPath_persistFalse", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -260,6 +277,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // persist_queue is true and storage type is file + // should directly switch to the default file storage for bulk it("10- CNR_persistTrue_file", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -279,6 +298,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // persist_queue is false and storage type is file + // storage type should overrule and switch into the default bulk file storage it("11- CNR_persistFalse_file", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -298,6 +319,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // persist_queue is true and storage type is file and custom path is given + // storage type should overrule and switch into the custom path file storage it("12- CNR_cPath_persistTrue_file", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -318,6 +341,8 @@ describe("Bulk Tests", () => { }, hp.mWait); }); + // persist_queue is false and storage type is file and custom path is given + // storage type should overrule and switch into the custom path file storage it("13- CNR_cPath_persistFalse_file", (done) => { var bulk = new CountlyBulk({ app_key: appKey, @@ -337,8 +362,4 @@ describe("Bulk Tests", () => { done(); }, hp.mWait); }); -}); - -// Currently tested: CNR, CNR_cPath_file, CNR_file -// TODO: Add tests for the following: -// - CR_CG for all of the above +}); \ No newline at end of file From fdb8b628900deb5f3f6aa005b3cea099582127e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Tue, 15 Oct 2024 16:50:22 +0300 Subject: [PATCH 57/58] refactor --- test/helpers/helper_functions.js | 14 +- test/helpers/test_utils.js | 38 ++ test/tests_storage.js | 658 +++++++------------------------ 3 files changed, 179 insertions(+), 531 deletions(-) diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index 2727cd2..9481b2e 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -26,9 +26,11 @@ const DIR_Bulk_request = (`${dir}/bulk_data/__cly_req_queue.json`); // Custom const DIR_Test = (`${dir_test}/customStorageDirectory`); +const DIR_Test_event = (`${dir_test}/customStorageDirectory/__cly_event.json`); +const DIR_Test_request = (`${dir_test}/customStorageDirectory/__cly_queue.json`); const DIR_Test_bulk = (`${dir_test}/customStorageDirectory/__cly_bulk_queue.json`); -const DIR_Test_event = (`${dir_test}/customStorageDirectory/ __cly_bulk_event.json`); -const DIR_Test_request = (`${dir_test}/customStorageDirectory/__cly_req_queue.json`); +const DIR_Test_bulk_event = (`${dir_test}/customStorageDirectory/__cly_bulk_event.json`); +const DIR_Test_bulk_request = (`${dir_test}/customStorageDirectory/__cly_req_queue.json`); // timeout variables const sWait = 50; @@ -48,10 +50,10 @@ function readEventQueue(givenPath = null, isBulk = false) { return a; } // parsing request queue -function readRequestQueue(givenPath = null, isBulk = false) { +function readRequestQueue(customPath = false, isBulk = false) { var destination = DIR_CLY_request; - if (givenPath !== null) { - destination = givenPath; + if (customPath) { + destination = DIR_Test_request; } var a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_queue; if (isBulk) { @@ -66,7 +68,7 @@ function doesFileStoragePathsExist(callback, isBulk = false, testPath = false) { paths = [DIR_Bulk_request, DIR_Bulk_event, DIR_Bulk_bulk]; } else if (testPath) { - paths = [DIR_Test_bulk, DIR_Test_event, DIR_Test_request]; + paths = [DIR_Test_bulk, DIR_Test_bulk_event, DIR_Test_bulk_request]; } let errors = 0; diff --git a/test/helpers/test_utils.js b/test/helpers/test_utils.js index 08af01e..ded2f53 100644 --- a/test/helpers/test_utils.js +++ b/test/helpers/test_utils.js @@ -41,6 +41,42 @@ var userDetailObj = { }, }; +const invalidStorageMethods = { + _storage: {}, + setInvalid: function() { + }, + getInvalid: function() { + }, + removeInvalid: function() { + }, +}; + +const customStorage = { + _storage: {}, + storeSet: function(key, value, callback) { + if (key) { + this._storage[key] = value; + if (typeof callback === "function") { + callback(null); + } + } + }, + storeGet: function(key, def) { + return typeof this._storage[key] !== "undefined" ? this._storage[key] : def; + }, + storeRemove: function(key) { + delete this._storage[key]; + }, +}; + +function getInvalidStorage() { + return invalidStorageMethods; +} + +function getCustomStorage() { + return customStorage; +} + var getUserDetailsObj = function() { return userDetailObj; }; @@ -57,4 +93,6 @@ module.exports = { getEventObj, getUserDetailsObj, getTimedEventObj, + getInvalidStorage, + getCustomStorage, }; \ No newline at end of file diff --git a/test/tests_storage.js b/test/tests_storage.js index 0ee9919..eaff2e1 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -1,567 +1,175 @@ /* eslint-disable no-console */ -const fs = require('fs'); -const path = require('path'); - +/* global runthis */ const assert = require("assert"); var Countly = require("../lib/countly"); var storage = require("../lib/countly-storage"); -var cc = require("../lib/countly-common"); var hp = require("./helpers/helper_functions"); var testUtils = require("./helpers/test_utils"); -const { StorageTypes } = Countly; - -// init function -function initMain(device_id) { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - interval: 10000, - max_events: -1, - device_id: device_id, - }); -} - -function recordValuesToStorageAndValidate(userPath, memoryOnly = false, isBulk = false, persistQueue = false) { - // Set values - var deviceIdType = cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED; - storage.initStorage(userPath, memoryOnly, isBulk, persistQueue); - storage.storeSet("cly_id", "SpecialDeviceId"); - storage.storeSet("cly_id_type", deviceIdType); - - // Set values with different data types - storage.storeSet("cly_count", 42); - storage.storeSet("cly_object", { key: "value" }); - storage.storeSet("cly_null", null); - - // Retrieve and assert values - assert.equal(storage.storeGet("cly_id"), "SpecialDeviceId"); - assert.equal(storage.storeGet("cly_id_type"), deviceIdType); - assert.equal(storage.storeGet("cly_count"), 42); - assert.deepEqual(storage.storeGet("cly_object"), { key: "value" }); - assert.equal(storage.storeGet("cly_null"), null); +var appKey = "YOUR_APP_KEY"; +var serverUrl = "https://your.server.ly"; - // Remove specific items by overriding with null or empty array - storage.storeSet("cly_id", null); - storage.storeSet("cly_object", []); - assert.equal(storage.storeGet("cly_id"), null); - assert.deepEqual(storage.storeGet("cly_object"), []); +const { StorageTypes } = Countly; - // Reset storage and check if it's empty again - storage.resetStorage(); +function shouldFilesExist(shouldExist, isCustomTest = false) { + hp.doesFileStoragePathsExist((exists) => { + assert.equal(shouldExist, exists); + }, true, isCustomTest); } -var __data = {}; -// technically same as defualt file storage method -const customFileStorage = { - storeSet: function(key, value, callback) { - __data[key] = value; - storage.writeFile(key, value, callback); - cc.log(cc.logLevelEnums.DEBUG, `storeSet, Setting item in storage with key: [${key}] and value: [${value}].`); - }, - storeGet: function(key, def) { - if (typeof __data[key] === "undefined") { - var ob = storage.readFile(key); - var obLen; - try { - obLen = Object.keys(ob).length; - } - catch (error) { - obLen = 0; - } - if (!ob || obLen === 0) { - __data[key] = def; - } - else { - __data[key] = ob[key]; - } +function validateStoragePathAndType(expectedStorageType, isCustomPath = false) { + if (expectedStorageType === StorageTypes.FILE) { + if (!isCustomPath) { + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); } - cc.log(cc.logLevelEnums.DEBUG, `storeGet, Fetching item from storage with key: [${key}] and value: [${__data[key]}].`); - return __data[key]; - }, - storeRemove: function(key) { - delete __data[key]; - var filePath = path.resolve(__dirname, `${storage.getStoragePath()}__${key}.json`); - fs.access(filePath, fs.constants.F_OK, (accessErr) => { - if (accessErr) { - cc.log(cc.logLevelEnums.WARNING, `storeRemove, No file found with key: [${key}]. Nothing to remove.`); - return; - } - fs.unlink(filePath, (err) => { - if (err) { - cc.log(cc.logLevelEnums.ERROR, `storeRemove, Failed to remove file with key: [${key}]. Error: [${err.message}].`); - } - else { - cc.log(cc.logLevelEnums.INFO, `storeRemove, Successfully removed file with key: [${key}].`); - } - }); - }); - }, -}; -const nonValidStorageMethods = { - _storage: {}, - - setInvalid: function() { - }, - getInvalid: function() { - }, - removeInvalid: function() { - }, -}; -const funkyMemoryStorage = { - _storage: {}, - - storeSet: function(key, value, callback) { - if (key) { - const existingValue = this._storage[key]; - if (typeof value === 'string' && typeof existingValue === 'string') { - this._storage[key] = existingValue + value; - } - else { - this._storage[key] = value; - } - if (typeof callback === "function") { - callback(null); - } + else { + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + assert.equal(storage.getStorageType(), StorageTypes.FILE); + } + } + else if (expectedStorageType === StorageTypes.MEMORY) { + assert.equal(storage.getStoragePath(), undefined); + assert.equal(storage.getStorageType(), StorageTypes.MEMORY); + } + // get storage type returns null in case of a custom method + else if (expectedStorageType === null) { + if (!isCustomPath) { + assert.equal(storage.getStoragePath(), "../data/"); + assert.equal(storage.getStorageType(), null); } - }, - storeGet: function(key, def) { - const value = this._storage[key]; - if (typeof value === 'string') { - return value.split('').reverse().join(''); + else { + assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); + assert.equal(storage.getStorageType(), null); } + } +} +function createData() { + // begin a session + Countly.begin_session(true); + // add an event + Countly.add_event(testUtils.getEventObj()); + // add user details + Countly.user_details(testUtils.getUserDetailsObj()); + // add crash + Countly.track_errors(); + try { + runthis(); + } + catch (ex) { + Countly.log_error(ex); + } +} +function validateData(isCustomPath = false) { + var beg = hp.readRequestQueue(isCustomPath)[0]; + hp.sessionRequestValidator(beg); - return value !== undefined ? value : def; - }, - storeRemove: function(key) { - delete this._storage[key]; - }, -}; -const customMemoryStorage = { - _storage: {}, - storeSet: function(key, value, callback) { - if (key) { - this._storage[key] = value; - if (typeof callback === "function") { - callback(null); - } - } - }, - storeGet: function(key, def) { - return typeof this._storage[key] !== "undefined" ? this._storage[key] : def; - }, - storeRemove: function(key) { - delete this._storage[key]; - }, -}; + var ud = hp.readRequestQueue(isCustomPath)[1]; + const isValid = hp.validateUserDetails(ud.user_details, testUtils.getUserDetailsObj()); + assert.equal(isValid, true); + + var crash = hp.readRequestQueue(isCustomPath)[2]; + hp.crashRequestValidator(crash, true); -// TODO: -// Other tests are testing default (FILE) already, so here focus should be: -// 1. MEMORY (checking it works with a set of events/situations, it does not saves something persistently) -// 2. Persistency (checking default saves persistently, checking default and FILE are same) -// 3. Custom (custom works with valid methods-memory or persistent-, does not work with invalid methods) -// 4. Their interaction with path -// here we test storage, not every + var ev = hp.readRequestQueue(isCustomPath)[3]; + var eventsArray = JSON.parse(ev.events); + hp.eventValidator(testUtils.getEventObj(), eventsArray[0]); +} +/* ++---------------------------------------------------+-------------------+ +| Configuration Option | Tested? (+/-) | ++---------------------------------------------------+-------------------+ +| 1. No Configuration Option Provided | + | ++---------------------------------------------------+-------------------+ +| 2. File Storage with Custom Path | + | ++---------------------------------------------------+-------------------+ +| 3. File Storage with Invalid Path | + | ++---------------------------------------------------+-------------------+ +| 4. File Storage while Custom Method Provided | | ++---------------------------------------------------+-------------------+ +| 5. Memory Storage with No Path | | ++---------------------------------------------------+-------------------+ +| 6. Memory Storage with Custom Path | | ++---------------------------------------------------+-------------------+ +| 7. Memory Storage while Custom Method Provided | | ++---------------------------------------------------+-------------------+ +| 8. Custom Storage Methods with No Path | | ++---------------------------------------------------+-------------------+ +| 9. Custom Storage Methods with Custom Path | | ++---------------------------------------------------+-------------------+ +| 10. Custom Storage with Invalid Path | | ++---------------------------------------------------+-------------------+ +| 11. Custom Storage Methods with Invalid Methods | | ++---------------------------------------------------+-------------------+ +| 12. File Storage without initializing SDK | | ++---------------------------------------------------+-------------------+ +| 13. Memory Storage without initializing SDK | | ++---------------------------------------------------+-------------------+ +| 14. Custom Storage without initializing SDK | | ++---------------------------------------------------+-------------------+ +*/ describe("Storage Tests", () => { beforeEach(async() => { await hp.clearStorage(); }); - // initialize the sdk and record an event - // event should be recorded stored and validated correctly - it("4- Record event and validate storage", (done) => { - initMain(); - Countly.add_event(testUtils.getEventObj()); - setTimeout(() => { - var storedEvents = hp.readEventQueue(); - assert.equal(storedEvents.length, 1); - var event = storedEvents[0]; - hp.eventValidator(testUtils.getEventObj(), event); - done(); - }, hp.mWait); - }); - - // initialize the sdk with providing no storage path or type - // if storage path is not provided it will be default "../data/" - it("5- Not provide storage path during init", (done) => { - initMain(); - assert.equal(storage.getStoragePath(), "../data/"); - done(); - }); - - // initialize the sdk with providing undefined storage path - // if set to undefined it should be set to default path - it("6- Set storage path to undefined", (done) => { + // if no config option provided sdk should init storage with default settings + // "../data/" as the storage path, FILE as the storage type + it("1- noConfigOption", (done) => { Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - storage_path: undefined, + app_key: appKey, + url: serverUrl, }); - assert.equal(storage.getStoragePath(), "../data/"); - done(); - }); + shouldFilesExist(true); + validateStoragePathAndType(StorageTypes.FILE); + createData(); - // initialize the sdk with providing null storage path - // if set to null it should be set to default path - it("7- Set storage path to null", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - storage_path: null, - }); - assert.equal(storage.getStoragePath(), "../data/"); - done(); + setTimeout(() => { + shouldFilesExist(true); + validateStoragePathAndType(StorageTypes.FILE); + validateData(); + done(); + }, hp.mWait); }); - // initialize the sdk with providing a custom storage path - // it should be set to the custom directory if provided a valid storage path - it("8- Set storage path to custom directory", (done) => { + // if custom path is provided sdk should init storage with using that path + it("2- file_cPath", (done) => { Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - interval: 10000, - max_events: -1, + app_key: appKey, + url: serverUrl, storage_path: "../test/customStorageDirectory/", + storage_type: StorageTypes.FILE, }); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - done(); - }); - - // resets the storage path to default and validates that it is set correctly, - // then resets it to undefined and confirms the reset. - it("9- Reset Storage While on Default Path /no-init", (done) => { - // will set to default storage path - storage.setStoragePath(); - assert.equal(storage.getStoragePath(), "../data/"); - // will set to undefined - storage.resetStorage(); - assert.equal(storage.getStoragePath(), undefined); - done(); - }); - - // sets the storage path to default and verifies it, - // then records values to storage and ensures they are stored correctly. - it("10- Recording to Storage with Default Storage Path /no-init", (done) => { - storage.initStorage(); - assert.equal(storage.getStoragePath(), "../data/"); - recordValuesToStorageAndValidate(); - done(); - }); - - // sets a custom storage path and verifies it, - // then records values to storage and ensures correct storage in the custom path. - it("11- Recording to Storage with Custom Storage Path /no-init", (done) => { - storage.initStorage("../test/customStorageDirectory/"); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - recordValuesToStorageAndValidate("../test/customStorageDirectory/"); - done(); - }); - - // sets the storage path to the default bulk storage path and verifies it, - // then records values to bulk storage and validates proper storage in bulk mode. - it("12- Recording to Bulk Storage with Default Bulk Data Path /no-init", (done) => { - storage.initStorage(null, StorageTypes.FILE, true); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(true, exists); - }, true); - assert.equal(storage.getStoragePath(), "../bulk_data/"); - recordValuesToStorageAndValidate(null, false, true, true); - done(); - }); - - // sets a custom bulk storage path and verifies it, - // then records values to bulk storage and ensures proper recording to the custom path. - it("13- Recording to Bulk Storage with Custom Bulk Storage Path /no-init", (done) => { - // will set to default storage path - storage.initStorage("../test/customStorageDirectory/", StorageTypes.FILE, true); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }, true); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - recordValuesToStorageAndValidate("../test/customStorageDirectory/", false, true); - done(); - }); - - // resetting and initializing the storage without initializing sdk - // without sdk initializing storage should be able to initialize - // if no path or type is provided storage should be set to default - it("14- Setting storage path to default path via initStorage /no-init", (done) => { - storage.initStorage(); - assert.equal(storage.getStoragePath(), "../data/"); - done(); - }); - - // resetting and initializing the bulk storage without initializing sdk - // without sdk initializing storage should be able to initialize - // if no path is provided storage should be set to default bulk path - it("15- Setting bulk storage path to default bulk file path via initStorage /no-init", (done) => { - storage.initStorage(null, StorageTypes.FILE, true); - assert.equal(storage.getStoragePath(), "../bulk_data/"); - done(); - }); - - // resetting and initializing the storage without initializing sdk - // without sdk initializing storage should be able to initialize - // if a valid path is provided storage should be set to that path - it("16- Setting custom storage path via initStorage /no-init", (done) => { - storage.initStorage("../test/customStorageDirectory/"); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - done(); - }); - - // resetting and initializing the storage in memory mode - // without sdk initializing storage should be able to initialize in memory mode - // since initialized in memory mode sdk should have no storage path or storage files should not exist - it("17- Setting storage method to memory only and checking storage path /no-init", (done) => { - storage.initStorage(null, StorageTypes.MEMORY); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - assert.equal(storage.getStoragePath(), undefined); - done(); - }); - - // recording device-id in memory only mode - // initializes the SDK in memory only mode, validates that file storage files does not exist - // retrieve the developer supplied device id and id type from storage - it("18- Memory only storage Device-Id", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - storage_type: StorageTypes.MEMORY, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - assert.equal(storage.getStoragePath(), undefined); - assert.equal(storage.storeGet("cly_id", null), "Test-Device-Id"); - assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - done(); - }); + shouldFilesExist(true); + validateStoragePathAndType(StorageTypes.FILE, true); + createData(); - // recording event in memory only mode - // initializes the SDK in memory only mode, validates that file storage files does not exist - // records an event and validates the recorded event - it("19- Record event in memory only mode and validate the record", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - storage_type: StorageTypes.MEMORY, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - Countly.add_event(testUtils.getEventObj()); setTimeout(() => { - const storedData = storage.storeGet("cly_queue", null); - const eventArray = JSON.parse(storedData[0].events); - const eventFromQueue = eventArray[0]; - hp.eventValidator(testUtils.getEventObj(), eventFromQueue); + shouldFilesExist(true); + validateStoragePathAndType(StorageTypes.FILE, true); + validateData(true); done(); }, hp.mWait); }); - // recording user details in memory only mode - // initializes the SDK in memory only mode, validates that file storage files does not exist - // records user details and validates the recorded details - it("20- Record and validate user details in memory only mode", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - storage_type: StorageTypes.MEMORY, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - Countly.user_details(testUtils.getUserDetailsObj); - const storedData = storage.storeGet("cly_queue", null); - const userDetailsReq = storedData[0]; - hp.validateUserDetails(testUtils.getUserDetailsObj, userDetailsReq); - done(); - }); - - // tests device id changes in memory only storage - // initialize the SDK in memory only mode, check the device id and switch it - // SDK and storage should function properly - it("21- Memory only storage, change SDK Generated Device-Id", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - storage_type: StorageTypes.MEMORY, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - assert.equal(storage.getStoragePath(), undefined); - assert.equal(storage.storeGet("cly_id", null), Countly.get_device_id()); - assert.equal(storage.storeGet("cly_id_type", null), Countly.get_device_id_type()); - - Countly.change_id("Test-Id-2"); - assert.equal(storage.storeGet("cly_id", null), "Test-Id-2"); - assert.equal(storage.storeGet("cly_id_type", null), cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED); - done(); - }); - - // tests switching between storage types after initializing SDK - // passing memory storage type during init and initializing storage afterwards - // SDK should switch to file storage - it("22- Switch to file storage after init", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - storage_type: StorageTypes.MEMORY, - }); - assert.equal(storage.getStoragePath(), undefined); - assert.equal(storage.getStorageType(), StorageTypes.MEMORY); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - - storage.initStorage(); - assert.equal(storage.getStoragePath(), "../data/"); - assert.equal(storage.getStorageType(), StorageTypes.FILE); - done(); - }); - - // tests storeRemove function in CountlyStorage - // after initializing the memory storage, without initializing SDK, attempts to set, get and remove values - // without initializing SDK storage should function properly - it("23- storeRemove Memory Only /no-init", (done) => { - storage.initStorage(null, StorageTypes.MEMORY); - assert.equal(storage.getStoragePath(), undefined); - assert.equal(storage.getStorageType(), StorageTypes.MEMORY); - storage.storeSet("keyToStore", "valueToStore"); - assert.equal(storage.storeGet("keyToStore", null), "valueToStore"); - - storage.storeRemove("keyToStore"); - assert.equal(storage.storeGet("keyToStore", null), null); - done(); - }); - - // tests storeRemove function in CountlyStorage - // after initializing the file storage, without initializing SDK attempts to set, get and remove values - // without initializing SDK storage should function properly - it("24- storeRemove File Storage /no-init", (done) => { - storage.initStorage(); - assert.equal(storage.getStoragePath(), "../data/"); - assert.equal(storage.getStorageType(), StorageTypes.FILE); - storage.storeSet("keyToStore", "valueToStore"); - assert.equal(storage.storeGet("keyToStore", null), "valueToStore"); - - storage.storeRemove("keyToStore"); - assert.equal(storage.storeGet("keyToStore", null), null); - done(); - }); - - // tests init time storage config options - // choosing Custom storage type and passing null in storage methods - // passing null as storage method ends up with switching to default file storage - it("25- Null Custom Storage Method", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - custom_storage_method: null, - }); - assert.equal(storage.getStoragePath(), "../data/"); - assert.equal(storage.getStorageType(), StorageTypes.FILE); - done(); - }); - - // tests init time storage config options - // choosing Custom storage type and passing custom storage methods - // SDK should use custom methods as storage method, no File Storage should exist - it("26- Providing Custom Storage Method", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - custom_storage_method: customMemoryStorage, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - done(); - }); - - // tests init time storage config options - // Recording values in Custom Storage Methods - // SDK should use custom methods as storage methods and values should be recorded correctly - it("27- Record/Remove Values in Custom Storage Method", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - custom_storage_method: customMemoryStorage, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); - }); - storage.storeSet("CustomStorageKey", "CustomStorageValue"); - assert.equal(storage.storeGet("CustomStorageKey", null), "CustomStorageValue"); - storage.storeRemove("CustomStorageKey"); - assert.equal(storage.storeGet("CustomStorageKey", null), null); - done(); - }); - - // tests init time storage config options - // passes a funky storage method, which does store get as reversing string - // SDK should use custom methods as storage method - it("28- Record/Remove Values in Other Custom Storage Method", (done) => { + // if invalid path is provided such as null or undefined sdk should init storage with default path + // validateStoragePathAndType checks path as "../data/" if true is not passed as second param + it("3- file_invalidPath", (done) => { Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - custom_storage_method: funkyMemoryStorage, - }); - hp.doesFileStoragePathsExist((exists) => { - assert.equal(false, exists); + app_key: appKey, + url: serverUrl, + storage_path: undefined, + storage_type: StorageTypes.FILE, }); - storage.storeSet("CustomStorageKey", "CustomStorageValue"); - storage.storeSet("CustomStorageKey", "CustomStorageValue2"); - assert.equal("2eulaVegarotSmotsuCeulaVegarotSmotsuC", storage.storeGet("CustomStorageKey", null)); - done(); - }); + shouldFilesExist(true); + validateStoragePathAndType(StorageTypes.FILE); + createData(); - // tests init time storage config options - // choosing Custom storage type and passing invalid custom storage methods - // SDK should not use custom methods as storage method, and switch to File Storage - it("29- Providing Invalid Custom Storage Method", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "Test-Device-Id", - custom_storage_method: nonValidStorageMethods, - }); - assert.equal(storage.getStoragePath(), "../data/"); - assert.equal(storage.getStorageType(), StorageTypes.FILE); - done(); - }); - // tests init time storage config options - // choosing Custom storage type and passing custom file storage methods - // SDK should use custom methods as storage methods - it("30- Providing File Custom Storage Method", (done) => { - Countly.init({ - app_key: "YOUR_APP_KEY", - url: "https://test.url.ly", - device_id: "ID", - storage_path: "../test/customStorageDirectory/", - custom_storage_method: customFileStorage, - }); - Countly.begin_session(); setTimeout(() => { - assert.equal("ID", storage.storeGet("cly_id")); - assert.equal(cc.deviceIdTypeEnums.DEVELOPER_SUPPLIED, storage.storeGet("cly_id_type")); - assert.equal(storage.getStoragePath(), "../test/customStorageDirectory/"); - storage.storeSet("CustomStorageKey", "CustomStorageValue"); - assert.equal(storage.storeGet("CustomStorageKey", null), "CustomStorageValue"); - storage.storeRemove("CustomStorageKey"); - assert.equal(storage.storeGet("CustomStorageKey", null), null); - hp.clearStorage("../test/customStorageDirectory/"); + shouldFilesExist(true); + validateStoragePathAndType(StorageTypes.FILE); + validateData(); done(); - }, hp.sWait); + }, hp.mWait); }); }); \ No newline at end of file From 7c5c5232a41748e66c8f05b1077cf3a4658cdc0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ali=20R=C4=B1za=20Kat?= Date: Wed, 16 Oct 2024 15:10:58 +0300 Subject: [PATCH 58/58] storage tests done --- test/helpers/helper_functions.js | 13 +- test/tests_storage.js | 302 ++++++++++++++++++++++++++++--- 2 files changed, 286 insertions(+), 29 deletions(-) diff --git a/test/helpers/helper_functions.js b/test/helpers/helper_functions.js index 9481b2e..d8340f6 100644 --- a/test/helpers/helper_functions.js +++ b/test/helpers/helper_functions.js @@ -6,6 +6,7 @@ var assert = require("assert"); var fs = require("fs"); var fsp = require("fs/promises"); var Countly = require("../../lib/countly"); +var CountlyStorage = require("../../lib/countly-storage"); // paths for convenience var dir = path.resolve(__dirname, "../../"); @@ -50,15 +51,21 @@ function readEventQueue(givenPath = null, isBulk = false) { return a; } // parsing request queue -function readRequestQueue(customPath = false, isBulk = false) { +function readRequestQueue(customPath = false, isBulk = false, isMemory = false) { var destination = DIR_CLY_request; if (customPath) { destination = DIR_Test_request; } - var a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_queue; + var a; if (isBulk) { a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_req_queue; } + if (isMemory) { + a = CountlyStorage.storeGet("cly_queue"); + } + else { + a = JSON.parse(fs.readFileSync(destination, "utf-8")).cly_queue; + } return a; } function doesFileStoragePathsExist(callback, isBulk = false, testPath = false) { @@ -68,7 +75,7 @@ function doesFileStoragePathsExist(callback, isBulk = false, testPath = false) { paths = [DIR_Bulk_request, DIR_Bulk_event, DIR_Bulk_bulk]; } else if (testPath) { - paths = [DIR_Test_bulk, DIR_Test_bulk_event, DIR_Test_bulk_request]; + paths = [DIR_Test_event, DIR_Test_request]; } let errors = 0; diff --git a/test/tests_storage.js b/test/tests_storage.js index eaff2e1..2e3c85e 100644 --- a/test/tests_storage.js +++ b/test/tests_storage.js @@ -14,9 +14,21 @@ const { StorageTypes } = Countly; function shouldFilesExist(shouldExist, isCustomTest = false) { hp.doesFileStoragePathsExist((exists) => { assert.equal(shouldExist, exists); - }, true, isCustomTest); + }, false, isCustomTest); } -function validateStoragePathAndType(expectedStorageType, isCustomPath = false) { + +function validateStorageMethods() { + storage.storeSet("cly_count", 42); + storage.storeSet("cly_object", { key: "value" }); + storage.storeSet("cly_null", null); + + // Retrieve and assert values + assert.equal(storage.storeGet("cly_count"), 42); + assert.deepEqual(storage.storeGet("cly_object"), { key: "value" }); + assert.equal(storage.storeGet("cly_null"), null); +} + +function validateStorageTypeAndPath(expectedStorageType, isCustomPath = false) { if (expectedStorageType === StorageTypes.FILE) { if (!isCustomPath) { assert.equal(storage.getStoragePath(), "../data/"); @@ -59,18 +71,18 @@ function createData() { Countly.log_error(ex); } } -function validateData(isCustomPath = false) { - var beg = hp.readRequestQueue(isCustomPath)[0]; +function validateData(isCustomPath = false, isMemoryOrCustom = false) { + var beg = hp.readRequestQueue(isCustomPath, false, isMemoryOrCustom)[0]; hp.sessionRequestValidator(beg); - var ud = hp.readRequestQueue(isCustomPath)[1]; + var ud = hp.readRequestQueue(isCustomPath, false, isMemoryOrCustom)[1]; const isValid = hp.validateUserDetails(ud.user_details, testUtils.getUserDetailsObj()); assert.equal(isValid, true); - var crash = hp.readRequestQueue(isCustomPath)[2]; + var crash = hp.readRequestQueue(isCustomPath, false, isMemoryOrCustom)[2]; hp.crashRequestValidator(crash, true); - var ev = hp.readRequestQueue(isCustomPath)[3]; + var ev = hp.readRequestQueue(isCustomPath, false, isMemoryOrCustom)[3]; var eventsArray = JSON.parse(ev.events); hp.eventValidator(testUtils.getEventObj(), eventsArray[0]); } @@ -84,27 +96,33 @@ function validateData(isCustomPath = false) { +---------------------------------------------------+-------------------+ | 3. File Storage with Invalid Path | + | +---------------------------------------------------+-------------------+ -| 4. File Storage while Custom Method Provided | | +| 4. File Storage while Custom Method Provided | + | +---------------------------------------------------+-------------------+ -| 5. Memory Storage with No Path | | +| 5. Memory Storage with No Path | + | +---------------------------------------------------+-------------------+ -| 6. Memory Storage with Custom Path | | +| 6. Memory Storage with Custom Path | + | +---------------------------------------------------+-------------------+ -| 7. Memory Storage while Custom Method Provided | | +| 7. Memory Storage while Custom Method Provided | + | +---------------------------------------------------+-------------------+ -| 8. Custom Storage Methods with No Path | | +| 8. Custom Storage Methods with No Path | + | +---------------------------------------------------+-------------------+ -| 9. Custom Storage Methods with Custom Path | | +| 9. Custom Storage Methods with Custom Path | + | +---------------------------------------------------+-------------------+ -| 10. Custom Storage with Invalid Path | | +| 10. Custom Storage with Invalid Path | + | +---------------------------------------------------+-------------------+ -| 11. Custom Storage Methods with Invalid Methods | | +| 11. Custom Storage Methods with Invalid Methods | + | +---------------------------------------------------+-------------------+ -| 12. File Storage without initializing SDK | | +| 12. Init Storage default no SDK init | + | +---------------------------------------------------+-------------------+ -| 13. Memory Storage without initializing SDK | | +| 13. File Storage with null path no SDK init | + | +---------------------------------------------------+-------------------+ -| 14. Custom Storage without initializing SDK | | +| 14. File Storage with custom path no SDK init | + | ++---------------------------------------------------+-------------------+ +| 15. Memory Storage without no SDK init | + | ++---------------------------------------------------+-------------------+ +| 16. Custom Storage with null path no SDK init | + | ++---------------------------------------------------+-------------------+ +| 17. Custom Storage with custom path no SDK init | + | +---------------------------------------------------+-------------------+ */ @@ -121,12 +139,12 @@ describe("Storage Tests", () => { url: serverUrl, }); shouldFilesExist(true); - validateStoragePathAndType(StorageTypes.FILE); + validateStorageTypeAndPath(StorageTypes.FILE); createData(); setTimeout(() => { shouldFilesExist(true); - validateStoragePathAndType(StorageTypes.FILE); + validateStorageTypeAndPath(StorageTypes.FILE); validateData(); done(); }, hp.mWait); @@ -141,20 +159,20 @@ describe("Storage Tests", () => { storage_type: StorageTypes.FILE, }); shouldFilesExist(true); - validateStoragePathAndType(StorageTypes.FILE, true); + validateStorageTypeAndPath(StorageTypes.FILE, true); createData(); setTimeout(() => { shouldFilesExist(true); - validateStoragePathAndType(StorageTypes.FILE, true); + validateStorageTypeAndPath(StorageTypes.FILE, true); validateData(true); done(); }, hp.mWait); }); // if invalid path is provided such as null or undefined sdk should init storage with default path - // validateStoragePathAndType checks path as "../data/" if true is not passed as second param - it("3- file_invalidPath", (done) => { + // validateStorageTypeAndPath checks path as "../data/" if true is not passed as second param + it("3- file_invalid_path", (done) => { Countly.init({ app_key: appKey, url: serverUrl, @@ -162,14 +180,246 @@ describe("Storage Tests", () => { storage_type: StorageTypes.FILE, }); shouldFilesExist(true); - validateStoragePathAndType(StorageTypes.FILE); + validateStorageTypeAndPath(StorageTypes.FILE); createData(); setTimeout(() => { shouldFilesExist(true); - validateStoragePathAndType(StorageTypes.FILE); + validateStorageTypeAndPath(StorageTypes.FILE); validateData(); done(); }, hp.mWait); }); + + // since a custom method is provided sdk will switch to using that + // custom method will be applied, storage type will be null but sdk will create storage files anyway + it("4- file_cMethod", (done) => { + Countly.init({ + app_key: appKey, + url: serverUrl, + storage_type: StorageTypes.FILE, + custom_storage_method: testUtils.getCustomStorage(), + }); + shouldFilesExist(true); + // storage type will be null since custom method is provided + validateStorageTypeAndPath(null); + createData(); + + setTimeout(() => { + shouldFilesExist(true); + validateStorageTypeAndPath(null); + validateData(false, true); + done(); + }, hp.mWait); + }); + + // storage type will become memory, and storage files will not exist + it("5- memory_noPath", (done) => { + Countly.init({ + app_key: appKey, + url: serverUrl, + storage_type: StorageTypes.MEMORY, + }); + shouldFilesExist(false); + validateStorageTypeAndPath(StorageTypes.MEMORY); + createData(); + + setTimeout(() => { + shouldFilesExist(false); + validateStorageTypeAndPath(StorageTypes.MEMORY); + validateData(false, true); + done(); + }, hp.mWait); + }); + + // storage type will become memory, and storage files will not exist + // passing storage path will not affect how storage will initialize + it("6- memory_cPath", (done) => { + Countly.init({ + app_key: appKey, + url: serverUrl, + storage_type: StorageTypes.MEMORY, + storage_path: "../test/customStorageDirectory/", + }); + shouldFilesExist(false); + validateStorageTypeAndPath(StorageTypes.MEMORY); + createData(); + + setTimeout(() => { + shouldFilesExist(false); + validateStorageTypeAndPath(StorageTypes.MEMORY); + validateData(false, true); + done(); + }, hp.mWait); + }); + + // if custom method is provided with memory storage type, sdk will switch to custom method + it("7- memory_cMethod", (done) => { + Countly.init({ + app_key: appKey, + url: serverUrl, + storage_type: StorageTypes.MEMORY, + custom_storage_method: testUtils.getCustomStorage(), + }); + shouldFilesExist(false); + // storage type will be null since custom method is provided + validateStorageTypeAndPath(null); + createData(); + + setTimeout(() => { + shouldFilesExist(false); + validateStorageTypeAndPath(null); + validateData(false, true); + done(); + }, hp.mWait); + }); + + // custom method is provided without any path or storage type information + // storage files will be created and will set the path as the default but sdk will use custom methods + it("8- custom_noPath", (done) => { + Countly.init({ + app_key: appKey, + url: serverUrl, + custom_storage_method: testUtils.getCustomStorage(), + }); + shouldFilesExist(true); + // storage type will be null since custom method is provided + validateStorageTypeAndPath(null); + createData(); + + setTimeout(() => { + shouldFilesExist(true); + validateStorageTypeAndPath(null); + validateData(false, true); + done(); + }, hp.mWait); + }); + + // custom method is provided with custom path + // sdk will set the path as the custom and sdk will use the custom methods + it("9- custom_cPath", (done) => { + Countly.init({ + app_key: appKey, + url: serverUrl, + custom_storage_method: testUtils.getCustomStorage(), + storage_path: "../test/customStorageDirectory/", + }); + shouldFilesExist(true); + // storage type will be null since custom method is provided + // path will be the custom path in this case + validateStorageTypeAndPath(null, "../test/customStorageDirectory/"); + createData(); + + setTimeout(() => { + shouldFilesExist(true); + validateStorageTypeAndPath(null, "../test/customStorageDirectory/"); + validateData(false, true); + done(); + }, hp.mWait); + }); + + // custom storage method with invalid storage path + // sdk will not try to initialize the storage with invalid path and return to the default path + // storage files will exist, type will be null and use custom methods + it("10- custom_invalid_path", (done) => { + Countly.init({ + app_key: appKey, + url: serverUrl, + custom_storage_method: testUtils.getCustomStorage(), + storage_path: undefined, + }); + shouldFilesExist(true); + // storage type will be null since custom method is provided + validateStorageTypeAndPath(null); + createData(); + + setTimeout(() => { + shouldFilesExist(true); + validateStorageTypeAndPath(null); + validateData(false, true); + done(); + }, hp.mWait); + }); + + // custom storage method with invalid methods + // sdk should not use the invalid methods and switch back to file storage + it("11- custom_invalid_cMethod", (done) => { + Countly.init({ + app_key: appKey, + url: serverUrl, + custom_storage_method: testUtils.getInvalidStorage(), + }); + shouldFilesExist(true); + // storage type will be null since custom method is provided + validateStorageTypeAndPath(StorageTypes.FILE); + createData(); + + setTimeout(() => { + shouldFilesExist(true); + validateStorageTypeAndPath(StorageTypes.FILE); + validateData(); + done(); + }, hp.mWait); + }); + + // initStorage method without any parameters + // storage should initialize correctly and be ready to use + it("12- initStorage_noParams_noInit", (done) => { + storage.initStorage(); + shouldFilesExist(true); + // storage type will be File since it's default + validateStorageTypeAndPath(StorageTypes.FILE); + validateStorageMethods(); + done(); + }); + + // initStorage method with File storage type and null path + // storage should initialize correctly with default path + it("13- initStorage_file_nullPath_noInit", (done) => { + storage.initStorage(null, StorageTypes.FILE); + shouldFilesExist(true); + // storage type will be File since it's default + validateStorageTypeAndPath(StorageTypes.FILE); + validateStorageMethods(); + done(); + }); + + // initStorage method with File storage type and custom path + // storage should initialize correctly with custom path + it("14- initStorage_file_cPath_noInit", (done) => { + storage.initStorage("../test/customStorageDirectory/", StorageTypes.FILE); + shouldFilesExist(true); + // storage type will be File since it's default + validateStorageTypeAndPath(StorageTypes.FILE, true); + validateStorageMethods(); + done(); + }); + + // initStorage method with memory storage type and null path + // storage should initialize correctly with memory storage + it("15- initStorage_memory_noInit", (done) => { + storage.initStorage(null, StorageTypes.MEMORY); + shouldFilesExist(false); + validateStorageTypeAndPath(StorageTypes.MEMORY); + validateStorageMethods(); + done(); + }); + + // initStorage method with custom storage method and null path + it("16- initStorage_custom_nullPath_noInit", (done) => { + storage.initStorage(null, null, false, testUtils.getCustomStorage()); + shouldFilesExist(true); + validateStorageTypeAndPath(null); + validateStorageMethods(); + done(); + }); + + // initStorage method with custom storage method and custom path + it("17- initStorage_custom_cPath_noInit", (done) => { + storage.initStorage("../test/customStorageDirectory/", null, false, testUtils.getCustomStorage()); + shouldFilesExist(true); + validateStorageTypeAndPath(null, true); + validateStorageMethods(); + done(); + }); }); \ No newline at end of file