From e1232c30c16a7d2755ffa2f8c3807ef5a55efceb Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Tue, 23 Apr 2024 18:11:58 +0900 Subject: [PATCH 1/2] Added salt --- CHANGELOG.md | 6 + cypress/e2e/salt.cy.js | 88 +++++++++++++ cypress/e2e/web_worker_requests.cy.js | 43 +----- cypress/support/helper.js | 44 ++++++- examples/style/style.css | 1 - modules/Constants.js | 2 +- modules/CountlyClass.js | 182 +++++++++++++------------- modules/Utils.js | 35 ++++- package.json | 2 +- 9 files changed, 266 insertions(+), 137 deletions(-) create mode 100644 cypress/e2e/salt.cy.js diff --git a/CHANGELOG.md b/CHANGELOG.md index d00506b..157a96c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 24.4.0 + +! Minor breaking change ! For implementations using `salt` the browser compatability is tied to SubtleCrypto's `digest` method support + +* Added the `salt` init config flag to add checksums to requests (for secure web context only, no worker support yet) + ## 23.12.6 * Mitigated an issue where error tracking could prevent SDK initialization in async mode diff --git a/cypress/e2e/salt.cy.js b/cypress/e2e/salt.cy.js new file mode 100644 index 0000000..05ac9a5 --- /dev/null +++ b/cypress/e2e/salt.cy.js @@ -0,0 +1,88 @@ +/* eslint-disable cypress/no-unnecessary-waiting */ +/* eslint-disable require-jsdoc */ +var Countly = require("../../Countly.js"); +var Utils = require("../../modules/Utils.js"); +// import * as Countly from "../../dist/countly_umd.js"; +var hp = require("../support/helper.js"); +const crypto = require('crypto'); + +function initMain(salt) { + Countly.init({ + app_key: "YOUR_APP_KEY", + url: "https://your.domain.count.ly", + debug: true, + salt: salt + }); +} +const salt = "salt"; + +/** +* Tests for salt consists of: +* 1. Init without salt +* Create events and intercept the SDK requests. Request params should be normal and there should be no checksum +* 2. Init with salt +* Create events and intercept the SDK requests. Request params should be normal and there should be a checksum with length 64 +* 3. Node and Web Crypto comparison +* Compare the checksums calculated by node crypto api and SDK's web crypto api for the same data. Should be equal +*/ +describe("Salt Tests", () => { + it("Init without salt", () => { + hp.haltAndClearStorage(() => { + initMain(null); + var rqArray = []; + hp.events(); + cy.intercept("GET", "**/i?**", (req) => { + const { url } = req; + rqArray.push(url.split("?")[1]); // get the query string + }); + cy.wait(1000).then(() => { + cy.log(rqArray).then(() => { + for (const rq of rqArray) { + const paramsObject = hp.turnSearchStringToObject(rq); + hp.check_commons(paramsObject); + expect(paramsObject.checksum256).to.be.not.ok; + } + }); + }); + }); + }); + it("Init with salt", () => { + hp.haltAndClearStorage(() => { + initMain(salt); + var rqArray = []; + hp.events(); + cy.intercept("GET", "**/i?**", (req) => { + const { url } = req; + rqArray.push(url.split("?")[1]); + }); + cy.wait(1000).then(() => { + cy.log(rqArray).then(() => { + for (const rq of rqArray) { + const paramsObject = hp.turnSearchStringToObject(rq); + hp.check_commons(paramsObject); + expect(paramsObject.checksum256).to.be.ok; + expect(paramsObject.checksum256.length).to.equal(64); + // TODO: directly check the checksum with the node crypto api. Will need some extra decoding logic + } + }); + }); + }); + }); + it('Node and Web Crypto comparison', () => { + const hash = sha256("text" + salt); // node crypto api + Utils.calculateChecksum("text", salt).then((hash2) => { // SDK uses web crypto api + expect(hash2).to.equal(hash); + }); + }); +}); + +/** + * Calculate sha256 hash of given data + * @param {*} data - data to hash + * @returns {string} - sha256 hash + */ +function sha256(data) { + const hash = crypto.createHash('sha256'); + hash.update(data); + return hash.digest('hex'); +} \ No newline at end of file diff --git a/cypress/e2e/web_worker_requests.cy.js b/cypress/e2e/web_worker_requests.cy.js index ad166a9..d9b9ae7 100644 --- a/cypress/e2e/web_worker_requests.cy.js +++ b/cypress/e2e/web_worker_requests.cy.js @@ -1,4 +1,4 @@ -import { appKey } from "../support/helper"; +import { turnSearchStringToObject, check_commons } from "../support/helper"; const myEvent = { key: "buttonClick", @@ -83,44 +83,3 @@ describe("Web Worker Request Intercepting Tests", () => { }); }); }); - -/** - * Check common params for all requests - * @param {Object} paramsObject - object from search string - */ -function check_commons(paramsObject) { - expect(paramsObject.timestamp).to.be.ok; - expect(paramsObject.timestamp.toString().length).to.equal(13); - expect(paramsObject.hour).to.be.within(0, 23); - expect(paramsObject.dow).to.be.within(0, 7); - expect(paramsObject.app_key).to.equal(appKey); - expect(paramsObject.device_id).to.be.ok; - expect(paramsObject.sdk_name).to.equal("javascript_native_web"); - expect(paramsObject.sdk_version).to.be.ok; - expect(paramsObject.t).to.be.within(0, 3); - expect(paramsObject.av).to.equal(0); // av is 0 as we parsed parsable things - if (!paramsObject.hc) { // hc is direct request - expect(paramsObject.rr).to.be.above(-1); - } - expect(paramsObject.metrics._ua).to.be.ok; -} - -/** - * Turn search string into object with values parsed - * @param {String} searchString - search string - * @returns {object} - object from search string - */ -function turnSearchStringToObject(searchString) { - const searchParams = new URLSearchParams(searchString); - const paramsObject = {}; - for (const [key, value] of searchParams.entries()) { - try { - paramsObject[key] = JSON.parse(value); // try to parse value - } - catch (e) { - paramsObject[key] = value; - } - } - return paramsObject; -} - diff --git a/cypress/support/helper.js b/cypress/support/helper.js index a1b2baa..e85af32 100644 --- a/cypress/support/helper.js +++ b/cypress/support/helper.js @@ -274,6 +274,46 @@ function validateDefaultUtmTags(aq, source, medium, campaign, term, content) { } } +/** + * Check common params for all requests + * @param {Object} paramsObject - object from search string + */ +function check_commons(paramsObject) { + expect(paramsObject.timestamp).to.be.ok; + expect(paramsObject.timestamp.toString().length).to.equal(13); + expect(paramsObject.hour).to.be.within(0, 23); + expect(paramsObject.dow).to.be.within(0, 7); + expect(paramsObject.app_key).to.equal(appKey); + expect(paramsObject.device_id).to.be.ok; + expect(paramsObject.sdk_name).to.equal("javascript_native_web"); + expect(paramsObject.sdk_version).to.be.ok; + expect(paramsObject.t).to.be.within(0, 3); + expect(paramsObject.av).to.equal(0); // av is 0 as we parsed parsable things + if (!paramsObject.hc) { // hc is direct request + expect(paramsObject.rr).to.be.above(-1); + } + expect(paramsObject.metrics._ua).to.be.ok; +} + +/** + * Turn search string into object with values parsed + * @param {String} searchString - search string + * @returns {object} - object from search string + */ +function turnSearchStringToObject(searchString) { + const searchParams = new URLSearchParams(searchString); + const paramsObject = {}; + for (const [key, value] of searchParams.entries()) { + try { + paramsObject[key] = JSON.parse(decodeURIComponent(value)); // try to parse value + } + catch (e) { + paramsObject[key] = decodeURIComponent(value); + } + } + return paramsObject; +} + module.exports = { haltAndClearStorage, sWait, @@ -287,5 +327,7 @@ module.exports = { testNormalFlow, interceptAndCheckRequests, validateDefaultUtmTags, - userDetailObj + userDetailObj, + check_commons, + turnSearchStringToObject }; \ No newline at end of file diff --git a/examples/style/style.css b/examples/style/style.css index b716ee6..c7d47b4 100644 --- a/examples/style/style.css +++ b/examples/style/style.css @@ -11,7 +11,6 @@ body { a { text-decoration: none; color: #000; - padding: 20px; } #header { diff --git a/modules/Constants.js b/modules/Constants.js index 0ef2b4f..a37cdce 100644 --- a/modules/Constants.js +++ b/modules/Constants.js @@ -104,7 +104,7 @@ var healthCheckCounterEnum = Object.freeze({ errorMessage: "cly_hc_error_message", }); -var SDK_VERSION = "23.12.5"; +var SDK_VERSION = "24.4.0"; var SDK_NAME = "javascript_native_web"; // Using this on document.referrer would return an array with 17 elements in it. The 12th element (array[11]) would be the path we are looking for. Others would be things like password and such (use https://regex101.com/ to check more) diff --git a/modules/CountlyClass.js b/modules/CountlyClass.js index 15de017..70f02cd 100644 --- a/modules/CountlyClass.js +++ b/modules/CountlyClass.js @@ -140,6 +140,7 @@ class CountlyClass { this.maxStackTraceLinesPerThread = getConfig("max_stack_trace_lines_per_thread", ob, configurationDefaultValues.MAX_STACKTRACE_LINES_PER_THREAD); this.maxStackTraceLineLength = getConfig("max_stack_trace_line_length", ob, configurationDefaultValues.MAX_STACKTRACE_LINE_LENGTH); this.heatmapWhitelist = getConfig("heatmap_whitelist", ob, []); + self.salt = getConfig("salt", ob, null); self.hcErrorCount = getValueFromStorage(healthCheckCounterEnum.errorCount) || 0; self.hcWarningCount = getValueFromStorage(healthCheckCounterEnum.warningCount) || 0; self.hcStatusCode = getValueFromStorage(healthCheckCounterEnum.statusCode) || -1; @@ -280,6 +281,7 @@ class CountlyClass { if (ignoreReferrers) { log(logLevelEnums.DEBUG, "initialize, referrers to ignore :[" + JSON.stringify(ignoreReferrers) + "]"); } + log(logLevelEnums.DEBUG, "initialize, salt given:[" + !!self.salt + "]"); } catch (e) { log(logLevelEnums.ERROR, "initialize, Could not stringify some config object values"); @@ -626,6 +628,7 @@ class CountlyClass { self.track_domains = undefined; self.storage = undefined; self.enableOrientationTracking = undefined; + self.salt = undefined; self.maxKeyLength = undefined; self.maxValueSize = undefined; self.maxSegmentationValues = undefined; @@ -4048,57 +4051,58 @@ class CountlyClass { log(logLevelEnums.DEBUG, "Sending XML HTTP request"); var xhr = new XMLHttpRequest(); params = params || {}; - var data = prepareParams(params); - var method = "GET"; - if (self.force_post || data.length >= 2000) { - method = "POST"; - } - if (method === "GET") { - xhr.open("GET", url + "?" + data, true); - } - else { - xhr.open("POST", url, true); - xhr.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); - } - for (var header in self.headers) { - xhr.setRequestHeader(header, self.headers[header]); - } - // fallback on error - xhr.onreadystatechange = function () { - if (this.readyState === 4) { - log(logLevelEnums.DEBUG, functionName + " HTTP request completed with status code: [" + this.status + "] and response: [" + this.responseText + "]"); - // response validation function will be selected to also accept JSON arrays if useBroadResponseValidator is true - var isResponseValidated; - if (useBroadResponseValidator) { - // JSON array/object both can pass - isResponseValidated = isResponseValidBroad(this.status, this.responseText); - } - else { - // only JSON object can pass - isResponseValidated = isResponseValid(this.status, this.responseText); - } - if (isResponseValidated) { - if (typeof callback === "function") { - callback(false, params, this.responseText); + prepareParams(params, self.salt).then(saltedData => { + var method = "GET"; + if (self.force_post || saltedData.length >= 2000) { + method = "POST"; + } + if (method === "GET") { + xhr.open("GET", url + "?" + saltedData, true); + } + else { + xhr.open("POST", url, true); + xhr.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); + } + for (var header in self.headers) { + xhr.setRequestHeader(header, self.headers[header]); + } + // fallback on error + xhr.onreadystatechange = function () { + if (this.readyState === 4) { + log(logLevelEnums.DEBUG, functionName + " HTTP request completed with status code: [" + this.status + "] and response: [" + this.responseText + "]"); + // response validation function will be selected to also accept JSON arrays if useBroadResponseValidator is true + var isResponseValidated; + if (useBroadResponseValidator) { + // JSON array/object both can pass + isResponseValidated = isResponseValidBroad(this.status, this.responseText); } - } - else { - log(logLevelEnums.ERROR, functionName + " Invalid response from server"); - if (functionName === "send_request_queue") { - HealthCheck.saveRequestCounters(this.status, this.responseText); + else { + // only JSON object can pass + isResponseValidated = isResponseValid(this.status, this.responseText); } - if (typeof callback === "function") { - callback(true, params, this.status, this.responseText); + if (isResponseValidated) { + if (typeof callback === "function") { + callback(false, params, this.responseText); + } + } + else { + log(logLevelEnums.ERROR, functionName + " Invalid response from server"); + if (functionName === "send_request_queue") { + HealthCheck.saveRequestCounters(this.status, this.responseText); + } + if (typeof callback === "function") { + callback(true, params, this.status, this.responseText); + } } } + }; + if (method === "GET") { + xhr.send(); } - }; - if (method === "GET") { - xhr.send(); - } - else { - xhr.send(data); - } + else { + xhr.send(saltedData); + } + }); } catch (e) { // fallback @@ -4131,56 +4135,58 @@ class CountlyClass { var body = null; params = params || {}; - if (self.force_post || prepareParams(params).length >= 2000) { - method = "POST"; - body = prepareParams(params); - } - else { - url += "?" + prepareParams(params); - } - - // Add custom headers - for (var header in self.headers) { - headers[header] = self.headers[header]; - } - - // Make the fetch request - fetch(url, { - method: method, - headers: headers, - body: body, - }).then(function (res) { - response = res; - return response.text(); - }).then(function (data) { - log(logLevelEnums.DEBUG, functionName + " Fetch request completed wit status code: [" + response.status + "] and response: [" + data + "]"); - var isResponseValidated; - if (useBroadResponseValidator) { - isResponseValidated = isResponseValidBroad(response.status, data); + prepareParams(params, self.salt).then(saltedData => { + if (self.force_post || saltedData.length >= 2000) { + method = "POST"; + body = saltedData; } else { - isResponseValidated = isResponseValid(response.status, data); + url += "?" + saltedData; } - if (isResponseValidated) { - if (typeof callback === "function") { - callback(false, params, data); - } + // Add custom headers + for (var header in self.headers) { + headers[header] = self.headers[header]; } - else { - log(logLevelEnums.ERROR, functionName + " Invalid response from server"); - if (functionName === "send_request_queue") { - HealthCheck.saveRequestCounters(response.status, data); + + // Make the fetch request + fetch(url, { + method: method, + headers: headers, + body: body, + }).then(function (res) { + response = res; + return response.text(); + }).then(function (data) { + log(logLevelEnums.DEBUG, functionName + " Fetch request completed wit status code: [" + response.status + "] and response: [" + data + "]"); + var isResponseValidated; + if (useBroadResponseValidator) { + isResponseValidated = isResponseValidBroad(response.status, data); + } + else { + isResponseValidated = isResponseValid(response.status, data); } + + if (isResponseValidated) { + if (typeof callback === "function") { + callback(false, params, data); + } + } + else { + log(logLevelEnums.ERROR, functionName + " Invalid response from server"); + if (functionName === "send_request_queue") { + HealthCheck.saveRequestCounters(response.status, data); + } + if (typeof callback === "function") { + callback(true, params, response.status, data); + } + } + }).catch(function (error) { + log(logLevelEnums.ERROR, functionName + " Failed Fetch request: " + error); if (typeof callback === "function") { - callback(true, params, response.status, data); + callback(true, params); } - } - }).catch(function (error) { - log(logLevelEnums.ERROR, functionName + " Failed Fetch request: " + error); - if (typeof callback === "function") { - callback(true, params); - } + }); }); } catch (e) { diff --git a/modules/Utils.js b/modules/Utils.js index 80cae76..6470f0a 100644 --- a/modules/Utils.js +++ b/modules/Utils.js @@ -127,14 +127,22 @@ function dispatchErrors(error, fatality, segments) { * Convert JSON object to URL encoded query parameter string * @memberof Countly._internals * @param {Object} params - object with query parameters + * @param {String} salt - salt to be used for checksum calculation * @returns {String} URL encode query string */ -function prepareParams(params) { +function prepareParams(params, salt) { var str = []; for (var i in params) { str.push(i + "=" + encodeURIComponent(params[i])); } - return str.join("&"); + var data = str.join("&"); + if (salt && isBrowser) { // no web worker support (yet!) + return calculateChecksum(data, salt).then(checksum => { + data += "&checksum256=" + checksum; + return data; + }); + } + return Promise.resolve(data); } /** @@ -248,6 +256,26 @@ function truncateSingleValue(str, limit, errorLog, logCall) { return newStr; } +/** + * Calculates the checksum of the data with the given salt + * Uses SHA-256 algorithm with web crypto API + * Implementation based on https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest + * TODO: Turn to async function when we drop support for older browsers + * @param {string} data - data to be used for checksum calculation (concatenated query parameters) + * @param {string} salt - salt to be used for checksum calculation + * @returns {string} checksum in hex format + */ +function calculateChecksum(data, salt) { + const msgUint8 = new TextEncoder().encode(data + salt); // encode as (utf-8) Uint8Array + return crypto.subtle.digest("SHA-256", msgUint8).then((hashBuffer) => { // hash the message + const hashArray = Array.from(new Uint8Array(hashBuffer)); // convert buffer to byte array + const hashHex = hashArray + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); // convert bytes to hex string + return hashHex; + }); +} + /** * Polyfill to get closest parent matching nodeName * @param {HTMLElement} el - element from which to search @@ -640,5 +668,6 @@ export { showLoader, checkIfLoggingIsOn, hideLoader, - currentUserAgentDataString + currentUserAgentDataString, + calculateChecksum }; \ No newline at end of file diff --git a/package.json b/package.json index 6e86b99..7f349cc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "countly-sdk-js", - "version": "23.12.5", + "version": "24.4.0", "description": "Countly JavaScript SDK", "type": "module", "main": "Countly.js", From 00c5950cf37528925a497016b880d7c1174b7340 Mon Sep 17 00:00:00 2001 From: turtledreams <62231246+turtledreams@users.noreply.github.com> Date: Tue, 23 Apr 2024 18:55:34 +0900 Subject: [PATCH 2/2] worker --- CHANGELOG.md | 2 +- modules/CountlyClass.js | 4 +++- modules/Utils.js | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 157a96c..0936e9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ! Minor breaking change ! For implementations using `salt` the browser compatability is tied to SubtleCrypto's `digest` method support -* Added the `salt` init config flag to add checksums to requests (for secure web context only, no worker support yet) +* Added the `salt` init config flag to add checksums to requests (for secure contexts only) ## 23.12.6 diff --git a/modules/CountlyClass.js b/modules/CountlyClass.js index 70f02cd..b7396df 100644 --- a/modules/CountlyClass.js +++ b/modules/CountlyClass.js @@ -31,7 +31,8 @@ import { loadCSS, showLoader, checkIfLoggingIsOn, - hideLoader + hideLoader, + calculateChecksum } from "./Utils.js"; import { isBrowser, Countly } from "./Platform.js"; @@ -4647,6 +4648,7 @@ class CountlyClass { generateUUID: generateUUID, sendEventsForced: sendEventsForced, isUUID: isUUID, + calculateChecksum: calculateChecksum, isReferrerUsable: isReferrerUsable, getId: getStoredIdOrGenerateId, heartBeat: heartBeat, diff --git a/modules/Utils.js b/modules/Utils.js index 6470f0a..c9707b9 100644 --- a/modules/Utils.js +++ b/modules/Utils.js @@ -136,7 +136,7 @@ function prepareParams(params, salt) { str.push(i + "=" + encodeURIComponent(params[i])); } var data = str.join("&"); - if (salt && isBrowser) { // no web worker support (yet!) + if (salt) { return calculateChecksum(data, salt).then(checksum => { data += "&checksum256=" + checksum; return data;