diff --git a/.eslintignore b/.eslintignore index bc11e675809..303356d0450 100644 --- a/.eslintignore +++ b/.eslintignore @@ -26,4 +26,5 @@ scripts/skipPrepareScript.js .eslintignore .prettierignore *.json -Dockerfile* \ No newline at end of file +Dockerfile* +*.properties \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 89e496ad8b8..f150d914da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +## [1.89.0](https://github.com/rudderlabs/rudder-transformer/compare/v1.88.3...v1.89.0) (2025-01-27) + + +### Features + +* add DATA WAREHOUSE to integrations object in shopify pixel source ([#3980](https://github.com/rudderlabs/rudder-transformer/issues/3980)) ([3c20393](https://github.com/rudderlabs/rudder-transformer/commit/3c20393a0e4e3ec98d316820d187310bfec1faea)), closes [#3973](https://github.com/rudderlabs/rudder-transformer/issues/3973) [#3957](https://github.com/rudderlabs/rudder-transformer/issues/3957) [#3957](https://github.com/rudderlabs/rudder-transformer/issues/3957) +* add redis support in shopify pixel for id stitching ([#4001](https://github.com/rudderlabs/rudder-transformer/issues/4001)) ([23ad10a](https://github.com/rudderlabs/rudder-transformer/commit/23ad10a4bd470f09a75e9230d8c860b703a5a1f9)), closes [#3957](https://github.com/rudderlabs/rudder-transformer/issues/3957) +* add support for form format ([#4000](https://github.com/rudderlabs/rudder-transformer/issues/4000)) ([1fc15bf](https://github.com/rudderlabs/rudder-transformer/commit/1fc15bfd4b88e0b252c780a278376cea4c594057)) +* **http:** resolves bug fixes raised during testing ([#3991](https://github.com/rudderlabs/rudder-transformer/issues/3991)) ([a86f009](https://github.com/rudderlabs/rudder-transformer/commit/a86f0094cbef9428e7aeda4965b580f29c89f706)) + + +### Bug Fixes + +* add mappings to mp event ([#3997](https://github.com/rudderlabs/rudder-transformer/issues/3997)) ([2eab465](https://github.com/rudderlabs/rudder-transformer/commit/2eab46557743702a834597d0d0267a17e561516d)) +* sonar issues in adobe analytics ([#3999](https://github.com/rudderlabs/rudder-transformer/issues/3999)) ([d74c4ab](https://github.com/rudderlabs/rudder-transformer/commit/d74c4ab7ef582eb83d67c4b295aa5d3845c15166)) +* sonar issues in user transformations static lookup ([#3998](https://github.com/rudderlabs/rudder-transformer/issues/3998)) ([a7d6b8f](https://github.com/rudderlabs/rudder-transformer/commit/a7d6b8fca8681d2eb3bf1751bf7855a7cc220d3b)) +* sonar issues in various components ([#4006](https://github.com/rudderlabs/rudder-transformer/issues/4006)) ([454451d](https://github.com/rudderlabs/rudder-transformer/commit/454451dba3260a3664088e2bb009fd8a11cbf957)) + ### [1.88.3](https://github.com/rudderlabs/rudder-transformer/compare/v1.88.2...v1.88.3) (2025-01-24) diff --git a/benchmark/index.js b/benchmark/index.js deleted file mode 100644 index 919fa3c6de0..00000000000 --- a/benchmark/index.js +++ /dev/null @@ -1,193 +0,0 @@ -/* eslint-disable import/no-extraneous-dependencies */ -/* eslint-disable func-names */ -/* eslint-disable no-restricted-syntax */ -/* eslint-disable guard-for-in */ -/* eslint-disable no-await-in-loop */ -/* eslint-disable no-use-before-define */ -/* eslint-disable import/no-dynamic-require */ -/* eslint-disable global-require */ -const Benchmark = require('benchmark-suite'); -const fs = require('fs'); -const path = require('path'); -const Commander = require('commander'); -const logger = require('./metaLogger'); -const versionedRouter = require('../src/versionedRouter'); -const cdkV2Handler = require('../src/cdk/v2/handler'); - -const supportedDestinations = ['algolia', 'pinterest_tag']; - -logger.info(); - -const command = new Commander.Command(); -command - .allowUnknownOption() - .option( - '-d, --destinations ', - 'Enter destination names separated by comma', - supportedDestinations.toString(), - ) - .option( - '-bt, --benchmarktype ', - 'Enter the benchmark type (Operations or Memory)', - 'Operations', - ) - .option('-f, --feature ', 'Enter feature name (proc or rt)', 'proc') - .parse(); - -const getTestFileName = (intg, testSufix) => { - const featureSufix = cmdOpts.feature === 'rt' ? '_router' : ''; - return `${intg}${featureSufix}${testSufix}.json`; -}; - -const testDataDir = path.join(__dirname, '../test/__tests__/data'); -const getTestData = (intgList, fileNameSuffixes) => { - const intgTestData = {}; - intgList.forEach((intg) => { - // Use the last valid test data file - fileNameSuffixes.forEach((fileNameSuffix) => { - try { - intgTestData[intg] = JSON.parse( - fs.readFileSync(path.join(testDataDir, getTestFileName(intg, fileNameSuffix)), { - encoding: 'utf-8', - }), - ); - } catch (err) { - // logger.error( - // `Unable to load the data for: "${intg}" suffix: "${fileNameSuffix}"` - // ); - // logger.error(`Raw error: "${err}"`); - } - }); - }); - return intgTestData; -}; - -const cmdOpts = command.opts(); - -// Initialize data for destinations -const destinationsList = cmdOpts.destinations - .split(',') - .map((x) => x.trim()) - .filter((x) => x !== ''); -logger.info('Destinations:', destinationsList, 'feature:', cmdOpts.feature); -logger.info(); -const destDataset = getTestData(destinationsList, ['_input', '']); - -const nativeDestHandlers = {}; -const destCdKWorkflowEngines = {}; - -const benchmarkType = cmdOpts.benchmarktype.trim(); - -const getNativeHandleName = () => { - let handleName = 'process'; - if (cmdOpts.feature === 'rt') { - handleName = 'processRouterDest'; - } - return handleName; -}; - -async function initializeHandlers() { - for (const idx in destinationsList) { - const dest = destinationsList[idx]; - - // Native destination handler - nativeDestHandlers[dest] = versionedRouter.getDestHandler('v0', dest)[getNativeHandleName()]; - - // Get the CDK 2.0 workflow engine instance - destCdKWorkflowEngines[dest] = await cdkV2Handler.getWorkflowEngine(dest, cmdOpts.feature); - } -} - -async function runDataset(suitDesc, input, intg, params) { - logger.info('=========================================='); - logger.info(suitDesc); - logger.info('=========================================='); - - const results = {}; - const suite = new Benchmark(suitDesc, benchmarkType); - - Object.keys(params).forEach((opName) => { - const handler = params[opName].handlerResolver(intg); - const args = params[opName].argsResolver(intg, input); - suite.add(opName, async () => { - try { - await handler(...args); - } catch (err) { - // logger.info(err); - // Do nothing - } - }); - }); - - suite - .on('cycle', (result) => { - results[result.end.name] = { stats: result.end.stats }; - }) - .on('complete', (result) => { - logger.info( - benchmarkType === 'Operations' ? 'Fastest: ' : 'Memory intensive: ', - `"${result.end.name}"`, - ); - logger.info(); - Object.keys(results).forEach((impl) => { - logger.info(`"${impl}" - `, suite.formatStats(results[impl].stats)); - - if (result.end.name !== impl) { - if (benchmarkType === 'Operations') { - logger.info( - `-> "${result.end.name}" is faster by ${( - results[impl].stats.mean / result.end.stats.mean - ).toFixed(1)} times to "${impl}"`, - ); - } else { - logger.info( - `-> "${result.end.name}" consumed ${( - result.end.stats.mean / results[impl].stats.mean - ).toFixed(1)} times memory compared to "${impl}"`, - ); - } - } - - logger.info(); - }); - }); - - await suite.run({ time: 1000 }); -} - -async function runIntgDataset(dataset, type, params) { - for (const intg in dataset) { - for (const tc in dataset[intg]) { - const curTcData = dataset[intg][tc]; - let tcInput = curTcData; - let tcDesc = `${type} - ${intg} - ${cmdOpts.feature} - ${tc}`; - // New test data file structure - if ('description' in curTcData && 'input' in curTcData && 'output' in curTcData) { - tcInput = curTcData.input; - tcDesc += ` - "${curTcData.description}"`; - } - - await runDataset(tcDesc, tcInput, intg, params); - } - } -} - -async function run() { - // Initialize CDK and native handlers - await initializeHandlers(); - - // Destinations - await runIntgDataset(destDataset, 'Destination', { - native: { - handlerResolver: (intg) => nativeDestHandlers[intg], - argsResolver: (_intg, input) => [input], - }, - 'CDK 2.0': { - handlerResolver: () => cdkV2Handler.process, - argsResolver: (intg, input) => [destCdKWorkflowEngines[intg], input], - }, - }); -} - -// Start suites -run(); diff --git a/benchmark/metaLogger.js b/benchmark/metaLogger.js deleted file mode 100644 index b89ad71066d..00000000000 --- a/benchmark/metaLogger.js +++ /dev/null @@ -1,36 +0,0 @@ -/* istanbul ignore file */ - -const logger = require('../src/logger'); - -logger.setLogLevel('random'); - -const debug = (...args) => { - logger.setLogLevel('debug'); - logger.debug(...args); - logger.setLogLevel('random'); -}; - -const info = (...args) => { - logger.setLogLevel('info'); - logger.info(...args); - logger.setLogLevel('random'); -}; - -const warn = (...args) => { - logger.setLogLevel('warn'); - logger.warn(...args); - logger.setLogLevel('random'); -}; - -const error = (...args) => { - logger.setLogLevel('error'); - logger.error(...args); - logger.setLogLevel('random'); -}; - -module.exports = { - debug, - info, - warn, - error, -}; diff --git a/package-lock.json b/package-lock.json index 61fdfb9a63b..0a5bf4db982 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "rudder-transformer", - "version": "1.88.3", + "version": "1.89.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "rudder-transformer", - "version": "1.88.3", + "version": "1.89.0", "license": "ISC", "dependencies": { "@amplitude/ua-parser-js": "0.7.24", @@ -20,7 +20,7 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.13", - "@rudderstack/json-template-engine": "^0.18.0", + "@rudderstack/json-template-engine": "^0.19.4", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", @@ -32,6 +32,7 @@ "component-each": "^0.2.6", "crypto-js": "^4.2.0", "dotenv": "^16.0.3", + "fast-xml-parser": "^4.5.1", "flat": "^5.0.2", "form-data": "^4.0.0", "get-value": "^3.0.1", @@ -46,7 +47,6 @@ "json-diff": "^1.0.3", "json-size": "^1.0.0", "jsontoxml": "^1.0.1", - "jstoxml": "^5.0.2", "koa": "^2.15.3", "koa-bodyparser": "^4.4.0", "koa2-swagger-ui": "^5.7.0", @@ -1134,6 +1134,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-personalize/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/client-s3": { "version": "3.637.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.637.0.tgz", @@ -1570,6 +1591,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-s3/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/client-sso": { "version": "3.650.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.650.0.tgz", @@ -2014,6 +2056,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-sso-oidc/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/client-sso/node_modules/@aws-sdk/util-user-agent-browser": { "version": "3.649.0", "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.649.0.tgz", @@ -2481,6 +2544,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/client-sts/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/core": { "version": "3.649.0", "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.649.0.tgz", @@ -2502,6 +2586,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/core/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/credential-provider-cognito-identity": { "version": "3.650.0", "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.650.0.tgz", @@ -3604,6 +3709,27 @@ "node": ">=16.0.0" } }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/middleware-ssec": { "version": "3.609.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.609.0.tgz", @@ -6642,8 +6768,9 @@ } }, "node_modules/@rudderstack/json-template-engine": { - "version": "0.18.0", - "license": "MIT" + "version": "0.19.4", + "resolved": "https://registry.npmjs.org/@rudderstack/json-template-engine/-/json-template-engine-0.19.4.tgz", + "integrity": "sha512-/RTQeHCTBPsQt1U4bok7HBFPcIlc+1cMHYx6qCRhgCS+uhCmN84F9GI7K/H3y4nrn1CHctaHvXCwplZZA60FTQ==" }, "node_modules/@rudderstack/workflow-engine": { "version": "0.8.13", @@ -12330,7 +12457,9 @@ "license": "MIT" }, "node_modules/fast-xml-parser": { - "version": "4.4.1", + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.1.tgz", + "integrity": "sha512-y655CeyUQ+jj7KBbYMc4FG01V8ZQqjN+gDYGJ50RtfsUB8iG9AmwmwoAgeKLJdmueKKMrH1RJ7yXHTSoczdv5w==", "funding": [ { "type": "github", @@ -12341,7 +12470,6 @@ "url": "https://paypal.me/naturalintelligence" } ], - "license": "MIT", "dependencies": { "strnum": "^1.0.5" }, @@ -16385,10 +16513,6 @@ "node": ">=0.2.0" } }, - "node_modules/jstoxml": { - "version": "5.0.2", - "license": "MIT" - }, "node_modules/keygrip": { "version": "1.1.0", "license": "MIT", diff --git a/package.json b/package.json index 4485d386e80..460c0801a8e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rudder-transformer", - "version": "1.88.3", + "version": "1.89.0", "description": "", "homepage": "https://github.com/rudderlabs/rudder-transformer#readme", "bugs": { @@ -65,7 +65,7 @@ "@ndhoule/extend": "^2.0.0", "@pyroscope/nodejs": "^0.2.9", "@rudderstack/integrations-lib": "^0.2.13", - "@rudderstack/json-template-engine": "^0.18.0", + "@rudderstack/json-template-engine": "^0.19.4", "@rudderstack/workflow-engine": "^0.8.13", "@shopify/jest-koa-mocks": "^5.1.1", "ajv": "^8.12.0", @@ -77,6 +77,7 @@ "component-each": "^0.2.6", "crypto-js": "^4.2.0", "dotenv": "^16.0.3", + "fast-xml-parser": "^4.5.1", "flat": "^5.0.2", "form-data": "^4.0.0", "get-value": "^3.0.1", @@ -91,7 +92,6 @@ "json-diff": "^1.0.3", "json-size": "^1.0.0", "jsontoxml": "^1.0.1", - "jstoxml": "^5.0.2", "koa": "^2.15.3", "koa-bodyparser": "^4.4.0", "koa2-swagger-ui": "^5.7.0", diff --git a/sonar-project.properties b/sonar-project.properties index 6c45b0ce39d..284f159cb3d 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -18,7 +18,7 @@ sonar.testExecutionReportPaths=reports/sonar/results-report.xml sonar.eslint.reportPaths=reports/eslint.json # Path to sources -sonar.sources=src,benchmark +sonar.sources=src sonar.inclusions=**/*.js sonar.exclusions=**/*.json,**/*.html,**/*.png,**/*.jpg,**/*.gif,**/*.svg,**/*.yml,src/util/libExtractor.js,src/util/url-search-params.min.js,src/util/lodash-es-core.js diff --git a/src/adapters/network.js b/src/adapters/network.js index aeb1cc128b3..13ebc89aadc 100644 --- a/src/adapters/network.js +++ b/src/adapters/network.js @@ -323,7 +323,7 @@ function stringifyQueryParam(value) { * @param {Object} payload * @returns {String} */ -function getFormData(payload) { +function getFormData(payload = {}) { const data = new URLSearchParams(); Object.keys(payload).forEach((key) => { const payloadValStr = stringifyQueryParam(payload[key]); @@ -332,6 +332,22 @@ function getFormData(payload) { return data; } +function extractPayloadForFormat(payload, format) { + switch (format) { + case 'JSON_ARRAY': + return payload?.batch; + case 'JSON': + return payload; + case 'XML': + return payload?.payload; + case 'FORM': + return getFormData(payload); + default: + logger.debug(`Unknown payload format: ${format}`); + return undefined; + } +} + /** * Prepares the proxy request * @param {*} request @@ -340,33 +356,29 @@ function getFormData(payload) { const prepareProxyRequest = (request) => { const { body, method, params, endpoint, headers, destinationConfig: config } = request; const { payload, payloadFormat } = getPayloadData(body); - let data; - - switch (payloadFormat) { - case 'JSON_ARRAY': - data = payload.batch; - // TODO: add headers - break; - case 'JSON': - data = payload; - break; - case 'XML': - data = payload.payload; - break; - case 'FORM': - data = getFormData(payload); - break; - case 'MULTIPART-FORM': - // TODO: - break; - default: - logger.debug(`body format ${payloadFormat} not supported`); - } + const data = extractPayloadForFormat(payload, payloadFormat); // Ref: https://github.com/rudderlabs/rudder-server/blob/master/router/network.go#L164 headers['User-Agent'] = 'RudderLabs'; return removeUndefinedValues({ endpoint, data, params, headers, method, config }); }; +const getHttpWrapperMethod = (requestType) => { + switch (requestType) { + case 'get': + return httpGET; + case 'put': + return httpPUT; + case 'patch': + return httpPATCH; + case 'delete': + return httpDELETE; + case 'constructor': + return httpSend; + default: + return httpPOST; + } +}; + /** * handles http request and sends the response in a simple format that is followed in transformer * @@ -392,27 +404,7 @@ const prepareProxyRequest = (request) => { }) */ const handleHttpRequest = async (requestType = 'post', ...httpArgs) => { - let httpWrapperMethod; - switch (requestType.toLowerCase()) { - case 'get': - httpWrapperMethod = httpGET; - break; - case 'put': - httpWrapperMethod = httpPUT; - break; - case 'patch': - httpWrapperMethod = httpPATCH; - break; - case 'delete': - httpWrapperMethod = httpDELETE; - break; - case 'constructor': - httpWrapperMethod = httpSend; - break; - default: - httpWrapperMethod = httpPOST; - break; - } + const httpWrapperMethod = getHttpWrapperMethod(requestType.toLowerCase()); const httpResponse = await httpWrapperMethod(...httpArgs); const processedResponse = processAxiosResponse(httpResponse); return { httpResponse, processedResponse }; diff --git a/src/cdk/v2/destinations/bluecore/utils.js b/src/cdk/v2/destinations/bluecore/utils.js index 543b6de745f..e82a1355b51 100644 --- a/src/cdk/v2/destinations/bluecore/utils.js +++ b/src/cdk/v2/destinations/bluecore/utils.js @@ -19,74 +19,83 @@ const { EVENT_NAME_MAPPING, IDENTIFY_EXCLUSION_LIST, TRACK_EXCLUSION_LIST } = re const { EventType } = require('../../../../constants'); const { MAPPING_CONFIG, CONFIG_CATEGORIES } = require('./config'); -/** - * Verifies the correctness of payload for different events. - * - * @param {Object} payload - The payload object containing event information. - * @param {Object} message - The message object containing additional information. - * @throws {InstrumentationError} - Throws an error if required properties are missing. - * @returns {void} - */ -const verifyPayload = (payload, message) => { +const validateCustomerProperties = (payload, eventName) => { + if ( + !isDefinedAndNotNull(payload?.properties?.customer) || + Object.keys(payload.properties.customer).length === 0 + ) { + throw new InstrumentationError( + `[Bluecore] property:: No relevant trait to populate customer information, which is required for ${eventName} action`, + ); + } +}; + +const validateIdentifyAction = (message) => { if ( message.type === EventType.IDENTIFY && isDefinedNotNullNotEmpty(message.traits?.action) && message.traits?.action !== 'identify' ) { throw new InstrumentationError( - "[Bluecore] traits.action must be 'identify' for identify action", + "[Bluecore] traits.action must be 'identify' for identify action", ); } - switch (payload.event) { - case 'search': - if (!payload?.properties?.search_term) { - throw new InstrumentationError( - '[Bluecore] property:: search_query is required for search event', - ); - } - break; - case 'purchase': - if (!isDefinedAndNotNull(payload?.properties?.order_id)) { - throw new InstrumentationError( - '[Bluecore] property:: order_id is required for purchase event', - ); - } - if (!isDefinedAndNotNull(payload?.properties?.total)) { - throw new InstrumentationError( - '[Bluecore] property:: total is required for purchase event', - ); - } - if ( - !isDefinedAndNotNull(payload?.properties?.customer) || - Object.keys(payload.properties.customer).length === 0 - ) { - throw new InstrumentationError( - `[Bluecore] property:: No relevant trait to populate customer information, which is required for ${payload.event} event`, - ); - } - break; - case 'identify': - case 'optin': - case 'unsubscribe': - if (!isDefinedAndNotNullAndNotEmpty(getFieldValueFromMessage(message, 'email'))) { - throw new InstrumentationError( - `[Bluecore] property:: email is required for ${payload.event} action`, - ); - } - if ( - !isDefinedAndNotNull(payload?.properties?.customer) || - Object.keys(payload.properties.customer).length === 0 - ) { - throw new InstrumentationError( - `[Bluecore] property:: No relevant trait to populate customer information, which is required for ${payload.event} action`, - ); - } - break; - default: - break; +}; +const validateSearchEvent = (payload) => { + if (!payload?.properties?.search_term) { + throw new InstrumentationError( + '[Bluecore] property:: search_query is required for search event', + ); } }; +const validatePurchaseEvent = (payload) => { + if (!isDefinedAndNotNull(payload?.properties?.order_id)) { + throw new InstrumentationError('[Bluecore] property:: order_id is required for purchase event'); + } + if (!isDefinedAndNotNull(payload?.properties?.total)) { + throw new InstrumentationError('[Bluecore] property:: total is required for purchase event'); + } + validateCustomerProperties(payload, 'purchase'); +}; + +const validateCustomerEvent = (payload, message) => { + if (!isDefinedAndNotNullAndNotEmpty(getFieldValueFromMessage(message, 'email'))) { + throw new InstrumentationError( + `[Bluecore] property:: email is required for ${payload.event} action`, + ); + } + validateCustomerProperties(payload, payload.event); +}; + +const validateEventSpecificPayload = (payload, message) => { + const eventValidators = { + search: validateSearchEvent, + purchase: validatePurchaseEvent, + identify: validateCustomerEvent, + optin: validateCustomerEvent, + unsubscribe: validateCustomerEvent, + }; + + const validator = eventValidators[payload.event]; + if (validator) { + validator(payload, message); + } +}; + +/** + * Verifies the correctness of payload for different events. + * + * @param {Object} payload - The payload object containing event information. + * @param {Object} message - The message object containing additional information. + * @throws {InstrumentationError} - Throws an error if required properties are missing. + * @returns {void} + */ +const verifyPayload = (payload, message) => { + validateIdentifyAction(message); + validateEventSpecificPayload(payload, message); +}; + /** * Deduces the track event name based on the provided track event name and configuration. * diff --git a/src/cdk/v2/destinations/clicksend/utils.js b/src/cdk/v2/destinations/clicksend/utils.js index 797ea12025e..c6a8c281e3b 100644 --- a/src/cdk/v2/destinations/clicksend/utils.js +++ b/src/cdk/v2/destinations/clicksend/utils.js @@ -4,8 +4,11 @@ const { BatchUtils } = require('@rudderstack/workflow-engine'); const { SMS_SEND_ENDPOINT, MAX_BATCH_SIZE, COMMON_CONTACT_DOMAIN } = require('./config'); const { isDefinedAndNotNullAndNotEmpty, isDefinedAndNotNull } = require('../../../../v0/util'); -const getEndIdentifyPoint = (contactId, contactListId) => - `${COMMON_CONTACT_DOMAIN}/${contactListId}/contacts${isDefinedAndNotNullAndNotEmpty(contactId) ? `/${contactId}` : ''}`; +const getEndIdentifyPoint = (contactId, contactListId) => { + const basePath = `${COMMON_CONTACT_DOMAIN}/${contactListId}/contacts`; + const contactSuffix = isDefinedAndNotNullAndNotEmpty(contactId) ? `/${contactId}` : ''; + return basePath + contactSuffix; +}; const validateIdentifyPayload = (payload) => { if ( diff --git a/src/cdk/v2/destinations/http/procWorkflow.yaml b/src/cdk/v2/destinations/http/procWorkflow.yaml index 080dcdd80af..57483f31831 100644 --- a/src/cdk/v2/destinations/http/procWorkflow.yaml +++ b/src/cdk/v2/destinations/http/procWorkflow.yaml @@ -30,35 +30,38 @@ steps: - name: deduceBodyFormat template: | - $.context.format = .destination.Config.format ?? 'JSON'; + const format = .destination.Config.format ?? 'JSON'; + $.context.format = $.CONTENT_TYPES_MAP[format]; - name: buildHeaders template: | const configAuthHeaders = $.getAuthHeaders(.destination.Config); const additionalConfigHeaders = $.getCustomMappings(.message, .destination.Config.headers); + const metadataHeaders = $.metadataHeaders($.context.format); $.context.headers = { ...configAuthHeaders, - ...additionalConfigHeaders + ...additionalConfigHeaders, + ...metadataHeaders, } - name: prepareParams template: | - $.context.params = $.getCustomMappings(.message, .destination.Config.queryParams) + const params = $.getCustomMappings(.message, .destination.Config.queryParams); + $.context.params = $.encodeParamsObject(params); - name: deduceEndPoint template: | - $.context.endpoint = $.addPathParams(.message, .destination.Config.apiUrl); + $.context.endpoint = $.prepareEndpoint(.message, .destination.Config.apiUrl, .destination.Config.pathParams); - name: prepareBody template: | const payload = $.getCustomMappings(.message, .destination.Config.propertiesMapping); - $.context.payload = $.removeUndefinedAndNullValues($.excludeMappedFields(payload, .destination.Config.propertiesMapping)) - $.context.format === "XML" && !$.isEmptyObject($.context.payload) ? $.context.payload = {payload: $.getXMLPayload($.context.payload)}; + $.context.payload = $.prepareBody(payload, $.context.format); - name: buildResponseForProcessTransformation template: | const response = $.defaultRequestConfig(); - $.context.format === "JSON" ? response.body.JSON = $.context.payload: response.body.XML = $.context.payload; + response.body[$.context.format] = $.context.payload; response.endpoint = $.context.endpoint; response.headers = $.context.headers; response.method = $.context.method; diff --git a/src/cdk/v2/destinations/http/utils.js b/src/cdk/v2/destinations/http/utils.js index 355eb034870..613e19d4531 100644 --- a/src/cdk/v2/destinations/http/utils.js +++ b/src/cdk/v2/destinations/http/utils.js @@ -1,15 +1,22 @@ -const { toXML } = require('jstoxml'); +const { XMLBuilder } = require('fast-xml-parser'); const { groupBy } = require('lodash'); const { createHash } = require('crypto'); const { ConfigurationError } = require('@rudderstack/integrations-lib'); const { BatchUtils } = require('@rudderstack/workflow-engine'); +const jsonpath = require('rs-jsonpath'); const { base64Convertor, applyCustomMappings, isEmptyObject, - applyJSONStringTemplate, + removeUndefinedAndNullValues, } = require('../../../../v0/util'); +const CONTENT_TYPES_MAP = { + JSON: 'JSON', + XML: 'XML', + FORM: 'FORM', +}; + const getAuthHeaders = (config) => { let headers; switch (config.auth) { @@ -41,48 +48,101 @@ const getCustomMappings = (message, mapping) => { } }; -const addPathParams = (message, apiUrl) => { - try { - return applyJSONStringTemplate(message, `\`${apiUrl}\``); - } catch (e) { - throw new ConfigurationError(`Error in api url template: ${e.message}`); +const encodeParamsObject = (params) => { + if (!params || typeof params !== 'object') { + return {}; // Return an empty object if input is null, undefined, or not an object } + return Object.keys(params) + .filter((key) => params[key] !== undefined) + .reduce((acc, key) => { + acc[encodeURIComponent(key)] = encodeURIComponent(params[key]); + return acc; + }, {}); }; -const excludeMappedFields = (payload, mapping) => { - const rawPayload = { ...payload }; - if (mapping) { - mapping.forEach(({ from, to }) => { - // continue when from === to - if (from === to) return; - - // Remove the '$.' prefix and split the remaining string by '.' - const keys = from.replace(/^\$\./, '').split('.'); - let current = rawPayload; - - // Traverse to the parent of the key to be removed - keys.slice(0, -1).forEach((key) => { - if (current?.[key]) { - current = current[key]; - } else { - current = null; - } - }); - - if (current) { - // Remove the 'from' field from input payload - delete current[keys[keys.length - 1]]; - } - }); +const getPathValueFromJsonpath = (message, path) => { + let finalPath = path; + if (path.includes('/')) { + throw new ConfigurationError('Path value cannot contain "/"'); + } + if (path.includes('$')) { + try { + [finalPath = null] = jsonpath.query(message, path); + } catch (error) { + throw new ConfigurationError( + `An error occurred while querying the JSON path: ${error.message}`, + ); + } + if (finalPath === null) { + throw new ConfigurationError('Path not found in the object.'); + } } + return finalPath; +}; - return rawPayload; +const getPathParamsSubString = (message, pathParamsArray) => { + if (pathParamsArray.length === 0) { + return ''; + } + const pathParamsValuesArray = pathParamsArray.map((pathParam) => + encodeURIComponent(getPathValueFromJsonpath(message, pathParam.path)), + ); + return `/${pathParamsValuesArray.join('/')}`; }; -const getXMLPayload = (payload) => - toXML(payload, { - header: true, - }); +const prepareEndpoint = (message, apiUrl, pathParams) => { + if (!Array.isArray(pathParams)) { + return apiUrl; + } + const requestUrl = apiUrl.replace(/\/{1,10}$/, ''); + const pathParamsSubString = getPathParamsSubString(message, pathParams); + return `${requestUrl}${pathParamsSubString}`; +}; + +const sanitizeKey = (key) => + key + .replace(/[^\w.-]/g, '_') // Replace invalid characters with underscores + .replace(/^[^A-Z_a-z]/, '_'); // Ensure key starts with a letter or underscore + +const preprocessJson = (obj) => { + if (typeof obj !== 'object' || obj === null) { + // Handle null values: add xsi:nil attribute + if (obj === null) { + return { '@_xsi:nil': 'true' }; + } + return obj; // Return primitive values as is + } + + if (Array.isArray(obj)) { + return obj.map(preprocessJson); + } + + return Object.entries(obj).reduce((acc, [key, value]) => { + const sanitizedKey = sanitizeKey(key); + acc[sanitizedKey] = preprocessJson(value); + return acc; + }, {}); +}; + +const getXMLPayload = (payload) => { + const builderOptions = { + ignoreAttributes: false, // Include attributes if they exist + suppressEmptyNode: false, // Ensures that null or undefined values are not omitted + attributeNamePrefix: '@_', + }; + + if (Object.keys(payload).length !== 1) { + throw new ConfigurationError( + `Error: XML supports only one root key. Please update request body mappings accordingly`, + ); + } + const rootKey = Object.keys(payload)[0]; + + const builder = new XMLBuilder(builderOptions); + const processesPayload = preprocessJson(payload); + processesPayload[rootKey]['@_xmlns:xsi'] = 'http://www.w3.org/2001/XMLSchema-instance'; + return `${builder.build(processesPayload)}`; +}; const getMergedEvents = (batch) => { const events = []; @@ -94,6 +154,29 @@ const getMergedEvents = (batch) => { return events; }; +const metadataHeaders = (contentType) => { + switch (contentType) { + case CONTENT_TYPES_MAP.XML: + return { 'Content-Type': 'application/xml' }; + case CONTENT_TYPES_MAP.FORM: + return { 'Content-Type': 'application/x-www-form-urlencoded' }; + default: + return { 'Content-Type': 'application/json' }; + } +}; + +const prepareBody = (payload, contentType) => { + let responseBody; + if (contentType === CONTENT_TYPES_MAP.XML && !isEmptyObject(payload)) { + responseBody = { + payload: getXMLPayload(payload), + }; + } else { + responseBody = removeUndefinedAndNullValues(payload); + } + return responseBody; +}; + const mergeMetadata = (batch) => batch.map((event) => event.metadata[0]); const createHashKey = (endpoint, headers, params) => { @@ -147,10 +230,12 @@ const batchSuccessfulEvents = (events, batchSize) => { }; module.exports = { + CONTENT_TYPES_MAP, getAuthHeaders, getCustomMappings, - addPathParams, - excludeMappedFields, - getXMLPayload, + encodeParamsObject, + prepareEndpoint, + metadataHeaders, + prepareBody, batchSuccessfulEvents, }; diff --git a/src/cdk/v2/destinations/http/utils.test.js b/src/cdk/v2/destinations/http/utils.test.js new file mode 100644 index 00000000000..64fc87b66b4 --- /dev/null +++ b/src/cdk/v2/destinations/http/utils.test.js @@ -0,0 +1,97 @@ +const { encodeParamsObject, prepareEndpoint, prepareBody } = require('./utils'); + +const { XMLBuilder } = require('fast-xml-parser'); +const jsonpath = require('rs-jsonpath'); + +describe('Utils Functions', () => { + describe('encodeParamsObject', () => { + test('should return empty object for invalid inputs', () => { + expect(encodeParamsObject(null)).toEqual({}); + expect(encodeParamsObject(undefined)).toEqual({}); + expect(encodeParamsObject('string')).toEqual({}); + }); + + test('should encode object keys and values', () => { + const params = { key1: 'value1', key2: 'value2 3 4' }; + const expected = { key1: 'value1', key2: 'value2%203%204' }; + expect(encodeParamsObject(params)).toEqual(expected); + }); + }); + + describe('prepareEndpoint', () => { + test('should replace template variables in API URL', () => { + const message = { id: 123 }; + const apiUrl = 'https://api.example.com/resource/'; + expect(prepareEndpoint(message, apiUrl, [])).toBe('https://api.example.com/resource'); + }); + test('should replace template variables in API URL and add path params', () => { + const message = { id: 123, p2: 'P2' }; + const apiUrl = 'https://api.example.com/resource/'; + const pathParams = [ + { + path: 'p1', + }, + { + path: '$.p2', + }, + ]; + expect(prepareEndpoint(message, apiUrl, pathParams)).toBe( + 'https://api.example.com/resource/p1/P2', + ); + }); + test('should add path params after uri encoding', () => { + const message = { id: 123, p2: 'P2%&' }; + const apiUrl = 'https://api.example.com/resource/'; + const pathParams = [ + { + path: 'p1', + }, + { + path: '$.p2', + }, + ]; + expect(prepareEndpoint(message, apiUrl, pathParams)).toBe( + 'https://api.example.com/resource/p1/P2%25%26', + ); + }); + test('should throw error as path contains slash', () => { + const message = { id: 123, p2: 'P2%&' }; + const apiUrl = 'https://api.example.com/resource/${$.id}'; + const pathParams = [ + { + path: 'p1/', + }, + { + path: '$.p2', + }, + ]; + expect(() => prepareEndpoint(message, apiUrl, pathParams)).toThrowError( + 'Path value cannot contain "/"', + ); + }); + }); + + describe('prepareBody', () => { + test('should prepare XML payload when content type is XML', () => { + const payload = { root: { key: 'value', key2: null } }; + const expectedXML = + 'value'; + const result = prepareBody(payload, 'XML'); + expect(result).toEqual({ payload: expectedXML }); + }); + + test('should prepare FORM-URLENCODED payload when content type is FORM-URLENCODED', () => { + const payload = { key1: 'value1', key2: 'value2' }; + const expectedFORM = { key1: 'value1', key2: 'value2' }; + const result = prepareBody(payload, 'FORM-URLENCODED'); + expect(result).toEqual(expectedFORM); + }); + + test('should return original payload without null or undefined values for other content types', () => { + const payload = { key1: 'value1', key2: null, key3: undefined, key4: 'value4' }; + const expected = { key1: 'value1', key4: 'value4' }; + const result = prepareBody(payload, 'JSON'); + expect(result).toEqual(expected); + }); + }); +}); diff --git a/src/middlewares/stats.ts b/src/middlewares/stats.ts new file mode 100644 index 00000000000..6fe0cc1c2cd --- /dev/null +++ b/src/middlewares/stats.ts @@ -0,0 +1,15 @@ +import { Context, Next } from 'koa'; + +export class StatsMiddleware { + private static instanceID: string = process.env.INSTANCE_ID || 'default'; + + private static workerID: string = process.env.WORKER_ID || 'master'; + + public static async executionStats(ctx: Context, next: Next) { + const start = Date.now(); + await next(); + const ms = Date.now() - start; + ctx.set('X-Response-Time', `${ms}ms`); + ctx.set('X-Instance-ID', `${StatsMiddleware.instanceID}/${StatsMiddleware.workerID}`); + } +} diff --git a/src/routes/userTransform.ts b/src/routes/userTransform.ts index fc61ab7b941..e2883bdc22f 100644 --- a/src/routes/userTransform.ts +++ b/src/routes/userTransform.ts @@ -1,7 +1,8 @@ import Router from '@koa/router'; -import { RouteActivationMiddleware } from '../middlewares/routeActivation'; -import { FeatureFlagMiddleware } from '../middlewares/featureFlag'; import { UserTransformController } from '../controllers/userTransform'; +import { FeatureFlagMiddleware } from '../middlewares/featureFlag'; +import { RouteActivationMiddleware } from '../middlewares/routeActivation'; +import { StatsMiddleware } from '../middlewares/stats'; const router = new Router(); @@ -15,6 +16,7 @@ router.post( '/customTransform', RouteActivationMiddleware.isUserTransformRouteActive, FeatureFlagMiddleware.handle, + StatsMiddleware.executionStats, UserTransformController.transform, ); router.post( diff --git a/src/util/cluster.js b/src/util/cluster.js index b9b86cd3c61..6be25018f1d 100644 --- a/src/util/cluster.js +++ b/src/util/cluster.js @@ -28,6 +28,7 @@ async function shutdownWorkers() { } function start(port, app, metricsApp) { + if (cluster.isMaster) { logger.info(`Master (pid: ${process.pid}) has started`); @@ -44,7 +45,9 @@ function start(port, app, metricsApp) { // Fork workers. for (let i = 0; i < numWorkers; i += 1) { - cluster.fork(); + cluster.fork({ + WORKER_ID: `worker-${i + 1}`, + }); } cluster.on('online', (worker) => { diff --git a/src/util/customTransformer-faas.js b/src/util/customTransformer-faas.js index 0f59f5db60f..d8bf556d893 100644 --- a/src/util/customTransformer-faas.js +++ b/src/util/customTransformer-faas.js @@ -34,7 +34,6 @@ function generateFunctionName(userTransformation, libraryVersionIds, testMode, h ids = ids.concat([hashSecret]); } - // FIXME: Why the id's are sorted ?! const hash = crypto.createHash('md5').update(`${ids}`).digest('hex'); return `fn-${userTransformation.workspaceId}-${hash}`.substring(0, 63).toLowerCase(); } diff --git a/src/util/prometheus.js b/src/util/prometheus.js index 2a3a1fb22a8..18cbcdf6b7e 100644 --- a/src/util/prometheus.js +++ b/src/util/prometheus.js @@ -445,6 +445,12 @@ class Prometheus { type: 'counter', labelNames: ['event', 'writeKey'], }, + { + name: 'shopify_pixel_cart_token_not_found_server_side', + help: 'shopify_pixel_cart_token_not_found_server_side', + type: 'counter', + labelNames: ['event', 'writeKey'], + }, { name: 'shopify_pixel_cart_token_set', help: 'shopify_pixel_cart_token_set', diff --git a/src/util/utils.js b/src/util/utils.js index 82c85b41a58..36554ff5d0a 100644 --- a/src/util/utils.js +++ b/src/util/utils.js @@ -44,35 +44,35 @@ const fetchAddressFromHostName = async (hostname) => { return { address, cacheHit: false }; }; -const staticLookup = (transformationTags) => async (hostname, _, cb) => { - let ip; - const resolveStartTime = new Date(); - try { - const { address, cacheHit } = await fetchAddressFromHostName(hostname); - ip = address; - stats.timing('fetch_dns_resolve_time', resolveStartTime, { ...transformationTags, cacheHit }); - } catch (error) { - logger.error(`DNS Error Code: ${error.code} | Message : ${error.message}`); - stats.timing('fetch_dns_resolve_time', resolveStartTime, { - ...transformationTags, - error: 'true', - }); - cb(null, `unable to resolve IP address for ${hostname}`, RECORD_TYPE_A); - return; - } - - if (!ip) { - cb(null, `resolved empty list of IP address for ${hostname}`, RECORD_TYPE_A); - return; - } - - if (ip.startsWith(LOCALHOST_OCTET)) { - cb(null, `cannot use ${ip} as IP address`, RECORD_TYPE_A); - return; - } - - cb(null, ip, RECORD_TYPE_A); -}; +const staticLookup = + (transformationTags, fetchAddress = fetchAddressFromHostName) => + (hostname, _, cb) => { + const resolveStartTime = new Date(); + + fetchAddress(hostname) + .then(({ address, cacheHit }) => { + stats.timing('fetch_dns_resolve_time', resolveStartTime, { + ...transformationTags, + cacheHit, + }); + + if (!address) { + cb(null, `resolved empty list of IP address for ${hostname}`, RECORD_TYPE_A); + } else if (address.startsWith(LOCALHOST_OCTET)) { + cb(null, `cannot use ${address} as IP address`, RECORD_TYPE_A); + } else { + cb(null, address, RECORD_TYPE_A); + } + }) + .catch((error) => { + logger.error(`DNS Error Code: ${error.code} | Message : ${error.message}`); + stats.timing('fetch_dns_resolve_time', resolveStartTime, { + ...transformationTags, + error: 'true', + }); + cb(null, `unable to resolve IP address for ${hostname}`, RECORD_TYPE_A); + }); + }; const httpAgentWithDnsLookup = (scheme, transformationTags) => { const httpModule = scheme === 'http' ? http : https; @@ -226,4 +226,5 @@ module.exports = { logProcessInfo, extractStackTraceUptoLastSubstringMatch, fetchWithDnsWrapper, + staticLookup, }; diff --git a/src/util/utils.test.js b/src/util/utils.test.js new file mode 100644 index 00000000000..d7974e3ff61 --- /dev/null +++ b/src/util/utils.test.js @@ -0,0 +1,71 @@ +const { staticLookup } = require('./utils'); + +describe('staticLookup', () => { + const transformationTags = { tag: 'value' }; + const RECORD_TYPE_A = 4; + const HOST_NAME = 'example.com'; + const fetchAddressFromHostName = jest.fn(); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should resolve the hostname and return the IP address', async () => { + const mockAddress = '192.168.1.1'; + fetchAddressFromHostName.mockResolvedValueOnce({ address: mockAddress, cacheHit: true }); + + const resolve = staticLookup(transformationTags, fetchAddressFromHostName); + const callback = (args) => { + expect(fetchAddressFromHostName).toHaveBeenCalledWith(HOST_NAME); + expect(args).toEqual(null, mockAddress, RECORD_TYPE_A); + }; + resolve(HOST_NAME, null, callback); + }); + + it('should handle errors from fetchAddressFromHostName', async () => { + const error = new Error('DNS error'); + error.code = 'ENOTFOUND'; + fetchAddressFromHostName.mockRejectedValueOnce(error); + + const resolve = staticLookup(transformationTags, fetchAddressFromHostName); + const callback = (args) => { + expect(fetchAddressFromHostName).toHaveBeenCalledWith(HOST_NAME); + expect(args).toEqual(null, `unable to resolve IP address for ${HOST_NAME}`, RECORD_TYPE_A); + }; + resolve(HOST_NAME, null, callback); + }); + + it('should handle empty address', async () => { + fetchAddressFromHostName.mockResolvedValueOnce({ address: '', cacheHit: true }); + + const resolve = staticLookup(transformationTags, fetchAddressFromHostName); + const callback = (args) => { + expect(fetchAddressFromHostName).toHaveBeenCalledWith(HOST_NAME); + expect(args).toEqual( + null, + `resolved empty list of IP address for ${HOST_NAME}`, + RECORD_TYPE_A, + ); + }; + resolve(HOST_NAME, null, callback); + }); + + it('should handle localhost address', async () => { + const LOCALHOST_OCTET = '127'; + fetchAddressFromHostName.mockResolvedValueOnce({ + address: `${LOCALHOST_OCTET}.0.0.1`, + cacheHit: true, + }); + + const resolve = staticLookup(transformationTags, fetchAddressFromHostName); + const callback = (args) => { + expect(fetchAddressFromHostName).toHaveBeenCalledWith(HOST_NAME); + expect(args).toEqual( + null, + `cannot use ${LOCALHOST_OCTET}.0.0.1 as IP address`, + RECORD_TYPE_A, + ); + }; + resolve(HOST_NAME, null, callback); + }); +}); diff --git a/src/v0/destinations/active_campaign/transform.js b/src/v0/destinations/active_campaign/transform.js index f21bb1a70d4..3fd5229c371 100644 --- a/src/v0/destinations/active_campaign/transform.js +++ b/src/v0/destinations/active_campaign/transform.js @@ -95,7 +95,7 @@ const customTagProcessor = async ({ message, destination, metadata }, category, // Step - 1 // Fetch already created tags from dest, so that we avoid duplicate tag creation request // Ref - https://developers.activecampaign.com/reference/retrieve-all-tags - endpoint = `${destination.Config.apiUrl}${`${tagEndPoint}?limit=100`}`; + endpoint = `${destination.Config.apiUrl}${tagEndPoint}?limit=100`; requestOptions = { headers: getHeader(destination), }; diff --git a/src/v0/destinations/adobe_analytics/utils.js b/src/v0/destinations/adobe_analytics/utils.js index ceba177ff14..e5196a30d48 100644 --- a/src/v0/destinations/adobe_analytics/utils.js +++ b/src/v0/destinations/adobe_analytics/utils.js @@ -97,6 +97,25 @@ function escapeToHTML(inputString) { ); } +/** + * Tries to find a value from alternative sources based on the source key. + * @param {Object} message - The input message object. + * @param {string} sourceKey - The key to look for. + * @returns {any} - The found value or null. + */ +function findValueFromSources(message, sourceKey) { + // Try alternative sources defined in SOURCE_KEYS + for (const source of SOURCE_KEYS) { + const value = getMappingFieldValueFormMessage(message, source, sourceKey); + if (isDefinedAndNotNull(value)) { + return value; + } + } + + // Fallback to value retrieval by path + return getValueByPath(message, sourceKey); +} + /** * This function is used for populating the eVars and hVars in the payload * @param {*} destVarMapping @@ -106,31 +125,28 @@ function escapeToHTML(inputString) { * @returns updated paylaod with eVars and hVars added */ function rudderPropToDestMap(destVarMapping, message, payload, destVarStrPrefix) { - const mappedVar = {}; - // pass the Rudder Property mapped in the ui whose evar you want to map - Object.keys(destVarMapping).forEach((key) => { - let val = get(message, `properties.${key}`); - if (isDefinedAndNotNull(val)) { - const destVarKey = destVarStrPrefix + destVarMapping[key]; - mappedVar[destVarKey] = escapeToHTML(val); - } else { - SOURCE_KEYS.some((sourceKey) => { - val = getMappingFieldValueFormMessage(message, sourceKey, key); - if (isDefinedAndNotNull(val)) { - mappedVar[`${destVarStrPrefix}${[destVarMapping[key]]}`] = escapeToHTML(val); - } else { - val = getValueByPath(message, key); - if (isDefinedAndNotNull(val)) { - mappedVar[`${destVarStrPrefix}${[destVarMapping[key]]}`] = escapeToHTML(val); - } - } - }); + const mappedVariables = {}; + + // Iterate over each key in the destination variable mapping + Object.keys(destVarMapping).forEach((sourceKey) => { + let value = message?.properties?.[sourceKey]; + + if (!isDefinedAndNotNull(value)) { + // Try getting the value from alternative sources + value = findValueFromSources(message, sourceKey); + } + + if (isDefinedAndNotNull(value)) { + const destinationKey = `${destVarStrPrefix}${destVarMapping[sourceKey]}`; + mappedVariables[destinationKey] = escapeToHTML(value); } }); - if (Object.keys(mappedVar).length > 0) { - // non-empty object - Object.assign(payload, mappedVar); + + // Add non-empty mapped variables to the payload + if (Object.keys(mappedVariables).length > 0) { + Object.assign(payload, mappedVariables); } + return payload; } diff --git a/src/v0/destinations/mp/data/MPEventPropertiesConfig.json b/src/v0/destinations/mp/data/MPEventPropertiesConfig.json index 956b60248eb..bed246bc7be 100644 --- a/src/v0/destinations/mp/data/MPEventPropertiesConfig.json +++ b/src/v0/destinations/mp/data/MPEventPropertiesConfig.json @@ -76,11 +76,11 @@ "destKey": "mp_lib" }, { - "sourceKeys": "context.page.initialReferrer", + "sourceKeys": ["context.page.initialReferrer", "context.page.initial_referrer"], "destKey": "$initial_referrer" }, { - "sourceKeys": "context.page.initialReferringDomain", + "sourceKeys": ["context.page.initialReferringDomain", "context.page.initial_referring_domain"], "destKey": "$initial_referring_domain" }, { diff --git a/src/v0/sources/shopify/util.js b/src/v0/sources/shopify/util.js index b7e79e35a16..3665dfff203 100644 --- a/src/v0/sources/shopify/util.js +++ b/src/v0/sources/shopify/util.js @@ -272,6 +272,7 @@ module.exports = { createPropertiesForEcomEvent, extractEmailFromPayload, getAnonymousIdAndSessionId, + getCartToken, checkAndUpdateCartItems, getHashLineItems, getDataFromRedis, diff --git a/src/v1/sources/shopify/config.js b/src/v1/sources/shopify/config.js index 9cb11e471f5..20db7be331e 100644 --- a/src/v1/sources/shopify/config.js +++ b/src/v1/sources/shopify/config.js @@ -35,11 +35,20 @@ const PIXEL_EVENT_MAPPING = { search_submitted: 'Search Submitted', }; +const ECOM_TOPICS = { + CHECKOUTS_CREATE: 'checkouts_create', + CHECKOUTS_UPDATE: 'checkouts_update', + ORDERS_UPDATE: 'orders_updated', + ORDERS_CREATE: 'orders_create', + ORDERS_CANCELLED: 'orders_cancelled', +}; + const RUDDER_ECOM_MAP = { - checkouts_create: 'Checkout Started - Webhook', + checkouts_create: 'Checkout Started Webhook', checkouts_update: 'Checkout Updated', orders_updated: 'Order Updated', orders_create: 'Order Created', + orders_cancelled: 'Order Cancelled', }; const contextualFieldMappingJSON = JSON.parse( @@ -94,6 +103,7 @@ module.exports = { INTEGERATION, PIXEL_EVENT_TOPICS, PIXEL_EVENT_MAPPING, + ECOM_TOPICS, RUDDER_ECOM_MAP, contextualFieldMappingJSON, cartViewedEventMappingJSON, diff --git a/src/v1/sources/shopify/pixelEventsMappings/campaignObjectMappings.json b/src/v1/sources/shopify/pixelEventsMappings/campaignObjectMappings.json new file mode 100644 index 00000000000..319502e377b --- /dev/null +++ b/src/v1/sources/shopify/pixelEventsMappings/campaignObjectMappings.json @@ -0,0 +1,18 @@ +[ + { + "sourceKeys": "utm_campaign", + "destKeys": "name" + }, + { + "sourceKeys": "utm_medium", + "destKeys": "medium" + }, + { + "sourceKeys": "utm_term", + "destKeys": "term" + }, + { + "sourceKeys": "utm_content", + "destKeys": "content" + } +] diff --git a/src/v1/sources/shopify/transform.js b/src/v1/sources/shopify/transform.js index 5ebf4a34fc5..23228ecc10f 100644 --- a/src/v1/sources/shopify/transform.js +++ b/src/v1/sources/shopify/transform.js @@ -1,29 +1,13 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -const { processPixelWebEvents } = require('./webpixelTransformations/pixelTransform'); -const { process: processWebhookEvents } = require('../../../v0/sources/shopify/transform'); -const { - process: processPixelWebhookEvents, -} = require('./webhookTransformations/serverSideTransform'); +const { process: processV0 } = require('../../../v0/sources/shopify/transform'); +const { processV1Events } = require('./transformV1'); +const { isShopifyV1Event } = require('./utils'); const process = async (inputEvent) => { const { event } = inputEvent; - const { query_parameters } = event; - // check identify the event is from the web pixel based on the pixelEventLabel property. - const { pixelEventLabel: pixelClientEventLabel } = event; - if (pixelClientEventLabel) { - // this is a event fired from the web pixel loaded on the browser - // by the user interactions with the store. - const pixelWebEventResponse = await processPixelWebEvents(event); - return pixelWebEventResponse; + if (isShopifyV1Event(event)) { + return processV1Events(event); } - if (query_parameters && query_parameters?.version?.[0] === 'pixel') { - // this is a server-side event from the webhook subscription made by the pixel app. - const pixelWebhookEventResponse = await processPixelWebhookEvents(event); - return pixelWebhookEventResponse; - } - // this is a server-side event from the webhook subscription made by the legacy tracker-based app. - const response = await processWebhookEvents(event); - return response; + return processV0(event); }; module.exports = { process }; diff --git a/src/v1/sources/shopify/transformV1.js b/src/v1/sources/shopify/transformV1.js new file mode 100644 index 00000000000..b3e01d95c67 --- /dev/null +++ b/src/v1/sources/shopify/transformV1.js @@ -0,0 +1,36 @@ +const { PlatformError } = require('@rudderstack/integrations-lib'); +const { isIdentifierEvent, processIdentifierEvent } = require('./utils'); +const { processWebhookEvents } = require('./webhookTransformations/serverSideTransform'); +const { processPixelWebEvents } = require('./webpixelTransformations/pixelTransform'); + +const processV1Events = async (event) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { query_parameters } = event; + + // these are the events from the front-end tracking, viz. web-pixel or theme-app extension. + const { pixelEventLabel: clientSideEvent } = event; + const isServerSideEvent = query_parameters && query_parameters?.version?.[0] === 'pixel'; + + if (clientSideEvent) { + // check if the event is an identifier event, used to set the anonymousId in the redis for identity stitching. + if (isIdentifierEvent(event)) { + return processIdentifierEvent(event); + } + // handle events from the app pixel. + const pixelWebEventResponse = processPixelWebEvents(event); + return pixelWebEventResponse; + } + if (isServerSideEvent) { + // this is a server-side event from the webhook subscription made by the pixel app. + const pixelWebhookEventResponse = await processWebhookEvents(event); + return pixelWebhookEventResponse; + } + throw new PlatformError( + 'Invalid Event for Shopiyf V1 (not matching client or server side event requirements)', + 500, + ); +}; + +module.exports = { + processV1Events, +}; diff --git a/src/v1/sources/shopify/utils.js b/src/v1/sources/shopify/utils.js new file mode 100644 index 00000000000..9502b2ee595 --- /dev/null +++ b/src/v1/sources/shopify/utils.js @@ -0,0 +1,34 @@ +const { RedisDB } = require('../../../util/redis/redisConnector'); + +const NO_OPERATION_SUCCESS = { + outputToSource: { + body: Buffer.from('OK').toString('base64'), + contentType: 'text/plain', + }, + statusCode: 200, +}; + +const isIdentifierEvent = (payload) => ['rudderIdentifier'].includes(payload?.event); + +const processIdentifierEvent = async (event) => { + const { cartToken, anonymousId } = event; + await RedisDB.setVal(`pixel:${cartToken}`, ['anonymousId', anonymousId]); + return NO_OPERATION_SUCCESS; +}; + +const isShopifyV1Event = (event) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { query_parameters } = event; + const { pixelEventLabel: pixelClientEventLabel } = event; + + return !!( + (query_parameters && query_parameters?.version?.[0] === 'pixel') || + pixelClientEventLabel + ); +}; + +module.exports = { + processIdentifierEvent, + isIdentifierEvent, + isShopifyV1Event, +}; diff --git a/src/v1/sources/shopify/utils.test.js b/src/v1/sources/shopify/utils.test.js new file mode 100644 index 00000000000..558cdec7c52 --- /dev/null +++ b/src/v1/sources/shopify/utils.test.js @@ -0,0 +1,48 @@ +const { isIdentifierEvent, processIdentifierEvent } = require('./utils'); +const { RedisDB } = require('../../../util/redis/redisConnector'); + +describe('Identifier Utils Tests', () => { + describe('test isIdentifierEvent', () => { + it('should return true if the event is rudderIdentifier', () => { + const event = { event: 'rudderIdentifier' }; + expect(isIdentifierEvent(event)).toBe(true); + }); + + it('should return false if the event is not rudderIdentifier', () => { + const event = { event: 'checkout started' }; + expect(isIdentifierEvent(event)).toBe(false); + }); + }); + + describe('test processIdentifierEvent', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should set the anonymousId in redis and return NO_OPERATION_SUCCESS', async () => { + const setValSpy = jest.spyOn(RedisDB, 'setVal').mockResolvedValue('OK'); + const event = { cartToken: 'cartTokenTest1', anonymousId: 'anonymousIdTest1' }; + + const response = await processIdentifierEvent(event); + + expect(setValSpy).toHaveBeenCalledWith('pixel:cartTokenTest1', [ + 'anonymousId', + 'anonymousIdTest1', + ]); + expect(response).toEqual({ + outputToSource: { + body: Buffer.from('OK').toString('base64'), + contentType: 'text/plain', + }, + statusCode: 200, + }); + }); + + it('should handle redis errors', async () => { + jest.spyOn(RedisDB, 'setVal').mockRejectedValue(new Error('Redis connection failed')); + const event = { cartToken: 'cartTokenTest1', anonymousId: 'anonymousIdTest1' }; + + await expect(processIdentifierEvent(event)).rejects.toThrow('Redis connection failed'); + }); + }); +}); diff --git a/src/v1/sources/shopify/webhookEventsMapping/productMapping.json b/src/v1/sources/shopify/webhookEventsMapping/productMapping.json index e78ed50dfc4..7bd20498286 100644 --- a/src/v1/sources/shopify/webhookEventsMapping/productMapping.json +++ b/src/v1/sources/shopify/webhookEventsMapping/productMapping.json @@ -8,7 +8,10 @@ }, { "sourceKeys": "total_price", - "destKey": "value" + "destKey": "value", + "metadata": { + "type": "toNumber" + } }, { "sourceKeys": "total_tax", diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js index d0221f6950a..1fe92bbee0d 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideTransform.js @@ -1,25 +1,24 @@ -/* eslint-disable @typescript-eslint/naming-convention */ const lodash = require('lodash'); const get = require('get-value'); const stats = require('../../../../util/stats'); -const { getShopifyTopic, extractEmailFromPayload } = require('../../../../v0/sources/shopify/util'); -const { removeUndefinedAndNullValues, isDefinedAndNotNull } = require('../../../../v0/util'); +const { getShopifyTopic } = require('../../../../v0/sources/shopify/util'); +const { removeUndefinedAndNullValues } = require('../../../../v0/util'); const Message = require('../../../../v0/sources/message'); const { EventType } = require('../../../../constants'); const { INTEGERATION, MAPPING_CATEGORIES, IDENTIFY_TOPICS, - ECOM_TOPICS, SUPPORTED_TRACK_EVENTS, SHOPIFY_TRACK_MAP, lineItemsMappingJSON, } = require('../../../../v0/sources/shopify/config'); -const { RUDDER_ECOM_MAP } = require('../config'); +const { ECOM_TOPICS, RUDDER_ECOM_MAP } = require('../config'); const { createPropertiesForEcomEventFromWebhook, getProductsFromLineItems, - getAnonymousIdFromAttributes, + setAnonymousId, + handleCommonProperties, } = require('./serverSideUtlis'); const NO_OPERATION_SUCCESS = { @@ -65,9 +64,6 @@ const ecomPayloadBuilder = (event, shopifyTopic) => { if (event.billing_address) { message.setProperty('traits.billingAddress', event.billing_address); } - if (!message.userId && event.user_id) { - message.setProperty('userId', event.user_id); - } return message; }; @@ -96,6 +92,7 @@ const processEvent = async (inputEvent, metricMetadata) => { case ECOM_TOPICS.ORDERS_UPDATE: case ECOM_TOPICS.CHECKOUTS_CREATE: case ECOM_TOPICS.CHECKOUTS_UPDATE: + case ECOM_TOPICS.ORDERS_CANCELLED: message = ecomPayloadBuilder(event, shopifyTopic); break; default: @@ -110,42 +107,16 @@ const processEvent = async (inputEvent, metricMetadata) => { message = trackPayloadBuilder(event, shopifyTopic); break; } - - if (message.userId) { - message.userId = String(message.userId); - } - if (!get(message, 'traits.email')) { - const email = extractEmailFromPayload(event); - if (email) { - message.setProperty('traits.email', email); - } - } // attach anonymousId if the event is track event using note_attributes if (message.type !== EventType.IDENTIFY) { - const anonymousId = getAnonymousIdFromAttributes(event); - if (isDefinedAndNotNull(anonymousId)) { - message.setProperty('anonymousId', anonymousId); - } - } - message.setProperty(`integrations.${INTEGERATION}`, true); - message.setProperty('context.library', { - eventOrigin: 'server', - name: 'RudderStack Shopify Cloud', - version: '2.0.0', - }); - message.setProperty('context.topic', shopifyTopic); - // attaching cart, checkout and order tokens in context object - message.setProperty(`context.cart_token`, event.cart_token); - message.setProperty(`context.checkout_token`, event.checkout_token); - // raw shopify payload passed inside context object under shopifyDetails - message.setProperty('context.shopifyDetails', event); - if (shopifyTopic === 'orders_updated') { - message.setProperty(`context.order_token`, event.token); + await setAnonymousId(message, event, metricMetadata); } + // attach userId, email and other contextual properties + message = handleCommonProperties(message, event, shopifyTopic); message = removeUndefinedAndNullValues(message); return message; }; -const process = async (event) => { +const processWebhookEvents = async (event) => { const metricMetadata = { writeKey: event.query_parameters?.writeKey?.[0], source: 'SHOPIFY', @@ -155,7 +126,7 @@ const process = async (event) => { }; module.exports = { - process, + processWebhookEvents, processEvent, identifyPayloadBuilder, ecomPayloadBuilder, diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js index b94ec9c3ddf..070fdafdd71 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js @@ -1,15 +1,13 @@ +const { processEvent } = require('./serverSideTransform'); const { getProductsFromLineItems, createPropertiesForEcomEventFromWebhook, getAnonymousIdFromAttributes, + getCartToken, } = require('./serverSideUtlis'); +const { RedisDB } = require('../../../../util/redis/redisConnector'); -const { constructPayload } = require('../../../../v0/util'); - -const { - lineItemsMappingJSON, - productMappingJSON, -} = require('../../../../v0/sources/shopify/config'); +const { lineItemsMappingJSON } = require('../../../../v0/sources/shopify/config'); const Message = require('../../../../v0/sources/message'); jest.mock('../../../../v0/sources/message'); @@ -63,7 +61,6 @@ describe('serverSideUtils.js', () => { }); it('should return array of products', () => { - const mapping = {}; const result = getProductsFromLineItems(LINEITEMS, lineItemsMappingJSON); expect(result).toEqual([ { brand: 'Hydrogen Vendor', price: '600.00', product_id: 7234590408818, quantity: 1 }, @@ -115,7 +112,13 @@ describe('serverSideUtils.js', () => { // Handles empty note_attributes array gracefully it('should return null when note_attributes is an empty array', async () => { const event = { note_attributes: [] }; - const result = await getAnonymousIdFromAttributes(event); + const result = getAnonymousIdFromAttributes(event); + expect(result).toBeNull(); + }); + + it('should return null when note_attributes is not present', async () => { + const event = {}; + const result = getAnonymousIdFromAttributes(event); expect(result).toBeNull(); }); @@ -123,8 +126,48 @@ describe('serverSideUtils.js', () => { const event = { note_attributes: [{ name: 'rudderAnonymousId', value: '123456' }], }; - const result = await getAnonymousIdFromAttributes(event); + const result = getAnonymousIdFromAttributes(event); expect(result).toEqual('123456'); }); }); + + describe('getCartToken', () => { + it('should return null if cart_token is not present', () => { + const event = {}; + const result = getCartToken(event); + expect(result).toBeNull(); + }); + + it('should return cart_token if it is present', () => { + const event = { cart_token: 'cartTokenTest1' }; + const result = getCartToken(event); + expect(result).toEqual('cartTokenTest1'); + }); + }); +}); + +describe('Redis cart token tests', () => { + it('should get anonymousId property from redis', async () => { + const getValSpy = jest + .spyOn(RedisDB, 'getVal') + .mockResolvedValue({ anonymousId: 'anonymousIdTest1' }); + const event = { + cart_token: `cartTokenTest1`, + id: 5778367414385, + line_items: [ + { + id: 14234727743601, + }, + ], + query_parameters: { + topic: ['orders_updated'], + version: ['pixel'], + writeKey: ['dummy-write-key'], + }, + }; + const message = await processEvent(event); + expect(getValSpy).toHaveBeenCalledTimes(1); + expect(getValSpy).toHaveBeenCalledWith('pixel:cartTokenTest1'); + expect(message.anonymousId).toEqual('anonymousIdTest1'); + }); }); diff --git a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js index df33d7a347a..0d81de99ac3 100644 --- a/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js +++ b/src/v1/sources/shopify/webhookTransformations/serverSideUtlis.js @@ -1,7 +1,11 @@ +/* eslint-disable no-param-reassign */ +const get = require('get-value'); const { isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); +const { extractEmailFromPayload } = require('../../../../v0/sources/shopify/util'); const { constructPayload } = require('../../../../v0/util'); -const { lineItemsMappingJSON, productMappingJSON } = require('../config'); - +const { INTEGERATION, lineItemsMappingJSON, productMappingJSON } = require('../config'); +const { RedisDB } = require('../../../../util/redis/redisConnector'); +const stats = require('../../../../util/stats'); /** * Returns an array of products from the lineItems array received from the webhook event * @param {Array} lineItems @@ -54,8 +58,84 @@ const getAnonymousIdFromAttributes = (event) => { return rudderAnonymousIdObject ? rudderAnonymousIdObject.value : null; }; +/** + * Returns the cart_token from the event message + * @param {Object} event + * @returns {String} cart_token + */ +const getCartToken = (event) => event?.cart_token || null; + +/** + * Handles the anonymousId assignment for the message, based on the event attributes and redis data + * @param {Object} message rudderstack message object + * @param {Object} event raw shopify event payload + * @param {Object} metricMetadata metric metadata object + */ +const setAnonymousId = async (message, event, metricMetadata) => { + const anonymousId = getAnonymousIdFromAttributes(event); + if (isDefinedAndNotNull(anonymousId)) { + message.anonymousId = anonymousId; + } else { + // if anonymousId is not present in note_attributes or note_attributes is not present, query redis for anonymousId + const cartToken = getCartToken(event); + if (cartToken) { + const redisData = await RedisDB.getVal(`pixel:${cartToken}`); + if (redisData?.anonymousId) { + message.anonymousId = redisData.anonymousId; + } + } else { + stats.increment('shopify_pixel_cart_token_not_found_server_side', { + source: metricMetadata.source, + writeKey: metricMetadata.writeKey, + }); + } + } +}; + +/** + Handles userId, email and contextual properties enrichment for the message payload + * @param {Object} message rudderstack message object + * @param {Object} event raw shopify event payload + * @param {String} shopifyTopic shopify event topic +*/ +const handleCommonProperties = (message, event, shopifyTopic) => { + if (message.userId) { + message.userId = String(message.userId); + } + if (!get(message, 'traits.email')) { + const email = extractEmailFromPayload(event); + if (email) { + message.setProperty('traits.email', email); + } + } + message.setProperty(`integrations.${INTEGERATION}`, true); + message.setProperty('context.library', { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }); + message.setProperty('context.topic', shopifyTopic); + // attaching cart, checkout and order tokens in context object + message.setProperty(`context.cart_token`, event.cart_token); + message.setProperty(`context.checkout_token`, event.checkout_token); + // raw shopify payload passed inside context object under shopifyDetails + message.setProperty('context.shopifyDetails', event); + if (shopifyTopic === 'orders_updated') { + message.setProperty(`context.order_token`, event.token); + } + message.setProperty('integrations.DATA_WAREHOUSE', { + options: { + jsonPaths: [`${message.type}.context.shopifyDetails`], + }, + }); + return message; +}; + module.exports = { createPropertiesForEcomEventFromWebhook, + getCartToken, getProductsFromLineItems, getAnonymousIdFromAttributes, + setAnonymousId, + handleCommonProperties, }; diff --git a/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js index b1d1c8b2fac..44928686f83 100644 --- a/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelTransform.js @@ -15,7 +15,9 @@ const { checkoutEventBuilder, checkoutStepEventBuilder, searchEventBuilder, + extractCampaignParams, } = require('./pixelUtils'); +const campaignObjectMappings = require('../pixelEventsMappings/campaignObjectMappings.json'); const { INTEGERATION, PIXEL_EVENT_TOPICS, @@ -68,7 +70,7 @@ const handleCartTokenRedisOperations = async (inputEvent, clientId) => { const cartToken = extractCartToken(inputEvent); try { if (isDefinedNotNullNotEmpty(clientId) && isDefinedNotNullNotEmpty(cartToken)) { - await RedisDB.setVal(cartToken, ['anonymousId', clientId]); + await RedisDB.setVal(`pixel:${cartToken}`, ['anonymousId', clientId]); stats.increment('shopify_pixel_cart_token_set', { event: inputEvent.name, writeKey: inputEvent.query_parameters.writeKey, @@ -85,7 +87,7 @@ const handleCartTokenRedisOperations = async (inputEvent, clientId) => { function processPixelEvent(inputEvent) { // eslint-disable-next-line @typescript-eslint/naming-convention - const { name, query_parameters, clientId, data, id } = inputEvent; + const { name, query_parameters, context, clientId, data, id } = inputEvent; const shopifyDetails = { ...inputEvent }; delete shopifyDetails.context; delete shopifyDetails.query_parameters; @@ -140,6 +142,11 @@ function processPixelEvent(inputEvent) { } message.anonymousId = clientId; message.setProperty(`integrations.${INTEGERATION}`, true); + message.setProperty('integrations.DATA_WAREHOUSE', { + options: { + jsonPaths: [`${message.type}.context.shopifyDetails`], + }, + }); message.setProperty('context.library', { name: 'RudderStack Shopify Cloud', eventOrigin: 'client', @@ -147,12 +154,18 @@ function processPixelEvent(inputEvent) { }); message.setProperty('context.topic', name); message.setProperty('context.shopifyDetails', shopifyDetails); + + // adding campaign object with utm parameters to the message context + const campaignParams = extractCampaignParams(context, campaignObjectMappings); + if (campaignParams) { + message.context.campaign = campaignParams; + } message.messageId = id; message = removeUndefinedAndNullValues(message); return message; } -const processPixelWebEvents = async (event) => { +const processPixelWebEvents = (event) => { const pixelEvent = processPixelEvent(event); return removeUndefinedAndNullValues(pixelEvent); }; diff --git a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js index 46ae59e0cf8..388be2a16e8 100644 --- a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.js @@ -1,4 +1,5 @@ /* eslint-disable no-param-reassign */ +const { isDefinedAndNotNull } = require('@rudderstack/integrations-lib'); const Message = require('../../../../v0/sources/message'); const { EventType } = require('../../../../constants'); const { @@ -203,6 +204,41 @@ const searchEventBuilder = (inputEvent) => { ); }; +/** + * Extracts UTM parameters from the context object + * @param {*} context context object from the event + * @param {*} campaignMappings mappings for UTM parameters + * @returns campaignParams, an object containing UTM parameters + */ +const extractCampaignParams = (context, campaignMappings) => { + if (context?.document?.location?.href) { + const url = new URL(context.document.location.href); + const campaignParams = {}; + + // Loop through mappings and extract UTM parameters + campaignMappings.forEach((mapping) => { + const value = url.searchParams.get(mapping.sourceKeys); + if (isDefinedAndNotNull(value)) { + campaignParams[mapping.destKeys] = value; + } + }); + + // Extract any UTM parameters not in the mappings + const campaignObjectSourceKeys = campaignMappings.flatMap((mapping) => mapping.sourceKeys); + url.searchParams.forEach((value, key) => { + if (key.startsWith('utm_') && !campaignObjectSourceKeys.includes(key)) { + campaignParams[key] = value; + } + }); + + // Only return campaign object if we have any UTM parameters + if (Object.keys(campaignParams).length > 0) { + return campaignParams; + } + } + return null; +}; + module.exports = { pageViewedEventBuilder, cartViewedEventBuilder, @@ -212,4 +248,5 @@ module.exports = { checkoutEventBuilder, checkoutStepEventBuilder, searchEventBuilder, + extractCampaignParams, }; diff --git a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js index e8f53a5f153..fc82404ee73 100644 --- a/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js +++ b/src/v1/sources/shopify/webpixelTransformations/pixelUtils.test.js @@ -7,7 +7,9 @@ const { checkoutEventBuilder, checkoutStepEventBuilder, searchEventBuilder, + extractCampaignParams, } = require('./pixelUtils'); +const campaignObjectMappings = require('../pixelEventsMappings/campaignObjectMappings.json'); const Message = require('../../../../v0/sources/message'); jest.mock('ioredis', () => require('../../../../test/__mocks__/redis')); jest.mock('../../../../v0/sources/message'); @@ -787,4 +789,59 @@ describe('utilV2.js', () => { expect(message.context).toEqual({ userAgent: 'Mozilla/5.0' }); }); }); + + describe('extractCampaignParams', () => { + it('should extract campaign parameters from URL', () => { + const context = { + document: { + location: { + href: 'https://example.com?utm_source=google&utm_medium=cpc&utm_campaign=spring_sale', + }, + }, + }; + + const result = extractCampaignParams(context, campaignObjectMappings); + expect(result).toEqual({ + utm_source: 'google', + medium: 'cpc', + name: 'spring_sale', + }); + }); + + it('should return null if no campaign parameters are found', () => { + const context = { + document: { + location: { + href: 'https://example.com', + }, + }, + }; + + const result = extractCampaignParams(context, campaignObjectMappings); + expect(result).toBeNull(); + }); + + it('should extract additional UTM parameters not in mappings', () => { + const context = { + document: { + location: { + href: 'https://example.com?utm_source=google&utm_term=shoes', + }, + }, + }; + + const result = extractCampaignParams(context, campaignObjectMappings); + expect(result).toEqual({ + utm_source: 'google', + term: 'shoes', + }); + }); + + it('should handle missing context or location gracefully', () => { + const context = {}; + + const result = extractCampaignParams(context, campaignObjectMappings); + expect(result).toBeNull(); + }); + }); }); diff --git a/src/warehouse/index.js b/src/warehouse/index.js index ea663c9b2fc..c5f167909f7 100644 --- a/src/warehouse/index.js +++ b/src/warehouse/index.js @@ -11,6 +11,7 @@ const { validTimestamp, getVersionedUtils, isRudderSourcesEvent, + mergeJSONPathsFromDataWarehouse, } = require('./util'); const { getMergeRuleEvent } = require('./identity'); @@ -307,8 +308,8 @@ function isStringLikeObject(obj) { let minKey = Infinity; let maxKey = -Infinity; - for (let i = 0; i < keys.length; i++) { - const key = keys[i]; + for (const element of keys) { + const key = element; const value = obj[key]; if (!isNonNegativeInteger(key)) return false; @@ -336,8 +337,8 @@ function stringLikeObjectToString(obj) { .sort((a, b) => a - b); let result = ''; - for (let i = 0; i < keys.length; i++) { - result += obj[keys[i].toString()]; + for (const element of keys) { + result += obj[element.toString()]; } return result; @@ -655,6 +656,8 @@ function processWarehouseMessage(message, options) { const skipReservedKeywordsEscaping = options.integrationOptions.skipReservedKeywordsEscaping || false; + mergeJSONPathsFromDataWarehouse(message, options); + // underscoreDivideNumbers when set to false, if a column has a format like "_v_3_", it will be formatted to "_v3_" // underscoreDivideNumbers when set to true, if a column has a format like "_v_3_", we keep it like that // For older destinations, it will come as true and for new destinations this config will not be present which means we will treat it as false. diff --git a/src/warehouse/util.js b/src/warehouse/util.js index 7f4e224a349..1f2d9215f2c 100644 --- a/src/warehouse/util.js +++ b/src/warehouse/util.js @@ -136,6 +136,27 @@ const getRecordIDForExtract = (message) => { return recordId; }; +function mergeJSONPathsFromDataWarehouse(message, options) { + const dataWarehouseOptions = message.integrations?.['DATA_WAREHOUSE']?.options; + if (!dataWarehouseOptions?.jsonPaths) return; + + const dataWarehouseJSONPaths = Array.isArray(dataWarehouseOptions.jsonPaths) + ? dataWarehouseOptions.jsonPaths + : []; + const currentJSONPaths = Array.isArray(options.integrationOptions?.jsonPaths) + ? options.integrationOptions.jsonPaths + : []; + + switch (options.provider) { + case 'rs': + case 'postgres': + case 'snowflake': + case 'bq': + options.integrationOptions.jsonPaths = [...dataWarehouseJSONPaths, ...currentJSONPaths]; + break; + } +} + module.exports = { isObject, isValidJsonPathKey, @@ -148,4 +169,5 @@ module.exports = { sourceCategoriesToUseRecordId, getCloudRecordID, getRecordIDForExtract, + mergeJSONPathsFromDataWarehouse, }; diff --git a/test/__tests__/shopify_warehouse.test.js b/test/__tests__/shopify_warehouse.test.js new file mode 100644 index 00000000000..080022680bb --- /dev/null +++ b/test/__tests__/shopify_warehouse.test.js @@ -0,0 +1,103 @@ +const event = { + "request": { + "query": { + "whSchemaVersion": "v1" + } + }, + "message": { + "context": { + "shopifyDetails": { + "id": 5778367414385, + "current_total_tax": "10.00", + "current_total_tax_set": { + "shop_money": { + "amount": "10.00", + "currency_code": "USD" + }, + }, + "name": "#1017", + "phone": null, + } + }, + "integrations": { + "SHOPIFY": true, + "DATA_WAREHOUSE": { + "options": { + "jsonPaths": [ + "track.context.shopifyDetails" + ] + } + } + }, + "type": "track", + "event": "Order Updated", + "properties": { + "order_id": "5778367414385", + "currency": "USD", + "products": [ + { + "product_id": "7234590408817", + "price": 600, + "quantity": 1 + } + ] + }, + "userId": "123321", + "traits": {}, + "timestamp": "2024-01-01T01:23:45.678Z", + }, + "destination": { + "Config": {}, + } +}; + +/* + Test for warehouse agnostic DATA_WAREHOUSE JSON column support for Shopify source +*/ +describe('DATA_WAREHOUSE integrations', () => { + it('should process event and return responses for common providers for agnostic support', () => { + const responses = require('../../src/v0/destinations/snowflake/transform').process(event); + expect(responses).toHaveLength(2); + expect(responses[0].metadata.table).toBe('TRACKS'); + expect(responses[1].metadata.table).toBe('ORDER_UPDATED'); + + expect(responses[0].metadata.columns.CONTEXT_SHOPIFY_DETAILS).toBe('json'); + expect(responses[0].data.CONTEXT_SHOPIFY_DETAILS).toBe('{"id":5778367414385,"current_total_tax":"10.00","current_total_tax_set":{"shop_money":{"amount":"10.00","currency_code":"USD"}},"name":"#1017","phone":null}'); + + expect(responses[1].metadata.columns.CONTEXT_SHOPIFY_DETAILS).toBe('json'); + expect(responses[1].data.CONTEXT_SHOPIFY_DETAILS).toBe('{"id":5778367414385,"current_total_tax":"10.00","current_total_tax_set":{"shop_money":{"amount":"10.00","currency_code":"USD"}},"name":"#1017","phone":null}'); + }); + + it('should process event and return response for other providers like mssql', () => { + const responses = require('../../src/v0/destinations/mssql/transform').process(event); + expect(responses).toHaveLength(2); + expect(responses[0].metadata.table).toBe('tracks'); + expect(responses[1].metadata.table).toBe('order_updated'); + + expect(responses[0].metadata.columns.context_shopify_details).toBe(undefined); + expect(responses[0].metadata.columns.context_shopify_details_id).toBe('int'); + expect(responses[0].metadata.columns.context_shopify_details_current_total_tax).toBe('string'); + expect(responses[0].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('string'); + expect(responses[0].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('string'); + expect(responses[0].metadata.columns.context_shopify_details_name).toBe('string'); + expect(responses[0].data.context_shopify_details).toBe(undefined); + expect(responses[0].data.context_shopify_details_id).toBe(5778367414385); + expect(responses[0].data.context_shopify_details_current_total_tax).toBe('10.00'); + expect(responses[0].data.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('10.00'); + expect(responses[0].data.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('USD'); + expect(responses[0].data.context_shopify_details_name).toBe('#1017'); + + expect(responses[1].metadata.columns.context_shopify_details).toBe(undefined); + expect(responses[1].metadata.columns.context_shopify_details_id).toBe('int'); + expect(responses[1].metadata.columns.context_shopify_details_current_total_tax).toBe('string'); + expect(responses[1].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('string'); + expect(responses[1].metadata.columns.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('string'); + expect(responses[1].metadata.columns.context_shopify_details_name).toBe('string'); + expect(responses[1].data.context_shopify_details).toBe(undefined); + expect(responses[1].data.context_shopify_details_id).toBe(5778367414385); + expect(responses[1].data.context_shopify_details_current_total_tax).toBe('10.00'); + expect(responses[1].data.context_shopify_details_current_total_tax_set_shop_money_amount).toBe('10.00'); + expect(responses[1].data.context_shopify_details_current_total_tax_set_shop_money_currency_code).toBe('USD'); + expect(responses[1].data.context_shopify_details_name).toBe('#1017'); + }); +}); \ No newline at end of file diff --git a/test/integrations/component.test.ts b/test/integrations/component.test.ts index 040843d8110..27007b240dd 100644 --- a/test/integrations/component.test.ts +++ b/test/integrations/component.test.ts @@ -254,7 +254,7 @@ describe.each(allTestDataFilePaths)('%s Tests', (testDataPath) => { describe(`${testData[0].name} ${testData[0].module}`, () => { test.each(extendedTestData)( - '$feature -> $description$descriptionSuffix (index: $#)', + '$tcData.feature -> $tcData.description $descriptionSuffix (index: $#)', async ({ tcData, sourceTransformV2Flag }) => { tcData?.mockFns?.(mockAdapter); diff --git a/test/integrations/destinations/bluecore/identifyTestData.ts b/test/integrations/destinations/bluecore/identifyTestData.ts index fee27ccf0fa..dbc6c3967c3 100644 --- a/test/integrations/destinations/bluecore/identifyTestData.ts +++ b/test/integrations/destinations/bluecore/identifyTestData.ts @@ -237,7 +237,7 @@ export const identifyData = [ body: [ { error: - "[Bluecore] traits.action must be 'identify' for identify action: Workflow: procWorkflow, Step: prepareIdentifyPayload, ChildStep: undefined, OriginalError: [Bluecore] traits.action must be 'identify' for identify action", + "[Bluecore] traits.action must be 'identify' for identify action: Workflow: procWorkflow, Step: prepareIdentifyPayload, ChildStep: undefined, OriginalError: [Bluecore] traits.action must be 'identify' for identify action", metadata: { destinationId: '', destinationType: '', diff --git a/test/integrations/destinations/http/common.ts b/test/integrations/destinations/http/common.ts index f0c8bc8a337..4dc9d4daf27 100644 --- a/test/integrations/destinations/http/common.ts +++ b/test/integrations/destinations/http/common.ts @@ -92,13 +92,18 @@ const destinations: Destination[] = [ }, { Config: { - apiUrl: 'http://abc.com/contacts/{{$.traits.email}}/', + apiUrl: 'http://abc.com/contacts/', auth: 'apiKeyAuth', apiKeyName: 'x-api-key', apiKeyValue: 'test-api-key', method: 'DELETE', isBatchingEnabled: true, maxBatchSize: 4, + pathParams: [ + { + path: '$.traits.email', + }, + ], }, DestinationDefinition: { DisplayName: displayName, @@ -114,13 +119,18 @@ const destinations: Destination[] = [ }, { Config: { - apiUrl: 'http://abc.com/contacts/{{$.traits.email}}/', + apiUrl: 'http://abc.com/contacts/', auth: 'apiKeyAuth', apiKeyName: 'x-api-key', apiKeyValue: 'test-api-key', method: 'GET', isBatchingEnabled: true, maxBatchSize: 4, + pathParams: [ + { + path: '$.traits.email', + }, + ], }, DestinationDefinition: { DisplayName: displayName, @@ -158,27 +168,27 @@ const destinations: Destination[] = [ propertiesMapping: [ { from: '$.event', - to: '$.event', + to: '$.body.event', }, { from: '$.properties.currency', - to: '$.currency', + to: '$.body.currency', }, { from: '$.userId', - to: '$.userId', + to: '$.body.userId', }, { from: '$.properties.products[*].product_id', - to: '$.properties.items[*].item_id', + to: '$.body.properties.items[*].item_id', }, { from: '$.properties.products[*].name', - to: '$.properties.items[*].name', + to: '$.body.properties.items[*].name', }, { from: '$.properties.products[*].price', - to: '$.properties.items[*].price', + to: '$.body.properties.items[*].price', }, ], }, @@ -249,7 +259,7 @@ const destinations: Destination[] = [ }, { Config: { - apiUrl: 'http://abc.com/contacts/{{$.traits.phone}}', + apiUrl: 'http://abc.com/contacts/', auth: 'noAuth', method: 'POST', format: 'JSON', @@ -265,6 +275,305 @@ const destinations: Destination[] = [ from: '.traits.key', }, ], + pathParams: [ + { + path: '$.traits.phone', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/contacts', + auth: 'basicAuth', + username: 'test-user', + password: '', + method: 'GET', + format: 'JSON', + isBatchingEnabled: true, + maxBatchSize: 2, + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '2', + }, + { + to: "$.'content-type'", + from: "'application/json'", + }, + { + to: '$.h3', + from: '$.traits.firstName', + }, + ], + queryParams: [ + { + to: "$['q1']", + from: "'val1'", + }, + { + to: '$.q2', + from: '$.traits.email', + }, + ], + pathParams: [ + { + path: '$.userId', + }, + { + path: 'c1', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/contacts', + auth: 'basicAuth', + username: 'test-user', + password: '', + method: 'GET', + format: 'JSON', + isBatchingEnabled: true, + maxBatchSize: 2, + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '2', + }, + { + to: "$.'content-type'", + from: "'application/json'", + }, + { + to: '$.h3', + from: '$.traits.firstName', + }, + ], + queryParams: [ + { + to: 'user name', + from: "'val1'", + }, + { + to: '$.q2', + from: '$.traits.email', + }, + ], + pathParams: [ + { + path: '$.userId', + }, + { + path: 'c1', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/events', + auth: 'bearerTokenAuth', + bearerToken: 'test-token', + method: 'POST', + format: 'XML', + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '$.key1', + }, + { + to: "$.'content-type'", + from: "'application/json'", + }, + ], + propertiesMapping: [ + { + from: '$.properties', + to: '$', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/events', + auth: 'bearerTokenAuth', + bearerToken: 'test-token', + method: 'POST', + format: 'FORM', + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '$.key1', + }, + { + to: "$.'content-type'", + from: "'application/json'", + }, + ], + propertiesMapping: [ + { + from: '$.event', + to: '$.event', + }, + { + from: '$.properties.currency', + to: '$.currency', + }, + { + from: '$.userId', + to: '$.userId', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/events', + auth: 'bearerTokenAuth', + bearerToken: 'test-token', + method: 'POST', + format: 'FORM', + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '$.key1', + }, + ], + propertiesMapping: [ + { + from: '$.event', + to: '$.event', + }, + { + from: '$.properties.currency', + to: '$.currency', + }, + { + from: '$.userId', + to: '$.userId', + }, + ], + }, + DestinationDefinition: { + DisplayName: displayName, + ID: '123', + Name: destTypeInUpperCase, + Config: { cdkV2Enabled: true }, + }, + Enabled: true, + ID: '123', + Name: destTypeInUpperCase, + Transformations: [], + WorkspaceID: 'test-workspace-id', + }, + { + Config: { + apiUrl: 'http://abc.com/events', + auth: 'bearerTokenAuth', + bearerToken: 'test-token', + method: 'POST', + format: 'FORM', + headers: [ + { + to: '$.h1', + from: "'val1'", + }, + { + to: '$.h2', + from: '$.key1', + }, + ], + propertiesMapping: [ + { + from: '$.event', + to: '$.event', + }, + { + from: '$.properties.currency', + to: '$.currency', + }, + { + from: '$.userId', + to: '$.userId', + }, + ], }, DestinationDefinition: { DisplayName: displayName, @@ -329,7 +638,7 @@ const properties = { const processorInstrumentationErrorStatTags = { destType: destTypeInUpperCase, errorCategory: 'dataValidation', - errorType: 'instrumentation', + errorType: 'configuration', feature: 'processor', implementation: 'cdkV2', module: 'destination', diff --git a/test/integrations/destinations/http/processor/configuration.ts b/test/integrations/destinations/http/processor/configuration.ts index 43d39952b9a..54673bbae28 100644 --- a/test/integrations/destinations/http/processor/configuration.ts +++ b/test/integrations/destinations/http/processor/configuration.ts @@ -1,6 +1,12 @@ import { ProcessorTestData } from '../../../testTypes'; import { generateMetadata, transformResultBuilder } from '../../../testUtils'; -import { destType, destinations, properties, traits } from '../common'; +import { + destType, + destinations, + properties, + traits, + processorInstrumentationErrorStatTags, +} from '../common'; export const configuration: ProcessorTestData[] = [ { @@ -38,6 +44,9 @@ export const configuration: ProcessorTestData[] = [ method: 'POST', userId: '', endpoint: destinations[0].Config.apiUrl, + headers: { + 'Content-Type': 'application/json', + }, JSON: { contacts: { first_name: 'John', @@ -89,8 +98,9 @@ export const configuration: ProcessorTestData[] = [ output: transformResultBuilder({ method: 'DELETE', userId: '', - endpoint: 'http://abc.com/contacts/john.doe@example.com/', + endpoint: 'http://abc.com/contacts/john.doe%40example.com', headers: { + 'Content-Type': 'application/json', 'x-api-key': 'test-api-key', }, }), @@ -137,6 +147,7 @@ export const configuration: ProcessorTestData[] = [ userId: '', endpoint: destinations[1].Config.apiUrl, headers: { + 'Content-Type': 'application/json', Authorization: 'Basic dGVzdC11c2VyOg==', h1: 'val1', h2: 2, @@ -191,14 +202,321 @@ export const configuration: ProcessorTestData[] = [ userId: '', endpoint: destinations[4].Config.apiUrl, headers: { + 'Content-Type': 'application/xml', Authorization: 'Bearer test-token', h1: 'val1', 'content-type': 'application/json', }, XML: { payload: - 'Order CompletedUSDuserId123622c6f5d5cf86a4c77358033Cones of Dunshire40577c6f5d5cf86a4c7735ba03Five Crowns5', + 'Order CompletedUSDuserId123622c6f5d5cf86a4c77358033Cones of Dunshire40577c6f5d5cf86a4c7735ba03Five Crowns5', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-5', + name: destType, + description: 'Track call with pathParams mapping', + scenario: 'Business', + successCriteria: 'Response should have the give paths added in the endpoint', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: destinations[7], + message: { + type: 'track', + userId: 'userId123', + event: 'Order Completed', + properties, + }, + metadata: generateMetadata(1), + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'GET', + userId: '', + endpoint: 'http://abc.com/contacts/userId123/c1', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC11c2VyOg==', + h1: 'val1', + h2: 2, + 'content-type': 'application/json', + }, + params: { + q1: 'val1', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-6', + name: destType, + description: 'Track call with query params keys containing space', + scenario: 'Business', + successCriteria: 'Response should contain query params with URI encoded keys', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: destinations[8], + message: { + type: 'track', + userId: 'userId123', + event: 'Order Completed', + properties, + }, + metadata: generateMetadata(1), + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'GET', + userId: '', + endpoint: 'http://abc.com/contacts/userId123/c1', + headers: { + 'Content-Type': 'application/json', + Authorization: 'Basic dGVzdC11c2VyOg==', + h1: 'val1', + h2: 2, + 'content-type': 'application/json', + }, + params: { + 'user%20name': 'val1', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-7', + name: destType, + description: 'Track call with xml format with multiple keys', + scenario: 'Business', + successCriteria: 'Should throw error as the body have multiple root keys', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + body: [ + { + destination: destinations[9], + message: { + type: 'track', + userId: 'userId123', + event: 'Order Completed', + properties: { + name: "Rubik's Cube", + "1revenue-wdfqwe'": 4.99, + brand: null, + }, + }, + metadata: generateMetadata(1), + }, + ], + method: 'POST', + }, + }, + output: { + response: { + status: 200, + body: [ + { + error: + 'Error: XML supports only one root key. Please update request body mappings accordingly: Workflow: procWorkflow, Step: prepareBody, ChildStep: undefined, OriginalError: Error: XML supports only one root key. Please update request body mappings accordingly', + statusCode: 400, + metadata: generateMetadata(1), + statTags: { ...processorInstrumentationErrorStatTags }, + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-8', + name: destType, + description: + 'Track call with bearer token, form format, post method, additional headers and properties mapping', + scenario: 'Business', + successCriteria: + 'Response should be in form format with post method, headers and properties mapping', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + method: 'POST', + body: [ + { + destination: destinations[10], + message: { + type: 'track', + userId: 'userId123', + event: 'Order Completed', + properties, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destinations[10].Config.apiUrl, + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + Authorization: 'Bearer test-token', + h1: 'val1', + 'content-type': 'application/json', + }, + FORM: { + currency: 'USD', + event: 'Order Completed', + userId: 'userId123', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-9', + name: destType, + description: 'Track call with bearer token, form url encoded format', + scenario: 'Business', + successCriteria: + 'Response should be in form format with post method, headers and properties mapping', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + method: 'POST', + body: [ + { + destination: destinations[11], + message: { + type: 'track', + userId: 'userId123', + event: 'Order Completed', + properties, + }, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destinations[11].Config.apiUrl, + headers: { + Authorization: 'Bearer test-token', + h1: 'val1', + 'Content-Type': 'application/x-www-form-urlencoded', + }, + FORM: { + currency: 'USD', + event: 'Order Completed', + userId: 'userId123', + }, + }), + statusCode: 200, + metadata: generateMetadata(1), + }, + ], + }, + }, + }, + { + id: 'http-configuration-test-10', + name: destType, + description: 'empty body', + scenario: 'Business', + successCriteria: + 'Response should be in form format with post method, headers and properties mapping', + feature: 'processor', + module: 'destination', + version: 'v0', + input: { + request: { + method: 'POST', + body: [ + { + destination: destinations[12], + message: {}, + metadata: generateMetadata(1), + }, + ], + }, + }, + output: { + response: { + status: 200, + body: [ + { + output: transformResultBuilder({ + method: 'POST', + userId: '', + endpoint: destinations[12].Config.apiUrl, + headers: { + Authorization: 'Bearer test-token', + h1: 'val1', + 'Content-Type': 'application/x-www-form-urlencoded', }, + FORM: {}, }), statusCode: 200, metadata: generateMetadata(1), diff --git a/test/integrations/destinations/http/router/data.ts b/test/integrations/destinations/http/router/data.ts index ea14ec73418..8fd3a1584cf 100644 --- a/test/integrations/destinations/http/router/data.ts +++ b/test/integrations/destinations/http/router/data.ts @@ -182,9 +182,10 @@ export const data = [ version: '1', type: 'REST', method: 'GET', - endpoint: 'http://abc.com/contacts/john.doe@example.com/', + endpoint: 'http://abc.com/contacts/john.doe%40example.com', headers: { 'x-api-key': 'test-api-key', + 'Content-Type': 'application/json', }, params: {}, body: { @@ -233,6 +234,7 @@ export const data = [ method: 'GET', endpoint: 'http://abc.com/contacts', headers: { + 'Content-Type': 'application/json', Authorization: 'Basic dGVzdC11c2VyOg==', 'content-type': 'application/json', h1: 'val1', @@ -241,7 +243,7 @@ export const data = [ }, params: { q1: 'val1', - q2: 'john.doe@example.com', + q2: 'john.doe%40example.com', }, body: { JSON: {}, @@ -265,6 +267,7 @@ export const data = [ method: 'GET', endpoint: 'http://abc.com/contacts', headers: { + 'Content-Type': 'application/json', Authorization: 'Basic dGVzdC11c2VyOg==', 'content-type': 'application/json', h1: 'val1', @@ -273,7 +276,7 @@ export const data = [ }, params: { q1: 'val1', - q2: 'john.doe@example.com', + q2: 'john.doe%40example.com', }, body: { JSON: {}, @@ -297,6 +300,7 @@ export const data = [ method: 'GET', endpoint: 'http://abc.com/contacts', headers: { + 'Content-Type': 'application/json', Authorization: 'Basic dGVzdC11c2VyOg==', 'content-type': 'application/json', h1: 'val1', @@ -305,7 +309,7 @@ export const data = [ }, params: { q1: 'val1', - q2: 'alex.t@example.com', + q2: 'alex.t%40example.com', }, body: { JSON: {}, @@ -355,6 +359,7 @@ export const data = [ endpoint: 'http://abc.com/events', params: {}, headers: { + 'Content-Type': 'application/json', 'content-type': 'application/json', }, body: { @@ -409,6 +414,7 @@ export const data = [ method: 'POST', endpoint: 'http://abc.com/contacts/1234567890', headers: { + 'Content-Type': 'application/json', 'content-type': 'application/json', key: 'value1', }, @@ -433,6 +439,7 @@ export const data = [ method: 'POST', endpoint: 'http://abc.com/contacts/1234567890', headers: { + 'Content-Type': 'application/json', 'content-type': 'application/json', }, params: {}, @@ -456,6 +463,7 @@ export const data = [ method: 'POST', endpoint: 'http://abc.com/contacts/2234567890', headers: { + 'Content-Type': 'application/json', 'content-type': 'application/json', }, params: {}, diff --git a/test/integrations/destinations/mp/processor/data.ts b/test/integrations/destinations/mp/processor/data.ts index d13cf64cae2..9c385daee4b 100644 --- a/test/integrations/destinations/mp/processor/data.ts +++ b/test/integrations/destinations/mp/processor/data.ts @@ -97,7 +97,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","campaign_id":"test_name","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1579847342402,"utm_campaign":"test_name","utm_source":"rudder","utm_medium":"test_medium","utm_term":"test_tem","utm_content":"test_content","utm_test":"test","utm_keyword":"test_keyword","name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","campaign_id":"test_name","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1579847342402,"utm_campaign":"test_name","utm_source":"rudder","utm_medium":"test_medium","utm_term":"test_tem","utm_content":"test_content","utm_test":"test","utm_keyword":"test_keyword","name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -204,7 +204,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"Viewed a Contact Us page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1579847342402,"name":"Contact Us","category":"Contact","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Viewed a Contact Us page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1579847342402,"name":"Contact Us","category":"Contact","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -726,7 +726,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"test revenue MIXPANEL","properties":{"currency":"USD","revenue":45.89,"counter":1,"item_purchased":"2","number_of_logins":"","city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","campaign_id":"test_name","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"a6a0ad5a-bd26-4f19-8f75-38484e580fc7","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342403,"utm_campaign":"test_name","utm_source":"rudder","utm_medium":"test_medium","utm_term":"test_tem","utm_content":"test_content","utm_test":"test","utm_keyword":"test_keyword","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"test revenue MIXPANEL","properties":{"currency":"USD","revenue":45.89,"counter":1,"item_purchased":"2","number_of_logins":"","city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","campaign_id":"test_name","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"a6a0ad5a-bd26-4f19-8f75-38484e580fc7","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342403,"utm_campaign":"test_name","utm_source":"rudder","utm_medium":"test_medium","utm_term":"test_tem","utm_content":"test_content","utm_test":"test","utm_keyword":"test_keyword","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -979,7 +979,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"revenue":25,"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"revenue":25,"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -1138,7 +1138,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","revenue":34,"key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","first_name":"Mickey","lastName":"Mouse","name":"Mickey Mouse","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","revenue":34,"key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","first_name":"Mickey","lastName":"Mouse","name":"Mickey Mouse","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -1272,7 +1272,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":" new Order Completed totally","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"total":23,"order_id":"50314b8e9bcf000000000000","key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":" new Order Completed totally","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"total":23,"order_id":"50314b8e9bcf000000000000","key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -1406,7 +1406,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":" Order Completed ","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"total":23,"order_id":"50314b8e9bcf000000000000","key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"Billing Amount":"77","city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":" Order Completed ","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"total":23,"order_id":"50314b8e9bcf000000000000","key_1":{"child_key1":"child_value1","child_key2":{"child_key21":"child_value21","child_key22":"child_value22"}},"products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"shipping":3,"subtotal":22.5,"tax":2,"Billing Amount":"77","city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -2077,7 +2077,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"revenue":25,"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","firstname":"Mickey","lastname":"Mouse","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"KM Order Completed","properties":{"affiliation":"Google Store","checkout_id":"fksdjfsdjfisjf9sdfjsd9f","coupon":"hasbros","currency":"USD","discount":2.5,"order_id":"50314b8e9bcf000000000000","products":[{"category":"Games","image_url":"https:///www.example.com/product/path.jpg","name":"Monopoly: 3rd Edition","price":19,"product_id":"507f1f77bcf86cd799439011","quantity":1,"sku":"45790-32","url":"https://www.example.com/product/path"},{"category":"Games","name":"Uno Card Game","price":3,"product_id":"505bd76785ebb509fc183733","quantity":2,"sku":"46493-32"}],"revenue":25,"shipping":3,"subtotal":22.5,"tax":2,"total":27.5,"city":"Disney","country":"USA","email":"mickey@disney.com","firstname":"Mickey","lastname":"Mouse","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"aa5f5e44-8756-40ad-ad1e-b0d3b9fa710a","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -2374,7 +2374,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"Loaded a Page","properties":{"path":"/tests/html/index2.html","referrer":"","search":"","title":"","url":"http://localhost/tests/html/index2.html","category":"communication","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Loaded a Page","properties":{"path":"/tests/html/index2.html","referrer":"","search":"","title":"","url":"http://localhost/tests/html/index2.html","category":"communication","ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1579847342402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -3558,7 +3558,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"FirstTrackCall12","properties":{"foo":"bar","$deviceId":"nkasdnkasd","anonymousId":"ea776ad0-3136-44fb-9216-5b1578609a2b","userId":"as09sufa09usaf09as0f9uasf","id":"as09sufa09usaf09as0f9uasf","firstName":"Bob","lastName":"Marley","name":"Bob Marley","age":43,"email":"bob@marleymail.com","phone":"+447748544123","birthday":"1987-01-01T20:08:59+0000","createdAt":"2022-01-21T14:10:12+0000","address":"51,B.L.T road, Kolkata-700060","description":"I am great","gender":"male","title":"Founder","username":"bobm","website":"https://bobm.com","randomProperty":"randomValue","$user_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","$current_url":"http://127.0.0.1:7307/Testing/App_for_testingTool/","$referrer":"http://127.0.0.1:7307/Testing/","$screen_height":900,"$screen_width":1440,"$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.1.18","$insert_id":"0d5c1a4a-27e4-41da-a246-4d01f44e74bd","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1632986123523,"$browser":"Chrome","$browser_version":"93.0.4577.82"}}]', + '[{"event":"FirstTrackCall12","properties":{"foo":"bar","$deviceId":"nkasdnkasd","anonymousId":"ea776ad0-3136-44fb-9216-5b1578609a2b","userId":"as09sufa09usaf09as0f9uasf","id":"as09sufa09usaf09as0f9uasf","firstName":"Bob","lastName":"Marley","name":"Bob Marley","age":43,"email":"bob@marleymail.com","phone":"+447748544123","birthday":"1987-01-01T20:08:59+0000","createdAt":"2022-01-21T14:10:12+0000","address":"51,B.L.T road, Kolkata-700060","description":"I am great","gender":"male","title":"Founder","username":"bobm","website":"https://bobm.com","randomProperty":"randomValue","$user_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","$current_url":"http://127.0.0.1:7307/Testing/App_for_testingTool/","$referrer":"http://127.0.0.1:7307/Testing/","$screen_height":900,"$screen_width":1440,"$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"http://127.0.0.1:7307/Testing/","$initial_referring_domain":"127.0.0.1:7307","$app_build_number":"1.0.0","$app_version_string":"1.1.18","$insert_id":"0d5c1a4a-27e4-41da-a246-4d01f44e74bd","token":"test_api_token","distinct_id":"e6ab2c5e-2cda-44a9-a962-e2f67df78bca","time":1632986123523,"$browser":"Chrome","$browser_version":"93.0.4577.82"}}]', }, XML: {}, FORM: {}, @@ -5174,7 +5174,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"test revenue MIXPANEL","properties":{"currency":"USD","revenue":18.9,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$user_id":"userId01","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"a6a0ad5a-bd26-4f19-8f75-38484e580fc7","token":"test_api_token","distinct_id":"userId01","time":1579847342403,"$device_id":"anonId01","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"test revenue MIXPANEL","properties":{"currency":"USD","revenue":18.9,"city":"Disney","country":"USA","email":"mickey@disney.com","firstName":"Mickey","ip":"0.0.0.0","$user_id":"userId01","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"a6a0ad5a-bd26-4f19-8f75-38484e580fc7","token":"test_api_token","distinct_id":"userId01","time":1579847342403,"$device_id":"anonId01","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -5281,7 +5281,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"$device:anonId01","time":1579847342402,"$device_id":"anonId01","name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"$device:anonId01","time":1579847342402,"$device_id":"anonId01","name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, diff --git a/test/integrations/destinations/mp/router/data.ts b/test/integrations/destinations/mp/router/data.ts index 8716c9daa09..67f4d006c28 100644 --- a/test/integrations/destinations/mp/router/data.ts +++ b/test/integrations/destinations/mp/router/data.ts @@ -450,7 +450,7 @@ export const data = [ JSON_ARRAY: {}, GZIP: { payload: - '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1688624942402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1688624942402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, @@ -1173,7 +1173,7 @@ export const data = [ JSON: {}, JSON_ARRAY: { batch: - '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1688624942402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', + '[{"event":"Loaded a Page","properties":{"ip":"0.0.0.0","$user_id":"hjikl","$current_url":"https://docs.rudderstack.com/destinations/mixpanel","$screen_dpi":2,"mp_lib":"RudderLabs JavaScript SDK","$initial_referrer":"https://docs.rudderstack.com","$initial_referring_domain":"docs.rudderstack.com","$app_build_number":"1.0.0","$app_version_string":"1.0.5","$insert_id":"dd266c67-9199-4a52-ba32-f46ddde67312","token":"test_api_token","distinct_id":"hjikl","time":1688624942402,"name":"Contact Us","$browser":"Chrome","$browser_version":"79.0.3945.117"}}]', }, XML: {}, FORM: {}, diff --git a/test/integrations/sources/shopify/constants.ts b/test/integrations/sources/shopify/constants.ts index af53a3180e8..cd362adaec3 100644 --- a/test/integrations/sources/shopify/constants.ts +++ b/test/integrations/sources/shopify/constants.ts @@ -1,3 +1,58 @@ +const dummyResponseCommonPayload = { + navigator: { + language: 'en-US', + cookieEnabled: true, + languages: ['en-US', 'en'], + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + }, + window: { + innerHeight: 1028, + innerWidth: 1362, + outerHeight: 1080, + outerWidth: 1728, + pageXOffset: 0, + pageYOffset: 0, + location: { + href: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + hash: '', + host: 'store.myshopify.com', + hostname: 'store.myshopify.com', + origin: 'https://store.myshopify.com', + pathname: '/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + port: '', + protocol: 'https:', + search: '', + }, + origin: 'https://store.myshopify.com', + screen: { + height: 1117, + width: 1728, + }, + screenX: 0, + screenY: 37, + scrollX: 0, + scrollY: 0, + }, + page: { + title: 'Checkout - pixel-testing-rs', + url: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + path: '/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + search: '', + }, + userAgent: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', + screen: { + height: 1117, + width: 1728, + }, + library: { + name: 'RudderStack Shopify Cloud', + eventOrigin: 'client', + version: '2.0.0', + }, +}; + export const dummySourceConfig = { ID: 'dummy-source-id', OriginalID: '', @@ -83,25 +138,10 @@ export const dummyContext = { }, }; -export const note_attributes = [ - { - name: 'cartId', - value: '9c623f099fc8819aa4d6a958b65dfe7d', - }, - { - name: 'cartToken', - value: 'Z2NwLXVzLWVhc3QxOjAxSkQzNUFXVEI4VkVUNUpTTk1LSzBCMzlF', - }, - { - name: 'rudderAnonymousId', - value: '50ead33e-d763-4854-b0ab-765859ef05cb', - }, -]; - -export const responseDummyContext = { +export const dummyContextwithCampaign = { document: { location: { - href: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + href: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU?checkout%5Bpayment_gateway%5D=shopify_payments&utm_campaign=shopifySale&utm_medium=checkout&utm_term=term_checkout&utm_content=web&utm_custom1=customutm&tag=tag', hash: '', host: 'store.myshopify.com', hostname: 'store.myshopify.com', @@ -150,21 +190,60 @@ export const responseDummyContext = { scrollX: 0, scrollY: 0, }, - page: { - title: 'Checkout - pixel-testing-rs', - url: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', - path: '/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', - search: '', +}; + +export const note_attributes = [ + { + name: 'cartId', + value: '9c623f099fc8819aa4d6a958b65dfe7d', }, - userAgent: - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36', - screen: { - height: 1117, - width: 1728, + { + name: 'cartToken', + value: 'Z2NwLXVzLWVhc3QxOjAxSkQzNUFXVEI4VkVUNUpTTk1LSzBCMzlF', }, - library: { - name: 'RudderStack Shopify Cloud', - eventOrigin: 'client', - version: '2.0.0', + { + name: 'rudderAnonymousId', + value: '50ead33e-d763-4854-b0ab-765859ef05cb', + }, +]; + +export const responseDummyContext = { + document: { + location: { + href: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + hash: '', + host: 'store.myshopify.com', + hostname: 'store.myshopify.com', + origin: 'https://store.myshopify.com', + pathname: '/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + port: '', + protocol: 'https:', + search: '', + }, + referrer: 'https://store.myshopify.com/cart', + characterSet: 'UTF-8', + title: 'Checkout - pixel-testing-rs', + }, + ...dummyResponseCommonPayload, +}; + +export const responseDummyContextwithCampaign = { + document: { + location: { + href: 'https://store.myshopify.com/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU?checkout%5Bpayment_gateway%5D=shopify_payments&utm_campaign=shopifySale&utm_medium=checkout&utm_term=term_checkout&utm_content=web&utm_custom1=customutm&tag=tag', + hash: '', + host: 'store.myshopify.com', + hostname: 'store.myshopify.com', + origin: 'https://store.myshopify.com', + pathname: '/checkouts/cn/Z2NwLXVzLWVhc3QxOjAxSjY5OVpIRURQNERFMDBKUTVaRkI4UzdU', + port: '', + protocol: 'https:', + search: '', + }, + referrer: 'https://store.myshopify.com/cart', + title: 'Checkout - pixel-testing-rs', + characterSet: 'UTF-8', }, + // title: 'Checkout - pixel-testing-rs', + ...dummyResponseCommonPayload, }; diff --git a/test/integrations/sources/shopify/pixelTestScenarios/CheckoutEventsTests.ts b/test/integrations/sources/shopify/pixelTestScenarios/CheckoutEventsTests.ts index ff1ea39ed13..2b04a33fb8b 100644 --- a/test/integrations/sources/shopify/pixelTestScenarios/CheckoutEventsTests.ts +++ b/test/integrations/sources/shopify/pixelTestScenarios/CheckoutEventsTests.ts @@ -336,6 +336,11 @@ export const pixelCheckoutEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Checkout Started', @@ -775,6 +780,11 @@ export const pixelCheckoutEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Order Completed', diff --git a/test/integrations/sources/shopify/pixelTestScenarios/CheckoutStepsTests.ts b/test/integrations/sources/shopify/pixelTestScenarios/CheckoutStepsTests.ts index 3db2e3ab101..95fd2ea26bd 100644 --- a/test/integrations/sources/shopify/pixelTestScenarios/CheckoutStepsTests.ts +++ b/test/integrations/sources/shopify/pixelTestScenarios/CheckoutStepsTests.ts @@ -387,6 +387,11 @@ export const pixelCheckoutStepsScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Checkout Address Info Submitted', @@ -921,6 +926,11 @@ export const pixelCheckoutStepsScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Checkout Contact Info Submitted', @@ -1470,6 +1480,11 @@ export const pixelCheckoutStepsScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Checkout Shipping Info Submitted', @@ -2035,6 +2050,11 @@ export const pixelCheckoutStepsScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Payment Info Entered', diff --git a/test/integrations/sources/shopify/pixelTestScenarios/ProductEventsTests.ts b/test/integrations/sources/shopify/pixelTestScenarios/ProductEventsTests.ts index 46bd4f96151..7427ea5ae76 100644 --- a/test/integrations/sources/shopify/pixelTestScenarios/ProductEventsTests.ts +++ b/test/integrations/sources/shopify/pixelTestScenarios/ProductEventsTests.ts @@ -1,5 +1,11 @@ // This file contains the test scenarios related to Shopify pixel events, emitted from web pixel on the browser. -import { dummyContext, dummySourceConfig, responseDummyContext } from '../constants'; +import { + dummyContext, + dummyContextwithCampaign, + dummySourceConfig, + responseDummyContext, + responseDummyContextwithCampaign, +} from '../constants'; export const pixelEventsTestScenarios = [ { @@ -18,7 +24,7 @@ export const pixelEventsTestScenarios = [ type: 'standard', clientId: 'c7b3f99b-4d34-463b-835f-c879482a7750', timestamp: '2024-09-15T17:24:30.373Z', - context: dummyContext, + context: dummyContextwithCampaign, pixelEventLabel: true, query_parameters: { topic: ['page_viewed'], @@ -42,7 +48,14 @@ export const pixelEventsTestScenarios = [ batch: [ { context: { - ...responseDummyContext, + ...responseDummyContextwithCampaign, + campaign: { + content: 'web', + medium: 'checkout', + name: 'shopifySale', + term: 'term_checkout', + utm_custom1: 'customutm', + }, shopifyDetails: { clientId: 'c7b3f99b-4d34-463b-835f-c879482a7750', data: {}, @@ -55,6 +68,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['page.context.shopifyDetails'], + }, + }, }, name: 'Page View', type: 'page', @@ -166,6 +184,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Product Viewed', @@ -330,6 +353,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Cart Viewed', @@ -555,6 +583,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Product List Viewed', @@ -725,6 +758,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Product Added', @@ -866,6 +904,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Product Removed', @@ -955,6 +998,11 @@ export const pixelEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Search Submitted', diff --git a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts index b6fa5be322e..4938294ef99 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/CheckoutEventsTests.ts @@ -215,12 +215,17 @@ export const checkoutEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', - event: 'Checkout Started - Webhook', + event: 'Checkout Started Webhook', properties: { order_id: '35550298931313', - value: '600.00', + value: 600, tax: 0, currency: 'USD', products: [ @@ -530,6 +535,11 @@ export const checkoutEventsTestScenarios = [ event: 'Checkout Updated', integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, properties: { currency: 'USD', @@ -1202,6 +1212,11 @@ export const checkoutEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Order Updated', @@ -1592,12 +1607,17 @@ export const checkoutEventsTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Order Created', properties: { order_id: '5778367414385', - value: '600.00', + value: 600, tax: 0, currency: 'USD', products: [ @@ -1683,4 +1703,139 @@ export const checkoutEventsTestScenarios = [ }, }, }, + { + id: 'c005', + name: 'shopify', + description: 'Track Call -> Order Cancelled event from Pixel app', + module: 'source', + version: 'v1', + input: { + request: { + body: [ + { + event: { + email: 'henry@wfls.com', + total_price: '600.00', + total_tax: '0.00', + updated_at: '2024-11-05T21:54:50-05:00', + line_items: [ + { + id: 14234727743601, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + sku: '', + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + variant_id: 41327142600817, + vendor: 'Hydrogen Vendor', + }, + ], + shipping_address: { + first_name: 'henry', + address1: 'Yuimaru Kitchen', + city: 'Johnson City', + zip: '37604', + }, + query_parameters: { + topic: ['orders_cancelled'], + version: ['pixel'], + writeKey: ['2mw9SN679HngnZkCHT4oSVVBVmb'], + }, + }, + source: dummySourceConfig, + }, + ], + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + }, + pathSuffix: '', + }, + output: { + response: { + status: 200, + body: [ + { + output: { + batch: [ + { + context: { + integration: { + name: 'SHOPIFY', + }, + library: { + eventOrigin: 'server', + name: 'RudderStack Shopify Cloud', + version: '2.0.0', + }, + shopifyDetails: { + email: 'henry@wfls.com', + line_items: [ + { + id: 14234727743601, + name: 'The Collection Snowboard: Hydrogen', + price: '600.00', + product_id: 7234590408817, + quantity: 1, + sku: '', + title: 'The Collection Snowboard: Hydrogen', + total_discount: '0.00', + variant_id: 41327142600817, + vendor: 'Hydrogen Vendor', + }, + ], + shipping_address: { + address1: 'Yuimaru Kitchen', + city: 'Johnson City', + first_name: 'henry', + zip: '37604', + }, + total_price: '600.00', + total_tax: '0.00', + updated_at: '2024-11-05T21:54:50-05:00', + }, + topic: 'orders_cancelled', + }, + event: 'Order Cancelled', + integrations: { + SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, + }, + properties: { + products: [ + { + brand: 'Hydrogen Vendor', + price: 600, + product_id: '7234590408817', + quantity: 1, + title: 'The Collection Snowboard: Hydrogen', + }, + ], + tax: 0, + value: 600, + }, + timestamp: '2024-11-06T02:54:50.000Z', + traits: { + email: 'henry@wfls.com', + shippingAddress: { + address1: 'Yuimaru Kitchen', + city: 'Johnson City', + first_name: 'henry', + zip: '37604', + }, + }, + type: 'track', + }, + ], + }, + }, + ], + }, + }, + }, ]; diff --git a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts index d68d0a8f59c..422fe0135a2 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/GenericTrackTests.ts @@ -68,6 +68,11 @@ export const genericTrackTestScenarios = [ event: 'Cart Update', integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, properties: { products: [], @@ -530,6 +535,11 @@ export const genericTrackTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['track.context.shopifyDetails'], + }, + }, }, type: 'track', event: 'Order Paid', diff --git a/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts b/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts index b03f5635b6a..346485fe1e6 100644 --- a/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts +++ b/test/integrations/sources/shopify/webhookTestScenarios/IdentifyTests.ts @@ -187,6 +187,11 @@ export const identityTestScenarios = [ }, integrations: { SHOPIFY: true, + DATA_WAREHOUSE: { + options: { + jsonPaths: ['identify.context.shopifyDetails'], + }, + }, }, type: 'identify', userId: '7358220173425',