From 104181130a3f12a0de2bf5e82935d0419a847992 Mon Sep 17 00:00:00 2001 From: Arvind Chembarpu Date: Wed, 27 Oct 2021 13:57:12 +0000 Subject: [PATCH] Fix deployment for new cloud function changes Change-Id: Id5bcbf31f0b642c47aeb19542b40254fb6fadce2 --- README.md | 179 +++++++++++++++++++++++++++++---------------------- cm.js | 2 +- index.js | 3 + package.json | 14 ++-- 4 files changed, 113 insertions(+), 85 deletions(-) create mode 100644 index.js diff --git a/README.md b/README.md index 12275d6..d224336 100644 --- a/README.md +++ b/README.md @@ -15,35 +15,41 @@ ingestions. ### Google Cloud Project -* Setup a Google Cloud project. -* Create a BigQuery dataset - tables will be created automatically per report. -* Create a new IAM Service Account for Argon, with the BigQuery Admin role. -* Enable the necessary APIs in API Explorer, or via `gcloud services enable` : - * DV: DoubleClick Bid Manager API (`doubleclickbidmanager.googleapis.com`) - * CM: DCM/DFA Reporting And Trafficking API (`dfareporting.googleapis.com`) -* Clone this repository and deploy Argon code to your cloud project: - ``` - gcloud functions deploy argon \ - --runtime nodejs10 \ - --memory 512MB \ - --timeout 540s \ - --trigger-http \ - --service-account "[SERVICE_ACCOUNT_EMAIL]" - ``` +- Setup a Google Cloud project. +- Create a BigQuery dataset - tables will be created automatically per report. +- Create a new IAM Service Account for Argon, with the BigQuery Admin role. +- Give your own account "Service Account User" permissions for this service account. +- Enable the necessary APIs in API Explorer, or via `gcloud services enable` : + - GCP: + - Cloud Functions API (`cloudfunctions.googleapis.com`) + - Cloud Build API (`cloudbuild.googleapis.com`) + - DV: DoubleClick Bid Manager API (`doubleclickbidmanager.googleapis.com`) + - CM: DCM/DFA Reporting And Trafficking API (`dfareporting.googleapis.com`) +- Clone this repository and deploy Argon code to your cloud project: + + ```sh + gcloud functions deploy argon \ + --runtime nodejs12 \ + --memory 512MB \ + --timeout 540s \ + --trigger-http \ + --service-account "[SERVICE_ACCOUNT_EMAIL]" + ``` ### Google Marketing Platform #### Accounts -* Ensure that the CM Account has the following Permissions: - * Properties > Enable account for API access - * Reporting > View all generated reports -* Create a CM/ DV User Profile with the service account's email address - with the respective role: - * DV: Reporting only - * CM: Advanced Agency Admin, with permissions: - * View all generated files - * View all saved reports +- Ensure that the CM Account has the following Permissions: + - Properties > Enable account for API access + - Reporting > View all generated reports + +- Create a CM/ DV User Profile with the service account's email address + with the respective role: + - DV: Reporting only + - CM: Advanced Agency Admin, with permissions: + - View all generated files + - View all saved reports #### Report @@ -55,62 +61,81 @@ change the report template. All columns are string type, and Argon will append an additional column (`file_id`), to keep track of ingested files. If you change the schema in Bigquery, Argon's schema check will fail. -* Choose the necessary report template in "Offline Reporting". -* Choose the "CSV" File type. -* Select the required Dimensions, Metrics, and Rich Media Events. -* Add the service account's email address to the "Share with > +add people", - and use the "Link" option. -* If you want historical data to be backfilled for the first time, - select the appropriate backfill Date Range with "Custom". -* If this range is significant, break it up into much smaller chunks, - otherwise ingestion timeouts will result in partial uploads. -* Save and run the report, for each chunk, if necessary. -* Now, edit the report again, and select a Date Range of "Yesterday". -* Activate the Schedule for repeats "Daily" every "1 day" and choose a - far-off in the future "Expiry" date. -* Save (and do not run) the report. +- Choose the necessary report template in "Offline Reporting". + +- Choose the "CSV" File type. + +- Select the required Dimensions, Metrics, and Rich Media Events. + +- Add the service account's email address to the "Share with > +add people", + and use the "Link" option. + +- If you want historical data to be backfilled for the first time, + select the appropriate backfill Date Range with "Custom". + +- If this range is significant, break it up into much smaller chunks, + otherwise ingestion timeouts will result in partial uploads. + +- Save and run the report, for each chunk, if necessary. + +- Now, edit the report again, and select a Date Range of "Yesterday". + +- Activate the Schedule for repeats "Daily" every "1 day" and choose a + far-off in the future "Expiry" date. + +- Save (and do not run) the report. ### Google Cloud Scheduler -* Create a Scheduler Job with: - * Frequency: `0 */12 * * *` (repeating every 12 hours) - * Target: HTTP - * URL: Cloud Function URL - * HTTP Method: POST - * Body: - ```json5 - { - "product": "[PRODUCT]", // required: CM or DV - "reportId": [REPORT_ID], - "profileId": [PROFILE_ID], // only for CM - "datasetName": "[DATASET_NAME]", - "projectId": "[BIGQUERY_PROJECT]", // default: current cloud project - "single": [SINGLE_FILE_MODE], // default: false - "ignore": [IGNORE_FILE_IDS] // default: [] - } - ``` - * Notes: - * Use `projectId` if the output BigQuery dataset lives outside the - currently deployed cloud project. - * Set `single` to true, to process only one file per run. This is - useful if your reports are multiple GBs large, as Cloud Functions - will timeout in 540s. - * Set `ignore` to a list of Report File IDs, to skip wrongly generated - or unnecessary report files. -* Save the job and run once to ingest the initially generated - backfill historical data file. -* If it fails, check the logs for error messages and ensure all the above - steps have been appropriately followed, with the correct permissions. -* Moving forward, Cloud Scheduler will trigger Argon for regular ingestion. -* #### Cloud Scheduler with OIDC Token #### - * Add the following roles in IAM to the service account that will run the - Cloud Scheduler and call Argon: - * Cloud Functions Invoker - * Cloud Scheduler Job Runner - * While setting up the Cloud Scheduler job click on `SHOW MORE`. - * Select `Add OIDC token` under Auth header. - * Set service account email address. - * Paste the Argon Cloud Function URL under `Audience`. +- Create a Scheduler Job with: + - Frequency: `0 */12 * * *` (repeating every 12 hours) + - Target: HTTP + - URL: Cloud Function URL + - HTTP Method: POST + - Body: + + ```json5 + { + "product": "[PRODUCT]", // required: CM or DV + "reportId": [REPORT_ID], + "profileId": [PROFILE_ID], // only for CM + "datasetName": "[DATASET_NAME]", + "projectId": "[BIGQUERY_PROJECT]", // default: current cloud project + "single": [SINGLE_FILE_MODE], // default: false + "ignore": [IGNORE_FILE_IDS] // default: [] + } + ``` + +- Notes: + - Use `projectId` if the output BigQuery dataset lives outside the + currently deployed cloud project. + + - Set `single` to true, to process only one file per run. This is + useful if your reports are multiple GBs large, as Cloud Functions + will timeout in 540s. + + - Set `ignore` to a list of Report File IDs, to skip wrongly generated + or unnecessary report files. + +- Save the job and run once to ingest the initially generated + backfill historical data file. + +- If it fails, check the logs for error messages and ensure all the above + steps have been appropriately followed, with the correct permissions. + +- Moving forward, Cloud Scheduler will trigger Argon for regular ingestion. + +### Cloud Scheduler with OIDC Token + +- Add the following roles in IAM to the service account that will run the + Cloud Scheduler and call Argon: + - Cloud Functions Invoker + - Cloud Scheduler Job Runner + +- While setting up the Cloud Scheduler job click on `SHOW MORE`. +- Select `Add OIDC token` under Auth header. +- Set service account email address. +- Paste the Argon Cloud Function URL under `Audience`. ## Development diff --git a/cm.js b/cm.js index 4f193a3..be9f26e 100644 --- a/cm.js +++ b/cm.js @@ -19,7 +19,7 @@ const {CSVExtractorBase} = require('./helpers.js'); const REPORTING_SCOPES = ['https://www.googleapis.com/auth/dfareporting']; -const REPORTING_BASE_URL = 'https://www.googleapis.com/dfareporting/v3.4'; +const REPORTING_BASE_URL = 'https://www.googleapis.com/dfareporting/v3.5'; const REPORT_AVAILABLE = 'REPORT_AVAILABLE'; const DCM_FIELDS_INDICATOR = 'Report Fields'; diff --git a/index.js b/index.js new file mode 100644 index 0000000..eabb1ef --- /dev/null +++ b/index.js @@ -0,0 +1,3 @@ +const {argon} = require('./argon.js'); + +module.exports = {argon}; diff --git a/package.json b/package.json index 7e8f275..877bed2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "argon", - "version": "3.3.1", + "version": "3.4.0", "author": "Google Inc.", "license": "Apache-2.0", "description": "Google Marketing Platform Reporting to BigQuery connector", @@ -9,7 +9,7 @@ "type": "git", "url": "https://github.com/google/argon.git" }, - "main": "argon.js", + "main": "index.js", "engines": { "node": ">=10", "npm": ">=6" @@ -21,17 +21,17 @@ "test": "echo 'Error: No tests defined' && exit 1" }, "dependencies": { - "@google-cloud/bigquery": "^5.6.0", + "@google-cloud/bigquery": "^5.9.1", "express": "^4.17.1", - "google-auth-library": "^7.0.3", + "google-auth-library": "^7.10.1", "got": "^11.8.2", "split2": "^3.2.2" }, "devDependencies": { - "eslint": "^7.25.0", + "eslint": "^7.32.0", "eslint-config-google": "^0.14.0", - "nodemon": "^2.0.7", - "prettier": "^2.2.1", + "nodemon": "^2.0.14", + "prettier": "^2.4.1", "prettier-eslint": "^12.0.0", "prettier-eslint-cli": "^5.0.1" }