Skip to content
This repository has been archived by the owner on Aug 16, 2023. It is now read-only.

Commit

Permalink
Fix DV failures
Browse files Browse the repository at this point in the history
Change-Id: I2f62153db38a06c07aca2e994fbae8de3300d26e
  • Loading branch information
achembarpu committed Apr 30, 2021
1 parent d772428 commit 8cd053a
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 9 deletions.
5 changes: 3 additions & 2 deletions argon.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const util = require('util');
const {BigQuery} = require('@google-cloud/bigquery');
const {GoogleAuth} = require('google-auth-library');
const {pipeline: pipeline_} = require('stream');
const got = require('got');

const pipeline = util.promisify(pipeline_);

Expand Down Expand Up @@ -115,6 +116,7 @@ async function argon(req, res) {
info('Initializing the API client.');
const auth = new GoogleAuth({scopes: REPORTING_SCOPES});
const client = await auth.getClient();
const headers = await client.getRequestHeaders();

info(`Checking for existence of Report ${reportId}.`);
const reportName = await getReportName({client, profileId, reportId});
Expand Down Expand Up @@ -180,8 +182,7 @@ async function argon(req, res) {
info(`Fetching report file ${fileId}.`);
try {
const url = reports.get(fileId);
const fileOpts = {responseType: 'stream'};
const {data: file} = await client.request({url, ...fileOpts});
const file = got.stream(url, {headers});
if (!file) {
warn('Report file not found.');
continue;
Expand Down
16 changes: 12 additions & 4 deletions bq.js
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,16 @@ function compareSchema(left, right) {
return true;
}

/**
* Deep copies a JSON-encodable object.
*
* @param {!object} obj JSON encodable object
* @return {!object} Deep copy of original
*/
function deepCopy(obj) {
return JSON.parse(JSON.stringify(obj));
}

/**
* Transform column names in schema using regex.
*
Expand All @@ -79,7 +89,7 @@ function compareSchema(left, right) {
* @return {?object} New schema, if renamed, else null
*/
function transformSchema(schema, patternMap) {
const fields = schema.fields.slice();
const fields = deepCopy(schema.fields);
let renamed = false;

for (const i in fields) {
Expand All @@ -93,9 +103,7 @@ function transformSchema(schema, patternMap) {
}

if (renamed) {
// update schema with new fields
schema.fields = fields;
return schema;
return {fields};
} else {
return null;
}
Expand Down
7 changes: 4 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "argon",
"version": "3.3.0",
"version": "3.3.1",
"author": "Google Inc.",
"license": "Apache-2.0",
"description": "Google Marketing Platform Reporting to BigQuery connector",
Expand All @@ -21,13 +21,14 @@
"test": "echo 'Error: No tests defined' && exit 1"
},
"dependencies": {
"@google-cloud/bigquery": "^5.5.0",
"@google-cloud/bigquery": "^5.6.0",
"express": "^4.17.1",
"google-auth-library": "^7.0.3",
"got": "^11.8.2",
"split2": "^3.2.2"
},
"devDependencies": {
"eslint": "^7.22.0",
"eslint": "^7.25.0",
"eslint-config-google": "^0.14.0",
"nodemon": "^2.0.7",
"prettier": "^2.2.1",
Expand Down

0 comments on commit 8cd053a

Please sign in to comment.