diff --git a/README.md b/README.md index 122f298..ff4c7cf 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ If you change the schema in Bigquery, Argon's schema check will fail. "projectId": "[BIGQUERY_PROJECT]", // default: current cloud project "single": [SINGLE_FILE_MODE], // default: true "ignore": [IGNORE_FILE_IDS], // default: [] + "newest": [ORDERING_MODE], // default: false "replace": [REPLACE_TABLE_MODE], // default: false, append only "email": "[EMAIL_ADDRESS]" // default: no impersonation } @@ -156,6 +157,9 @@ If you change the schema in Bigquery, Argon's schema check will fail. - Set `ignore` to a list of Report File IDs, to skip wrongly generated or unnecessary report files. + - Set `newest` to true, to order report files by most recent first, + instead of ordering by oldest first. + - Set `replace` to true, to replace the BigQuery table on running, instead of appending to it. @@ -164,7 +168,7 @@ If you change the schema in Bigquery, Argon's schema check will fail. - Save the job and run once to ingest any initially generated historical data files. Alternatively, you can run Argon on your local machine to - ingest larger files [[how-to](#ingest-large-data-files)]. + ingest larger files [how-to](#ingest-large-report-files). - If it fails, check the logs for error messages and ensure all the above steps have been appropriately followed, with the correct permissions. diff --git a/package.json b/package.json index 7f92bca..9407c61 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "argon", - "version": "4.3.0", + "version": "4.4.0", "author": "Google Inc.", "license": "Apache-2.0", "description": "Google Marketing Platform Reporting to BigQuery connector", diff --git a/src/helpers.ts b/src/helpers.ts index 291b061..1d96345 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -62,12 +62,23 @@ export class StreamLogger extends Transform { * * @param {number} a * @param {number} b - * @return {number} a - b + * @return {number} 0 if a=b, +ve if a > b, -ve if a < b */ export function ascendingComparator(a: number, b: number): number { return a - b; } +/** + * Descending sort comparator function for numbers. + * + * @param {number} a + * @param {number} b + * @return {number} 0 if a=b, +ve if a < b, -ve if a > b + */ +export function descendingComparator(a: number, b: number): number { + return b - a; +} + /** * Sleeps for given milliseconds. * @@ -121,7 +132,7 @@ function isSupportedProduct(product: string): product is SupportedProduct { type DefaultArgonOpts = Pick< ArgonOpts, - 'projectId' | 'single' | 'ignore' | 'replace' + 'projectId' | 'single' | 'ignore' | 'newest' | 'replace' >; /** @@ -135,6 +146,7 @@ async function getDefaultArgonOpts(): Promise { projectId: await getProjectId(), single: true, ignore: [], + newest: false, replace: false, }; } @@ -234,6 +246,16 @@ export async function parseBody(body: ParsedJSON): Promise { warn(`Ignoring file IDs: ${[...ignore]}`); } + let newest: ArgonOpts['newest'] = defaults.newest; + if ('newest' in body) { + newest = Boolean(body.newest); + } + if (newest) { + info('Ordering Mode: newest'); + } else { + info('Ordering Mode: oldest'); + } + let replace: ArgonOpts['replace'] = defaults.replace; if ('replace' in body) { replace = Boolean(body.replace); @@ -265,6 +287,7 @@ export async function parseBody(body: ParsedJSON): Promise { projectId, single, ignore, + newest, replace, email, }; diff --git a/src/index.ts b/src/index.ts index 8d3b6e3..bff1f86 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,25 +24,26 @@ import {GoogleAuth, Impersonated} from 'google-auth-library'; import got from 'got'; import split from 'split2'; -import {CM_REPORTING_SCOPES, CMReportFetcher, CMCSVExtractor} from './cm'; -import {DV_REPORTING_SCOPES, DVReportFetcher, DVCSVExtractor} from './dv'; +import {version} from '../package.json'; import { - FILE_ID_COLUMN, buildLookbackQuery, buildValidBQName, + FILE_ID_COLUMN, getNames, } from './bq'; +import {CMCSVExtractor, CMReportFetcher, CM_REPORTING_SCOPES} from './cm'; +import {DVCSVExtractor, DVReportFetcher, DV_REPORTING_SCOPES} from './dv'; import { ascendingComparator, decodeBody, - parseBody, + descendingComparator, error, info, + parseBody, warn, } from './helpers'; import {CSVExtractorBase} from './reports'; import {ArgonOpts, GoogleAuthClient} from './typings'; -import {version} from '../package.json'; export const argon: HttpFunction = async (req, res) => { info(`Connector version: ${version}`); @@ -156,8 +157,8 @@ export const argon: HttpFunction = async (req, res) => { const pendingIds = [...reports.keys()] // remove ingested & ignored .filter(id => !ingestedIds.has(id) && !ignoredIds.has(id)) - // sort by oldest first - .sort(ascendingComparator); + // sort by selected order + .sort(opts.newest ? descendingComparator : ascendingComparator); if (pendingIds.length === 0) { return resolve('No files to ingest.'); diff --git a/src/typings.d.ts b/src/typings.d.ts index 30252bb..75871e1 100644 --- a/src/typings.d.ts +++ b/src/typings.d.ts @@ -42,6 +42,7 @@ export interface ArgonOpts { projectId: string; single: boolean; ignore: number[]; + newest: boolean; replace: boolean; email: string | null; }