Add option to order files by newest

Change-Id: Iec9052b5095d3b8ed2d7d8bfce9f0270ea6296e1

README.md

@@ -139,6 +139,7 @@ If you change the schema in Bigquery, Argon's schema check will fail.
       "projectId": "[BIGQUERY_PROJECT]", // default: current cloud project
       "single": [SINGLE_FILE_MODE], // default: true
       "ignore": [IGNORE_FILE_IDS], // default: []
+      "newest": [ORDERING_MODE], // default: false
       "replace": [REPLACE_TABLE_MODE], // default: false, append only
       "email": "[EMAIL_ADDRESS]" // default: no impersonation
     }
@@ -156,6 +157,9 @@ If you change the schema in Bigquery, Argon's schema check will fail.
   - Set `ignore` to a list of Report File IDs, to skip wrongly generated
     or unnecessary report files.
 
+  - Set `newest` to true, to order report files by most recent first,
+    instead of ordering by oldest first.
+
   - Set `replace` to true, to replace the BigQuery table on running,
     instead of appending to it.
 
@@ -164,7 +168,7 @@ If you change the schema in Bigquery, Argon's schema check will fail.
 
 - Save the job and run once to ingest any initially generated historical
   data files. Alternatively, you can run Argon on your local machine to
-  ingest larger files [[how-to](#ingest-large-data-files)].
+  ingest larger files [how-to](#ingest-large-report-files).
 
 - If it fails, check the logs for error messages and ensure all the above
   steps have been appropriately followed, with the correct permissions.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "argon",
-  "version": "4.3.0",
+  "version": "4.4.0",
   "author": "Google Inc.",
   "license": "Apache-2.0",
   "description": "Google Marketing Platform Reporting to BigQuery connector",

src/helpers.ts

@@ -62,12 +62,23 @@ export class StreamLogger extends Transform {
  *
  * @param {number} a
  * @param {number} b
- * @return {number} a - b
+ * @return {number} 0 if a=b, +ve if a > b, -ve if a < b
  */
 export function ascendingComparator(a: number, b: number): number {
   return a - b;
 }
 
+/**
+ * Descending sort comparator function for numbers.
+ *
+ * @param {number} a
+ * @param {number} b
+ * @return {number} 0 if a=b, +ve if a < b, -ve if a > b
+ */
+export function descendingComparator(a: number, b: number): number {
+  return b - a;
+}
+
 /**
  * Sleeps for given milliseconds.
  *
@@ -121,7 +132,7 @@ function isSupportedProduct(product: string): product is SupportedProduct {
 
 type DefaultArgonOpts = Pick<
   ArgonOpts,
-  'projectId' | 'single' | 'ignore' | 'replace'
+  'projectId' | 'single' | 'ignore' | 'newest' | 'replace'
 >;
 
 /**
@@ -135,6 +146,7 @@ async function getDefaultArgonOpts(): Promise<DefaultArgonOpts> {
     projectId: await getProjectId(),
     single: true,
     ignore: [],
+    newest: false,
     replace: false,
   };
 }
@@ -234,6 +246,16 @@ export async function parseBody(body: ParsedJSON): Promise<ArgonOpts> {
     warn(`Ignoring file IDs: ${[...ignore]}`);
   }
 
+  let newest: ArgonOpts['newest'] = defaults.newest;
+  if ('newest' in body) {
+    newest = Boolean(body.newest);
+  }
+  if (newest) {
+    info('Ordering Mode: newest');
+  } else {
+    info('Ordering Mode: oldest');
+  }
+
   let replace: ArgonOpts['replace'] = defaults.replace;
   if ('replace' in body) {
     replace = Boolean(body.replace);
@@ -265,6 +287,7 @@ export async function parseBody(body: ParsedJSON): Promise<ArgonOpts> {
     projectId,
     single,
     ignore,
+    newest,
     replace,
     email,
   };

src/index.ts

@@ -24,25 +24,26 @@ import {GoogleAuth, Impersonated} from 'google-auth-library';
 import got from 'got';
 import split from 'split2';
 
-import {CM_REPORTING_SCOPES, CMReportFetcher, CMCSVExtractor} from './cm';
-import {DV_REPORTING_SCOPES, DVReportFetcher, DVCSVExtractor} from './dv';
+import {version} from '../package.json';
 import {
-  FILE_ID_COLUMN,
   buildLookbackQuery,
   buildValidBQName,
+  FILE_ID_COLUMN,
   getNames,
 } from './bq';
+import {CMCSVExtractor, CMReportFetcher, CM_REPORTING_SCOPES} from './cm';
+import {DVCSVExtractor, DVReportFetcher, DV_REPORTING_SCOPES} from './dv';
 import {
   ascendingComparator,
   decodeBody,
-  parseBody,
+  descendingComparator,
   error,
   info,
+  parseBody,
   warn,
 } from './helpers';
 import {CSVExtractorBase} from './reports';
 import {ArgonOpts, GoogleAuthClient} from './typings';
-import {version} from '../package.json';
 
 export const argon: HttpFunction = async (req, res) => {
   info(`Connector version: ${version}`);
@@ -156,8 +157,8 @@ export const argon: HttpFunction = async (req, res) => {
     const pendingIds = [...reports.keys()]
       // remove ingested & ignored
       .filter(id => !ingestedIds.has(id) && !ignoredIds.has(id))
-      // sort by oldest first
-      .sort(ascendingComparator);
+      // sort by selected order
+      .sort(opts.newest ? descendingComparator : ascendingComparator);
 
     if (pendingIds.length === 0) {
       return resolve('No files to ingest.');

src/typings.d.ts

@@ -42,6 +42,7 @@ export interface ArgonOpts {
   projectId: string;
   single: boolean;
   ignore: number[];
+  newest: boolean;
   replace: boolean;
   email: string | null;
 }