FE-6300 Accept S3 URI as input for export and show S3 URI in output. (#…

…584) * FE-6300 Accept s3 URI as input for export and show s3 URI in output. * Use is_terminal rather than waiting on states. * is_terminal tests * Get rid of artificial destionation_uri. * Tests for invalid S3 URI, and redundant destination inputs.
fauna · Jan 27, 2025 · ea81a92 · ea81a92
1 parent 2331aa0
commit ea81a92
Show file tree

Hide file tree

Showing 9 changed files with 256 additions and 283 deletions.
diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs
@@ -1,7 +1,6 @@
 // @ts-check
 
 import { container } from "../../config/container.mjs";
-import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs";
 import { ValidationError } from "../../lib/errors.mjs";
 import { colorize, Format } from "../../lib/formatting/colorize.mjs";
 import { DATABASE_PATH_OPTIONS } from "../../lib/options.mjs";
@@ -19,23 +18,28 @@ async function createS3Export(argv) {
     wait,
     maxWait,
     quiet,
+    destination,
   } = argv;
   const logger = container.resolve("logger");
   const { createExport } = container.resolve("accountAPI");
-
-  let createdExport = await createExport({
-    database,
-    collections,
-    destination: {
+  let destinationInput = destination;
+  if (!destinationInput) {
+    destinationInput = {
       s3: {
         bucket,
         path,
       },
-    },
+    };
+  }
+
+  let createdExport = await createExport({
+    database,
+    collections,
+    destination: destinationInput,
     format,
   });
 
-  if (wait && !EXPORT_TERMINAL_STATES.includes(createdExport.state)) {
+  if (wait && !createdExport.is_terminal) {
     createdExport = await waitUntilExportIsReady({
       id: createdExport.id,
       opts: {
@@ -44,7 +48,6 @@ async function createS3Export(argv) {
       },
     });
   }
-
   if (json) {
     logger.stdout(colorize(createdExport, { color, format: Format.JSON }));
   } else {
@@ -54,39 +57,52 @@ async function createS3Export(argv) {
 
 const sharedExamples = [
   [
-    "$0 export create s3 --database us/my_db --bucket doc-example-bucket --path exports/my_db",
-    "Export the 'us-std/my_db' database to the 'exports/my_db' path of the 'doc-example-bucket' S3 bucket. Outputs the export ID.",
+    "$0 export create s3 --destination s3://doc-example-bucket/exports/my_db",
+    "Export the 'us-std/my_db' database to the S3 URI 's3://doc-example-bucket/exports/my_db'.",
   ],
   [
-    "$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --json",
+    "$0 export create s3 --bucket doc-example-bucket --path exports/my_db",
+    "You can also specify the S3 location using --bucket and --path options rather than --destination.",
+  ],
+  [
+    "$0 export create s3 --destination s3://doc-example-bucket/my-prefix --json",
     "Output the full JSON of the export request.",
   ],
   [
-    "$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --collection my-collection",
+    "$0 export create s3 --destination s3://doc-example-bucket/my-prefix --collection my-collection",
     "Export the 'my-collection' collection only.",
   ],
   [
-    "$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --format tagged",
+    "$0 export create s3 --destination s3://doc-example-bucket/my-prefix --format tagged",
     "Encode the export's document data using the 'tagged' format.",
   ],
   [
-    "$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --wait --max-wait 180",
+    "$0 export create s3 --destination s3://doc-example-bucket/my-prefix --wait --max-wait 180",
     "Wait for the export to complete or fail before exiting. Waits up to 180 minutes.",
   ],
 ];
 
+const S3_URI_REGEX = /^s3:\/\/[^/]+\/.+$/;
+
 function buildCreateS3ExportCommand(yargs) {
   return yargs
     .options({
+      destination: {
+        alias: ["uri", "destination-uri"],
+        type: "string",
+        required: false,
+        description: "S3 URI in the format s3://bucket/path.",
+        group: "API:",
+      },
       bucket: {
         type: "string",
-        required: true,
+        required: false,
         description: "Name of the S3 bucket where the export will be stored.",
         group: "API:",
       },
       path: {
         type: "string",
-        required: true,
+        required: false,
         description:
           "Path prefix for the S3 bucket. Separate subfolders using a slash (`/`).",
         group: "API:",
@@ -108,7 +124,22 @@ function buildCreateS3ExportCommand(yargs) {
           "--database is required to create an export.",
         );
       }
-
+      if (argv.destination) {
+        if (argv.bucket || argv.path) {
+          throw new ValidationError(
+            "Cannot specify --destination with --bucket or --path. Use either --destination or both --bucket and --path.",
+          );
+        }
+        if (!S3_URI_REGEX.test(argv.destination)) {
+          throw new ValidationError(
+            "Invalid S3 URI format. Expected format: s3://bucket/path",
+          );
+        }
+      } else if (!argv.bucket || !argv.path) {
+        throw new ValidationError(
+          "Either --destination or both --bucket and --path are required to create an export.",
+        );
+      }
       return true;
     })
     .example(sharedExamples);

diff --git a/src/commands/export/get.mjs b/src/commands/export/get.mjs
@@ -1,5 +1,4 @@
 import { container } from "../../config/container.mjs";
-import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs";
 import { colorize, Format } from "../../lib/formatting/colorize.mjs";
 import { WAIT_OPTIONS, waitUntilExportIsReady } from "./wait.mjs";
 
@@ -9,7 +8,7 @@ async function getExport(argv) {
   const { exportId, json, color, wait, maxWait, quiet } = argv;
 
   let response = await getExport({ exportId });
-  if (wait && !EXPORT_TERMINAL_STATES.includes(response.state)) {
+  if (wait && !response.is_terminal) {
     response = await waitUntilExportIsReady({
       id: exportId,
       opts: {

diff --git a/src/commands/export/list.mjs b/src/commands/export/list.mjs
@@ -27,7 +27,7 @@ async function listExports(argv) {
         r.id,
         r.database,
         (r.collections ?? []).join(COLLECTION_SEPARATOR),
-        r.destination_uri,
+        r.destination.uri,
         r.state,
       ];
       logger.stdout(

diff --git a/src/commands/export/wait.mjs b/src/commands/export/wait.mjs
@@ -1,7 +1,6 @@
 // @ts-check
 
 import { container } from "../../config/container.mjs";
-import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs";
 import { CommandError } from "../../lib/errors.mjs";
 import { colorize, Format } from "../../lib/formatting/colorize.mjs";
 import { isTTY } from "../../lib/utils.mjs";
@@ -169,7 +168,7 @@ export async function waitAndCheckExportState({
   const data = await getExport({ exportId: id });
 
   // If the export is ready, return the data
-  if (EXPORT_TERMINAL_STATES.includes(data.state)) {
+  if (data.is_terminal) {
     statusHandler(
       colorize(`${id} has a terminal state of ${data.state}.`, {
         format: Format.LOG,

diff --git a/src/lib/account-api.mjs b/src/lib/account-api.mjs
@@ -22,10 +22,6 @@ export const ExportState = {
 };
 
 export const EXPORT_STATES = Object.values(ExportState);
-export const EXPORT_TERMINAL_STATES = [
-  ExportState.Complete,
-  ExportState.Failed,
-];
 
 let accountUrl = process.env.FAUNA_ACCOUNT_URL ?? "https://account.fauna.com";
 
@@ -406,15 +402,6 @@ async function createKey({ path, role, ttl, name }) {
   return await responseHandler(response);
 }
 
-const getExportUri = (data) => {
-  const { destination, state } = data;
-  if (!destination || !state) {
-    return "";
-  }
-  const path = destination.s3.path.replace(/^\/+/, "");
-  return `s3://${destination.s3.bucket}/${path}`;
-};
-
 /**
  * Creates an export for a given database.
  *
@@ -450,7 +437,7 @@ async function createExport({
   });
 
   const data = await responseHandler(response);
-  return { ...data.response, destination_uri: getExportUri(data.response) }; // eslint-disable-line camelcase
+  return data.response;
 }
 
 /**
@@ -481,12 +468,6 @@ async function listExports({ maxResults = 100, nextToken, state } = {}) {
   });
   const { response: data } = await responseHandler(response);
 
-  if (data.results && Array.isArray(data.results)) {
-    data.results.forEach((r) => {
-      r.destination_uri = getExportUri(r); // eslint-disable-line camelcase
-    });
-  }
-
   return data;
 }
 
@@ -505,10 +486,7 @@ async function getExport({ exportId }) {
   });
   const response = await fetchWithAccountKey(url, { method: "GET" });
   const data = await responseHandler(response);
-  return {
-    ...data.response,
-    destination_uri: getExportUri(data.response), // eslint-disable-line camelcase
-  };
+  return data.response;
 }
 
 /**