Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FE-6300 Accept S3 URI as input for export and show S3 URI in output. #584

Merged
merged 5 commits into from
Jan 27, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 49 additions & 18 deletions src/commands/export/create.mjs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// @ts-check

import { container } from "../../config/container.mjs";
import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs";
import { ValidationError } from "../../lib/errors.mjs";
import { colorize, Format } from "../../lib/formatting/colorize.mjs";
import { DATABASE_PATH_OPTIONS } from "../../lib/options.mjs";
Expand All @@ -19,23 +18,28 @@ async function createS3Export(argv) {
wait,
maxWait,
quiet,
destination,
} = argv;
const logger = container.resolve("logger");
const { createExport } = container.resolve("accountAPI");

let createdExport = await createExport({
database,
collections,
destination: {
let destinationInput = destination;
if (!destinationInput) {
destinationInput = {
s3: {
bucket,
path,
},
},
};
}

let createdExport = await createExport({
database,
collections,
destination: destinationInput,
format,
});

if (wait && !EXPORT_TERMINAL_STATES.includes(createdExport.state)) {
if (wait && !createdExport.is_terminal) {
createdExport = await waitUntilExportIsReady({
id: createdExport.id,
opts: {
Expand All @@ -44,7 +48,6 @@ async function createS3Export(argv) {
},
});
}

if (json) {
logger.stdout(colorize(createdExport, { color, format: Format.JSON }));
} else {
Expand All @@ -54,39 +57,52 @@ async function createS3Export(argv) {

const sharedExamples = [
[
"$0 export create s3 --database us/my_db --bucket doc-example-bucket --path exports/my_db",
"Export the 'us-std/my_db' database to the 'exports/my_db' path of the 'doc-example-bucket' S3 bucket. Outputs the export ID.",
"$0 export create s3 --destination s3://doc-example-bucket/exports/my_db",
"Export the 'us-std/my_db' database to the S3 URI 's3://doc-example-bucket/exports/my_db'.",
],
[
"$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --json",
"$0 export create s3 --bucket doc-example-bucket --path exports/my_db",
"You can also specify the S3 location using --bucket and --path options rather than --destination.",
],
[
"$0 export create s3 --destination s3://doc-example-bucket/my-prefix --json",
"Output the full JSON of the export request.",
],
[
"$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --collection my-collection",
"$0 export create s3 --destination s3://doc-example-bucket/my-prefix --collection my-collection",
"Export the 'my-collection' collection only.",
],
[
"$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --format tagged",
"$0 export create s3 --destination s3://doc-example-bucket/my-prefix --format tagged",
"Encode the export's document data using the 'tagged' format.",
],
[
"$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --wait --max-wait 180",
"$0 export create s3 --destination s3://doc-example-bucket/my-prefix --wait --max-wait 180",
"Wait for the export to complete or fail before exiting. Waits up to 180 minutes.",
],
];

const S3_URI_REGEX = /^s3:\/\/[^/]+\/.+$/;

function buildCreateS3ExportCommand(yargs) {
return yargs
.options({
destination: {
alias: ["uri", "destination-uri"],
type: "string",
required: false,
description: "S3 URI in the format s3://bucket/path.",
group: "API:",
},
bucket: {
type: "string",
required: true,
required: false,
description: "Name of the S3 bucket where the export will be stored.",
group: "API:",
},
path: {
type: "string",
required: true,
required: false,
description:
"Path prefix for the S3 bucket. Separate subfolders using a slash (`/`).",
group: "API:",
Expand All @@ -108,7 +124,22 @@ function buildCreateS3ExportCommand(yargs) {
"--database is required to create an export.",
);
}

if (argv.destination) {
if (argv.bucket || argv.path) {
throw new ValidationError(
"Cannot specify --destination with --bucket or --path. Use either --destination or both --bucket and --path.",
);
}
if (!S3_URI_REGEX.test(argv.destination)) {
cleve-fauna marked this conversation as resolved.
Show resolved Hide resolved
throw new ValidationError(
"Invalid S3 URI format. Expected format: s3://bucket/path",
);
}
} else if (!argv.bucket || !argv.path) {
throw new ValidationError(
"Either --destination or both --bucket and --path are required to create an export.",
);
}
return true;
})
.example(sharedExamples);
Expand Down
3 changes: 1 addition & 2 deletions src/commands/export/get.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { container } from "../../config/container.mjs";
import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs";
import { colorize, Format } from "../../lib/formatting/colorize.mjs";
import { WAIT_OPTIONS, waitUntilExportIsReady } from "./wait.mjs";

Expand All @@ -9,7 +8,7 @@ async function getExport(argv) {
const { exportId, json, color, wait, maxWait, quiet } = argv;

let response = await getExport({ exportId });
if (wait && !EXPORT_TERMINAL_STATES.includes(response.state)) {
if (wait && !response.is_terminal) {
response = await waitUntilExportIsReady({
id: exportId,
opts: {
Expand Down
2 changes: 1 addition & 1 deletion src/commands/export/list.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ async function listExports(argv) {
r.id,
r.database,
(r.collections ?? []).join(COLLECTION_SEPARATOR),
r.destination_uri,
r.destination.uri,
r.state,
];
logger.stdout(
Expand Down
3 changes: 1 addition & 2 deletions src/commands/export/wait.mjs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// @ts-check

import { container } from "../../config/container.mjs";
import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs";
import { CommandError } from "../../lib/errors.mjs";
import { colorize, Format } from "../../lib/formatting/colorize.mjs";
import { isTTY } from "../../lib/utils.mjs";
Expand Down Expand Up @@ -169,7 +168,7 @@ export async function waitAndCheckExportState({
const data = await getExport({ exportId: id });

// If the export is ready, return the data
if (EXPORT_TERMINAL_STATES.includes(data.state)) {
if (data.is_terminal) {
statusHandler(
colorize(`${id} has a terminal state of ${data.state}.`, {
format: Format.LOG,
Expand Down
26 changes: 2 additions & 24 deletions src/lib/account-api.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,6 @@ export const ExportState = {
};

export const EXPORT_STATES = Object.values(ExportState);
export const EXPORT_TERMINAL_STATES = [
ExportState.Complete,
ExportState.Failed,
];

let accountUrl = process.env.FAUNA_ACCOUNT_URL ?? "https://account.fauna.com";

Expand Down Expand Up @@ -406,15 +402,6 @@ async function createKey({ path, role, ttl, name }) {
return await responseHandler(response);
}

const getExportUri = (data) => {
const { destination, state } = data;
if (!destination || !state) {
return "";
}
const path = destination.s3.path.replace(/^\/+/, "");
return `s3://${destination.s3.bucket}/${path}`;
};

/**
* Creates an export for a given database.
*
Expand Down Expand Up @@ -450,7 +437,7 @@ async function createExport({
});

const data = await responseHandler(response);
return { ...data.response, destination_uri: getExportUri(data.response) }; // eslint-disable-line camelcase
return data.response;
}

/**
Expand Down Expand Up @@ -481,12 +468,6 @@ async function listExports({ maxResults = 100, nextToken, state } = {}) {
});
const { response: data } = await responseHandler(response);

if (data.results && Array.isArray(data.results)) {
data.results.forEach((r) => {
r.destination_uri = getExportUri(r); // eslint-disable-line camelcase
});
}

return data;
}

Expand All @@ -505,10 +486,7 @@ async function getExport({ exportId }) {
});
const response = await fetchWithAccountKey(url, { method: "GET" });
const data = await responseHandler(response);
return {
...data.response,
destination_uri: getExportUri(data.response), // eslint-disable-line camelcase
};
return data.response;
}

/**
Expand Down
Loading
Loading