diff --git a/src/commands/export/create.mjs b/src/commands/export/create.mjs index 4aa15df5..216d0495 100644 --- a/src/commands/export/create.mjs +++ b/src/commands/export/create.mjs @@ -1,7 +1,6 @@ // @ts-check import { container } from "../../config/container.mjs"; -import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs"; import { ValidationError } from "../../lib/errors.mjs"; import { colorize, Format } from "../../lib/formatting/colorize.mjs"; import { DATABASE_PATH_OPTIONS } from "../../lib/options.mjs"; @@ -19,23 +18,28 @@ async function createS3Export(argv) { wait, maxWait, quiet, + destination, } = argv; const logger = container.resolve("logger"); const { createExport } = container.resolve("accountAPI"); - - let createdExport = await createExport({ - database, - collections, - destination: { + let destinationInput = destination; + if (!destinationInput) { + destinationInput = { s3: { bucket, path, }, - }, + }; + } + + let createdExport = await createExport({ + database, + collections, + destination: destinationInput, format, }); - if (wait && !EXPORT_TERMINAL_STATES.includes(createdExport.state)) { + if (wait && !createdExport.is_terminal) { createdExport = await waitUntilExportIsReady({ id: createdExport.id, opts: { @@ -44,7 +48,6 @@ async function createS3Export(argv) { }, }); } - if (json) { logger.stdout(colorize(createdExport, { color, format: Format.JSON })); } else { @@ -54,39 +57,52 @@ async function createS3Export(argv) { const sharedExamples = [ [ - "$0 export create s3 --database us/my_db --bucket doc-example-bucket --path exports/my_db", - "Export the 'us-std/my_db' database to the 'exports/my_db' path of the 'doc-example-bucket' S3 bucket. Outputs the export ID.", + "$0 export create s3 --destination s3://doc-example-bucket/exports/my_db", + "Export the 'us-std/my_db' database to the S3 URI 's3://doc-example-bucket/exports/my_db'.", ], [ - "$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --json", + "$0 export create s3 --bucket doc-example-bucket --path exports/my_db", + "You can also specify the S3 location using --bucket and --path options rather than --destination.", + ], + [ + "$0 export create s3 --destination s3://doc-example-bucket/my-prefix --json", "Output the full JSON of the export request.", ], [ - "$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --collection my-collection", + "$0 export create s3 --destination s3://doc-example-bucket/my-prefix --collection my-collection", "Export the 'my-collection' collection only.", ], [ - "$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --format tagged", + "$0 export create s3 --destination s3://doc-example-bucket/my-prefix --format tagged", "Encode the export's document data using the 'tagged' format.", ], [ - "$0 export create s3 --database us/my_db --bucket doc-example-bucket --path my-prefix --wait --max-wait 180", + "$0 export create s3 --destination s3://doc-example-bucket/my-prefix --wait --max-wait 180", "Wait for the export to complete or fail before exiting. Waits up to 180 minutes.", ], ]; +const S3_URI_REGEX = /^s3:\/\/[^/]+\/.+$/; + function buildCreateS3ExportCommand(yargs) { return yargs .options({ + destination: { + alias: ["uri", "destination-uri"], + type: "string", + required: false, + description: "S3 URI in the format s3://bucket/path.", + group: "API:", + }, bucket: { type: "string", - required: true, + required: false, description: "Name of the S3 bucket where the export will be stored.", group: "API:", }, path: { type: "string", - required: true, + required: false, description: "Path prefix for the S3 bucket. Separate subfolders using a slash (`/`).", group: "API:", @@ -108,7 +124,22 @@ function buildCreateS3ExportCommand(yargs) { "--database is required to create an export.", ); } - + if (argv.destination) { + if (argv.bucket || argv.path) { + throw new ValidationError( + "Cannot specify --destination with --bucket or --path. Use either --destination or both --bucket and --path.", + ); + } + if (!S3_URI_REGEX.test(argv.destination)) { + throw new ValidationError( + "Invalid S3 URI format. Expected format: s3://bucket/path", + ); + } + } else if (!argv.bucket || !argv.path) { + throw new ValidationError( + "Either --destination or both --bucket and --path are required to create an export.", + ); + } return true; }) .example(sharedExamples); diff --git a/src/commands/export/get.mjs b/src/commands/export/get.mjs index 43e00fbe..ea1f29aa 100644 --- a/src/commands/export/get.mjs +++ b/src/commands/export/get.mjs @@ -1,5 +1,4 @@ import { container } from "../../config/container.mjs"; -import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs"; import { colorize, Format } from "../../lib/formatting/colorize.mjs"; import { WAIT_OPTIONS, waitUntilExportIsReady } from "./wait.mjs"; @@ -9,7 +8,7 @@ async function getExport(argv) { const { exportId, json, color, wait, maxWait, quiet } = argv; let response = await getExport({ exportId }); - if (wait && !EXPORT_TERMINAL_STATES.includes(response.state)) { + if (wait && !response.is_terminal) { response = await waitUntilExportIsReady({ id: exportId, opts: { diff --git a/src/commands/export/list.mjs b/src/commands/export/list.mjs index 276ffda5..be1cd484 100644 --- a/src/commands/export/list.mjs +++ b/src/commands/export/list.mjs @@ -27,7 +27,7 @@ async function listExports(argv) { r.id, r.database, (r.collections ?? []).join(COLLECTION_SEPARATOR), - r.destination_uri, + r.destination.uri, r.state, ]; logger.stdout( diff --git a/src/commands/export/wait.mjs b/src/commands/export/wait.mjs index 1125ffe6..b155115d 100644 --- a/src/commands/export/wait.mjs +++ b/src/commands/export/wait.mjs @@ -1,7 +1,6 @@ // @ts-check import { container } from "../../config/container.mjs"; -import { EXPORT_TERMINAL_STATES } from "../../lib/account-api.mjs"; import { CommandError } from "../../lib/errors.mjs"; import { colorize, Format } from "../../lib/formatting/colorize.mjs"; import { isTTY } from "../../lib/utils.mjs"; @@ -169,7 +168,7 @@ export async function waitAndCheckExportState({ const data = await getExport({ exportId: id }); // If the export is ready, return the data - if (EXPORT_TERMINAL_STATES.includes(data.state)) { + if (data.is_terminal) { statusHandler( colorize(`${id} has a terminal state of ${data.state}.`, { format: Format.LOG, diff --git a/src/lib/account-api.mjs b/src/lib/account-api.mjs index be103489..9f282650 100644 --- a/src/lib/account-api.mjs +++ b/src/lib/account-api.mjs @@ -22,10 +22,6 @@ export const ExportState = { }; export const EXPORT_STATES = Object.values(ExportState); -export const EXPORT_TERMINAL_STATES = [ - ExportState.Complete, - ExportState.Failed, -]; let accountUrl = process.env.FAUNA_ACCOUNT_URL ?? "https://account.fauna.com"; @@ -406,15 +402,6 @@ async function createKey({ path, role, ttl, name }) { return await responseHandler(response); } -const getExportUri = (data) => { - const { destination, state } = data; - if (!destination || !state) { - return ""; - } - const path = destination.s3.path.replace(/^\/+/, ""); - return `s3://${destination.s3.bucket}/${path}`; -}; - /** * Creates an export for a given database. * @@ -450,7 +437,7 @@ async function createExport({ }); const data = await responseHandler(response); - return { ...data.response, destination_uri: getExportUri(data.response) }; // eslint-disable-line camelcase + return data.response; } /** @@ -481,12 +468,6 @@ async function listExports({ maxResults = 100, nextToken, state } = {}) { }); const { response: data } = await responseHandler(response); - if (data.results && Array.isArray(data.results)) { - data.results.forEach((r) => { - r.destination_uri = getExportUri(r); // eslint-disable-line camelcase - }); - } - return data; } @@ -505,10 +486,7 @@ async function getExport({ exportId }) { }); const response = await fetchWithAccountKey(url, { method: "GET" }); const data = await responseHandler(response); - return { - ...data.response, - destination_uri: getExportUri(data.response), // eslint-disable-line camelcase - }; + return data.response; } /** diff --git a/test/commands/export/create.mjs b/test/commands/export/create.mjs index 4282b532..ebdcd87c 100644 --- a/test/commands/export/create.mjs +++ b/test/commands/export/create.mjs @@ -20,7 +20,6 @@ const createExportStub = (opts) => ({ }, created_at: "2025-01-02T22:59:51", updated_at: "2025-01-02T22:59:51", - destination_uri: "", ...opts, }); @@ -34,113 +33,159 @@ describe("export create s3", () => { ({ createExport } = container.resolve("accountAPI")); }); - it("creates an export", async () => { - const database = "us-std/example"; - const bucket = "test-bucket"; - const path = "/test/key"; - const stubbedResponse = createExportStub({ - database, - destination: { + const scenarios = [ + { + description: "using --destination", + args: "--destination 's3://test-bucket/test/key'", + expectedDestination: { s3: { - path, - bucket, + bucket: "test-bucket", + path: "/test/key", }, + uri: "s3://test-bucket/test/key", }, - format: "simple", - }); - createExport.resolves(stubbedResponse); + expectedDestArgs: "s3://test-bucket/test/key", + }, + { + description: "using --bucket and --path", + args: "--bucket 'test-bucket' --path '/test/key'", + expectedDestination: { + s3: { + bucket: "test-bucket", + path: "/test/key", + }, + uri: "s3://test-bucket/test/key", + }, + expectedDestArgs: { s3: { bucket: "test-bucket", path: "/test/key" } }, + }, + ]; - await run( - `export create s3 --database '${database}' --bucket '${bucket}' --path '${path}'`, - container, - ); - await stdout.waitForWritten(); + scenarios.forEach( + ({ description, args, expectedDestination, expectedDestArgs }) => { + it(`creates an export ${description}`, async () => { + const database = "us-std/example"; + const stubbedResponse = createExportStub({ + database, + destination: expectedDestination, + format: "simple", + }); + createExport.resolves(stubbedResponse); - expect(stdout.getWritten()).to.equal(`id: test-export-id + await run( + `export create s3 --database '${database}' ${args}`, + container, + ); + await stdout.waitForWritten(); + + expect(stdout.getWritten()).to.equal(`id: test-export-id state: Pending database: us-std/example format: simple destination: s3: - path: /test/key bucket: test-bucket + path: /test/key + uri: s3://test-bucket/test/key created_at: 2025-01-02T22:59:51 updated_at: 2025-01-02T22:59:51 -destination_uri: "" `); - expect(createExport).to.have.been.calledWith({ - database, - collections: [], - destination: { - s3: { - bucket, - path, - }, - }, - format: "simple", - }); - }); + expect(createExport).to.have.been.calledWith({ + database, + collections: [], + destination: expectedDestArgs, + format: "simple", + }); + }); - it("outputs the full response with --json", async () => { - const database = "us-std/example"; - const bucket = "test-bucket"; - const path = "/test/key"; - const stubbedResponse = createExportStub({ - database, - destination: { - s3: { - path, - bucket, - }, - }, - format: "simple", - }); - createExport.resolves(stubbedResponse); + it(`outputs the full response with --json ${description}`, async () => { + const database = "us-std/example"; + const stubbedResponse = createExportStub({ + database, + destination: expectedDestination, + format: "simple", + }); + createExport.resolves(stubbedResponse); - await run( - `export create s3 --database '${database}' --bucket '${bucket}' --path '${path}' --json`, - container, - ); - await stdout.waitForWritten(); + await run( + `export create s3 --database '${database}' ${args} --json`, + container, + ); + await stdout.waitForWritten(); - expect(stdout.getWritten()).to.equal( - `${colorize(stubbedResponse, { format: Format.JSON })}\n`, - ); - }); + expect(stdout.getWritten()).to.equal( + `${colorize(stubbedResponse, { format: Format.JSON })}\n`, + ); + }); - it("passes the format to the account api", async () => { - createExport.resolves(createExportStub({ format: "tagged" })); - await run( - `export create s3 --database 'us-std/example' --bucket 'test-bucket' --path 'test/key' --format 'tagged'`, - container, - ); - expect(createExport).to.have.been.calledWith( - sinon.match({ - format: "tagged", - }), - ); - }); + it(`passes the format to the account api ${description}`, async () => { + createExport.resolves(createExportStub({ format: "tagged" })); + await run( + `export create s3 --database 'us-std/example' ${args} --format 'tagged'`, + container, + ); + expect(createExport).to.have.been.calledWith( + sinon.match({ + format: "tagged", + }), + ); + }); - it("should allow providing multiple collections", async () => { - createExport.resolves(createExportStub({ collections: ["foo", "bar"] })); - await run( - `export create s3 --database 'us-std/example' --bucket 'test-bucket' --path 'test/key' --collection foo --collection bar`, - container, - ); - expect(createExport).to.have.been.calledWith( - sinon.match({ - database: "us-std/example", - collections: ["foo", "bar"], - }), - ); + it(`should allow providing multiple collections ${description}`, async () => { + createExport.resolves( + createExportStub({ collections: ["foo", "bar"] }), + ); + await run( + `export create s3 --database 'us-std/example' ${args} --collection foo --collection bar`, + container, + ); + expect(createExport).to.have.been.calledWith( + sinon.match({ + database: "us-std/example", + collections: ["foo", "bar"], + }), + ); + }); + }, + ); + + const invalidScenarios = [ + { + description: "an invalid S3 URI is given as the --destination input", + args: "--destination invalid-uri", + expectedError: "Invalid S3 URI format. Expected format: s3://bucket/path", + }, + { + description: "both --destination and --bucket are given", + args: "--destination s3://test-bucket/test/key --bucket test-bucket", + expectedError: + "Cannot specify --destination with --bucket or --path. Use either --destination or both --bucket and --path.", + }, + { + description: "both --destination and --path are given", + args: "--destination s3://test-bucket/test/key --path /test/key", + expectedError: + "Cannot specify --destination with --bucket or --path. Use either --destination or both --bucket and --path.", + }, + ]; + + invalidScenarios.forEach(({ description, args, expectedError }) => { + it(`should display an error when ${description}`, async () => { + try { + await run( + `export create s3 --database us-std/example ${args}`, + container, + ); + } catch {} + + await stderr.waitForWritten(); + expect(stderr.getWritten()).to.contain(expectedError); + }); }); it("should output an error if --database is not provided", async () => { + const destination = "s3://test-bucket/test/key"; try { - await run( - "export create s3 --bucket test-bucket --path test/key", - container, - ); + await run(`export create s3 --destination ${destination}`, container); } catch {} await stderr.waitForWritten(); diff --git a/test/commands/export/get.mjs b/test/commands/export/get.mjs index 90f52689..37341ac0 100644 --- a/test/commands/export/get.mjs +++ b/test/commands/export/get.mjs @@ -13,10 +13,10 @@ const getExportStub = (opts) => ({ bucket: "test-bucket", path: "some/key/prefix", }, + uri: "s3://test-bucket/some/key/prefix", }, created_at: "2025-01-09T19:07:25.642703Z", updated_at: "2025-01-09T19:07:25.642703Z", - destination_uri: "", ...opts, }); @@ -51,9 +51,9 @@ destination: s3: bucket: test-bucket path: some/key/prefix + uri: s3://test-bucket/some/key/prefix created_at: 2025-01-09T19:07:25.642703Z updated_at: 2025-01-09T19:07:25.642703Z -destination_uri: "" failure: code: validation_error message: "failed to get bucket region: bucket not found" diff --git a/test/commands/export/wait.mjs b/test/commands/export/wait.mjs index 4dbd0663..688a2c8c 100644 --- a/test/commands/export/wait.mjs +++ b/test/commands/export/wait.mjs @@ -24,7 +24,11 @@ describe("export wait helpers", () => { describe("waitUntilExportIsReady", () => { it("should return export data when export completes successfully", async () => { const exportId = "test-export-id"; - const exportData = { id: exportId, state: ExportState.Complete }; + const exportData = { + id: exportId, + is_terminal: true, + state: ExportState.Complete, + }; const statusHandler = sinon.stub(); getExport.resolves(exportData); @@ -40,9 +44,6 @@ describe("export wait helpers", () => { expect(statusHandler).to.have.been.calledWith( `test-export-id is Pending and not yet started.`, ); - expect(statusHandler).to.have.been.calledWith( - "test-export-id is Pending and not yet started.", - ); expect(statusHandler).to.have.been.calledWith( "test-export-id has a terminal state of Complete.", ); @@ -50,7 +51,7 @@ describe("export wait helpers", () => { it("should not print status when quiet is true", async () => { const exportId = "test-export-id"; - const exportData = { id: exportId, state: ExportState.Complete }; + const exportData = { id: exportId, is_terminal: true }; const statusHandler = sinon.stub(); getExport.resolves(exportData); @@ -92,9 +93,17 @@ describe("export wait helpers", () => { getExport .onFirstCall() - .resolves({ id: exportId, state: ExportState.Pending }) + .resolves({ + id: exportId, + is_terminal: false, + state: ExportState.Pending, + }) .onSecondCall() - .resolves({ id: exportId, state: ExportState.Complete }); + .resolves({ + id: exportId, + is_terminal: true, + state: ExportState.Complete, + }); const result = await waitAndCheckExportState({ id: exportId, @@ -134,11 +143,11 @@ describe("export wait helpers", () => { getExport .onFirstCall() - .resolves({ id: exportId, state: ExportState.Pending }) + .resolves({ id: exportId, is_terminal: false }) .onSecondCall() - .resolves({ id: exportId, state: ExportState.Pending }) + .resolves({ id: exportId, is_terminal: false }) .onThirdCall() - .resolves({ id: exportId, state: ExportState.Complete }); + .resolves({ id: exportId, is_terminal: true }); await waitAndCheckExportState({ id: exportId, diff --git a/test/lib/account-api/account-api.mjs b/test/lib/account-api/account-api.mjs index 228a4f4e..7957cf76 100644 --- a/test/lib/account-api/account-api.mjs +++ b/test/lib/account-api/account-api.mjs @@ -130,153 +130,69 @@ describe("accountAPI", () => { }); }); - describe("createExport", () => { - const testExport = { - id: "419633606504219216", - state: "Pending", - database: "us-std/demo", - format: "simple", + const scenarios = [ + { + description: "using destination URI", + destination: "s3://test-bucket/test/key", + }, + { + description: "using bucket and path", destination: { s3: { bucket: "test-bucket", - path: "some/key", + path: "test/key", }, }, - created_at: "2025-01-09T19:57:22.735201Z", - }; - - it("should call the endpoint", async () => { - fetch - .withArgs( - sinon.match({ href: "https://account.fauna.com/v2/exports" }), - sinon.match({ method: "POST" }), - ) - .resolves(f({ response: testExport }, 201)); - - const data = await accountAPI.createExport({ - database: "us/demo", - format: "simple", - destination: { - s3: { - bucket: "test-bucket", - path: "some/key", - }, - }, - }); - - expect(fetch).to.have.been.calledWith( - sinon.match({ href: "https://account.fauna.com/v2/exports" }), - sinon.match({ - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: "Bearer some-account-key", - }, - body: JSON.stringify({ - database: "us-std/demo", - destination: { - s3: { bucket: "test-bucket", path: "some/key" }, - }, - format: "simple", - }), - }), - ); - expect(data).to.deep.equal({ - ...testExport, - destination_uri: "s3://test-bucket/some/key", - }); - }); - - it("should support collections", async () => { - fetch - .withArgs( - sinon.match({ href: "https://account.fauna.com/v2/exports" }), - sinon.match({ method: "POST" }), - ) - .resolves( - f( - { response: { ...testExport, collections: ["test-collection"] } }, - 201, - ), - ); - - const data = await accountAPI.createExport({ - database: "us/demo", + }, + ]; + + scenarios.forEach(({ description, destination }) => { + describe(`createExport ${description}`, () => { + const testExport = { + id: "419633606504219216", + state: "Pending", + database: "us-std/demo", format: "simple", destination: { s3: { bucket: "test-bucket", - path: "some/key", + path: "test/key", }, + uri: "s3://test-bucket/test/key", }, - collections: ["test-collection"], - }); - - expect(fetch).to.have.been.calledWith( - sinon.match({ href: "https://account.fauna.com/v2/exports" }), - sinon.match({ - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: "Bearer some-account-key", - }, - body: JSON.stringify({ - database: "us-std/demo", - destination: { - s3: { bucket: "test-bucket", path: "some/key" }, - }, - format: "simple", - collections: ["test-collection"], - }), - }), - ); - expect(data).to.deep.equal({ - ...testExport, - collections: ["test-collection"], - destination_uri: "s3://test-bucket/some/key", - }); - }); - - it("should support tagged format", async () => { - fetch - .withArgs( + created_at: "2025-01-09T19:57:22.735201Z", + }; + + it("should call the endpoint", async () => { + fetch + .withArgs( + sinon.match({ href: "https://account.fauna.com/v2/exports" }), + sinon.match({ method: "POST" }), + ) + .resolves(f({ response: testExport }, 201)); + + const data = await accountAPI.createExport({ + database: "us/demo", + format: "simple", + destination, + }); + + expect(fetch).to.have.been.calledWith( sinon.match({ href: "https://account.fauna.com/v2/exports" }), - sinon.match({ method: "POST" }), - ) - .resolves(f({ response: { ...testExport, format: "tagged" } }, 201)); - - const data = await accountAPI.createExport({ - database: "us/demo", - format: "tagged", - destination: { - s3: { - bucket: "test-bucket", - path: "some/key", - }, - }, - }); - - expect(fetch).to.have.been.calledWith( - sinon.match({ href: "https://account.fauna.com/v2/exports" }), - sinon.match({ - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: "Bearer some-account-key", - }, - body: JSON.stringify({ - database: "us-std/demo", - destination: { - s3: { bucket: "test-bucket", path: "some/key" }, + sinon.match({ + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer some-account-key", }, - format: "tagged", + body: JSON.stringify({ + database: "us-std/demo", + destination, + format: "simple", + }), }), - }), - ); - expect(data).to.deep.equal({ - ...testExport, - format: "tagged", - destination_uri: "s3://test-bucket/some/key", + ); + expect(data).to.deep.equal(testExport); }); }); }); @@ -333,11 +249,10 @@ describe("accountAPI", () => { expect(data).to.deep.equal({ results: [ - { ...testExport, destination_uri: "s3://test-bucket/some/key" }, + testExport, { ...testExport, state: "Complete", - destination_uri: "s3://test-bucket/some/key", }, ], next_token: "456", @@ -420,10 +335,7 @@ describe("accountAPI", () => { }, }), ); - expect(data).to.deep.equal({ - ...testExport, - destination_uri: "s3://test-bucket/some/key", - }); + expect(data).to.deep.equal(testExport); }); }); });