From de46a4c61e8517751aa7508265c9a11771311d40 Mon Sep 17 00:00:00 2001 From: Doug Newman Date: Fri, 22 Nov 2024 14:06:15 -0500 Subject: [PATCH] CMR-10238: Adding the virtual catalog 'ALL' (#379) * Added ALL catalog and functionality for ALL/collections, ALL/collections/, ALL/collections/search etc. * testing for /stac and /stac/all routes, debug logs for all external calls * Added tests for all routes related to ALL * Fixed merge problem with stringifyQuery * Linted * Update src/routes/browse.ts Co-authored-by: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> * LPFALL provider name fix * Linting * Added test for LPALL provider, removed multiple providers from test --------- Co-authored-by: doug-newman-nasa Co-authored-by: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> --- src/@types/StacCollection.d.ts | 2 +- src/__tests__/items.spec.ts | 30 ++++++++ src/__tests__/providerCatalog.spec.ts | 94 ++++++++++++++++++++++++ src/__tests__/providerCollection.spec.ts | 68 +++++++++++++++++ src/__tests__/providerSearch.spec.ts | 34 +++++++++ src/__tests__/rootCatalog.spec.ts | 18 ++++- src/domains/collections.ts | 15 ++-- src/domains/providers.ts | 10 ++- src/domains/stac.ts | 2 +- src/middleware/index.ts | 30 +++++++- src/routes/__tests__/browse.spec.ts | 52 +++++++++++-- src/routes/browse.ts | 65 +++++++++++++--- src/routes/catalog.ts | 45 +++++++----- src/routes/healthcheck.ts | 2 + src/routes/index.ts | 20 ++++- src/utils/testUtils.ts | 6 ++ 16 files changed, 448 insertions(+), 45 deletions(-) diff --git a/src/@types/StacCollection.d.ts b/src/@types/StacCollection.d.ts index 12273480..b6ed395c 100644 --- a/src/@types/StacCollection.d.ts +++ b/src/@types/StacCollection.d.ts @@ -187,7 +187,7 @@ export type STACCollection = { description: Description; keywords?: Keywords; license: CollectionLicenseName; - providers?: { + providers: { name: OrganizationName; description?: OrganizationDescription; roles?: OrganizationRoles; diff --git a/src/__tests__/items.spec.ts b/src/__tests__/items.spec.ts index fd5043b6..e22e1a1a 100644 --- a/src/__tests__/items.spec.ts +++ b/src/__tests__/items.spec.ts @@ -172,3 +172,33 @@ describe("GET /PROVIDER/collections/COLLECTION/items/ITEM", () => { }); }); }); + +describe("GET /ALL/collections/:collection/items/", () => { + it("should return a 404", async () => { + sandbox + .stub(Providers, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); + + const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo/items"); + + expect(statusCode).to.equal(404); + expect(body).to.deep.equal({ + errors: ["This operation is not allowed for the ALL Catalog."], + }); + }); +}); + +describe("GET /ALL/collections/:collection/items/:item", () => { + it("should return a 404", async () => { + sandbox + .stub(Providers, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); + + const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo/items/bar"); + + expect(statusCode).to.equal(404); + expect(body).to.deep.equal({ + errors: ["This operation is not allowed for the ALL Catalog."], + }); + }); +}); diff --git a/src/__tests__/providerCatalog.spec.ts b/src/__tests__/providerCatalog.spec.ts index bd1ecf2f..05f41678 100644 --- a/src/__tests__/providerCatalog.spec.ts +++ b/src/__tests__/providerCatalog.spec.ts @@ -143,6 +143,7 @@ describe("GET /:provider", () => { items: mockCollections.map((coll) => ({ id: `${coll.id}`, title: coll.title ?? faker.random.words(4), + provider: `TEST`, })), }); @@ -179,6 +180,7 @@ describe("GET /:provider", () => { items: mockCollections.map((coll) => ({ id: `${coll.id}`, title: coll.title ?? faker.random.words(4), + provider: "TEST", })), }); @@ -209,6 +211,7 @@ describe("GET /:provider", () => { items: mockCollections.map((coll) => ({ id: `${coll.id}`, title: coll.title ?? faker.random.words(4), + provider: "TEST`", })), }); @@ -232,6 +235,7 @@ describe("GET /:provider", () => { items: mockCollections.map((coll) => ({ id: `${coll.id}`, title: coll.title ?? faker.random.words(4), + provider: "TEST", })), }); @@ -287,4 +291,94 @@ describe("GET /:provider", () => { expect(res.statusCode).to.equal(404); }); }); + + describe("given the ALL provider/catalog", () => { + it("should call the graphql API with no provider search clause", async () => { + sandbox + .stub(Provider, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); + + const getCollectionsSpy = sandbox + .stub(Collections, "getCollectionIds") + .resolves({ count: 0, cursor: null, items: [] }); + + const res = await request(stacApp).get("/stac/ALL"); + expect(res.statusCode).to.equal(200); + // getCollectionIds should have no provider clause in query argument. + // If this was any provider other than 'ALL', this method would be + // called with { provider: 'TEST', cursor: undefined, limit: NaN } + expect(getCollectionsSpy).to.have.been.calledWith({ cursor: undefined, limit: NaN }); + }); + it("should return rel=child links whose href contains a provider rather than 'ALL'", async () => { + sandbox + .stub(Provider, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); + + const mockCollections = generateSTACCollections(5); + sandbox.stub(Collections, "getCollectionIds").resolves({ + count: mockCollections.length, + cursor: "foundCursor", + items: mockCollections.map((coll) => ({ + id: `${coll.id}`, + title: coll.title ?? faker.random.words(4), + provider: `TEST`, + })), + }); + + const { body: catalog, statusCode } = await request(stacApp).get("/stac/ALL"); + + const children = catalog.links.filter((l: Link) => l.rel === "child"); + expect(children).to.have.length(mockCollections.length); + + mockCollections.forEach((collection) => { + const childLink = children.find((l: Link) => l.href.endsWith(collection.id)); + + expect(childLink.href).to.endWith(`/TEST/collections/${collection.id}`); + expect(childLink.href).to.not.contain("/ALL/"); + }); + + expect(statusCode).to.equal(200); + }); + it("should not return any links of rel=search", async () => { + sandbox + .stub(Provider, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); + + sandbox.stub(Collections, "getCollectionIds").resolves({ count: 0, cursor: null, items: [] }); + + const { body: catalog, statusCode } = await request(stacApp).get("/stac/ALL"); + + const children = catalog.links.filter((l: Link) => l.rel === "search"); + expect(children).to.have.length(0); + expect(statusCode).to.equal(200); + }); + it("should be able to handle providers whose name contains the text 'ALL'", async () => { + sandbox + .stub(Provider, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "LPALL" }]]); + + const mockCollections = generateSTACCollections(1); + + sandbox.stub(Collections, "getCollectionIds").resolves({ + count: mockCollections.length, + cursor: "foundCursor", + items: mockCollections.map((coll) => ({ + id: `${coll.id}`, + title: coll.title ?? faker.random.words(4), + provider: "LPALL", + })), + }); + + const { body: catalog, statusCode } = await request(stacApp).get("/stac/ALL"); + + const children = catalog.links.filter((l: Link) => l.rel === "child"); + + mockCollections.forEach((collection) => { + const childLink = children.find((l: Link) => l.href.endsWith(collection.id)); + + expect(childLink.href).to.endWith(`/LPALL/collections/${collection.id}`); + expect(childLink.href).to.not.contain("/ALL/"); + }); + }); + }); }); diff --git a/src/__tests__/providerCollection.spec.ts b/src/__tests__/providerCollection.spec.ts index 5ab3091d..a164c7c6 100644 --- a/src/__tests__/providerCollection.spec.ts +++ b/src/__tests__/providerCollection.spec.ts @@ -134,6 +134,7 @@ describe("GET /:provider/collections", () => { // Expect the href to match the generic STAC API. const link1: Link = body.collections[1].links.find((l: Link) => l.rel === "items"); expect(link1.href).to.contain("/stac/TEST/collections/"); + expect(link1.href).to.endsWith("/items"); }); }); @@ -470,3 +471,70 @@ describe("GET /:provider/collections/:collectionId", () => { }); }); }); + +describe("GET /ALL/collections", () => { + describe("given the ALL catalog", () => { + it("returns item links relative to the provider catalogs rather than ALL", async () => { + sandbox + .stub(Providers, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); + + const mockCollections = generateSTACCollections(1); + + sandbox.stub(Collections, "getCollections").resolves({ + count: 1, + cursor: null, + items: mockCollections, + }); + + const { statusCode, body } = await request(app).get("/stac/ALL/collections"); + + expect(statusCode).to.equal(200); + expect(body.collections).to.have.lengthOf(1); + // Make sure that we are determining the 'provider' element of the items path from the provider detailed + // in the collection metadata rather than the ALL route. + expect(body.collections[0].links.find((l: Link) => l.rel === "items").href).to.include( + "/PROV1/" + ); + expect(body.collections[0].links.find((l: Link) => l.rel === "items").href).to.not.include( + "/ALL/" + ); + }); + it("returns collection items links that end in 'items", async () => { + sandbox + .stub(Providers, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); + + const mockCollections = generateSTACCollections(1); + + sandbox.stub(Collections, "getCollections").resolves({ + count: 1, + cursor: null, + items: mockCollections, + }); + + const { statusCode, body } = await request(app).get("/stac/ALL/collections"); + + expect(statusCode).to.equal(200); + expect(body.collections).to.have.lengthOf(1); + expect(body.collections[0].links.find((l: Link) => l.rel === "items").href).to.endsWith( + "/items" + ); + }); + }); +}); + +describe("GET /ALL/collections/:collectionId", () => { + it("should return a 404", async () => { + sandbox + .stub(Providers, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); + + const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo"); + + expect(statusCode).to.equal(404); + expect(body).to.deep.equal({ + errors: ["This operation is not allowed for the ALL Catalog."], + }); + }); +}); diff --git a/src/__tests__/providerSearch.spec.ts b/src/__tests__/providerSearch.spec.ts index 35d2b68b..b0e46e98 100644 --- a/src/__tests__/providerSearch.spec.ts +++ b/src/__tests__/providerSearch.spec.ts @@ -305,3 +305,37 @@ describe("Query validation", () => { }); }); }); + +describe("GET /ALL/search", () => { + describe("given an 'ALL' provider", () => { + it("should return a 404", async () => { + sandbox + .stub(Providers, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); + + const { statusCode, body } = await request(app).get("/stac/ALL/search"); + + expect(statusCode).to.equal(404); + expect(body).to.deep.equal({ + errors: ["This operation is not allowed for the ALL Catalog."], + }); + }); + }); +}); + +describe("POST /ALL/search", () => { + describe("given an 'ALL' provider", () => { + it("should return a 404", async () => { + sandbox + .stub(Providers, "getProviders") + .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]); + + const { statusCode, body } = await request(app).post("/stac/ALL/search"); + + expect(statusCode).to.equal(404); + expect(body).to.deep.equal({ + errors: ["This operation is not allowed for the ALL Catalog."], + }); + }); + }); +}); diff --git a/src/__tests__/rootCatalog.spec.ts b/src/__tests__/rootCatalog.spec.ts index 61a1c87c..b1ca2456 100644 --- a/src/__tests__/rootCatalog.spec.ts +++ b/src/__tests__/rootCatalog.spec.ts @@ -93,6 +93,22 @@ describe("GET /stac", () => { expect(providerLink.title).to.equal(provider["provider-id"]); }); }); + + it("should have an entry for the 'ALL' catalog", async () => { + const { statusCode, body } = await request(app).get("/stac"); + + expect(statusCode).to.equal(200); + const [, expectedProviders] = cmrProvidersResponse; + + const allLink = body.links.find((l: Link) => l.title === "all"); + + expect(allLink.href).to.match(/^(http)s?:\/\/.*\w+/); + expect(allLink.href).to.endWith("/stac/ALL"); + expect(allLink.href).to.not.contain("?param=value"); + expect(allLink.rel).to.equal("child"); + expect(allLink.type).to.equal("application/json"); + expect(allLink.title).to.equal("all"); + }); }); describe("given CMR providers endpoint responds with an error", () => { @@ -147,6 +163,6 @@ describe("/cloudstac", () => { expect(body.links.find((l: { title: string }) => l.title === "NOT_CLOUD")).to.be.undefined; - expect(mockCmrHits).to.have.been.calledTwice; + expect(mockCmrHits).to.have.been.calledThrice; }); }); diff --git a/src/domains/collections.ts b/src/domains/collections.ts index aaade788..e4963304 100644 --- a/src/domains/collections.ts +++ b/src/domains/collections.ts @@ -57,6 +57,7 @@ const collectionIdsQuery = gql` conceptId entryId title + provider } } } @@ -242,7 +243,7 @@ export const collectionToStac = (collection: Collection): STACCollection => { const extent = createExtent(collection); const keywords = createKeywords(collection); const links = generateCollectionLinks(collection, [licenseLink]); - const provider = generateProviders(collection); + const providers = generateProviders(collection); const summaries = createSummaries(collection); return { @@ -253,7 +254,7 @@ export const collectionToStac = (collection: Collection): STACCollection => { stac_version: STAC_VERSION, extent, assets, - provider, + providers, links, license, keywords, @@ -334,14 +335,18 @@ export const getCollectionIds = async ( ): Promise<{ count: number; cursor: string | null; - items: { id: string; title: string }[]; + items: { id: string; title: string; provider: string }[]; }> => { const { cursor, count, items: collectionIds, } = await paginateQuery(collectionIdsQuery, params, opts, collectionIdsHandler); - return { cursor, count, items: collectionIds as { id: string; title: string }[] }; + return { + cursor, + count, + items: collectionIds as { id: string; title: string; provider: string }[], + }; }; /** @@ -356,7 +361,7 @@ export const getAllCollectionIds = async ( ): Promise<{ count: number; cursor: string | null; - items: { id: string; title: string }[]; + items: { id: string; title: string; provider: string }[]; }> => { params.limit = CMR_QUERY_MAX; diff --git a/src/domains/providers.ts b/src/domains/providers.ts index 98a608db..0ca970a1 100644 --- a/src/domains/providers.ts +++ b/src/domains/providers.ts @@ -8,6 +8,12 @@ const CMR_LB_INGEST = `${CMR_LB_URL}/ingest`; const CMR_LB_SEARCH = `${CMR_LB_URL}/search`; const CMR_LB_SEARCH_COLLECTIONS = `${CMR_LB_SEARCH}/collections`; +export const ALL_PROVIDER = "ALL"; +export const ALL_PROVIDERS = { + "provider-id": ALL_PROVIDER.toUpperCase(), + "short-name": ALL_PROVIDER.toLowerCase(), +}; + export const conformance = [ "https://api.stacspec.org/v1.0.0-rc.2/core", "https://api.stacspec.org/v1.0.0-rc.2/item-search", @@ -31,6 +37,7 @@ export const conformance = [ */ export const getProviders = async (): Promise<[string, null] | [null, Provider[]]> => { try { + console.debug(`GET ${CMR_LB_INGEST}/providers`); const { data: providers } = await axios.get(`${CMR_LB_INGEST}/providers`); return [null, providers]; } catch (err) { @@ -50,7 +57,7 @@ export const getProvider = async ( if (errs) { return [errs as string, null]; } - + providers?.push(ALL_PROVIDERS); return [ null, (providers ?? []).find((provider) => provider["provider-id"] === providerId) ?? null, @@ -79,6 +86,7 @@ export const getCloudProviders = async ( await Promise.all( (candidates ?? []).map(async (provider) => { try { + console.debug(`GET ${CMR_LB_SEARCH_COLLECTIONS}`); const { headers } = await axios.get(CMR_LB_SEARCH_COLLECTIONS, { headers: mergeMaybe({}, { authorization }), params: { provider: provider["short-name"], cloud_hosted: true }, diff --git a/src/domains/stac.ts b/src/domains/stac.ts index d2e5823c..90b19dcd 100644 --- a/src/domains/stac.ts +++ b/src/domains/stac.ts @@ -563,6 +563,7 @@ export const paginateQuery = async ( try { console.info(timingMessage); + console.debug(`GET ${GRAPHQL_URL}`); const response = await request(GRAPHQL_URL, gqlQuery, variables, requestHeaders); // use the passed in results handler const [errors, data] = handler(response); @@ -570,7 +571,6 @@ export const paginateQuery = async ( if (errors) throw new Error(errors); if (!data) throw new Error("No data returned from GraphQL during paginated query"); const { count, cursor, items } = data; - return { items: items, count, cursor }; } catch (err: unknown) { if ( diff --git a/src/middleware/index.ts b/src/middleware/index.ts index dd7212f2..47f93548 100644 --- a/src/middleware/index.ts +++ b/src/middleware/index.ts @@ -11,7 +11,7 @@ import { import { WarmProviderCache } from "../domains/cache"; import { getCollections } from "../domains/collections"; import { parseOrdinateString } from "../domains/bounding-box"; -import { getProviders, getCloudProviders } from "../domains/providers"; +import { ALL_PROVIDER, getProviders, getCloudProviders, ALL_PROVIDERS } from "../domains/providers"; import { scrubTokens, mergeMaybe, ERRORS } from "../utils"; import { validDateTime } from "../utils/datetime"; @@ -94,7 +94,8 @@ export const refreshProviderCache = async (req: Request, _res: Response, next: N if (errs || !updatedProviders) { return next(new ServiceUnavailableError(ERRORS.serviceUnavailable)); } - + updatedProviders; + updatedProviders.push(ALL_PROVIDERS); updatedProviders.forEach((provider) => { cachedProviders.set(provider["provider-id"], provider); }); @@ -139,7 +140,8 @@ export const validateProvider = async (req: Request, _res: Response, next: NextF `Provider [${providerId}] not found or does not have any visible cloud hosted collections.` ) ); - } else if (!provider) { + // If it's not the 'ALL' provider and the provider ID cannot be found then throw an error + } else if (!provider && providerId != ALL_PROVIDER.toString()) { next(new ItemNotFound(`Provider [${providerId}] not found.`)); } else { req.provider = provider; @@ -147,6 +149,28 @@ export const validateProvider = async (req: Request, _res: Response, next: NextF } }; +/** + * Middleware validates the provider in the route is not ALL. + * + * This validation is only required for routes that will search CMR based on the provider + * supplied. + * + * If the provider is not 'ALL' then validation passes. + * If the provider is 'ALL', it exits early with a 404. + + */ +export const validateNotAllProvider = async (req: Request, _res: Response, next: NextFunction) => { + const { providerId } = req.params; + + if (providerId == ALL_PROVIDER.toString()) { + next( + new ItemNotFound(`This operation is not allowed for the ${ALL_PROVIDER.toString()} Catalog.`) + ); + } else { + next(); + } +}; + /** * Middleware that adds the `collection` to the request object. * Should be used after the provider has been validated. diff --git a/src/routes/__tests__/browse.spec.ts b/src/routes/__tests__/browse.spec.ts index c0b22888..873b6582 100644 --- a/src/routes/__tests__/browse.spec.ts +++ b/src/routes/__tests__/browse.spec.ts @@ -7,10 +7,16 @@ chai.use(sinonChai); const { expect } = chai; import * as gql from "graphql-request"; -import { collectionHandler, collectionsHandler, addItemLinkIfPresent } from "../browse"; +import { + collectionHandler, + collectionsHandler, + addItemLinkIfNotPresent, + generateBaseUrlForCollection, + generateCollectionResponse, +} from "../browse"; import { generateSTACCollections } from "../../utils/testUtils"; -describe("addItemLinkIfPresent", () => { +describe("addItemLinkIfNotPresent", () => { it("will add an item link if no item link is present", async () => { // Create a STACCollection with no item link let stacCollection = generateSTACCollections(1)[0]; @@ -24,7 +30,7 @@ describe("addItemLinkIfPresent", () => { const numberoOfLinks = stacCollection.links.length; // Invoke method - addItemLinkIfPresent(stacCollection, "https://foo.com"); + addItemLinkIfNotPresent(stacCollection, "https://foo.com"); // Observe an addiitonal link in the STAC Collection with rel=items etc. expect(stacCollection.links.length).to.equal(numberoOfLinks + 1); expect(stacCollection).to.have.deep.property("links", [ @@ -62,8 +68,8 @@ describe("addItemLinkIfPresent", () => { }); const numberoOfLinks = stacCollection.links.length; // Invoke method - addItemLinkIfPresent(stacCollection, "https://foo.com/items"); - // Observe no additional link in the STAC Collection and that the item link remains a CMR link + addItemLinkIfNotPresent(stacCollection, "https://foo.com/items"); + // Observe no addiitonal link in the STAC Collection and that the item link remains a CMR link expect(stacCollection.links.length).to.equal(numberoOfLinks); expect(stacCollection).to.have.deep.property("links", [ { @@ -81,3 +87,39 @@ describe("addItemLinkIfPresent", () => { ]); }); }); + +describe("generateBaseUrlForCollection", () => { + it("will use the provider name for an ALL collection result", async () => { + let stacCollection = generateSTACCollections(1)[0]; + + const baseUrl = generateBaseUrlForCollection( + "http://localhost:3000/stac/ALL/collections/Test%201_1.2", + stacCollection + ); + expect(baseUrl).to.equal("http://localhost:3000/stac/PROV1/collections/Test%201_1.2"); + }); + it("will use the same provider name for any other collection result", async () => { + let stacCollection = generateSTACCollections(1)[0]; + + const baseUrl = generateBaseUrlForCollection( + "http://localhost:3000/stac/PROV1/collections/Test%201_1.2", + stacCollection + ); + expect(baseUrl).to.equal("http://localhost:3000/stac/PROV1/collections/Test%201_1.2"); + }); +}); + +describe("generateCollectionResponse", () => { + it("will add the correct description if the provider is 'ALL'", async () => { + let stacCollections = generateSTACCollections(1); + const baseUrl = "http://localhost:3000/stac/ALL/collections"; + const collectionsResponse = generateCollectionResponse(baseUrl, [], stacCollections); + expect(collectionsResponse.description).to.equal("All collections provided by CMR"); + }); + it("will add the correct description if the provider is a real provider", async () => { + let stacCollections = generateSTACCollections(1); + const baseUrl = "http://localhost:3000/stac/PROV1/collections"; + const collectionsResponse = generateCollectionResponse(baseUrl, [], stacCollections); + expect(collectionsResponse.description).to.equal("All collections provided by PROV1"); + }); +}); diff --git a/src/routes/browse.ts b/src/routes/browse.ts index bf86c8a9..f7716757 100644 --- a/src/routes/browse.ts +++ b/src/routes/browse.ts @@ -8,6 +8,7 @@ import { buildQuery } from "../domains/stac"; import { ItemNotFound } from "../models/errors"; import { getBaseUrl, mergeMaybe, stacContext } from "../utils"; import { STACCollection } from "../@types/StacCollection"; +import { ALL_PROVIDER } from "../domains/providers"; const collectionLinks = (req: Request, nextCursor?: string | null): Links => { const { stacRoot, self } = stacContext(req); @@ -53,6 +54,11 @@ export const collectionsHandler = async (req: Request, res: Response): Promise { + const baseUrl = generateBaseUrlForCollection(getBaseUrl(self), collection); collection.links.push({ rel: "self", - href: `${getBaseUrl(self)}/${encodeURIComponent(collection.id)}`, + href: `${baseUrl}/${encodeURIComponent(collection.id)}`, type: "application/json", }); collection.links.push({ @@ -70,16 +77,12 @@ export const collectionsHandler = async (req: Request, res: Response): Promise p.roles?.includes("producer")); + // Construct the items url from that provider + if (provider) baseUrl = baseUrl.replace("/ALL/", `/${provider.name}/`); + return baseUrl; +} + /** * A CMR collection can now indicate to consumers that it has a STAC API. * If that is the case then we use that link instead of a generic CMR one. @@ -119,7 +166,7 @@ export const collectionHandler = async (req: Request, res: Response): Promise link.rel === "items"); if (!itemsLink) { diff --git a/src/routes/catalog.ts b/src/routes/catalog.ts index 080e1675..7d229862 100644 --- a/src/routes/catalog.ts +++ b/src/routes/catalog.ts @@ -8,6 +8,7 @@ import { conformance } from "../domains/providers"; import { ServiceUnavailableError } from "../models/errors"; import { getBaseUrl, mergeMaybe, stacContext } from "../utils"; import { CMR_QUERY_MAX } from "../domains/stac"; +import { ALL_PROVIDER } from "../domains/providers"; const STAC_VERSION = process.env.STAC_VERSION ?? "1.0.0"; @@ -41,20 +42,6 @@ const generateSelfLinks = (req: Request, nextCursor?: string | null, count?: num title: "Provider Collections", method: "POST", }, - { - rel: "search", - href: `${path}/search`, - type: "application/geo+json", - title: "Provider Item Search", - method: "GET", - }, - { - rel: "search", - href: `${path}/search`, - type: "application/geo+json", - title: "Provider Item Search", - method: "POST", - }, { rel: "conformance", href: `${path}/conformance`, @@ -75,6 +62,24 @@ const generateSelfLinks = (req: Request, nextCursor?: string | null, count?: num }, ]; + const { provider } = req; + if (provider && provider["provider-id"] != ALL_PROVIDER) { + links.push({ + rel: "search", + href: `${path}/search`, + type: "application/geo+json", + title: "Provider Item Search", + method: "GET", + }); + links.push({ + rel: "search", + href: `${path}/search`, + type: "application/geo+json", + title: "Provider Item Search", + method: "POST", + }); + } + const originalQuery = mergeMaybe(req.query, req.body); // Add a 'next' link if there are more results available @@ -98,7 +103,9 @@ const generateSelfLinks = (req: Request, nextCursor?: string | null, count?: num const providerCollections = async ( req: Request -): Promise<[null, { id: string; title: string }[], string | null] | [string, null]> => { +): Promise< + [null, { id: string; title: string; provider: string }[], string | null] | [string, null] +> => { const { headers, provider, query } = req; const cloudOnly = headers["cloud-stac"] === "true" ? { cloudHosted: true } : {}; @@ -112,6 +119,8 @@ const providerCollections = async ( ); try { + if ("provider" in mergedQuery && mergedQuery.provider == ALL_PROVIDER) + delete mergedQuery.provider; const { items, cursor } = await getAllCollectionIds(mergedQuery, { headers }); return [null, items, cursor]; } catch (err) { @@ -133,9 +142,11 @@ export const providerCatalogHandler = async (req: Request, res: Response) => { const selfLinks = generateSelfLinks(req, cursor, collections?.length); - const childLinks = (collections ?? []).map(({ id, title }) => ({ + const childLinks = (collections ?? []).map(({ id, title, provider }) => ({ rel: "child", - href: `${getBaseUrl(self)}/collections/${encodeURIComponent(id)}`, + href: `${getBaseUrl(self) + .concat("/") + .replace("/ALL/", "/" + provider + "/")}collections/${encodeURIComponent(id)}`, title, type: "application/json", })); diff --git a/src/routes/healthcheck.ts b/src/routes/healthcheck.ts index 9a320b39..d0f7b2e2 100644 --- a/src/routes/healthcheck.ts +++ b/src/routes/healthcheck.ts @@ -6,7 +6,9 @@ const CMR_INGEST_HEALTH = `${CMR_LB_URL}/ingest/health`; const CMR_SEARCH_HEALTH = `${CMR_LB_URL}/search/health`; export const healthcheckHandler = async (_req: Request, res: Response) => { + console.debug(`GET ${CMR_INGEST_HEALTH}`); const { status: ingestStatus } = await axios.get(CMR_INGEST_HEALTH); + console.debug(`GET ${CMR_SEARCH_HEALTH}`); const { status: searchStatus } = await axios.get(CMR_SEARCH_HEALTH); if ([ingestStatus, searchStatus].every((status) => status === 200)) { diff --git a/src/routes/index.ts b/src/routes/index.ts index 6188e834..5b7b26de 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -18,6 +18,7 @@ import { validateCollection, validateProvider, validateStacQuery, + validateNotAllProvider, } from "../middleware"; const router = express.Router(); @@ -43,8 +44,20 @@ router.get( router .route("/:providerId/search") - .get(refreshProviderCache, validateProvider, validateStacQuery, wrapErrorHandler(searchHandler)) - .post(refreshProviderCache, validateProvider, validateStacQuery, wrapErrorHandler(searchHandler)); + .get( + refreshProviderCache, + validateNotAllProvider, + validateProvider, + validateStacQuery, + wrapErrorHandler(searchHandler) + ) + .post( + refreshProviderCache, + validateNotAllProvider, + validateProvider, + validateStacQuery, + wrapErrorHandler(searchHandler) + ); router .route("/:providerId/collections") @@ -64,6 +77,7 @@ router router.get( "/:providerId/collections/:collectionId", refreshProviderCache, + validateNotAllProvider, validateProvider, validateStacQuery, validateCollection, @@ -73,6 +87,7 @@ router.get( router.get( "/:providerId/collections/:collectionId/items", refreshProviderCache, + validateNotAllProvider, validateProvider, validateStacQuery, validateCollection, @@ -82,6 +97,7 @@ router.get( router.get( "/:providerId/collections/:collectionId/items/:itemId", refreshProviderCache, + validateNotAllProvider, validateProvider, validateStacQuery, validateCollection, diff --git a/src/utils/testUtils.ts b/src/utils/testUtils.ts index 7e1c2b9d..61013ff1 100644 --- a/src/utils/testUtils.ts +++ b/src/utils/testUtils.ts @@ -14,6 +14,12 @@ export const generateSTACCollections = (quantity: number) => { type: "Collection", description: faker.hacker.phrase(), license: "proprietary", + providers: [ + { + name: "PROV1", + roles: ["producer"], + }, + ], extent: { spatial: { bbox: [