CMR-10238: Adding the virtual catalog 'ALL'

src/@types/StacCollection.d.ts

@@ -187,7 +187,7 @@ export type STACCollection = {
   description: Description;
   keywords?: Keywords;
   license: CollectionLicenseName;
-  providers?: {
+  providers: {
     name: OrganizationName;
     description?: OrganizationDescription;
     roles?: OrganizationRoles;

src/__tests__/items.spec.ts

@@ -172,3 +172,33 @@ describe("GET /PROVIDER/collections/COLLECTION/items/ITEM", () => {
     });
   });
 });
+
+describe("GET /ALL/collections/:collection/items/", () => {
+  it("should return a 404", async () => {
+    sandbox
+      .stub(Providers, "getProviders")
+      .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+    const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo/items");
+
+    expect(statusCode).to.equal(404);
+    expect(body).to.deep.equal({
+      errors: ["This operation is not allowed for the ALL Catalog."],
+    });
+  });
+});
+
+describe("GET /ALL/collections/:collection/items/:item", () => {
+  it("should return a 404", async () => {
+    sandbox
+      .stub(Providers, "getProviders")
+      .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+    const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo/items/bar");
+
+    expect(statusCode).to.equal(404);
+    expect(body).to.deep.equal({
+      errors: ["This operation is not allowed for the ALL Catalog."],
+    });
+  });
+});

src/__tests__/providerCatalog.spec.ts

@@ -143,6 +143,7 @@ describe("GET /:provider", () => {
             items: mockCollections.map((coll) => ({
               id: `${coll.id}`,
               title: coll.title ?? faker.random.words(4),
+              provider: `TEST`,
             })),
           });
 
@@ -179,6 +180,7 @@ describe("GET /:provider", () => {
           items: mockCollections.map((coll) => ({
             id: `${coll.id}`,
             title: coll.title ?? faker.random.words(4),
+            provider: "TEST",
           })),
         });
 
@@ -209,6 +211,7 @@ describe("GET /:provider", () => {
           items: mockCollections.map((coll) => ({
             id: `${coll.id}`,
             title: coll.title ?? faker.random.words(4),
+            provider: "TEST`",
           })),
         });
 
@@ -232,6 +235,7 @@ describe("GET /:provider", () => {
           items: mockCollections.map((coll) => ({
             id: `${coll.id}`,
             title: coll.title ?? faker.random.words(4),
+            provider: "TEST",
           })),
         });
 
@@ -287,4 +291,66 @@ describe("GET /:provider", () => {
       expect(res.statusCode).to.equal(404);
     });
   });
+
+  describe("given the ALL provider/catalog", () => {
+    it("should call the graphql API with no provider search clause", async () => {
+      sandbox
+        .stub(Provider, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const getCollectionsSpy = sandbox
+        .stub(Collections, "getCollectionIds")
+        .resolves({ count: 0, cursor: null, items: [] });
+
+      const res = await request(stacApp).get("/stac/ALL");
+      expect(res.statusCode).to.equal(200);
+      // getCollectionIds should have no provider clause in query argument.
+      // If this was any provider other than 'ALL', this method would be
+      // called with { provider: 'TEST', cursor: undefined, limit: NaN }
+      expect(getCollectionsSpy).to.have.been.calledWith({ cursor: undefined, limit: NaN });
+    });
+    it("should return rel=child links whose href contains a provider rather than 'ALL'", async () => {
+      sandbox
+        .stub(Provider, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const mockCollections = generateSTACCollections(5);
+      sandbox.stub(Collections, "getCollectionIds").resolves({
+        count: mockCollections.length,
+        cursor: "foundCursor",
+        items: mockCollections.map((coll) => ({
+          id: `${coll.id}`,
+          title: coll.title ?? faker.random.words(4),
+          provider: `TEST`,
+        })),
+      });
+
+      const { body: catalog, statusCode } = await request(stacApp).get("/stac/ALL");
+
+      const children = catalog.links.filter((l: Link) => l.rel === "child");
+      expect(children).to.have.length(mockCollections.length);
+
+      mockCollections.forEach((collection) => {
+        const childLink = children.find((l: Link) => l.href.endsWith(collection.id));
+
+        expect(childLink.href).to.endWith(`/TEST/collections/${collection.id}`);
+        expect(childLink.href).to.not.contain("/ALL/");
+      });
+
+      expect(statusCode).to.equal(200);
+    });
+    it("should not return any links of rel=search", async () => {
+      sandbox
+        .stub(Provider, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      sandbox.stub(Collections, "getCollectionIds").resolves({ count: 0, cursor: null, items: [] });
+
+      const { body: catalog, statusCode } = await request(stacApp).get("/stac/ALL");
+
+      const children = catalog.links.filter((l: Link) => l.rel === "search");
+      expect(children).to.have.length(0);
+      expect(statusCode).to.equal(200);
+    });
+  });
 });

src/__tests__/providerCollection.spec.ts

@@ -134,6 +134,7 @@ describe("GET /:provider/collections", () => {
       // Expect the href to match the generic STAC API.
       const link1: Link = body.collections[1].links.find((l: Link) => l.rel === "items");
       expect(link1.href).to.contain("/stac/TEST/collections/");
+      expect(link1.href).to.endsWith("/items");
     });
   });
 
@@ -470,3 +471,84 @@ describe("GET /:provider/collections/:collectionId", () => {
     });
   });
 });
+
+describe("GET /ALL/collections", () => {
+  describe("given the ALL catalog", () => {
+    it("returns collections from multiple providers", async () => {
+      sandbox
+        .stub(Providers, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const mockCollections = generateSTACCollections(3);
+
+      sandbox.stub(Collections, "getCollections").resolves({
+        count: 3,
+        cursor: null,
+        items: mockCollections,
+      });
+
+      const { statusCode, body } = await request(app).get("/stac/ALL/collections");
+
+      expect(statusCode).to.equal(200);
+      expect(body.collections).to.have.lengthOf(3);
+      // Make sure that we are determining the 'provider' element of the items path from the provider detailed
+      // in the collection metadata rather than the ALL route.
+      expect(body.collections[0].links.find((l: Link) => l.rel === "items").href).to.include(
+        "/PROV1/"
+      );
+      expect(body.collections[0].links.find((l: Link) => l.rel === "items").href).to.not.include(
+        "/ALL/"
+      );
+
+      expect(body.collections[1].links.find((l: Link) => l.rel === "items").href).to.include(
+        "/PROV1/"
+      );
+      expect(body.collections[1].links.find((l: Link) => l.rel === "items").href).to.not.include(
+        "/ALL/"
+      );
+
+      expect(body.collections[2].links.find((l: Link) => l.rel === "items").href).to.include(
+        "/PROV1/"
+      );
+      expect(body.collections[2].links.find((l: Link) => l.rel === "items").href).to.not.include(
+        "/ALL/"
+      );
+    });
+    it("returns collection items links that end in 'items", async () => {
+      sandbox
+        .stub(Providers, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const mockCollections = generateSTACCollections(1);
+
+      sandbox.stub(Collections, "getCollections").resolves({
+        count: 3,
+        cursor: null,
+        items: mockCollections,
+      });
+
+      const { statusCode, body } = await request(app).get("/stac/ALL/collections");
+
+      expect(statusCode).to.equal(200);
+      expect(body.collections).to.have.lengthOf(1);
+      expect(body.collections[0].links.find((l: Link) => l.rel === "items").href).to.endsWith(
+        "/items"
+      );
+    });
+  });
+});
+
+describe("GET /ALL/collections/:collectionId", () => {
+  it("should return a 404", async () => {
+    sandbox
+      .stub(Providers, "getProviders")
+      .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+    const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo");
+
+    expect(statusCode).to.equal(404);
+    expect(body).to.deep.equal({
+      errors: ["This operation is not allowed for the ALL Catalog."],
+    });
+  });
+});

src/__tests__/providerSearch.spec.ts

@@ -305,3 +305,37 @@ describe("Query validation", () => {
     });
   });
 });
+
+describe("GET /ALL/search", () => {
+  describe("given an 'ALL' provider", () => {
+    it("should return a 404", async () => {
+      sandbox
+        .stub(Providers, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const { statusCode, body } = await request(app).get("/stac/ALL/search");
+
+      expect(statusCode).to.equal(404);
+      expect(body).to.deep.equal({
+        errors: ["This operation is not allowed for the ALL Catalog."],
+      });
+    });
+  });
+});
+
+describe("POST /ALL/search", () => {
+  describe("given an 'ALL' provider", () => {
+    it("should return a 404", async () => {
+      sandbox
+        .stub(Providers, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const { statusCode, body } = await request(app).post("/stac/ALL/search");
+
+      expect(statusCode).to.equal(404);
+      expect(body).to.deep.equal({
+        errors: ["This operation is not allowed for the ALL Catalog."],
+      });
+    });
+  });
+});

src/__tests__/rootCatalog.spec.ts

@@ -93,6 +93,22 @@ describe("GET /stac", () => {
         expect(providerLink.title).to.equal(provider["provider-id"]);
       });
     });
+
+    it("should have an entry for the 'ALL' catalog", async () => {
+      const { statusCode, body } = await request(app).get("/stac");
+
+      expect(statusCode).to.equal(200);
+      const [, expectedProviders] = cmrProvidersResponse;
+
+      const allLink = body.links.find((l: Link) => l.title === "all");
+
+      expect(allLink.href).to.match(/^(http)s?:\/\/.*\w+/);
+      expect(allLink.href).to.endWith("/stac/ALL");
+      expect(allLink.href).to.not.contain("?param=value");
+      expect(allLink.rel).to.equal("child");
+      expect(allLink.type).to.equal("application/json");
+      expect(allLink.title).to.equal("all");
+    });
   });
 
   describe("given CMR providers endpoint responds with an error", () => {
@@ -147,6 +163,6 @@ describe("/cloudstac", () => {
 
     expect(body.links.find((l: { title: string }) => l.title === "NOT_CLOUD")).to.be.undefined;
 
-    expect(mockCmrHits).to.have.been.calledTwice;
+    expect(mockCmrHits).to.have.been.calledThrice;
   });
 });

src/domains/collections.ts

@@ -57,6 +57,7 @@ const collectionIdsQuery = gql`
         conceptId
         entryId
         title
+        provider
       }
     }
   }
@@ -242,7 +243,7 @@ export const collectionToStac = (collection: Collection): STACCollection => {
   const extent = createExtent(collection);
   const keywords = createKeywords(collection);
   const links = generateCollectionLinks(collection, [licenseLink]);
-  const provider = generateProviders(collection);
+  const providers = generateProviders(collection);
   const summaries = createSummaries(collection);
 
   return {
@@ -253,7 +254,7 @@ export const collectionToStac = (collection: Collection): STACCollection => {
     stac_version: STAC_VERSION,
     extent,
     assets,
-    provider,
+    providers,
     links,
     license,
     keywords,
@@ -334,14 +335,18 @@ export const getCollectionIds = async (
 ): Promise<{
   count: number;
   cursor: string | null;
-  items: { id: string; title: string }[];
+  items: { id: string; title: string; provider: string }[];
 }> => {
   const {
     cursor,
     count,
     items: collectionIds,
   } = await paginateQuery(collectionIdsQuery, params, opts, collectionIdsHandler);
-  return { cursor, count, items: collectionIds as { id: string; title: string }[] };
+  return {
+    cursor,
+    count,
+    items: collectionIds as { id: string; title: string; provider: string }[],
+  };
 };
 
 /**
@@ -356,7 +361,7 @@ export const getAllCollectionIds = async (
 ): Promise<{
   count: number;
   cursor: string | null;
-  items: { id: string; title: string }[];
+  items: { id: string; title: string; provider: string }[];
 }> => {
   params.limit = CMR_QUERY_MAX;
 

src/domains/providers.ts

@@ -8,6 +8,12 @@ const CMR_LB_INGEST = `${CMR_LB_URL}/ingest`;
 const CMR_LB_SEARCH = `${CMR_LB_URL}/search`;
 const CMR_LB_SEARCH_COLLECTIONS = `${CMR_LB_SEARCH}/collections`;
 
+export const ALL_PROVIDER = "ALL";
+export const ALL_PROVIDERS = {
+  "provider-id": ALL_PROVIDER.toUpperCase(),
+  "short-name": ALL_PROVIDER.toLowerCase(),
+};
+
 export const conformance = [
   "https://api.stacspec.org/v1.0.0-rc.2/core",
   "https://api.stacspec.org/v1.0.0-rc.2/item-search",
@@ -31,6 +37,7 @@ export const conformance = [
  */
 export const getProviders = async (): Promise<[string, null] | [null, Provider[]]> => {
   try {
+    console.debug(`GET ${CMR_LB_INGEST}/providers`);
     const { data: providers } = await axios.get(`${CMR_LB_INGEST}/providers`);
     return [null, providers];
   } catch (err) {
@@ -50,7 +57,7 @@ export const getProvider = async (
   if (errs) {
     return [errs as string, null];
   }
-
+  providers?.push(ALL_PROVIDERS);
   return [
     null,
     (providers ?? []).find((provider) => provider["provider-id"] === providerId) ?? null,
@@ -79,6 +86,7 @@ export const getCloudProviders = async (
   await Promise.all(
     (candidates ?? []).map(async (provider) => {
       try {
+        console.debug(`GET ${CMR_LB_SEARCH_COLLECTIONS}`);
         const { headers } = await axios.get(CMR_LB_SEARCH_COLLECTIONS, {
           headers: mergeMaybe({}, { authorization }),
           params: { provider: provider["short-name"], cloud_hosted: true },