CMR-10238: Adding the virtual catalog 'ALL' (#379)

* Added ALL catalog and functionality for ALL/collections, ALL/collections/<collection>, ALL/collections/search etc. * testing for /stac and /stac/all routes, debug logs for all external calls * Added tests for all routes related to ALL * Fixed merge problem with stringifyQuery * Linted * Update src/routes/browse.ts Co-authored-by: Ed Olivares <[email protected]> * LPFALL provider name fix * Linting * Added test for LPALL provider, removed multiple providers from test --------- Co-authored-by: doug-newman-nasa <[email protected]> Co-authored-by: Ed Olivares <[email protected]>
nasa · Nov 22, 2024 · de46a4c · de46a4c
1 parent a5aff1b
commit de46a4c
Show file tree

Hide file tree

Showing 16 changed files with 448 additions and 45 deletions.
diff --git a/src/@types/StacCollection.d.ts b/src/@types/StacCollection.d.ts
@@ -187,7 +187,7 @@ export type STACCollection = {
   description: Description;
   keywords?: Keywords;
   license: CollectionLicenseName;
-  providers?: {
+  providers: {
     name: OrganizationName;
     description?: OrganizationDescription;
     roles?: OrganizationRoles;

diff --git a/src/__tests__/items.spec.ts b/src/__tests__/items.spec.ts
@@ -172,3 +172,33 @@ describe("GET /PROVIDER/collections/COLLECTION/items/ITEM", () => {
     });
   });
 });
+
+describe("GET /ALL/collections/:collection/items/", () => {
+  it("should return a 404", async () => {
+    sandbox
+      .stub(Providers, "getProviders")
+      .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+    const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo/items");
+
+    expect(statusCode).to.equal(404);
+    expect(body).to.deep.equal({
+      errors: ["This operation is not allowed for the ALL Catalog."],
+    });
+  });
+});
+
+describe("GET /ALL/collections/:collection/items/:item", () => {
+  it("should return a 404", async () => {
+    sandbox
+      .stub(Providers, "getProviders")
+      .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+    const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo/items/bar");
+
+    expect(statusCode).to.equal(404);
+    expect(body).to.deep.equal({
+      errors: ["This operation is not allowed for the ALL Catalog."],
+    });
+  });
+});
diff --git a/src/__tests__/providerCatalog.spec.ts b/src/__tests__/providerCatalog.spec.ts
@@ -143,6 +143,7 @@ describe("GET /:provider", () => {
             items: mockCollections.map((coll) => ({
               id: `${coll.id}`,
               title: coll.title ?? faker.random.words(4),
+              provider: `TEST`,
             })),
           });
 
@@ -179,6 +180,7 @@ describe("GET /:provider", () => {
           items: mockCollections.map((coll) => ({
             id: `${coll.id}`,
             title: coll.title ?? faker.random.words(4),
+            provider: "TEST",
           })),
         });
 
@@ -209,6 +211,7 @@ describe("GET /:provider", () => {
           items: mockCollections.map((coll) => ({
             id: `${coll.id}`,
             title: coll.title ?? faker.random.words(4),
+            provider: "TEST`",
           })),
         });
 
@@ -232,6 +235,7 @@ describe("GET /:provider", () => {
           items: mockCollections.map((coll) => ({
             id: `${coll.id}`,
             title: coll.title ?? faker.random.words(4),
+            provider: "TEST",
           })),
         });
 
@@ -287,4 +291,94 @@ describe("GET /:provider", () => {
       expect(res.statusCode).to.equal(404);
     });
   });
+
+  describe("given the ALL provider/catalog", () => {
+    it("should call the graphql API with no provider search clause", async () => {
+      sandbox
+        .stub(Provider, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const getCollectionsSpy = sandbox
+        .stub(Collections, "getCollectionIds")
+        .resolves({ count: 0, cursor: null, items: [] });
+
+      const res = await request(stacApp).get("/stac/ALL");
+      expect(res.statusCode).to.equal(200);
+      // getCollectionIds should have no provider clause in query argument.
+      // If this was any provider other than 'ALL', this method would be
+      // called with { provider: 'TEST', cursor: undefined, limit: NaN }
+      expect(getCollectionsSpy).to.have.been.calledWith({ cursor: undefined, limit: NaN });
+    });
+    it("should return rel=child links whose href contains a provider rather than 'ALL'", async () => {
+      sandbox
+        .stub(Provider, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const mockCollections = generateSTACCollections(5);
+      sandbox.stub(Collections, "getCollectionIds").resolves({
+        count: mockCollections.length,
+        cursor: "foundCursor",
+        items: mockCollections.map((coll) => ({
+          id: `${coll.id}`,
+          title: coll.title ?? faker.random.words(4),
+          provider: `TEST`,
+        })),
+      });
+
+      const { body: catalog, statusCode } = await request(stacApp).get("/stac/ALL");
+
+      const children = catalog.links.filter((l: Link) => l.rel === "child");
+      expect(children).to.have.length(mockCollections.length);
+
+      mockCollections.forEach((collection) => {
+        const childLink = children.find((l: Link) => l.href.endsWith(collection.id));
+
+        expect(childLink.href).to.endWith(`/TEST/collections/${collection.id}`);
+        expect(childLink.href).to.not.contain("/ALL/");
+      });
+
+      expect(statusCode).to.equal(200);
+    });
+    it("should not return any links of rel=search", async () => {
+      sandbox
+        .stub(Provider, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      sandbox.stub(Collections, "getCollectionIds").resolves({ count: 0, cursor: null, items: [] });
+
+      const { body: catalog, statusCode } = await request(stacApp).get("/stac/ALL");
+
+      const children = catalog.links.filter((l: Link) => l.rel === "search");
+      expect(children).to.have.length(0);
+      expect(statusCode).to.equal(200);
+    });
+    it("should be able to handle providers whose name contains the text 'ALL'", async () => {
+      sandbox
+        .stub(Provider, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "LPALL" }]]);
+
+      const mockCollections = generateSTACCollections(1);
+
+      sandbox.stub(Collections, "getCollectionIds").resolves({
+        count: mockCollections.length,
+        cursor: "foundCursor",
+        items: mockCollections.map((coll) => ({
+          id: `${coll.id}`,
+          title: coll.title ?? faker.random.words(4),
+          provider: "LPALL",
+        })),
+      });
+
+      const { body: catalog, statusCode } = await request(stacApp).get("/stac/ALL");
+
+      const children = catalog.links.filter((l: Link) => l.rel === "child");
+
+      mockCollections.forEach((collection) => {
+        const childLink = children.find((l: Link) => l.href.endsWith(collection.id));
+
+        expect(childLink.href).to.endWith(`/LPALL/collections/${collection.id}`);
+        expect(childLink.href).to.not.contain("/ALL/");
+      });
+    });
+  });
 });
diff --git a/src/__tests__/providerCollection.spec.ts b/src/__tests__/providerCollection.spec.ts
@@ -134,6 +134,7 @@ describe("GET /:provider/collections", () => {
       // Expect the href to match the generic STAC API.
       const link1: Link = body.collections[1].links.find((l: Link) => l.rel === "items");
       expect(link1.href).to.contain("/stac/TEST/collections/");
+      expect(link1.href).to.endsWith("/items");
     });
   });
 
@@ -470,3 +471,70 @@ describe("GET /:provider/collections/:collectionId", () => {
     });
   });
 });
+
+describe("GET /ALL/collections", () => {
+  describe("given the ALL catalog", () => {
+    it("returns item links relative to the provider catalogs rather than ALL", async () => {
+      sandbox
+        .stub(Providers, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const mockCollections = generateSTACCollections(1);
+
+      sandbox.stub(Collections, "getCollections").resolves({
+        count: 1,
+        cursor: null,
+        items: mockCollections,
+      });
+
+      const { statusCode, body } = await request(app).get("/stac/ALL/collections");
+
+      expect(statusCode).to.equal(200);
+      expect(body.collections).to.have.lengthOf(1);
+      // Make sure that we are determining the 'provider' element of the items path from the provider detailed
+      // in the collection metadata rather than the ALL route.
+      expect(body.collections[0].links.find((l: Link) => l.rel === "items").href).to.include(
+        "/PROV1/"
+      );
+      expect(body.collections[0].links.find((l: Link) => l.rel === "items").href).to.not.include(
+        "/ALL/"
+      );
+    });
+    it("returns collection items links that end in 'items", async () => {
+      sandbox
+        .stub(Providers, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const mockCollections = generateSTACCollections(1);
+
+      sandbox.stub(Collections, "getCollections").resolves({
+        count: 1,
+        cursor: null,
+        items: mockCollections,
+      });
+
+      const { statusCode, body } = await request(app).get("/stac/ALL/collections");
+
+      expect(statusCode).to.equal(200);
+      expect(body.collections).to.have.lengthOf(1);
+      expect(body.collections[0].links.find((l: Link) => l.rel === "items").href).to.endsWith(
+        "/items"
+      );
+    });
+  });
+});
+
+describe("GET /ALL/collections/:collectionId", () => {
+  it("should return a 404", async () => {
+    sandbox
+      .stub(Providers, "getProviders")
+      .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+    const { statusCode, body } = await request(app).get("/stac/ALL/collections/foo");
+
+    expect(statusCode).to.equal(404);
+    expect(body).to.deep.equal({
+      errors: ["This operation is not allowed for the ALL Catalog."],
+    });
+  });
+});
diff --git a/src/__tests__/providerSearch.spec.ts b/src/__tests__/providerSearch.spec.ts
@@ -305,3 +305,37 @@ describe("Query validation", () => {
     });
   });
 });
+
+describe("GET /ALL/search", () => {
+  describe("given an 'ALL' provider", () => {
+    it("should return a 404", async () => {
+      sandbox
+        .stub(Providers, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const { statusCode, body } = await request(app).get("/stac/ALL/search");
+
+      expect(statusCode).to.equal(404);
+      expect(body).to.deep.equal({
+        errors: ["This operation is not allowed for the ALL Catalog."],
+      });
+    });
+  });
+});
+
+describe("POST /ALL/search", () => {
+  describe("given an 'ALL' provider", () => {
+    it("should return a 404", async () => {
+      sandbox
+        .stub(Providers, "getProviders")
+        .resolves([null, [{ "provider-id": "TEST", "short-name": "TEST" }]]);
+
+      const { statusCode, body } = await request(app).post("/stac/ALL/search");
+
+      expect(statusCode).to.equal(404);
+      expect(body).to.deep.equal({
+        errors: ["This operation is not allowed for the ALL Catalog."],
+      });
+    });
+  });
+});
diff --git a/src/__tests__/rootCatalog.spec.ts b/src/__tests__/rootCatalog.spec.ts
@@ -93,6 +93,22 @@ describe("GET /stac", () => {
         expect(providerLink.title).to.equal(provider["provider-id"]);
       });
     });
+
+    it("should have an entry for the 'ALL' catalog", async () => {
+      const { statusCode, body } = await request(app).get("/stac");
+
+      expect(statusCode).to.equal(200);
+      const [, expectedProviders] = cmrProvidersResponse;
+
+      const allLink = body.links.find((l: Link) => l.title === "all");
+
+      expect(allLink.href).to.match(/^(http)s?:\/\/.*\w+/);
+      expect(allLink.href).to.endWith("/stac/ALL");
+      expect(allLink.href).to.not.contain("?param=value");
+      expect(allLink.rel).to.equal("child");
+      expect(allLink.type).to.equal("application/json");
+      expect(allLink.title).to.equal("all");
+    });
   });
 
   describe("given CMR providers endpoint responds with an error", () => {
@@ -147,6 +163,6 @@ describe("/cloudstac", () => {
 
     expect(body.links.find((l: { title: string }) => l.title === "NOT_CLOUD")).to.be.undefined;
 
-    expect(mockCmrHits).to.have.been.calledTwice;
+    expect(mockCmrHits).to.have.been.calledThrice;
   });
 });
diff --git a/src/domains/collections.ts b/src/domains/collections.ts
@@ -57,6 +57,7 @@ const collectionIdsQuery = gql`
         conceptId
         entryId
         title
+        provider
       }
     }
   }
@@ -242,7 +243,7 @@ export const collectionToStac = (collection: Collection): STACCollection => {
   const extent = createExtent(collection);
   const keywords = createKeywords(collection);
   const links = generateCollectionLinks(collection, [licenseLink]);
-  const provider = generateProviders(collection);
+  const providers = generateProviders(collection);
   const summaries = createSummaries(collection);
 
   return {
@@ -253,7 +254,7 @@ export const collectionToStac = (collection: Collection): STACCollection => {
     stac_version: STAC_VERSION,
     extent,
     assets,
-    provider,
+    providers,
     links,
     license,
     keywords,
@@ -334,14 +335,18 @@ export const getCollectionIds = async (
 ): Promise<{
   count: number;
   cursor: string | null;
-  items: { id: string; title: string }[];
+  items: { id: string; title: string; provider: string }[];
 }> => {
   const {
     cursor,
     count,
     items: collectionIds,
   } = await paginateQuery(collectionIdsQuery, params, opts, collectionIdsHandler);
-  return { cursor, count, items: collectionIds as { id: string; title: string }[] };
+  return {
+    cursor,
+    count,
+    items: collectionIds as { id: string; title: string; provider: string }[],
+  };
 };
 
 /**
@@ -356,7 +361,7 @@ export const getAllCollectionIds = async (
 ): Promise<{
   count: number;
   cursor: string | null;
-  items: { id: string; title: string }[];
+  items: { id: string; title: string; provider: string }[];
 }> => {
   params.limit = CMR_QUERY_MAX;