From 060818424655233e3c694d8f29a6c32750aec227 Mon Sep 17 00:00:00 2001 From: Bruno Pacheco Date: Mon, 5 Aug 2024 18:17:56 +0200 Subject: [PATCH] chore: retrieve organization metadata --- _http/ckan.http | 3 + .../api/OrganizationQueryApiImpl.java | 8 +- .../CkanOrganizationsRepository.java | 9 +- .../RetrieveOrganizationsService.java | 16 ++- .../utils/DatasetOrganizationMapper.java | 19 ++- src/main/openapi/ckan.yaml | 21 ++- src/main/openapi/discovery.yaml | 22 +++- .../api/RetrieveOrganizationsTest.java | 78 ++++++++--- .../services/BeaconDatasetRepositoryTest.java | 4 - .../services/PackageShowMapperTest.java | 1 - .../resources/mappings/organization_list.json | 124 ++++++++++++++++-- 11 files changed, 251 insertions(+), 54 deletions(-) diff --git a/_http/ckan.http b/_http/ckan.http index f3ae4a1..e4c787a 100644 --- a/_http/ckan.http +++ b/_http/ckan.http @@ -15,3 +15,6 @@ GET https://catalogue.portal.dev.gdi.lu/api/3/action/enhanced_package_show?id=du ### GET https://catalogue.portal.dev.gdi.lu/dataset/vcf-and-phenotypic-data-of-stage-ii-and-iii-colorectal-cancer-patients-from-a-portuguese-use-ca.rdf + +### +GET https://catalogue.portal.dev.gdi.lu/api/3/action/organization_list?all_fields=true&limit=100 \ No newline at end of file diff --git a/src/main/java/io/github/genomicdatainfrastructure/discovery/api/OrganizationQueryApiImpl.java b/src/main/java/io/github/genomicdatainfrastructure/discovery/api/OrganizationQueryApiImpl.java index cb82a07..e71f8f5 100644 --- a/src/main/java/io/github/genomicdatainfrastructure/discovery/api/OrganizationQueryApiImpl.java +++ b/src/main/java/io/github/genomicdatainfrastructure/discovery/api/OrganizationQueryApiImpl.java @@ -7,7 +7,8 @@ import io.github.genomicdatainfrastructure.discovery.services.RetrieveOrganizationsService; import jakarta.ws.rs.core.Response; import lombok.RequiredArgsConstructor; -import java.util.List; + +import static java.util.Optional.ofNullable; @RequiredArgsConstructor public class OrganizationQueryApiImpl implements OrganizationQueryApi { @@ -15,8 +16,9 @@ public class OrganizationQueryApiImpl implements OrganizationQueryApi { private final RetrieveOrganizationsService retrieveOrganizationsService; @Override - public Response retrieveOrganizations() { - var content = retrieveOrganizationsService.get(); + public Response retrieveOrganizations(Integer limit) { + var nonNullLimit = ofNullable(limit).orElse(100); + var content = retrieveOrganizationsService.retrieve(nonNullLimit); return Response.ok(content).build(); } } diff --git a/src/main/java/io/github/genomicdatainfrastructure/discovery/repositories/CkanOrganizationsRepository.java b/src/main/java/io/github/genomicdatainfrastructure/discovery/repositories/CkanOrganizationsRepository.java index 087ef0f..5ea03b1 100644 --- a/src/main/java/io/github/genomicdatainfrastructure/discovery/repositories/CkanOrganizationsRepository.java +++ b/src/main/java/io/github/genomicdatainfrastructure/discovery/repositories/CkanOrganizationsRepository.java @@ -5,6 +5,7 @@ package io.github.genomicdatainfrastructure.discovery.repositories; import io.github.genomicdatainfrastructure.discovery.remote.ckan.api.CkanQueryApi; +import io.github.genomicdatainfrastructure.discovery.remote.ckan.model.CkanOrganization; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.eclipse.microprofile.rest.client.inject.RestClient; @@ -14,6 +15,8 @@ @ApplicationScoped public class CkanOrganizationsRepository { + private static final Boolean SHOW_ALL_FIELDS = true; + private final CkanQueryApi ckanQueryApi; @Inject @@ -23,7 +26,9 @@ public CkanOrganizationsRepository( this.ckanQueryApi = ckanQueryApi; } - public List retrieveOrganizations() { - return ckanQueryApi.organizationList().getResult(); + public List retrieveOrganizations(Integer limit) { + return ckanQueryApi + .organizationList(SHOW_ALL_FIELDS, limit) + .getResult(); } } diff --git a/src/main/java/io/github/genomicdatainfrastructure/discovery/services/RetrieveOrganizationsService.java b/src/main/java/io/github/genomicdatainfrastructure/discovery/services/RetrieveOrganizationsService.java index 5776c54..12eb579 100644 --- a/src/main/java/io/github/genomicdatainfrastructure/discovery/services/RetrieveOrganizationsService.java +++ b/src/main/java/io/github/genomicdatainfrastructure/discovery/services/RetrieveOrganizationsService.java @@ -4,19 +4,25 @@ package io.github.genomicdatainfrastructure.discovery.services; +import io.github.genomicdatainfrastructure.discovery.model.DatasetOrganization; import io.github.genomicdatainfrastructure.discovery.repositories.CkanOrganizationsRepository; +import io.github.genomicdatainfrastructure.discovery.utils.DatasetOrganizationMapper; import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; +import lombok.RequiredArgsConstructor; import java.util.List; @ApplicationScoped +@RequiredArgsConstructor public class RetrieveOrganizationsService { - @Inject - CkanOrganizationsRepository organizationsRepository; + private final CkanOrganizationsRepository organizationsRepository; - public List get() { - return organizationsRepository.retrieveOrganizations(); + public List retrieve(Integer limit) { + var ckanOrganizations = organizationsRepository.retrieveOrganizations(limit); + + return ckanOrganizations.stream() + .map(DatasetOrganizationMapper::from) + .toList(); } } diff --git a/src/main/java/io/github/genomicdatainfrastructure/discovery/utils/DatasetOrganizationMapper.java b/src/main/java/io/github/genomicdatainfrastructure/discovery/utils/DatasetOrganizationMapper.java index 32de814..cf7084b 100644 --- a/src/main/java/io/github/genomicdatainfrastructure/discovery/utils/DatasetOrganizationMapper.java +++ b/src/main/java/io/github/genomicdatainfrastructure/discovery/utils/DatasetOrganizationMapper.java @@ -8,19 +8,30 @@ import io.github.genomicdatainfrastructure.discovery.remote.ckan.model.*; import lombok.experimental.UtilityClass; +import java.util.Optional; +import java.util.function.Predicate; + @UtilityClass public class DatasetOrganizationMapper { public DatasetOrganization from(CkanOrganization organization) { if (organization == null) { - return DatasetOrganization.builder().build(); + return null; } return DatasetOrganization.builder() - .title(organization.getTitle()) + .id(organization.getId()) .name(organization.getName()) - .description(organization.getDescription()) - .imageUrl(organization.getImageUrl()) + .title(nullableString(organization.getTitle())) + .description(nullableString(organization.getDescription())) + .imageUrl(nullableString(organization.getImageUrl())) + .numberOfDatasets(organization.getPackageCount()) .build(); } + + private String nullableString(String value) { + return Optional.ofNullable(value) + .filter(Predicate.not(String::isBlank)) + .orElse(null); + } } diff --git a/src/main/openapi/ckan.yaml b/src/main/openapi/ckan.yaml index d36f5e9..f5de5b8 100644 --- a/src/main/openapi/ckan.yaml +++ b/src/main/openapi/ckan.yaml @@ -109,6 +109,21 @@ paths: get: summary: Retrieves a list of organizations operationId: organization_list + parameters: + - name: all_fields + in: query + description: if all metadata of organization is required + required: true + schema: + type: boolean + default: true + - name: limit + in: query + description: maximum number of organizations per query + required: true + schema: + type: integer + default: 100 tags: - "ckan-query" responses: @@ -333,10 +348,12 @@ components: type: string description: type: string + package_count: + type: integer required: - id - name - - title + - package_count CkanOrganizationsResponse: type: object properties: @@ -347,7 +364,7 @@ components: result: type: array items: - type: string + $ref: "#/components/schemas/CkanOrganization" CkanResource: type: object properties: diff --git a/src/main/openapi/discovery.yaml b/src/main/openapi/discovery.yaml index fb87e30..02e0411 100644 --- a/src/main/openapi/discovery.yaml +++ b/src/main/openapi/discovery.yaml @@ -79,6 +79,14 @@ paths: get: summary: Retrieves a list of organizations operationId: retrieve_organizations + parameters: + - name: limit + in: query + description: maximum number of organizations per query + required: true + schema: + type: integer + default: 100 tags: - "organization-query" responses: @@ -89,7 +97,7 @@ paths: schema: type: array items: - type: string + $ref: "#/components/schemas/DatasetOrganization" /api/v1/datasets/{id}.{format}: get: summary: Retrieves a dataset by its ID, in the requested format @@ -424,6 +432,9 @@ components: - relation DatasetOrganization: properties: + id: + type: string + title: ID of organization name: type: string title: Name of organization @@ -433,9 +444,16 @@ components: description: type: string title: Description of organization - image_url: + imageUrl: type: string title: Image url of organization + numberOfDatasets: + type: integer + title: Number of datasets + required: + - id + - name + - numberOfDatasets DatasetDictionaryEntry: properties: name: diff --git a/src/test/java/io/github/genomicdatainfrastructure/discovery/api/RetrieveOrganizationsTest.java b/src/test/java/io/github/genomicdatainfrastructure/discovery/api/RetrieveOrganizationsTest.java index d16bc6c..35ef2cc 100644 --- a/src/test/java/io/github/genomicdatainfrastructure/discovery/api/RetrieveOrganizationsTest.java +++ b/src/test/java/io/github/genomicdatainfrastructure/discovery/api/RetrieveOrganizationsTest.java @@ -5,42 +5,78 @@ package io.github.genomicdatainfrastructure.discovery.api; import io.github.genomicdatainfrastructure.discovery.BaseTest; +import io.github.genomicdatainfrastructure.discovery.model.DatasetOrganization; import io.quarkus.test.junit.QuarkusTest; +import io.restassured.common.mapper.TypeRef; import org.junit.jupiter.api.Test; import java.util.List; import static io.restassured.RestAssured.given; -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.Assertions.assertThat; @QuarkusTest class RetrieveOrganizationsTest extends BaseTest { @Test void can_retrieve_organizations() { - given() - .when() - .get("/api/v1/organizations") - .then() - .statusCode(200); - } - - @Test - void can_retrieve_correct_organizations_data() { var response = given() .when() .get("/api/v1/organizations"); - var body = response.getBody().as(List.class); - assertThat(body).isEqualTo(List.of( - "cscfi", - "ega", - "instituto-superior-tecnico", - "lnds", - "nbis", - "radboud", - "umcg", - "university-of-oslo" - )); + var actual = response.getBody() + .as(new TypeRef>() { + }); + + assertThat(actual) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyInAnyOrder( + DatasetOrganization.builder() + .id("3e0a548c-eef4-404b-96ec-dd95b549643c") + .name("instituto-superior-tecnico") + .title("Instituto Superior Técnico") + .numberOfDatasets(1) + .build(), + DatasetOrganization.builder() + .id("d9133f3e-8747-4764-a3c6-27f93a37c38d") + .title("CSC FI") + .name("csc-fi") + .description( + "The tools CSC provides for scientific computing and data management create a stepping stone for breakthroughs.") + .imageUrl( + "https://csc.fi/app/uploads/2023/09/CSC_logo_no_tagline.svg") + .numberOfDatasets(1) + .build(), + DatasetOrganization.builder() + .id("da348a59-8632-4c28-af3e-786e72448b8c") + .name("ega") + .title("EGA") + .numberOfDatasets(4) + .build(), + DatasetOrganization.builder() + .id("1acb508d-371c-4a43-b143-65ec2e0f1f82") + .name("lnds") + .title("LNDS") + .numberOfDatasets(1) + .build(), + DatasetOrganization.builder() + .id("d4f783b7-64e9-4552-9667-c81a56259bbb") + .name("nbis") + .title("NBIS") + .numberOfDatasets(3) + .build(), + DatasetOrganization.builder() + .id("c24f44ff-a0e0-4171-bf33-a9b6e2ea9298") + .name("umcg") + .title("UMCG") + .numberOfDatasets(5) + .build(), + DatasetOrganization.builder() + .id("5e48036d-4e38-40d8-ba8e-8cce6faeb3be") + .name("university-of-oslo") + .title("University of Oslo") + .numberOfDatasets(1) + .build() + ); } } diff --git a/src/test/java/io/github/genomicdatainfrastructure/discovery/services/BeaconDatasetRepositoryTest.java b/src/test/java/io/github/genomicdatainfrastructure/discovery/services/BeaconDatasetRepositoryTest.java index afebeed..8ce0d82 100644 --- a/src/test/java/io/github/genomicdatainfrastructure/discovery/services/BeaconDatasetRepositoryTest.java +++ b/src/test/java/io/github/genomicdatainfrastructure/discovery/services/BeaconDatasetRepositoryTest.java @@ -94,7 +94,6 @@ void doesnt_call_beacon_if_access_token_is_null() { SearchedDataset.builder() .id("id") .title("title") - .organization(DatasetOrganization.builder().build()) .themes(List.of()) .build() )) @@ -140,7 +139,6 @@ void doesnt_call_beacon_if_keycloak_throws_expected_4xx_errors(Integer statusCod SearchedDataset.builder() .id("id") .title("title") - .organization(DatasetOrganization.builder().build()) .themes(List.of()) .build() )) @@ -199,7 +197,6 @@ void doesnt_call_beacon_if_there_are_no_beacon_filters() { SearchedDataset.builder() .id("id") .title("title") - .organization(DatasetOrganization.builder().build()) .themes(List.of()) .build() )) @@ -424,7 +421,6 @@ void calls_ckan_and_beacon() { SearchedDataset.builder() .id("id") .title("title") - .organization(DatasetOrganization.builder().build()) .themes(List.of()) .build() )) diff --git a/src/test/java/io/github/genomicdatainfrastructure/discovery/services/PackageShowMapperTest.java b/src/test/java/io/github/genomicdatainfrastructure/discovery/services/PackageShowMapperTest.java index 5fc3120..e4f969f 100644 --- a/src/test/java/io/github/genomicdatainfrastructure/discovery/services/PackageShowMapperTest.java +++ b/src/test/java/io/github/genomicdatainfrastructure/discovery/services/PackageShowMapperTest.java @@ -43,7 +43,6 @@ void accepts_empty_package() { .keywords(List.of()) .contacts(List.of()) .creators(List.of()) - .organization(DatasetOrganization.builder().build()) .datasetRelationships(List.of()) .dataDictionary(List.of()) .build(); diff --git a/src/test/resources/mappings/organization_list.json b/src/test/resources/mappings/organization_list.json index 30b5e3b..f01e95f 100644 --- a/src/test/resources/mappings/organization_list.json +++ b/src/test/resources/mappings/organization_list.json @@ -1,7 +1,7 @@ { "request": { "method": "GET", - "url": "/api/3/action/organization_list" + "url": "/api/3/action/organization_list?all_fields=true&limit=100" }, "response": { "status": 200, @@ -9,17 +9,121 @@ "Content-Type": "application/json" }, "jsonBody": { - "help": "https://ckan-test.healthdata.nl/api/3/action/help_show?name=organization_list", + "help": "https://catalogue.portal.dev.gdi.lu/api/3/action/help_show?name=organization_list", "success": true, "result": [ - "cscfi", - "ega", - "instituto-superior-tecnico", - "lnds", - "nbis", - "radboud", - "umcg", - "university-of-oslo" + { + "approval_status": "approved", + "created": "2024-04-25T12:01:57.722274", + "description": "", + "display_name": " Instituto Superior Técnico", + "id": "3e0a548c-eef4-404b-96ec-dd95b549643c", + "image_display_url": "", + "image_url": "", + "is_organization": true, + "name": "instituto-superior-tecnico", + "num_followers": 0, + "package_count": 1, + "state": "active", + "title": "Instituto Superior Técnico", + "type": "organization" + }, + { + "approval_status": "approved", + "created": "2024-04-19T12:55:51.635047", + "description": "The tools CSC provides for scientific computing and data management create a stepping stone for breakthroughs.", + "display_name": "CSC FI", + "id": "d9133f3e-8747-4764-a3c6-27f93a37c38d", + "image_display_url": "https://csc.fi/app/uploads/2023/09/CSC_logo_no_tagline.svg", + "image_url": "https://csc.fi/app/uploads/2023/09/CSC_logo_no_tagline.svg", + "is_organization": true, + "name": "csc-fi", + "num_followers": 0, + "package_count": 1, + "state": "active", + "title": "CSC FI", + "type": "organization" + }, + { + "approval_status": "approved", + "created": "2024-04-19T13:09:30.488193", + "description": "", + "display_name": "EGA", + "id": "da348a59-8632-4c28-af3e-786e72448b8c", + "image_display_url": "", + "image_url": "", + "is_organization": true, + "name": "ega", + "num_followers": 0, + "package_count": 4, + "state": "active", + "title": "EGA", + "type": "organization" + }, + { + "approval_status": "approved", + "created": "2024-04-16T11:04:15.333600", + "description": "", + "display_name": "LNDS", + "id": "1acb508d-371c-4a43-b143-65ec2e0f1f82", + "image_display_url": "", + "image_url": "", + "is_organization": true, + "name": "lnds", + "num_followers": 0, + "package_count": 1, + "state": "active", + "title": "LNDS", + "type": "organization" + }, + { + "approval_status": "approved", + "created": "2024-04-19T13:46:17.885858", + "description": "", + "display_name": "NBIS", + "id": "d4f783b7-64e9-4552-9667-c81a56259bbb", + "image_display_url": "", + "image_url": "", + "is_organization": true, + "name": "nbis", + "num_followers": 0, + "package_count": 3, + "state": "active", + "title": "NBIS", + "type": "organization" + }, + { + "approval_status": "approved", + "created": "2024-04-16T18:43:42.174385", + "description": "", + "display_name": "UMCG", + "id": "c24f44ff-a0e0-4171-bf33-a9b6e2ea9298", + "image_display_url": "", + "image_url": "", + "is_organization": true, + "name": "umcg", + "num_followers": 0, + "package_count": 5, + "state": "active", + "title": "UMCG", + "type": "organization" + }, + { + "approval_status": "approved", + "created": "2024-04-19T12:55:38.657883", + "description": "", + "display_name": "University of Oslo", + "id": "5e48036d-4e38-40d8-ba8e-8cce6faeb3be", + "image_display_url": "", + "image_url": "", + "is_organization": true, + "name": "university-of-oslo", + "num_followers": 0, + "package_count": 1, + "state": "active", + "title": "University of Oslo", + "type": "organization" + } ] } }