diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkService.java index b1fa301b..a6a57d5a 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkService.java @@ -2,6 +2,8 @@ import org.springframework.http.ResponseEntity; +import java.io.IOException; + public interface GeoNetworkService { void setServer(String server); String getServer(); @@ -10,6 +12,8 @@ public interface GeoNetworkService { String getIndexName(); String searchRecordBy(String uuid); + String findGroupById(String uuid) throws IOException; + /** * Return Iterable of records, noted that the item inside can be null, so please check null on each item * @return diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java index 108ff886..1ff69e58 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java @@ -51,6 +51,9 @@ protected HttpEntity getRequestEntity(MediaType accept, String body) { return body == null ? new HttpEntity<>(headers) : new HttpEntity<>(body, headers); } + protected final static String UUID = "uuid"; + protected final static String GEONETWORK_GROUP = "groupOwner"; + public GeoNetworkServiceImpl( @Value("${geonetwork.host}") String server, @Value("${geonetwork.search.api.index}") String indexName) { @@ -59,7 +62,7 @@ public GeoNetworkServiceImpl( .index(indexName) .query(new MatchAllQuery.Builder().build()._toQuery()) // Match all .source(s -> s - .filter(f -> f.includes("uuid")))// Only select uuid field + .filter(f -> f.includes(UUID)))// Only select uuid field // TODO: redesign the iterator to be more efficient /* by default ES will return just 10 top hits (10 records of the thousands available records), the iterator implementation in getAllMetadataRecords() method will help saving memory but process those 10 records only, @@ -73,13 +76,52 @@ the iterator implementation in getAllMetadataRecords() method will help saving m setServer(server); } + public String findGroupById(String uuid) throws IOException { + SearchRequest request = new SearchRequest.Builder() + .index(indexName) + .query(q -> q.bool(b -> b.filter(f -> f.matchPhrase(p -> p.field(UUID).query(uuid))))) + .source(s -> s + .filter(f -> f.includes(GEONETWORK_GROUP))) + .size(1) + .build(); + + final SearchResponse response = gn4ElasticClient.search(request, ObjectNode.class); + + if(response.hits() != null && response.hits().hits() != null && !response.hits().hits().isEmpty()) { + // UUID should result in only 1 record, hence get(0) is ok. + String group = response.hits().hits().get(0).source().get(GEONETWORK_GROUP).asText(); + + Map params = new HashMap<>(); + params.put("id", group); + + HttpEntity requestEntity = getRequestEntity(MediaType.APPLICATION_JSON, null); + + ResponseEntity responseEntity = restTemplate.exchange( + getGeoNetworkGroupsEndpoint(), + HttpMethod.GET, + requestEntity, + JsonNode.class, params); + + if(responseEntity.getStatusCode().is2xxSuccessful()) { + return Objects.requireNonNull(responseEntity.getBody()).get("name").asText(); + } + else { + return null; + } + } + else { + return null; + } + + } + protected String findFormatterId(String uuid) { try { HttpEntity requestEntity = getRequestEntity(MediaType.APPLICATION_JSON, null); Map params = new HashMap<>(); params.put("indexName", getIndexName()); - params.put("uuid", uuid); + params.put(UUID, uuid); ResponseEntity responseEntity = restTemplate.exchange( getGeoNetworkRecordsEndpoint(), @@ -110,7 +152,7 @@ public String searchRecordBy(String uuid) { Map params = new HashMap<>(); params.put("indexName", getIndexName()); - params.put("uuid", uuid); + params.put(UUID, uuid); params.put("formatterId", this.findFormatterId(uuid)); ResponseEntity responseEntity = restTemplate.exchange( @@ -185,7 +227,7 @@ public boolean hasNext() { @Override public String next() { // TODO: Potential problem with edge case where the list is bigger then size set Integer.MAX - String uuid = response.hits().hits().get(index++).source().get("uuid").asText(); + String uuid = response.hits().hits().get(index++).source().get(UUID).asText(); try { return GeoNetworkServiceImpl.this.searchRecordBy(uuid); } @@ -222,6 +264,10 @@ protected String getGeoNetworkRecordsEndpoint() { return getServer() + "/geonetwork/srv/api/{indexName}/{uuid}"; } + protected String getGeoNetworkGroupsEndpoint() { + return getServer() + "/geonetwork/srv/api/groups/{id}"; + } + protected String getReIndexEndpoint() { return getServer() + "/geonetwork/srv/api/site/index?reset=false&asynchronous=false"; } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java index 61a8ea59..b03058bc 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/StacCollectionMapperService.java @@ -14,9 +14,11 @@ import org.mapstruct.Named; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import java.io.IOException; import java.math.BigDecimal; import java.time.ZoneId; import java.time.ZoneOffset; @@ -45,6 +47,7 @@ public abstract class StacCollectionMapperService { @Mapping(target="summaries.temporal", source = "source", qualifiedByName = "mapSummaries.temporal") @Mapping(target="summaries.updateFrequency", source = "source", qualifiedByName = "mapSummaries.updateFrequency") @Mapping(target="summaries.datasetProvider", source = "source", qualifiedByName = "mapSummaries.datasetProvider") + @Mapping(target="summaries.datasetGroup", source = "source", qualifiedByName = "mapSummaries.datasetGroup") @Mapping(target="extent.bbox", source = "source", qualifiedByName = "mapExtentBbox") @Mapping(target="extent.temporal", source = "source", qualifiedByName = "mapExtentTemporal") @Mapping(target="contacts", source = "source", qualifiedByName = "mapContacts") @@ -61,6 +64,9 @@ public abstract class StacCollectionMapperService { @Value("${spring.jpa.properties.hibernate.jdbc.time_zone}") private String timeZoneId; + @Autowired + private GeoNetworkService geoNetworkService; + @Named("mapUUID") String mapUUID(MDMetadataType source) { return source.getMetadataIdentifier().getMDIdentifier().getCode().getCharacterString().getValue().toString(); @@ -295,6 +301,17 @@ String mapDatasetOwner(MDMetadataType source) { return providers.stream().anyMatch(p -> p.getName().contains("IMOS")) ? "IMOS" : null; } + @Named("mapSummaries.datasetGroup") + String mapGeoNetworkGroup(MDMetadataType source) { + try { + String group = geoNetworkService.findGroupById(mapUUID(source)); + return group != null ? group.toLowerCase() : null; + } + catch (IOException e) { + return null; + } + } + protected List> mapThemesConcepts(MDKeywordsPropertyType descriptiveKeyword) { List> keywords = new ArrayList<>(); descriptiveKeyword.getMDKeywords().getKeyword().forEach(keyword -> { diff --git a/indexer/src/main/resources/config_files/portal_records_index_schema.json b/indexer/src/main/resources/config_files/portal_records_index_schema.json index 6d2732be..2d3c3c97 100644 --- a/indexer/src/main/resources/config_files/portal_records_index_schema.json +++ b/indexer/src/main/resources/config_files/portal_records_index_schema.json @@ -56,6 +56,19 @@ "properties" : { "score": { "type": "long" }, "status": { "type": "text" }, + "scope" : { + "type": "nested", + "properties" : { + "code" : { + "type": "text" + }, + "name" : { + "type": "text" + } + } + }, + "dataset_provider": { "type": "text" }, + "dataset_group": { "type": "text" }, "creation": { "type": "date" }, "proj:geometry": { "type": "geo_shape" diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/GeoNetworkServiceTests.java b/indexer/src/test/java/au/org/aodn/esindexer/service/GeoNetworkServiceTests.java index fe7b7dc5..e6f7070a 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/GeoNetworkServiceTests.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/GeoNetworkServiceTests.java @@ -99,7 +99,18 @@ public void verifyFindFormatterId() throws IOException, InterruptedException { } @Test - public void verifySearchRecordBy() throws IOException, InterruptedException { + public void verifyFindGroupById() throws IOException { + // By default, record will assign to group with group id = 2 + insertMetadataRecords("9e5c3031-a026-48b3-a153-a70c2e2b78b9", "classpath:canned/sample1.xml"); + String group = geoNetworkService.findGroupById("9e5c3031-a026-48b3-a153-a70c2e2b78b9"); + + assertEquals("Default group equals", "sample", group); + + deleteRecord("9e5c3031-a026-48b3-a153-a70c2e2b78b9"); + } + + @Test + public void verifySearchRecordBy() throws IOException { String content = insertMetadataRecords("9e5c3031-a026-48b3-a153-a70c2e2b78b9", "classpath:canned/sample1.xml"); String xml = geoNetworkService.searchRecordBy("9e5c3031-a026-48b3-a153-a70c2e2b78b9"); diff --git a/indexer/src/test/resources/canned/sample3_stac.json b/indexer/src/test/resources/canned/sample3_stac.json index eaeb990b..73e40397 100644 --- a/indexer/src/test/resources/canned/sample3_stac.json +++ b/indexer/src/test/resources/canned/sample3_stac.json @@ -21,6 +21,7 @@ "name": "IMOS dataset level record" }, "dataset_provider": "IMOS", + "dataset_group":"sample", "temporal": [ { "start": "2007-03-26T06:00:00Z", diff --git a/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java b/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java index a583528f..36b81f95 100644 --- a/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java +++ b/stacmodel/src/main/java/au/org/aodn/stac/model/SummariesModel.java @@ -14,7 +14,11 @@ public class SummariesModel { protected Integer score; protected String status; protected Map scope; - + /** + * Group info as setup in geonetwork + */ + @JsonProperty("dataset_group") + protected String datasetGroup; /** * It is used to flag who owns the dataset, right now this field appears if it is from IMOS */