From b2fb0cc6603572d586ee3c94e1d28bffdfe83f8e Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 16 Dec 2024 08:11:39 -0500 Subject: [PATCH] extened search api --- conf/solr/schema.xml | 4 +++ .../iq/dataverse/search/IndexServiceBean.java | 4 +++ .../iq/dataverse/search/SearchFields.java | 3 ++ .../dataverse/search/SearchServiceBean.java | 17 +++++++++ .../iq/dataverse/search/SolrSearchResult.java | 35 +++++++++++++++++- .../util/json/NullSafeJsonBuilder.java | 5 ++- .../harvard/iq/dataverse/api/SearchIT.java | 36 +++++++++++++++++-- 7 files changed, 100 insertions(+), 4 deletions(-) diff --git a/conf/solr/schema.xml b/conf/solr/schema.xml index d5c789c7189..380e4fc4da5 100644 --- a/conf/solr/schema.xml +++ b/conf/solr/schema.xml @@ -167,6 +167,8 @@ + + @@ -201,6 +203,8 @@ + + diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index 4efd339ee46..4290a58bd00 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -1580,6 +1580,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set variables = fileMetadata.getDataFile().getDataTable().getDataVariables(); + Long observations = fileMetadata.getDataFile().getDataTable().getCaseQuantity(); + datafileSolrInputDocument.addField(SearchFields.OBSERVATIONS, observations); + datafileSolrInputDocument.addField(SearchFields.VARIABLE_COUNT, variables.size()); Map variableMap = null; List variablesByMetadata = variableService.findVarMetByFileMetaId(fileMetadata.getId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java index 1f1137016f2..712f90186f5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java @@ -171,6 +171,7 @@ public class SearchFields { public static final String FILE_CHECKSUM_TYPE = "fileChecksumType"; public static final String FILE_CHECKSUM_VALUE = "fileChecksumValue"; public static final String FILENAME_WITHOUT_EXTENSION = "fileNameWithoutExtension"; + public static final String FILE_RESTRICTED = "fileRestricted"; /** * Indexed as a string so we can facet on it. */ @@ -270,6 +271,8 @@ more targeted results for just datasets. The format is YYYY (i.e. */ public static final String DATASET_TYPE = "datasetType"; + public static final String OBSERVATIONS = "observations"; + public static final String VARIABLE_COUNT = "variableCount"; public static final String VARIABLE_NAME = "variableName"; public static final String VARIABLE_LABEL = "variableLabel"; public static final String LITERAL_QUESTION = "literalQuestion"; diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index 3fd97d418c0..60bcc9f846e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.search; import edu.harvard.iq.dataverse.*; +import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.groups.Group; import edu.harvard.iq.dataverse.authorization.groups.GroupServiceBean; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -18,6 +19,7 @@ import java.util.Calendar; import java.util.Collections; import java.util.Date; +import java.util.EnumSet; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -75,6 +77,8 @@ public class SearchServiceBean { SystemConfig systemConfig; @EJB SolrClientService solrClientService; + @EJB + PermissionServiceBean permissionService; @Inject ThumbnailServiceWrapper thumbnailServiceWrapper; @@ -677,6 +681,15 @@ public SolrQueryResponse search( logger.info("Exception setting setFileChecksumType: " + ex); } solrSearchResult.setFileChecksumValue((String) solrDocument.getFieldValue(SearchFields.FILE_CHECKSUM_VALUE)); + + if (solrDocument.getFieldValue(SearchFields.FILE_RESTRICTED) != null) { + solrSearchResult.setFileRestricted((Boolean) solrDocument.getFieldValue(SearchFields.FILE_RESTRICTED)); + } + + if (solrSearchResult.getEntity() != null) { + solrSearchResult.setCanDownloadFile(permissionService.hasPermissionsFor(dataverseRequest, solrSearchResult.getEntity(), EnumSet.of(Permission.DownloadFile))); + } + solrSearchResult.setUnf((String) solrDocument.getFieldValue(SearchFields.UNF)); solrSearchResult.setDatasetVersionId(datasetVersionId); List fileCategories = (List) solrDocument.getFieldValues(SearchFields.FILE_TAG); @@ -688,6 +701,10 @@ public SolrQueryResponse search( Collections.sort(tabularDataTags); solrSearchResult.setTabularDataTags(tabularDataTags); } + Long observations = (Long) solrDocument.getFieldValue(SearchFields.OBSERVATIONS); + solrSearchResult.setObservations(observations); + Long tabCount = (Long) solrDocument.getFieldValue(SearchFields.VARIABLE_COUNT); + solrSearchResult.setTabularDataCount(tabCount); String filePID = (String) solrDocument.getFieldValue(SearchFields.FILE_PERSISTENT_ID); if(null != filePID && !"".equals(filePID) && !"".equals("null")) { solrSearchResult.setFilePersistentId(filePID); diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 8802555affd..70e9a549554 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -97,6 +97,8 @@ public class SolrSearchResult { private String fileMd5; private DataFile.ChecksumType fileChecksumType; private String fileChecksumValue; + private Boolean fileRestricted; + private Boolean canDownloadFile; private String dataverseAlias; private String dataverseParentAlias; private String dataverseParentName; @@ -122,6 +124,8 @@ public class SolrSearchResult { private String harvestingDescription = null; private List fileCategories = null; private List tabularDataTags = null; + private Long tabularDataCount; + private Long observations; private String identifierOfDataverse = null; private String nameOfDataverse = null; @@ -565,7 +569,12 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool .add("citationHtml", this.citationHtml) .add("identifier_of_dataverse", this.identifierOfDataverse) .add("name_of_dataverse", this.nameOfDataverse) - .add("citation", this.citation); + .add("citation", this.citation) + .add("restricted", this.fileRestricted) + .add("variables", this.tabularDataCount) + .add("observations", this.observations) + .add("canDownloadFile", this.canDownloadFile); + // Now that nullSafeJsonBuilder has been instatiated, check for null before adding to it! if (showRelevance) { nullSafeJsonBuilder.add("matches", getRelevance()); @@ -579,6 +588,9 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool if (!getPublicationStatuses().isEmpty()) { nullSafeJsonBuilder.add("publicationStatuses", getPublicationStatusesAsJSON()); } + if (this.fileCategories != null && !this.fileCategories.isEmpty()) { + nullSafeJsonBuilder.add("categories", JsonPrinter.asJsonArray(this.fileCategories)); + } if (this.entity == null) { @@ -956,6 +968,18 @@ public List getTabularDataTags() { public void setTabularDataTags(List tabularDataTags) { this.tabularDataTags = tabularDataTags; } + public void setTabularDataCount(Long tabularDataCount) { + this.tabularDataCount = tabularDataCount; + } + public Long getTabularDataCount() { + return tabularDataCount; + } + public Long getObservations() { + return observations; + } + public void setObservations(Long observations) { + this.observations = observations; + } public Map getParent() { return parent; @@ -1078,6 +1102,15 @@ public void setFileChecksumValue(String fileChecksumValue) { this.fileChecksumValue = fileChecksumValue; } + public Boolean getFileRestricted() { return fileRestricted; } + public void setFileRestricted(Boolean fileRestricted) { + this.fileRestricted = fileRestricted; + } + public Boolean getCanDownloadFile() { return canDownloadFile; } + public void setCanDownloadFile(Boolean canDownloadFile) { + this.canDownloadFile = canDownloadFile; + } + public String getNameSort() { return nameSort; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java b/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java index ef8ab39122f..21360fcd708 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/NullSafeJsonBuilder.java @@ -85,7 +85,10 @@ public NullSafeJsonBuilder add(String name, boolean value) { delegate.add(name, value); return this; } - + public NullSafeJsonBuilder add(String name, Boolean value) { + return (value != null) ? add(name, value.booleanValue()) : this; + } + @Override public NullSafeJsonBuilder addNull(String name) { delegate.addNull(name); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index b03c23cd1e2..8338caff9ef 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -4,6 +4,7 @@ import io.restassured.path.json.JsonPath; import io.restassured.response.Response; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; import jakarta.json.Json; @@ -1300,8 +1301,12 @@ public void testSearchFilesAndUrlImages() { System.out.println("id: " + datasetId); String datasetPid = JsonPath.from(createDatasetResponse.getBody().asString()).getString("data.persistentId"); System.out.println("datasetPid: " + datasetPid); - String pathToFile = "src/main/webapp/resources/images/dataverseproject.png"; + Response logoResponse = UtilIT.uploadDatasetLogo(datasetPid, pathToFile, apiToken); + logoResponse.prettyPrint(); + logoResponse.then().assertThat() + .statusCode(200); + Response uploadImage = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, apiToken); uploadImage.prettyPrint(); uploadImage.then().assertThat() @@ -1311,7 +1316,16 @@ public void testSearchFilesAndUrlImages() { uploadFile.prettyPrint(); uploadFile.then().assertThat() .statusCode(200); - + pathToFile = "src/test/resources/tab/test.tab"; + String searchableUniqueId = "testtab"+ UUID.randomUUID().toString().substring(0, 8); // so the search only returns 1 file + JsonObjectBuilder json = Json.createObjectBuilder() + .add("description", searchableUniqueId) + .add("restrict", "true") + .add("categories", Json.createArrayBuilder().add("Data")); + Response uploadTabFile = UtilIT.uploadFileViaNative(datasetId.toString(), pathToFile, json.build(), apiToken); + uploadTabFile.prettyPrint(); + uploadTabFile.then().assertThat() + .statusCode(200); Response publishDataverse = UtilIT.publishDataverseViaSword(dataverseAlias, apiToken); publishDataverse.prettyPrint(); publishDataverse.then().assertThat() @@ -1339,6 +1353,13 @@ public void testSearchFilesAndUrlImages() { .body("data.items[0].url", CoreMatchers.containsString("/dataverse/")) .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("image_url"))); + searchResp = UtilIT.search(datasetPid, apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("dataset")) + .body("data.items[0].image_url", CoreMatchers.containsString("/logo")); + searchResp = UtilIT.search("mydata", apiToken); searchResp.prettyPrint(); searchResp.then().assertThat() @@ -1346,5 +1367,16 @@ public void testSearchFilesAndUrlImages() { .body("data.items[0].type", CoreMatchers.is("file")) .body("data.items[0].url", CoreMatchers.containsString("/datafile/")) .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("image_url"))); + searchResp = UtilIT.search(searchableUniqueId, apiToken); + searchResp.prettyPrint(); + searchResp.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.items[0].type", CoreMatchers.is("file")) + .body("data.items[0].url", CoreMatchers.containsString("/datafile/")) + .body("data.items[0].variables", CoreMatchers.is(3)) + .body("data.items[0].observations", CoreMatchers.is(10)) + .body("data.items[0].restricted", CoreMatchers.is(true)) + .body("data.items[0].canDownloadFile", CoreMatchers.is(true)) + .body("data.items[0]", CoreMatchers.not(CoreMatchers.hasItem("image_url"))); } }