From e9cf04178e8a492bc1f1bda7f3113dc1364f6f59 Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 28 Aug 2023 18:45:59 +0100 Subject: [PATCH 01/19] Stash: getVersionFileCounts endpoint WIP. Pending access type count, test coverage and refactor --- .../DatasetVersionFilesServiceBean.java | 66 +++++++++++++++++-- .../harvard/iq/dataverse/api/Datasets.java | 34 +++++++++- .../harvard/iq/dataverse/api/DatasetsIT.java | 47 ++++++++++--- .../edu/harvard/iq/dataverse/api/UtilIT.java | 6 ++ 4 files changed, 138 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java index 751da6daba8..364ac36652c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java @@ -5,6 +5,7 @@ import edu.harvard.iq.dataverse.QEmbargo; import edu.harvard.iq.dataverse.QFileMetadata; +import com.querydsl.core.Tuple; import com.querydsl.core.types.dsl.BooleanExpression; import com.querydsl.core.types.dsl.CaseBuilder; import com.querydsl.core.types.dsl.DateExpression; @@ -21,7 +22,9 @@ import java.io.Serializable; import java.sql.Timestamp; import java.time.LocalDate; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Stateless @Named @@ -48,6 +51,57 @@ public enum DataFileAccessStatus { Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic } + /** + * Given a DatasetVersion, returns its total file metadata count + * + * @param datasetVersion the DatasetVersion to access + * @return long value of total file metadata count + */ + public long getFileMetadataCount(DatasetVersion datasetVersion) { + JPAQueryFactory queryFactory = new JPAQueryFactory(em); + return queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())).stream().count(); + } + + /** + * Given a DatasetVersion, returns its file metadata count per content type + * + * @param datasetVersion the DatasetVersion to access + * @return Map of file metadata counts per content type + */ + public Map getFileMetadataCountPerContentType(DatasetVersion datasetVersion) { + JPAQueryFactory queryFactory = new JPAQueryFactory(em); + List contentTypeOccurrences = queryFactory + .select(fileMetadata.dataFile.contentType, fileMetadata.count()) + .from(fileMetadata) + .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())) + .groupBy(fileMetadata.dataFile.contentType).fetch(); + Map result = new HashMap<>(); + for (Tuple occurrence : contentTypeOccurrences) { + result.put(occurrence.get(fileMetadata.dataFile.contentType), occurrence.get(fileMetadata.count())); + } + return result; + } + + /** + * Given a DatasetVersion, returns its file metadata count per category name + * + * @param datasetVersion the DatasetVersion to access + * @return Map of file metadata counts per category name + */ + public Map getFileMetadataCountPerCategoryName(DatasetVersion datasetVersion) { + JPAQueryFactory queryFactory = new JPAQueryFactory(em); + List categoryNameOccurrences = queryFactory + .select(dataFileCategory.name, fileMetadata.count()) + .from(dataFileCategory, fileMetadata) + .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(fileMetadata.fileCategories.contains(dataFileCategory))) + .groupBy(dataFileCategory.name).fetch(); + Map result = new HashMap<>(); + for (Tuple occurrence : categoryNameOccurrences) { + result.put(occurrence.get(dataFileCategory.name), occurrence.get(fileMetadata.count())); + } + return result; + } + /** * Returns a FileMetadata list of files in the specified DatasetVersion * @@ -62,13 +116,13 @@ public enum DataFileAccessStatus { * @return a FileMetadata list from the specified DatasetVersion */ public List getFileMetadatas(DatasetVersion datasetVersion, Integer limit, Integer offset, String contentType, DataFileAccessStatus accessStatus, String categoryName, String searchText, FileMetadatasOrderCriteria orderCriteria) { - JPAQuery baseQuery = createBaseQuery(datasetVersion, orderCriteria); + JPAQuery baseQuery = createGetFileMetadatasBaseQuery(datasetVersion, orderCriteria); if (contentType != null) { baseQuery.where(fileMetadata.dataFile.contentType.eq(contentType)); } if (accessStatus != null) { - baseQuery.where(createAccessStatusExpression(accessStatus)); + baseQuery.where(createGetFileMetadatasAccessStatusExpression(accessStatus)); } if (categoryName != null) { baseQuery.from(dataFileCategory).where(dataFileCategory.name.eq(categoryName).and(fileMetadata.fileCategories.contains(dataFileCategory))); @@ -78,7 +132,7 @@ public List getFileMetadatas(DatasetVersion datasetVersion, Intege baseQuery.where(fileMetadata.label.lower().contains(searchText).or(fileMetadata.description.lower().contains(searchText))); } - applyOrderCriteriaToQuery(baseQuery, orderCriteria); + applyOrderCriteriaToGetFileMetadatasQuery(baseQuery, orderCriteria); if (limit != null) { baseQuery.limit(limit); @@ -90,7 +144,7 @@ public List getFileMetadatas(DatasetVersion datasetVersion, Intege return baseQuery.fetch(); } - private JPAQuery createBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) { + private JPAQuery createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) { JPAQueryFactory queryFactory = new JPAQueryFactory(em); JPAQuery baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())); if (orderCriteria == FileMetadatasOrderCriteria.Newest || orderCriteria == FileMetadatasOrderCriteria.Oldest) { @@ -99,7 +153,7 @@ private JPAQuery createBaseQuery(DatasetVersion datasetVersion, Fi return baseQuery; } - private BooleanExpression createAccessStatusExpression(DataFileAccessStatus accessStatus) { + private BooleanExpression createGetFileMetadatasAccessStatusExpression(DataFileAccessStatus accessStatus) { QEmbargo embargo = fileMetadata.dataFile.embargo; BooleanExpression activelyEmbargoedExpression = embargo.dateAvailable.goe(DateExpression.currentDate(LocalDate.class)); BooleanExpression inactivelyEmbargoedExpression = embargo.isNull(); @@ -123,7 +177,7 @@ private BooleanExpression createAccessStatusExpression(DataFileAccessStatus acce return accessStatusExpression; } - private void applyOrderCriteriaToQuery(JPAQuery query, FileMetadatasOrderCriteria orderCriteria) { + private void applyOrderCriteriaToGetFileMetadatasQuery(JPAQuery query, FileMetadatasOrderCriteria orderCriteria) { DateTimeExpression orderByLifetimeExpression = new CaseBuilder().when(dvObject.publicationDate.isNotNull()).then(dvObject.publicationDate).otherwise(dvObject.createDate); switch (orderCriteria) { case NameZA: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index dc192281695..297cffaa06f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -520,7 +520,39 @@ public Response getVersionFiles(@Context ContainerRequestContext crc, return ok(jsonFileMetadatas(datasetVersionFilesServiceBean.getFileMetadatas(datasetVersion, limit, offset, contentType, dataFileAccessStatus, categoryName, searchText, fileMetadatasOrderCriteria))); }, getRequestUser(crc)); } - + + // TODO: Finalize and refine + @GET + @AuthRequired + @Path("{id}/versions/{versionId}/files/counts") + public Response getVersionFileCounts(@Context ContainerRequestContext crc, @PathParam("id") String datasetId, @PathParam("versionId") String versionId, @Context UriInfo uriInfo, @Context HttpHeaders headers) { + return response(req -> { + DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers); + JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); + + jsonObjectBuilder.add("total", datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion)); + + Map fileMetadataCountsPerContentType = datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion); + JsonObjectBuilder fileMetadataCountsPerContentTypeJsonBuilder = Json.createObjectBuilder(); + for (Map.Entry fileMetadataCount : fileMetadataCountsPerContentType.entrySet()) { + fileMetadataCountsPerContentTypeJsonBuilder.add(fileMetadataCount.getKey(), fileMetadataCount.getValue()); + } + jsonObjectBuilder.add("perContentType", fileMetadataCountsPerContentTypeJsonBuilder); + + // TODO + jsonObjectBuilder.add("perAccessStatus", 3); + + Map fileMetadataCountsPerCategoryName = datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion); + JsonObjectBuilder fileMetadataCountsPerCategoryNameJsonBuilder = Json.createObjectBuilder(); + for (Map.Entry fileMetadataCount : fileMetadataCountsPerCategoryName.entrySet()) { + fileMetadataCountsPerCategoryNameJsonBuilder.add(fileMetadataCount.getKey(), fileMetadataCount.getValue()); + } + jsonObjectBuilder.add("perCategoryName", fileMetadataCountsPerCategoryNameJsonBuilder); + + return ok(jsonObjectBuilder); + }, getRequestUser(crc)); + } + @GET @AuthRequired @Path("{id}/dirindex") diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index c346228d034..efd4c31df57 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -11,6 +11,7 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.*; import java.util.logging.Logger; import org.junit.jupiter.api.AfterAll; @@ -19,8 +20,6 @@ import org.skyscreamer.jsonassert.JSONAssert; import org.junit.jupiter.api.Disabled; -import java.util.List; -import java.util.Map; import jakarta.json.JsonObject; import static jakarta.ws.rs.core.Response.Status.CREATED; @@ -41,8 +40,6 @@ import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; -import java.util.UUID; - import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -69,8 +66,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.Files; -import java.util.ArrayList; -import java.util.HashMap; + import jakarta.json.Json; import jakarta.json.JsonArray; import jakarta.json.JsonObjectBuilder; @@ -3327,8 +3323,6 @@ public void getVersionFiles() throws IOException { fileMetadatasCount = getVersionFilesResponsePaginated.jsonPath().getList("data").size(); assertEquals(testPageSize, fileMetadatasCount); - String testFileId3 = JsonPath.from(getVersionFilesResponsePaginated.body().asString()).getString("data[0].dataFile.id"); - // Test page 3 (last) getVersionFilesResponsePaginated = UtilIT.getVersionFiles(datasetId, testDatasetVersion, testPageSize, testPageSize * 2, null, null, null, null, null, apiToken); @@ -3490,4 +3484,41 @@ public void getVersionFiles() throws IOException { fileMetadatasCount = getVersionFilesResponseSearchText.jsonPath().getList("data").size(); assertEquals(1, fileMetadatasCount); } + + @Test + public void getVersionFileCounts() throws IOException { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String datasetPersistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); + Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + String testFileName1 = "test_1.txt"; + String testFileName2 = "test_2.txt"; + String testFileName3 = "test_3.png"; + + UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName1, new byte[50], apiToken); + UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName2, new byte[200], apiToken); + UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName3, new byte[100], apiToken); + + String testDatasetVersion = ":latest"; + + Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, apiToken); + + getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode()); + + JsonPath responseJsonPath = getVersionFileCountsResponse.jsonPath(); + LinkedHashMap responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType"); + + assertEquals((Integer) responseJsonPath.get("data.total"), 3); + assertEquals(responseCountPerContentTypeMap.get("image/png"), 1); + assertEquals(responseCountPerContentTypeMap.get("text/plain"), 2); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 7939fd7d88d..69549f2e4a8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3341,4 +3341,10 @@ static Response createFileEmbargo(Integer datasetId, Integer fileId, String date .urlEncodingEnabled(false) .post("/api/datasets/" + datasetId + "/files/actions/:set-embargo"); } + + static Response getVersionFileCounts(Integer datasetId, String version, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts"); + } } From cc5a1bf367105aade087add7bd442e2c9f9dd6e0 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 29 Aug 2023 10:43:33 +0100 Subject: [PATCH 02/19] Added: setFileCategories API endpoint --- .../edu/harvard/iq/dataverse/api/Files.java | 38 ++++++++++++---- .../edu/harvard/iq/dataverse/api/FilesIT.java | 44 +++++++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 14 ++++++ 3 files changed, 85 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 4c411a631f1..6712b68a09b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -53,6 +53,7 @@ import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -62,15 +63,12 @@ import jakarta.ejb.EJBException; import jakarta.inject.Inject; import jakarta.json.Json; +import jakarta.json.JsonArray; +import jakarta.json.JsonString; +import jakarta.json.JsonValue; +import jakarta.json.stream.JsonParsingException; import jakarta.servlet.http.HttpServletResponse; -import jakarta.ws.rs.Consumes; -import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.PUT; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.*; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.HttpHeaders; @@ -848,4 +846,28 @@ public Response getFileDataTables(@Context ContainerRequestContext crc, @PathPar return ok(jsonDT(dataFile.getDataTables())); }, getRequestUser(crc)); } + + @POST + @AuthRequired + @Path("{id}/metadata/categories") + @Produces(MediaType.APPLICATION_JSON) + public Response setFileCategories(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId, String jsonBody) { + return response(req -> { + DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId))); + jakarta.json.JsonObject jsonObject; + try (StringReader stringReader = new StringReader(jsonBody)) { + jsonObject = Json.createReader(stringReader).readObject(); + JsonArray requestedCategoriesJson = jsonObject.getJsonArray("categories"); + FileMetadata fileMetadata = dataFile.getFileMetadata(); + for (JsonValue jsonValue : requestedCategoriesJson) { + JsonString jsonString = (JsonString) jsonValue; + fileMetadata.addCategoryByName(jsonString.getString()); + } + execCommand(new UpdateDatasetVersionCommand(fileMetadata.getDataFile().getOwner(), req)); + return ok("Categories of file " + dataFileId + " updated."); + } catch (JsonParsingException jpe) { + return error(Response.Status.BAD_REQUEST, "Error parsing Json: " + jpe.getMessage()); + } + }, getRequestUser(crc)); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index 77519efa2cc..1004d3812d5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -2,6 +2,8 @@ import io.restassured.RestAssured; import io.restassured.response.Response; + +import java.util.List; import java.util.logging.Logger; import edu.harvard.iq.dataverse.api.auth.ApiKeyAuthMechanism; @@ -30,11 +32,10 @@ import static jakarta.ws.rs.core.Response.Status.*; import org.hamcrest.CoreMatchers; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.startsWith; -import static org.hamcrest.CoreMatchers.nullValue; import org.hamcrest.Matchers; +import static org.hamcrest.CoreMatchers.*; +import static org.hamcrest.CoreMatchers.hasItem; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -2184,4 +2185,41 @@ public void testGetFileDataTables() throws InterruptedException { int dataTablesNumber = JsonPath.from(getFileDataTablesForTabularFileResponse.body().asString()).getList("data").size(); assertTrue(dataTablesNumber > 0); } + + @Test + public void testSetFileCategories() { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + // Upload test file + String pathToTestFile = "src/test/resources/images/coffeeshop.png"; + Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + + String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id"); + + // Set categories + String testCategory1 = "testCategory1"; + String testCategory2 = "testCategory2"; + List testCategories = List.of(testCategory1, testCategory2); + Response setFileCategoriesResponse = UtilIT.setFileCategories(dataFileId, apiToken, testCategories); + setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Get file data and check for new categories + Response getFileDataResponse = UtilIT.getFileData(dataFileId, apiToken); + getFileDataResponse.prettyPrint(); + getFileDataResponse.then().assertThat() + .body("data.categories", hasItem(testCategory1)) + .body("data.categories", hasItem(testCategory2)) + .statusCode(OK.getStatusCode()); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 69549f2e4a8..0fcb7f4f287 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3347,4 +3347,18 @@ static Response getVersionFileCounts(Integer datasetId, String version, String a .header(API_TOKEN_HTTP_HEADER, apiToken) .get("/api/datasets/" + datasetId + "/versions/" + version + "/files/counts"); } + + static Response setFileCategories(String dataFileId, String apiToken, List categories) { + JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); + for (String category : categories) { + jsonArrayBuilder.add(category); + } + JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); + jsonObjectBuilder.add("categories", jsonArrayBuilder); + String jsonString = jsonObjectBuilder.build().toString(); + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .body(jsonString) + .post("/api/files/" + dataFileId + "/metadata/categories"); + } } From eadab48a4215b61198255f2ce93bc5088ae0e4d8 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 29 Aug 2023 10:51:19 +0100 Subject: [PATCH 03/19] Fixed: getVersionFilesIT test case for category filtering --- .../edu/harvard/iq/dataverse/api/DatasetsIT.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index efd4c31df57..5d4fad496d6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3406,13 +3406,21 @@ public void getVersionFiles() throws IOException { assertEquals(1, fileMetadatasCount); // Test Category Name - Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, "image/png", null, "nonExistentCategory", null, null, apiToken); + String testCategory = "testCategory"; + Response setFileCategoriesResponse = UtilIT.setFileCategories(testFileId1, apiToken, List.of(testCategory)); + setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode()); + setFileCategoriesResponse = UtilIT.setFileCategories(testFileId2, apiToken, List.of(testCategory)); + setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response getVersionFilesResponseCategoryName = UtilIT.getVersionFiles(datasetId, testDatasetVersion, null, null, null, null, testCategory, null, null, apiToken); getVersionFilesResponseCategoryName.then().assertThat() - .statusCode(OK.getStatusCode()); + .statusCode(OK.getStatusCode()) + .body("data[0].label", equalTo(testFileName1)) + .body("data[1].label", equalTo(testFileName2)); fileMetadatasCount = getVersionFilesResponseCategoryName.jsonPath().getList("data").size(); - assertEquals(0, fileMetadatasCount); + assertEquals(2, fileMetadatasCount); // Test Access Status Restricted Response restrictFileResponse = UtilIT.restrictFile(String.valueOf(testFileId1), true, apiToken); From 70b9193ff750e119b348f14ce938255cd7a8b497 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 29 Aug 2023 11:12:09 +0100 Subject: [PATCH 04/19] Added: getVersionFileCounts count per category test coverage --- .../harvard/iq/dataverse/api/DatasetsIT.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 5d4fad496d6..1b3811a6d66 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3506,8 +3506,9 @@ public void getVersionFileCounts() throws IOException { Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); String datasetPersistentId = JsonPath.from(createDatasetResponse.body().asString()).getString("data.persistentId"); - Integer datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + // Creating test files String testFileName1 = "test_1.txt"; String testFileName2 = "test_2.txt"; String testFileName3 = "test_3.png"; @@ -3516,17 +3517,26 @@ public void getVersionFileCounts() throws IOException { UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName2, new byte[200], apiToken); UtilIT.createAndUploadTestFile(datasetPersistentId, testFileName3, new byte[100], apiToken); - String testDatasetVersion = ":latest"; + // Creating a categorized test file + String pathToTestFile = "src/test/resources/images/coffeeshop.png"; + Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id"); + String testCategory = "testCategory"; + UtilIT.setFileCategories(dataFileId, apiToken, List.of(testCategory)); - Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, testDatasetVersion, apiToken); + // Getting the file counts and assert each count + Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, ":latest", apiToken); getVersionFileCountsResponse.then().assertThat().statusCode(OK.getStatusCode()); JsonPath responseJsonPath = getVersionFileCountsResponse.jsonPath(); LinkedHashMap responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType"); + LinkedHashMap responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName"); - assertEquals((Integer) responseJsonPath.get("data.total"), 3); - assertEquals(responseCountPerContentTypeMap.get("image/png"), 1); + assertEquals((Integer) responseJsonPath.get("data.total"), 4); + assertEquals(responseCountPerContentTypeMap.get("image/png"), 2); assertEquals(responseCountPerContentTypeMap.get("text/plain"), 2); + assertEquals(responseCountPerCategoryNameMap.get(testCategory), 1); } } From ace6783eb51c1de0995f3a1804a97446d85d801b Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 30 Aug 2023 11:33:25 +0100 Subject: [PATCH 05/19] Added: getVersionFileCounts count per access status --- .../DatasetVersionFilesServiceBean.java | 30 ++++++++++++++++ .../harvard/iq/dataverse/api/Datasets.java | 9 +++-- .../harvard/iq/dataverse/api/DatasetsIT.java | 34 ++++++++++++------- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java index 364ac36652c..a547a216ad5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java @@ -102,6 +102,21 @@ public Map getFileMetadataCountPerCategoryName(DatasetVersion data return result; } + /** + * Given a DatasetVersion, returns its file metadata count per DataFileAccessStatus + * + * @param datasetVersion the DatasetVersion to access + * @return Map of file metadata counts per DataFileAccessStatus + */ + public Map getFileMetadataCountPerAccessStatus(DatasetVersion datasetVersion) { + Map allCounts = new HashMap<>(); + addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.Public); + addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.Restricted); + addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.EmbargoedThenPublic); + addAccessStatusCountToTotal(datasetVersion, allCounts, DataFileAccessStatus.EmbargoedThenRestricted); + return allCounts; + } + /** * Returns a FileMetadata list of files in the specified DatasetVersion * @@ -144,6 +159,21 @@ public List getFileMetadatas(DatasetVersion datasetVersion, Intege return baseQuery.fetch(); } + private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map totalCounts, DataFileAccessStatus dataFileAccessStatus) { + long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus); + if (fileMetadataCount > 0) { + totalCounts.put(dataFileAccessStatus, fileMetadataCount); + } + } + + private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, DataFileAccessStatus accessStatus) { + JPAQueryFactory queryFactory = new JPAQueryFactory(em); + return queryFactory + .selectFrom(fileMetadata) + .where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId()).and(createGetFileMetadatasAccessStatusExpression(accessStatus))) + .stream().count(); + } + private JPAQuery createGetFileMetadatasBaseQuery(DatasetVersion datasetVersion, FileMetadatasOrderCriteria orderCriteria) { JPAQueryFactory queryFactory = new JPAQueryFactory(em); JPAQuery baseQuery = queryFactory.selectFrom(fileMetadata).where(fileMetadata.datasetVersion.id.eq(datasetVersion.getId())); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 297cffaa06f..b976d60b45f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -521,7 +521,6 @@ public Response getVersionFiles(@Context ContainerRequestContext crc, }, getRequestUser(crc)); } - // TODO: Finalize and refine @GET @AuthRequired @Path("{id}/versions/{versionId}/files/counts") @@ -539,8 +538,12 @@ public Response getVersionFileCounts(@Context ContainerRequestContext crc, @Path } jsonObjectBuilder.add("perContentType", fileMetadataCountsPerContentTypeJsonBuilder); - // TODO - jsonObjectBuilder.add("perAccessStatus", 3); + Map fileMetadataCountsPerAccessStatus = datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion); + JsonObjectBuilder fileMetadataCountsPerAccessStatusJsonBuilder = Json.createObjectBuilder(); + for (Map.Entry fileMetadataCount : fileMetadataCountsPerAccessStatus.entrySet()) { + fileMetadataCountsPerAccessStatusJsonBuilder.add(fileMetadataCount.getKey().toString(), fileMetadataCount.getValue()); + } + jsonObjectBuilder.add("perAccessStatus", fileMetadataCountsPerAccessStatusJsonBuilder); Map fileMetadataCountsPerCategoryName = datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion); JsonObjectBuilder fileMetadataCountsPerCategoryNameJsonBuilder = Json.createObjectBuilder(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 1b3811a6d66..6f103df3fe8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -3436,18 +3436,18 @@ public void getVersionFiles() throws IOException { fileMetadatasCount = getVersionFilesResponseRestricted.jsonPath().getList("data").size(); assertEquals(1, fileMetadatasCount); - // Test Access Status Embargoed Then Public + // Test Access Status Embargoed UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12"); String activeEmbargoDate = LocalDate.now().plusMonths(6).format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); - // Create embargo for test file 1 (Embargo and Restricted) - Response createExpiredFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId1), activeEmbargoDate, apiToken); + // Create embargo for test file 1 (Embargoed and Restricted) + Response createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId1), activeEmbargoDate, apiToken); - createExpiredFileEmbargoResponse.then().assertThat() + createActiveFileEmbargoResponse.then().assertThat() .statusCode(OK.getStatusCode()); - // Create embargo for test file 2 (Embargo and Public) - Response createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId2), activeEmbargoDate, apiToken); + // Create embargo for test file 2 (Embargoed and Public) + createActiveFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(testFileId2), activeEmbargoDate, apiToken); createActiveFileEmbargoResponse.then().assertThat() .statusCode(OK.getStatusCode()); @@ -3523,7 +3523,14 @@ public void getVersionFileCounts() throws IOException { uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id"); String testCategory = "testCategory"; - UtilIT.setFileCategories(dataFileId, apiToken, List.of(testCategory)); + Response setFileCategoriesResponse = UtilIT.setFileCategories(dataFileId, apiToken, List.of(testCategory)); + setFileCategoriesResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Setting embargo for file (Embargo and Public) + UtilIT.setSetting(SettingsServiceBean.Key.MaxEmbargoDurationInMonths, "12"); + String activeEmbargoDate = LocalDate.now().plusMonths(6).format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + Response createFileEmbargoResponse = UtilIT.createFileEmbargo(datasetId, Integer.parseInt(dataFileId), activeEmbargoDate, apiToken); + createFileEmbargoResponse.then().assertThat().statusCode(OK.getStatusCode()); // Getting the file counts and assert each count Response getVersionFileCountsResponse = UtilIT.getVersionFileCounts(datasetId, ":latest", apiToken); @@ -3533,10 +3540,13 @@ public void getVersionFileCounts() throws IOException { JsonPath responseJsonPath = getVersionFileCountsResponse.jsonPath(); LinkedHashMap responseCountPerContentTypeMap = responseJsonPath.get("data.perContentType"); LinkedHashMap responseCountPerCategoryNameMap = responseJsonPath.get("data.perCategoryName"); - - assertEquals((Integer) responseJsonPath.get("data.total"), 4); - assertEquals(responseCountPerContentTypeMap.get("image/png"), 2); - assertEquals(responseCountPerContentTypeMap.get("text/plain"), 2); - assertEquals(responseCountPerCategoryNameMap.get(testCategory), 1); + LinkedHashMap responseCountPerAccessStatusMap = responseJsonPath.get("data.perAccessStatus"); + + assertEquals(4, (Integer) responseJsonPath.get("data.total")); + assertEquals(2, responseCountPerContentTypeMap.get("image/png")); + assertEquals(2, responseCountPerContentTypeMap.get("text/plain")); + assertEquals(1, responseCountPerCategoryNameMap.get(testCategory)); + assertEquals(3, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.Public.toString())); + assertEquals(1, responseCountPerAccessStatusMap.get(DatasetVersionFilesServiceBean.DataFileAccessStatus.EmbargoedThenPublic.toString())); } } From a87136c1ae36ab83761e6cffdffe40c660b9296f Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 30 Aug 2023 11:54:16 +0100 Subject: [PATCH 06/19] Reefactor: new JsonPrinter methods for getVersionFileCounts response --- .../harvard/iq/dataverse/api/Datasets.java | 26 +++---------------- .../iq/dataverse/util/json/JsonPrinter.java | 16 ++++++++++++ 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index b976d60b45f..d082d9c29da 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -528,30 +528,10 @@ public Response getVersionFileCounts(@Context ContainerRequestContext crc, @Path return response(req -> { DatasetVersion datasetVersion = getDatasetVersionOrDie(req, versionId, findDatasetOrDie(datasetId), uriInfo, headers); JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); - jsonObjectBuilder.add("total", datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion)); - - Map fileMetadataCountsPerContentType = datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion); - JsonObjectBuilder fileMetadataCountsPerContentTypeJsonBuilder = Json.createObjectBuilder(); - for (Map.Entry fileMetadataCount : fileMetadataCountsPerContentType.entrySet()) { - fileMetadataCountsPerContentTypeJsonBuilder.add(fileMetadataCount.getKey(), fileMetadataCount.getValue()); - } - jsonObjectBuilder.add("perContentType", fileMetadataCountsPerContentTypeJsonBuilder); - - Map fileMetadataCountsPerAccessStatus = datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion); - JsonObjectBuilder fileMetadataCountsPerAccessStatusJsonBuilder = Json.createObjectBuilder(); - for (Map.Entry fileMetadataCount : fileMetadataCountsPerAccessStatus.entrySet()) { - fileMetadataCountsPerAccessStatusJsonBuilder.add(fileMetadataCount.getKey().toString(), fileMetadataCount.getValue()); - } - jsonObjectBuilder.add("perAccessStatus", fileMetadataCountsPerAccessStatusJsonBuilder); - - Map fileMetadataCountsPerCategoryName = datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion); - JsonObjectBuilder fileMetadataCountsPerCategoryNameJsonBuilder = Json.createObjectBuilder(); - for (Map.Entry fileMetadataCount : fileMetadataCountsPerCategoryName.entrySet()) { - fileMetadataCountsPerCategoryNameJsonBuilder.add(fileMetadataCount.getKey(), fileMetadataCount.getValue()); - } - jsonObjectBuilder.add("perCategoryName", fileMetadataCountsPerCategoryNameJsonBuilder); - + jsonObjectBuilder.add("perContentType", json(datasetVersionFilesServiceBean.getFileMetadataCountPerContentType(datasetVersion))); + jsonObjectBuilder.add("perCategoryName", json(datasetVersionFilesServiceBean.getFileMetadataCountPerCategoryName(datasetVersion))); + jsonObjectBuilder.add("perAccessStatus", jsonFileCountPerAccessStatusMap(datasetVersionFilesServiceBean.getFileMetadataCountPerAccessStatus(datasetVersion))); return ok(jsonObjectBuilder); }, getRequestUser(crc)); } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 36085f7ead7..fb8df057dcc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -1111,6 +1111,22 @@ public Set characteristics() { }; } + public static JsonObjectBuilder json(Map map) { + JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); + for (Map.Entry mapEntry : map.entrySet()) { + jsonObjectBuilder.add(mapEntry.getKey(), mapEntry.getValue()); + } + return jsonObjectBuilder; + } + + public static JsonObjectBuilder jsonFileCountPerAccessStatusMap(Map map) { + JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); + for (Map.Entry mapEntry : map.entrySet()) { + jsonObjectBuilder.add(mapEntry.getKey().toString(), mapEntry.getValue()); + } + return jsonObjectBuilder; + } + public static Collector, JsonArrayBuilder> toJsonArray() { return new Collector, JsonArrayBuilder>() { From e1913b3c024c009ad3dbbc6e1238a07c87c6863c Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 30 Aug 2023 12:19:12 +0100 Subject: [PATCH 07/19] Added: docs --- .../9834-files-api-extension-counts.md | 6 ++ doc/sphinx-guides/source/api/native-api.rst | 75 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 doc/release-notes/9834-files-api-extension-counts.md diff --git a/doc/release-notes/9834-files-api-extension-counts.md b/doc/release-notes/9834-files-api-extension-counts.md new file mode 100644 index 00000000000..3ec15d8bd36 --- /dev/null +++ b/doc/release-notes/9834-files-api-extension-counts.md @@ -0,0 +1,6 @@ +Implemented the following new endpoints: + +- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts): Given a dataset and its version, retrieves file counts based on different criteria (Total count, per content type, per access status and per category name). + + +- setFileCategories (/api/files/{id}/metadata/categories): Updates the categories (by name) for an existing file. If the specified categories do not exist, they will be created. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 96555b2ba5f..8b7d34425bc 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1022,6 +1022,32 @@ Please note that both filtering and ordering criteria values are case sensitive Keep in mind that you can combine all of the above query params depending on the results you are looking for. +Get File Counts in a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Get file counts, for the given dataset and version. + +The returned file counts are based on different criteria: + +- Total (The total file count) +- Per content type +- Per category name +- Per access status (Possible values: Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic) + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export ID=24 + export VERSION=1.0 + + curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION/files/counts" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts" + View Dataset Files and Folders as a Directory Index ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2895,6 +2921,55 @@ Also note that dataFileTags are not versioned and changes to these will update t .. _EditingVariableMetadata: +Updating File Metadata Categories +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Updates the categories for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the category names. + +Although updating categories can also be done with the previous endpoint, this has been created to be more practical when it is only necessary to update categories and not other metadata fields. + +A curl example using an ``ID`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=24 + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST \ + -F 'jsonData={"categories":["Category1","Category2"]}' \ + "$SERVER_URL/api/files/$ID/metadata/categories" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \ + -F 'jsonData={"categories":["Category1","Category2"]}' \ + "http://demo.dataverse.org/api/files/24/metadata/categories" + +A curl example using a ``PERSISTENT_ID`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_ID=doi:10.5072/FK2/AAA000 + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST \ + -F 'jsonData={"categories":["Category1","Category2"]}' \ + "$SERVER_URL/api/files/:persistentId/metadata/categories?persistentId=$PERSISTENT_ID" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \ + -F 'jsonData={"categories":["Category1","Category2"]}' \ + "https://demo.dataverse.org/api/files/:persistentId/metadata/categories?persistentId=doi:10.5072/FK2/AAA000" + +Note that if the specified categories do not exist, they will be created. + Editing Variable Level Metadata ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From aa60eae8826bf72309e595afe7dd96a59fb44c74 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 8 Sep 2023 17:01:37 +0100 Subject: [PATCH 08/19] Added: deleted, tabularData, and fileAccessRequest boolean fields to DataFile API payload --- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index fb8df057dcc..9bda1e24cfb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -688,9 +688,12 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo //--------------------------------------------- .add("md5", getMd5IfItExists(df.getChecksumType(), df.getChecksumValue())) .add("checksum", getChecksumTypeAndValue(df.getChecksumType(), df.getChecksumValue())) + .add("tabularData", df.isTabularData()) .add("tabularTags", getTabularFileTags(df)) .add("creationDate", df.getCreateDateFormattedYYYYMMDD()) - .add("publicationDate", df.getPublicationDateFormattedYYYYMMDD()); + .add("publicationDate", df.getPublicationDateFormattedYYYYMMDD()) + .add("deleted", df.getDeleted()) + .add("fileAccessRequest", df.getOwner().isFileAccessRequest()); /* * The restricted state was not included prior to #9175 so to avoid backward * incompatability, it is now only added when generating json for the From 312aedd3a2cc816e686c29e071c96a47081af29e Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 8 Sep 2023 17:28:33 +0100 Subject: [PATCH 09/19] Stash: userFileAccessRequested endpoint WIP --- .../edu/harvard/iq/dataverse/api/Access.java | 59 +++++++++++++------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index ccdec19456c..52d97703ff9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -1678,7 +1678,47 @@ public Response rejectFileAccess(@Context ContainerRequestContext crc, @PathPara return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.fileAccess.rejectFailure.noRequest", args)); } } - + + @GET + @AuthRequired + @Path("/datafile/{id}/userFileAccessRequested") + public Response userFileAccessRequested(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) { + DataFile dataFile; + AuthenticatedUser requestAuthenticatedUser; + try { + dataFile = findDataFileOrDie(dataFileId); + requestAuthenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + boolean fileAccessRequested = false; + List requests = dataFile.getFileAccessRequests(); + for (FileAccessRequest fileAccessRequest : requests) { + if (fileAccessRequest.getAuthenticatedUser().getId().equals(requestAuthenticatedUser.getId())) { + fileAccessRequested = true; + break; + } + } + return ok(fileAccessRequested); + } + + @GET + @AuthRequired + @Path("/datafile/{id}/userPermissions") + public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) { + DataFile dataFile; + try { + dataFile = findDataFileOrDie(dataFileId); + } catch (WrappedResponse wr) { + return wr.getResponse(); + } + JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); + User requestUser = getRequestUser(crc); + jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile)); + jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset)); + return ok(jsonObjectBuilder); + } + // checkAuthorization is a convenience method; it calls the boolean method // isAccessAuthorized(), the actual workhorse, tand throws a 403 exception if not. @@ -1946,21 +1986,4 @@ private URI handleCustomZipDownload(User user, String customZipServiceUrl, Strin } return redirectUri; } - - @GET - @AuthRequired - @Path("/datafile/{id}/userPermissions") - public Response getUserPermissionsOnFile(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) { - DataFile dataFile; - try { - dataFile = findDataFileOrDie(dataFileId); - } catch (WrappedResponse wr) { - return wr.getResponse(); - } - JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); - User requestUser = getRequestUser(crc); - jsonObjectBuilder.add("canDownloadFile", fileDownloadService.canDownloadFile(createDataverseRequest(requestUser), dataFile)); - jsonObjectBuilder.add("canEditOwnerDataset", permissionService.userOn(requestUser, dataFile.getOwner()).has(Permission.EditDataset)); - return ok(jsonObjectBuilder); - } } From 455cb2c950eab61bcaa25a8251cef186e8ea3837 Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 8 Sep 2023 18:10:49 +0100 Subject: [PATCH 10/19] Fixed: removed deleted field from DataFile payload which causes nullability issues --- .../java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 9bda1e24cfb..2b04bb3f657 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -692,7 +692,6 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo .add("tabularTags", getTabularFileTags(df)) .add("creationDate", df.getCreateDateFormattedYYYYMMDD()) .add("publicationDate", df.getPublicationDateFormattedYYYYMMDD()) - .add("deleted", df.getDeleted()) .add("fileAccessRequest", df.getOwner().isFileAccessRequest()); /* * The restricted state was not included prior to #9175 so to avoid backward From 55a81be1c6b279a253b12782a18e7b7c1db9bb9e Mon Sep 17 00:00:00 2001 From: GPortas Date: Fri, 8 Sep 2023 18:12:34 +0100 Subject: [PATCH 11/19] Refactor: simpler IT testGetUserPermissionsOnFile --- .../harvard/iq/dataverse/api/AccessIT.java | 23 ++----------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index 76012882ef5..dadd4093fc2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -633,27 +633,8 @@ public void testZipUploadAndDownload() throws IOException { @Test public void testGetUserPermissionsOnFile() { - Response createUser = UtilIT.createRandomUser(); - createUser.then().assertThat().statusCode(OK.getStatusCode()); - String apiToken = UtilIT.getApiTokenFromResponse(createUser); - - Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); - createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); - String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - - Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); - createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); - int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); - - // Upload test file - String pathToTestFile = "src/test/resources/images/coffeeshop.png"; - Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken); - uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); - - // Assert user permissions on file - int testFileId = JsonPath.from(uploadResponse.body().asString()).getInt("data.files[0].dataFile.id"); - Response getUserPermissionsOnFileResponse = UtilIT.getUserPermissionsOnFile(Integer.toString(testFileId), apiToken); - + // Call with valid file id + Response getUserPermissionsOnFileResponse = UtilIT.getUserPermissionsOnFile(Integer.toString(basicFileId), apiToken); getUserPermissionsOnFileResponse.then().assertThat().statusCode(OK.getStatusCode()); boolean canDownloadFile = JsonPath.from(getUserPermissionsOnFileResponse.body().asString()).getBoolean("data.canDownloadFile"); assertTrue(canDownloadFile); From 0248e1e0be00a332430563fcd762cbfdc26ac290 Mon Sep 17 00:00:00 2001 From: GPortas Date: Sat, 9 Sep 2023 12:02:01 +0100 Subject: [PATCH 12/19] Added: tests and tweaks for userFileAccessRequested API endpoint --- .../edu/harvard/iq/dataverse/api/Access.java | 2 +- .../harvard/iq/dataverse/api/AccessIT.java | 32 +++++++++++++++++-- .../edu/harvard/iq/dataverse/api/UtilIT.java | 6 ++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 52d97703ff9..256af8ec65e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -1682,7 +1682,7 @@ public Response rejectFileAccess(@Context ContainerRequestContext crc, @PathPara @GET @AuthRequired @Path("/datafile/{id}/userFileAccessRequested") - public Response userFileAccessRequested(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) { + public Response getUserFileAccessRequested(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) { DataFile dataFile; AuthenticatedUser requestAuthenticatedUser; try { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java index dadd4093fc2..48b6eee38e1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/AccessIT.java @@ -26,11 +26,9 @@ import static jakarta.ws.rs.core.Response.Status.*; import static org.hamcrest.MatcherAssert.*; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; +import static org.junit.jupiter.api.Assertions.*; /** * @@ -631,6 +629,34 @@ public void testZipUploadAndDownload() throws IOException { System.out.println("Zip upload-and-download round trip test: success!"); } + @Test + public void testGetUserFileAccessRequested() { + // Create new user + Response createUserResponse = UtilIT.createRandomUser(); + createUserResponse.then().assertThat().statusCode(OK.getStatusCode()); + String newUserApiToken = UtilIT.getApiTokenFromResponse(createUserResponse); + + String dataFileId = Integer.toString(tabFile3IdRestricted); + + // Call with new user and unrequested access file + Response getUserFileAccessRequestedResponse = UtilIT.getUserFileAccessRequested(dataFileId, newUserApiToken); + getUserFileAccessRequestedResponse.then().assertThat().statusCode(OK.getStatusCode()); + + boolean userFileAccessRequested = JsonPath.from(getUserFileAccessRequestedResponse.body().asString()).getBoolean("data"); + assertFalse(userFileAccessRequested); + + // Request file access for the new user + Response requestFileAccessResponse = UtilIT.requestFileAccess(dataFileId, newUserApiToken); + requestFileAccessResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Call with new user and requested access file + getUserFileAccessRequestedResponse = UtilIT.getUserFileAccessRequested(dataFileId, newUserApiToken); + getUserFileAccessRequestedResponse.then().assertThat().statusCode(OK.getStatusCode()); + + userFileAccessRequested = JsonPath.from(getUserFileAccessRequestedResponse.body().asString()).getBoolean("data"); + assertTrue(userFileAccessRequested); + } + @Test public void testGetUserPermissionsOnFile() { // Call with valid file id diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 973642b1617..164f3a66ffb 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3329,6 +3329,12 @@ static Response getFileDataTables(String dataFileId, String apiToken) { .get("/api/files/" + dataFileId + "/dataTables"); } + static Response getUserFileAccessRequested(String dataFileId, String apiToken) { + return given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("/api/access/datafile/" + dataFileId + "/userFileAccessRequested"); + } + static Response getUserPermissionsOnFile(String dataFileId, String apiToken) { return given() .header(API_TOKEN_HTTP_HEADER, apiToken) From d33e8f53ffa55c1d67711ca3ea7a833574c63d7a Mon Sep 17 00:00:00 2001 From: GPortas Date: Mon, 11 Sep 2023 11:13:29 +0100 Subject: [PATCH 13/19] Added: hasBeenDeleted files API endpoint. Pending IT --- .../java/edu/harvard/iq/dataverse/api/Files.java | 12 ++++++++++++ .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Files.java b/src/main/java/edu/harvard/iq/dataverse/api/Files.java index 6712b68a09b..6d60de18c70 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Files.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Files.java @@ -107,6 +107,8 @@ public class Files extends AbstractApiBean { MakeDataCountLoggingServiceBean mdcLogService; @Inject GuestbookResponseServiceBean guestbookResponseService; + @Inject + DataFileServiceBean dataFileServiceBean; private static final Logger logger = Logger.getLogger(Files.class.getName()); @@ -870,4 +872,14 @@ public Response setFileCategories(@Context ContainerRequestContext crc, @PathPar } }, getRequestUser(crc)); } + + @GET + @AuthRequired + @Path("{id}/hasBeenDeleted") + public Response getHasBeenDeleted(@Context ContainerRequestContext crc, @PathParam("id") String dataFileId) { + return response(req -> { + DataFile dataFile = execCommand(new GetDataFileCommand(req, findDataFileOrDie(dataFileId))); + return ok(dataFileServiceBean.hasBeenDeleted(dataFile)); + }, getRequestUser(crc)); + } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 164f3a66ffb..d243d3c47f2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3374,4 +3374,16 @@ static Response setFileCategories(String dataFileId, String apiToken, List Date: Tue, 12 Sep 2023 12:01:28 +0100 Subject: [PATCH 14/19] Added: IT for getHasBeenDeleted Files API endpoint --- .../edu/harvard/iq/dataverse/api/FilesIT.java | 50 +++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index bb6b261c387..7f1ca4c8d70 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -37,10 +37,7 @@ import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.CoreMatchers.hasItem; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; public class FilesIT { @@ -2235,4 +2232,49 @@ public void testSetFileCategories() { .body("data.categories", hasItem(testCategory2)) .statusCode(OK.getStatusCode()); } + + @Test + public void testGetHasBeenDeleted() { + Response createUser = UtilIT.createRandomUser(); + createUser.then().assertThat().statusCode(OK.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + createDataverseResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + createDatasetResponse.then().assertThat().statusCode(CREATED.getStatusCode()); + int datasetId = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + // Upload test file + String pathToTestFile = "src/test/resources/images/coffeeshop.png"; + Response uploadResponse = UtilIT.uploadFileViaNative(Integer.toString(datasetId), pathToTestFile, Json.createObjectBuilder().build(), apiToken); + uploadResponse.then().assertThat().statusCode(OK.getStatusCode()); + + String dataFileId = uploadResponse.getBody().jsonPath().getString("data.files[0].dataFile.id"); + + // Publish dataverse and dataset + Response publishDataverseResponse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + publishDataverseResponse.then().assertThat().statusCode(OK.getStatusCode()); + + Response publishDatasetResponse = UtilIT.publishDatasetViaNativeApi(datasetId, "major", apiToken); + publishDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Assert that the file has not been deleted + Response getHasBeenDeletedResponse = UtilIT.getHasBeenDeleted(dataFileId, apiToken); + getHasBeenDeletedResponse.then().assertThat().statusCode(OK.getStatusCode()); + boolean fileHasBeenDeleted = JsonPath.from(getHasBeenDeletedResponse.body().asString()).getBoolean("data"); + assertFalse(fileHasBeenDeleted); + + // Delete test file + Response deleteFileInDatasetResponse = UtilIT.deleteFileInDataset(Integer.parseInt(dataFileId), apiToken); + deleteFileInDatasetResponse.then().assertThat().statusCode(OK.getStatusCode()); + + // Assert that the file has been deleted + getHasBeenDeletedResponse = UtilIT.getHasBeenDeleted(dataFileId, apiToken); + getHasBeenDeletedResponse.then().assertThat().statusCode(OK.getStatusCode()); + fileHasBeenDeleted = JsonPath.from(getHasBeenDeletedResponse.body().asString()).getBoolean("data"); + assertTrue(fileHasBeenDeleted); + } } From c224af6fc4c3c31fe52523c5eb67b770060f8b79 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 12 Sep 2023 12:19:50 +0100 Subject: [PATCH 15/19] Added: docs for userFileAccessRequested endpoint --- doc/sphinx-guides/source/api/dataaccess.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index d714c90372a..0bfd29ed79d 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -404,6 +404,18 @@ A curl example using an ``id``:: curl -H "X-Dataverse-key:$API_TOKEN" -X GET http://$SERVER/api/access/datafile/{id}/listRequests +User Has Requested Access to a File: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``/api/access/datafile/{id}/userFileAccessRequested`` + +This method returns true or false depending on whether or not the calling user has requested access to a particular file. + +A curl example using an ``id``:: + + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "http://$SERVER/api/access/datafile/{id}/userFileAccessRequested" + + Get User Permissions on a File: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 578fdc5e4f63414b85507b6605f787630cdbdfe4 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 12 Sep 2023 12:27:35 +0100 Subject: [PATCH 16/19] Added: docs for hasBeenDeleted endpoint --- doc/sphinx-guides/source/api/native-api.rst | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 8b7d34425bc..bfadf1eb9a4 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2872,6 +2872,43 @@ The fully expanded example above (without environment variables) looks like this If you are interested in download counts for multiple files, see :doc:`/api/metrics`. +File Has Been Deleted +~~~~~~~~~~~~~~~~~~~~~ + +Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version. + +A curl example using an ``ID`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=24 + + curl "$SERVER_URL/api/files/$ID/hasBeenDeleted" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl "https://demo.dataverse.org/api/files/24/hasBeenDeleted" + +A curl example using a ``PERSISTENT_ID`` + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_ID=doi:10.5072/FK2/AAA000 + + curl "$SERVER_URL/api/files/:persistentId/hasBeenDeleted?persistentId=$PERSISTENT_ID" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl "https://demo.dataverse.org/api/files/:persistentId/hasBeenDeleted?persistentId=doi:10.5072/FK2/AAA000" + Updating File Metadata ~~~~~~~~~~~~~~~~~~~~~~ From 85b9139f94c364cf25bdf3590c5b769543f81a0f Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 12 Sep 2023 12:34:19 +0100 Subject: [PATCH 17/19] Added: release notes for #9851 --- ...-payload-extension-new-file-access-endpoints.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md diff --git a/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md b/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md new file mode 100644 index 00000000000..f306ae2ab80 --- /dev/null +++ b/doc/release-notes/9851-datafile-payload-extension-new-file-access-endpoints.md @@ -0,0 +1,14 @@ +Implemented the following new endpoints: + +- userFileAccessRequested (/api/access/datafile/{id}/userFileAccessRequested): Returns true or false depending on whether or not the calling user has requested access to a particular file. + + +- hasBeenDeleted (/api/files/{id}/hasBeenDeleted): Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version. + + +In addition, the DataFile API payload has been extended to include the following fields: + +- tabularData: Boolean field to know if the DataFile is of tabular type + + +- fileAccessRequest: Boolean field to know if the file access requests are enabled on the Dataset (DataFile owner) From aacbc643b4fe7827c14edd1ebf97c82c6e0bdc53 Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 12 Sep 2023 12:42:22 +0100 Subject: [PATCH 18/19] Fixed: curl call examples in files API docs --- doc/sphinx-guides/source/api/native-api.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index bfadf1eb9a4..90f4ad4e800 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2846,13 +2846,13 @@ A curl example using an ``ID`` export SERVER_URL=https://demo.dataverse.org export ID=24 - curl "$SERVER_URL/api/files/$ID/downloadCount" + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/downloadCount" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl "https://demo.dataverse.org/api/files/24/downloadCount" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/24/downloadCount" A curl example using a ``PERSISTENT_ID`` @@ -2862,13 +2862,13 @@ A curl example using a ``PERSISTENT_ID`` export SERVER_URL=https://demo.dataverse.org export PERSISTENT_ID=doi:10.5072/FK2/AAA000 - curl "$SERVER_URL/api/files/:persistentId/downloadCount?persistentId=$PERSISTENT_ID" + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/downloadCount?persistentId=$PERSISTENT_ID" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl "https://demo.dataverse.org/api/files/:persistentId/downloadCount?persistentId=doi:10.5072/FK2/AAA000" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/:persistentId/downloadCount?persistentId=doi:10.5072/FK2/AAA000" If you are interested in download counts for multiple files, see :doc:`/api/metrics`. @@ -2885,13 +2885,13 @@ A curl example using an ``ID`` export SERVER_URL=https://demo.dataverse.org export ID=24 - curl "$SERVER_URL/api/files/$ID/hasBeenDeleted" + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/hasBeenDeleted" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl "https://demo.dataverse.org/api/files/24/hasBeenDeleted" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/24/hasBeenDeleted" A curl example using a ``PERSISTENT_ID`` @@ -2901,13 +2901,13 @@ A curl example using a ``PERSISTENT_ID`` export SERVER_URL=https://demo.dataverse.org export PERSISTENT_ID=doi:10.5072/FK2/AAA000 - curl "$SERVER_URL/api/files/:persistentId/hasBeenDeleted?persistentId=$PERSISTENT_ID" + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/hasBeenDeleted?persistentId=$PERSISTENT_ID" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl "https://demo.dataverse.org/api/files/:persistentId/hasBeenDeleted?persistentId=doi:10.5072/FK2/AAA000" + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/:persistentId/hasBeenDeleted?persistentId=doi:10.5072/FK2/AAA000" Updating File Metadata ~~~~~~~~~~~~~~~~~~~~~~ From d9b3f547a14bd882e252ddc9d8060a7f40bfff3d Mon Sep 17 00:00:00 2001 From: GPortas Date: Tue, 12 Sep 2023 17:24:11 +0100 Subject: [PATCH 19/19] Fixed: null check for DataFile owner in JsonPrinter --- .../edu/harvard/iq/dataverse/util/json/JsonPrinter.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index 2b04bb3f657..e5cd72ff5fc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -691,8 +691,11 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo .add("tabularData", df.isTabularData()) .add("tabularTags", getTabularFileTags(df)) .add("creationDate", df.getCreateDateFormattedYYYYMMDD()) - .add("publicationDate", df.getPublicationDateFormattedYYYYMMDD()) - .add("fileAccessRequest", df.getOwner().isFileAccessRequest()); + .add("publicationDate", df.getPublicationDateFormattedYYYYMMDD()); + Dataset dfOwner = df.getOwner(); + if (dfOwner != null) { + builder.add("fileAccessRequest", dfOwner.isFileAccessRequest()); + } /* * The restricted state was not included prior to #9175 so to avoid backward * incompatability, it is now only added when generating json for the