diff --git a/.github/workflows/maven_cache_management.yml b/.github/workflows/maven_cache_management.yml new file mode 100644 index 00000000000..fedf63b7c54 --- /dev/null +++ b/.github/workflows/maven_cache_management.yml @@ -0,0 +1,101 @@ +name: Maven Cache Management + +on: + # Every push to develop should trigger cache rejuvenation (dependencies might have changed) + push: + branches: + - develop + # According to https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy + # all caches are deleted after 7 days of no access. Make sure we rejuvenate every 7 days to keep it available. + schedule: + - cron: '23 2 * * 0' # Run for 'develop' every Sunday at 02:23 UTC (3:23 CET, 21:23 ET) + # Enable manual cache management + workflow_dispatch: + # Delete branch caches once a PR is merged + pull_request: + types: + - closed + +env: + COMMON_CACHE_KEY: "dataverse-maven-cache" + COMMON_CACHE_PATH: "~/.m2/repository" + +jobs: + seed: + name: Drop and Re-Seed Local Repository + runs-on: ubuntu-latest + if: ${{ github.event_name != 'pull_request' }} + permissions: + # Write permission needed to delete caches + # See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id + actions: write + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Determine Java version from Parent POM + run: echo "JAVA_VERSION=$(grep '' modules/dataverse-parent/pom.xml | cut -f2 -d'>' | cut -f1 -d'<')" >> ${GITHUB_ENV} + - name: Set up JDK ${{ env.JAVA_VERSION }} + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: temurin + - name: Seed common cache + run: | + mvn -B -f modules/dataverse-parent dependency:go-offline dependency:resolve-plugins + # This non-obvious order is due to the fact that the download via Maven above will take a very long time (7-8 min). + # Jobs should not be left without a cache. Deleting and saving in one go leaves only a small chance for a cache miss. + - name: Drop common cache + run: | + gh extension install actions/gh-actions-cache + echo "🛒 Fetching list of cache keys" + cacheKeys=$(gh actions-cache list -R ${{ github.repository }} -B develop | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "🗑️ Deleting caches..." + for cacheKey in $cacheKeys + do + gh actions-cache delete $cacheKey -R ${{ github.repository }} -B develop --confirm + done + echo "✅ Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Save the common cache + uses: actions/cache@v4 + with: + path: ${{ env.COMMON_CACHE_PATH }} + key: ${{ env.COMMON_CACHE_KEY }} + enableCrossOsArchive: true + + # Let's delete feature branch caches once their PR is merged - we only have 10 GB of space before eviction kicks in + deplete: + name: Deplete feature branch caches + runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' }} + permissions: + # `actions:write` permission is required to delete caches + # See also: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-a-github-actions-cache-for-a-repository-using-a-cache-id + actions: write + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Cleanup caches + run: | + gh extension install actions/gh-actions-cache + + BRANCH=refs/pull/${{ github.event.pull_request.number }}/merge + echo "🛒 Fetching list of cache keys" + cacheKeysForPR=$(gh actions-cache list -R ${{ github.repository }} -B $BRANCH | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "🗑️ Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh actions-cache delete $cacheKey -R ${{ github.repository }} -B $BRANCH --confirm + done + echo "✅ Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/conf/solr/9.3.0/schema.xml b/conf/solr/9.3.0/schema.xml index 90e9287d659..521e7a7db72 100644 --- a/conf/solr/9.3.0/schema.xml +++ b/conf/solr/9.3.0/schema.xml @@ -157,7 +157,8 @@ - + + diff --git a/doc/release-notes/10015-RO-Crate-metadata-file.md b/doc/release-notes/10015-RO-Crate-metadata-file.md new file mode 100644 index 00000000000..4b018a634f7 --- /dev/null +++ b/doc/release-notes/10015-RO-Crate-metadata-file.md @@ -0,0 +1,10 @@ +Detection of mime-types based on a filename with extension and detection of the RO-Crate metadata files. + +From now on, filenames with extensions can be added into `MimeTypeDetectionByFileName.properties` file. Filenames added there will take precedence over simply recognizing files by extensions. For example, two new filenames are added into that file: +``` +ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +``` + +Therefore, files named `ro-crate-metadata.json` will be then detected as RO-Crated metadata files from now on, instead as generic `JSON` files. +For more information on the RO-Crate specifications, see https://www.researchobject.org/ro-crate diff --git a/doc/release-notes/10022_upload_redirect_without_tagging.md b/doc/release-notes/10022_upload_redirect_without_tagging.md new file mode 100644 index 00000000000..7ff17f08f4c --- /dev/null +++ b/doc/release-notes/10022_upload_redirect_without_tagging.md @@ -0,0 +1,5 @@ +If your S3 store does not support tagging and gives an error if you configure direct uploads, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. For more details see https://dataverse-guide--10029.org.readthedocs.build/en/10029/developers/big-data-support.html#s3-tags #10022 and #10029. + +## New config options + +- dataverse.files..disable-tagging diff --git a/doc/release-notes/10116-incomplete-metadata-label-setting.md b/doc/release-notes/10116-incomplete-metadata-label-setting.md new file mode 100644 index 00000000000..769100c3804 --- /dev/null +++ b/doc/release-notes/10116-incomplete-metadata-label-setting.md @@ -0,0 +1 @@ +Bug fixed for the ``incomplete metadata`` label being shown for published dataset with incomplete metadata in certain scenarios. This label will now be shown for draft versions of such datasets and published datasets that the user can edit. This label can also be made invisible for published datasets (regardless of edit rights) with the new option ``dataverse.ui.show-validity-label-when-published`` set to `false`. diff --git a/doc/release-notes/10242-add-feature-dv-api b/doc/release-notes/10242-add-feature-dv-api new file mode 100644 index 00000000000..5c786554ff9 --- /dev/null +++ b/doc/release-notes/10242-add-feature-dv-api @@ -0,0 +1 @@ +New api endpoints have been added to allow you to add or remove featured collections from a dataverse collection. diff --git a/doc/release-notes/10316_cvoc_http_headers.md b/doc/release-notes/10316_cvoc_http_headers.md new file mode 100644 index 00000000000..4b557383a2e --- /dev/null +++ b/doc/release-notes/10316_cvoc_http_headers.md @@ -0,0 +1,5 @@ +You are now able to add HTTP request headers required by the External Vocabulary Services you are implementing. + +A combined documentation can be found on pull request [#10404](https://github.com/IQSS/dataverse/pull/10404). + +For more information, see issue [#10316](https://github.com/IQSS/dataverse/issues/10316) and pull request [gddc/dataverse-external-vocab-support#19](https://github.com/gdcc/dataverse-external-vocab-support/pull/19). diff --git a/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md b/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md new file mode 100644 index 00000000000..e8840e9d4f7 --- /dev/null +++ b/doc/release-notes/10415-fix-api-performance-issues-on-large-datasets.md @@ -0,0 +1,4 @@ +For scenarios involving API calls related to large datasets (Numerous files, for example: ~10k) it has been optimized: + +- The search API endpoint. +- The permission checking logic present in PermissionServiceBean. diff --git a/doc/release-notes/10425-add-MIT-License.md b/doc/release-notes/10425-add-MIT-License.md new file mode 100644 index 00000000000..95d6fb38ded --- /dev/null +++ b/doc/release-notes/10425-add-MIT-License.md @@ -0,0 +1,3 @@ +A new file has been added to import the MIT License to Dataverse: licenseMIT.json. + +Documentation has been added to explain the procedure for adding new licenses to the guides. diff --git a/doc/release-notes/10477-metadatablocks-api-extension-input-levels.md b/doc/release-notes/10477-metadatablocks-api-extension-input-levels.md new file mode 100644 index 00000000000..77cc7f59773 --- /dev/null +++ b/doc/release-notes/10477-metadatablocks-api-extension-input-levels.md @@ -0,0 +1,3 @@ +Changed ``api/dataverses/{id}/metadatablocks`` so that setting the query parameter ``onlyDisplayedOnCreate=true`` also returns metadata blocks with dataset field type input levels configured as required on the General Information page of the collection, in addition to the metadata blocks and their fields with the property ``displayOnCreate=true`` (which was the original behavior). + +A new endpoint ``api/dataverses/{id}/inputLevels`` has been created for updating the dataset field type input levels of a collection via API. diff --git a/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md b/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md new file mode 100644 index 00000000000..5293c7267d0 --- /dev/null +++ b/doc/release-notes/10491-add-isreleased-to-get-dataverse-response.md @@ -0,0 +1,22 @@ +The Dataverse object returned by /api/dataverses has been extended to include "isReleased": {boolean}. +```javascript +{ + "status": "OK", + "data": { + "id": 32, + "alias": "dv6f645bb5", + "name": "dv6f645bb5", + "dataverseContacts": [ + { + "displayOrder": 0, + "contactEmail": "54180268@mailinator.com" + } + ], + "permissionRoot": true, + "dataverseType": "UNCATEGORIZED", + "ownerId": 1, + "creationDate": "2024-04-12T18:05:59Z", + "isReleased": true + } +} +``` \ No newline at end of file diff --git a/doc/release-notes/6.2-release-notes.md b/doc/release-notes/6.2-release-notes.md index f694703f0a6..0f076d32cf8 100644 --- a/doc/release-notes/6.2-release-notes.md +++ b/doc/release-notes/6.2-release-notes.md @@ -417,12 +417,16 @@ In the following commands we assume that Payara 6 is installed in `/usr/local/pa As noted above, deployment of the war file might take several minutes due a database migration script required for the new storage quotas feature. -6\. Restart Payara +6\. For installations with internationalization: + +- Please remember to update translations via [Dataverse language packs](https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs). + +7\. Restart Payara - `service payara stop` - `service payara start` -7\. Update the following Metadata Blocks to reflect the incremental improvements made to the handling of core metadata fields: +8\. Update the following Metadata Blocks to reflect the incremental improvements made to the handling of core metadata fields: ``` wget https://github.com/IQSS/dataverse/releases/download/v6.2/geospatial.tsv @@ -442,7 +446,7 @@ wget https://github.com/IQSS/dataverse/releases/download/v6.2/biomedical.tsv curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/biomedical.tsv ``` -8\. For installations with custom or experimental metadata blocks: +9\. For installations with custom or experimental metadata blocks: - Stop Solr instance (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.2/installation/prerequisites.html#solr-init-script)) @@ -455,7 +459,7 @@ curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/ta - Restart Solr instance (usually `service solr restart` depending on solr/OS) -9\. Reindex Solr: +10\. Reindex Solr: For details, see https://guides.dataverse.org/en/6.2/admin/solr-search-index.html but here is the reindex command: diff --git a/doc/release-notes/8655-re-add-cell-counting-biomedical-tsv.md b/doc/release-notes/8655-re-add-cell-counting-biomedical-tsv.md new file mode 100644 index 00000000000..295f206871f --- /dev/null +++ b/doc/release-notes/8655-re-add-cell-counting-biomedical-tsv.md @@ -0,0 +1,12 @@ +## Release Highlights + +### Life Science Metadata + +Re-adding value `cell counting` to Life Science metadatablock's Measurement Type vocabularies accidentally removed in `v5.1`. + +## Upgrade Instructions + +### Update the Life Science metadata block + +- `wget https://github.com/IQSS/dataverse/releases/download/v6.3/biomedical.tsv` +- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @biomedical.tsv -H "Content-type: text/tab-separated-values"` \ No newline at end of file diff --git a/doc/release-notes/8936-more-than-50000-entries-in-sitemap.md b/doc/release-notes/8936-more-than-50000-entries-in-sitemap.md new file mode 100644 index 00000000000..7b367e328c1 --- /dev/null +++ b/doc/release-notes/8936-more-than-50000-entries-in-sitemap.md @@ -0,0 +1,11 @@ +Dataverse can now handle more than 50,000 items when generating sitemap files, splitting the content across multiple files to comply with the Sitemap protocol. + +For details see https://dataverse-guide--10321.org.readthedocs.build/en/10321/installation/config.html#creating-a-sitemap-and-submitting-it-to-search-engines #8936 and #10321. + +## Upgrade instructions + +If your installation has more than 50,000 entries, you should re-submit your sitemap URL to Google or other search engines. The file in the URL will change from ``sitemap.xml`` to ``sitemap_index.xml``. + +As explained at https://dataverse-guide--10321.org.readthedocs.build/en/10321/installation/config.html#creating-a-sitemap-and-submitting-it-to-search-engines this is the command for regenerating your sitemap: + +`curl -X POST http://localhost:8080/api/admin/sitemap` diff --git a/doc/release-notes/9375-retention-period.md b/doc/release-notes/9375-retention-period.md new file mode 100644 index 00000000000..a088cabf138 --- /dev/null +++ b/doc/release-notes/9375-retention-period.md @@ -0,0 +1,8 @@ +The Dataverse Software now supports file-level retention periods. The ability to set retention periods, with a minimum duration (in months), can be configured by a Dataverse installation administrator. For more information, see the [Retention Periods section](https://guides.dataverse.org/en/6.3/user/dataset-management.html#retention-periods) of the Dataverse Software Guides. + +- Users can configure a specific retention period, defined by an end date and a short reason, on a set of selected files or an individual file, by selecting the 'Retention Period' menu item and entering information in a popup dialog. Retention Periods can only be set, changed, or removed before a file has been published. After publication, only Dataverse installation administrators can make changes, using an API. + +- After the retention period expires, files can not be previewed or downloaded (as if restricted, with no option to allow access requests). The file (landing) page and all the metadata remains available. + + +Release notes should mention that a Solr schema update is needed. diff --git a/doc/release-notes/9887-new-superuser-status-endpoint.md b/doc/release-notes/9887-new-superuser-status-endpoint.md new file mode 100644 index 00000000000..01b1f539f7a --- /dev/null +++ b/doc/release-notes/9887-new-superuser-status-endpoint.md @@ -0,0 +1 @@ +The existing API endpoint for toggling the superuser status of a user has been deprecated in favor of a new API endpoint that allows you to explicitly and idempotently set the status as true or false. For details, see [the guides](https://dataverse-guide--10440.org.readthedocs.build/en/10440/api/native-api.html), #9887 and #10440. \ No newline at end of file diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 66911aa0ad1..00d1e8a7831 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -552,6 +552,8 @@ Great care must be taken when reloading a metadata block. Matching is done on fi The ability to reload metadata blocks means that SQL update scripts don't need to be written for these changes. See also the :doc:`/developers/sql-upgrade-scripts` section of the Developer Guide. +.. _using-external-vocabulary-services: + Using External Vocabulary Services ---------------------------------- @@ -577,9 +579,9 @@ In general, the external vocabulary support mechanism may be a better choice for The specifics of the user interface for entering/selecting a vocabulary term and how that term is then displayed are managed by third-party Javascripts. The initial Javascripts that have been created provide auto-completion, displaying a list of choices that match what the user has typed so far, but other interfaces, such as displaying a tree of options for a hierarchical vocabulary, are possible. Similarly, existing scripts do relatively simple things for displaying a term - showing the term's name in the appropriate language and providing a link to an external URL with more information, but more sophisticated displays are possible. -Scripts supporting use of vocabularies from services supporting the SKOMOS protocol (see https://skosmos.org) and retrieving ORCIDs (from https://orcid.org) are available https://github.com/gdcc/dataverse-external-vocab-support. (Custom scripts can also be used and community members are encouraged to share new scripts through the dataverse-external-vocab-support repository.) +Scripts supporting use of vocabularies from services supporting the SKOMOS protocol (see https://skosmos.org), retrieving ORCIDs (from https://orcid.org), and using ROR (https://ror.org/) are available https://github.com/gdcc/dataverse-external-vocab-support. (Custom scripts can also be used and community members are encouraged to share new scripts through the dataverse-external-vocab-support repository.) -Configuration involves specifying which fields are to be mapped, whether free-text entries are allowed, which vocabulary(ies) should be used, what languages those vocabulary(ies) are available in, and several service protocol and service instance specific parameters. +Configuration involves specifying which fields are to be mapped, whether free-text entries are allowed, which vocabulary(ies) should be used, what languages those vocabulary(ies) are available in, and several service protocol and service instance specific parameters, including the ability to send HTTP headers on calls to the service. These are all defined in the :ref:`:CVocConf <:CVocConf>` setting as a JSON array. Details about the required elements as well as example JSON arrays are available at https://github.com/gdcc/dataverse-external-vocab-support, along with an example metadata block that can be used for testing. The scripts required can be hosted locally or retrieved dynamically from https://gdcc.github.io/ (similar to how dataverse-previewers work). diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index e33204e6354..db994a629b3 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -7,6 +7,11 @@ This API changelog is experimental and we would love feedback on its usefulness. :local: :depth: 1 +v6.3 +---- + +- **/api/admin/superuser/{identifier}**: The POST endpoint that toggles superuser status has been deprecated in favor of a new PUT endpoint that allows you to specify true or false. See :ref:`set-superuser-status`. + v6.2 ---- diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 0b9b042b052..f22f8727fb0 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -585,6 +585,70 @@ The fully expanded example above (without environment variables) looks like this Note: you must have "Add Dataset" permission in the given collection to invoke this endpoint. +List Featured Collections for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The response is a JSON array of the alias strings of the featured collections of a given Dataverse collection identified by ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/dataverses/$ID/featured" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/dataverses/root/featured" + + +Set Featured Collections for a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add featured collections to a given Dataverse collection identified by ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$ID/featured" --upload-file collection-alias.json + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/dataverses/root/featured" --upload-file collection-alias.json + +Where collection-alias.json contains a JSON encoded list of collections aliases to be featured (e.g. ``["collection1-alias","collection2-alias"]``). + +Note: You must have "Edit Dataverse" permission in the given Dataverse to invoke this endpoint. You may only feature collections that are published and owned by or linked to the featuring collection. Also, using this endpoint will only add new featured collections it will not remove collections that have already been featured. + +Remove Featured Collections from a Dataverse Collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Remove featured collections from a given Dataverse collection identified by ``id``: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + + curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/dataverses/$ID/featured" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/featured" + +Note: You must have "Edit Dataverse" permission in the given Dataverse to invoke this endpoint. + .. _create-dataset-command: Create a Dataset in a Dataverse Collection @@ -834,7 +898,46 @@ The following attributes are supported: * ``filePIDsEnabled`` ("true" or "false") Restricted to use by superusers and only when the :ref:`:AllowEnablingFilePIDsPerCollection <:AllowEnablingFilePIDsPerCollection>` setting is true. Enables or disables registration of file-level PIDs in datasets within the collection (overriding the instance-wide setting). .. _collection-storage-quotas: - + +Update Collection Input Levels +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Updates the dataset field type input levels in a collection. + +Please note that this endpoint overwrites all the input levels of the collection page, so if you want to keep the existing ones, you will need to add them to the JSON request body. + +If one of the input levels corresponds to a dataset field type belonging to a metadata block that does not exist in the collection, the metadata block will be added to the collection. + +This endpoint expects a JSON with the following format:: + + [ + { + "datasetFieldTypeName": "datasetFieldTypeName1", + "required": true, + "include": true + }, + { + "datasetFieldTypeName": "datasetFieldTypeName2", + "required": true, + "include": true + } + ] + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export ID=root + export JSON='[{"datasetFieldTypeName":"geographicCoverage", "required":true, "include":true}, {"datasetFieldTypeName":"country", "required":true, "include":true}]' + + curl -X PUT -H "X-Dataverse-key: $API_TOKEN" -H "Content-Type:application/json" "$SERVER_URL/api/dataverses/$ID/inputLevels" -d "$JSON" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X PUT -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -H "Content-Type:application/json" "https://demo.dataverse.org/api/dataverses/root/inputLevels" -d '[{"datasetFieldTypeName":"geographicCoverage", "required":true, "include":false}, {"datasetFieldTypeName":"country", "required":true, "include":false}]' + Collection Storage Quotas ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1164,6 +1267,7 @@ File access filtering is also optionally supported. In particular, by the follow * ``Restricted`` * ``EmbargoedThenRestricted`` * ``EmbargoedThenPublic`` +* ``RetentionPeriodExpired`` If no filter is specified, the files will match all of the above categories. @@ -1213,7 +1317,7 @@ The returned file counts are based on different criteria: - Per content type - Per category name - Per tabular tag name -- Per access status (Possible values: Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic) +- Per access status (Possible values: Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic, RetentionPeriodExpired) .. code-block:: bash @@ -1267,6 +1371,7 @@ File access filtering is also optionally supported. In particular, by the follow * ``Restricted`` * ``EmbargoedThenRestricted`` * ``EmbargoedThenPublic`` +* ``RetentionPeriodExpired`` If no filter is specified, the files will match all of the above categories. @@ -1949,7 +2054,7 @@ The fully expanded example above (without environment variables) looks like this .. _cleanup-storage-api: -Cleanup storage of a Dataset +Cleanup Storage of a Dataset ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This is an experimental feature and should be tested on your system before using it in production. @@ -2082,6 +2187,7 @@ File access filtering is also optionally supported. In particular, by the follow * ``Restricted`` * ``EmbargoedThenRestricted`` * ``EmbargoedThenPublic`` +* ``RetentionPeriodExpired`` If no filter is specified, the files will match all of the above categories. @@ -2519,7 +2625,38 @@ The API call requires a Json body that includes the list of the fileIds that the export JSON='{"fileIds":[300,301]}' curl -H "X-Dataverse-key: $API_TOKEN" -H "Content-Type:application/json" "$SERVER_URL/api/datasets/:persistentId/files/actions/:unset-embargo?persistentId=$PERSISTENT_IDENTIFIER" -d "$JSON" - + +Set a Retention Period on Files in a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``/api/datasets/$dataset-id/files/actions/:set-retention`` can be used to set a retention period on one or more files in a dataset. Retention periods can be set on files that are only in a draft dataset version (and are not in any previously published version) by anyone who can edit the dataset. The same API call can be used by a superuser to add a retention period to files that have already been released as part of a previously published dataset version. + +The API call requires a Json body that includes the retention period's end date (dateUnavailable), a short reason (optional), and a list of the fileIds that the retention period should be set on. The dateUnavailable must be after the current date and the duration (dateUnavailable - today's date) must be larger than the value specified by the :ref:`:MinRetentionDurationInMonths` setting. All files listed must be in the specified dataset. For example: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV + export JSON='{"dateUnavailable":"2051-12-31", "reason":"Standard project retention period", "fileIds":[300,301,302]}' + + curl -H "X-Dataverse-key: $API_TOKEN" -H "Content-Type:application/json" "$SERVER_URL/api/datasets/:persistentId/files/actions/:set-retention?persistentId=$PERSISTENT_IDENTIFIER" -d "$JSON" + +Remove a Retention Period on Files in a Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``/api/datasets/$dataset-id/files/actions/:unset-retention`` can be used to remove a retention period on one or more files in a dataset. Retention periods can be removed from files that are only in a draft dataset version (and are not in any previously published version) by anyone who can edit the dataset. The same API call can be used by a superuser to remove retention periods from files that have already been released as part of a previously published dataset version. + +The API call requires a Json body that includes the list of the fileIds that the retention period should be removed from. All files listed must be in the specified dataset. For example: + +.. code-block:: bash + + export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/7U7YBV + export JSON='{"fileIds":[300,301]}' + + curl -H "X-Dataverse-key: $API_TOKEN" -H "Content-Type:application/json" "$SERVER_URL/api/datasets/:persistentId/files/actions/:unset-retention?persistentId=$PERSISTENT_IDENTIFIER" -d "$JSON" .. _Archival Status API: @@ -5412,12 +5549,46 @@ Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST "https://demo.datavers This action changes the identifier of user johnsmith to jsmith. -Make User a SuperUser -~~~~~~~~~~~~~~~~~~~~~ +Toggle Superuser Status +~~~~~~~~~~~~~~~~~~~~~~~ -Toggles superuser mode on the ``AuthenticatedUser`` whose ``identifier`` (without the ``@`` sign) is passed. :: +Toggle the superuser status of a user. - POST http://$SERVER/api/admin/superuser/$identifier +.. note:: This endpoint is deprecated as explained in :doc:`/api/changelog`. Please use the :ref:`set-superuser-status` endpoint instead. + +.. code-block:: bash + + export SERVER_URL=http://localhost:8080 + export USERNAME=jdoe + curl -X POST "$SERVER_URL/api/admin/superuser/$USERNAME" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X POST "http://localhost:8080/api/admin/superuser/jdoe" + +.. _set-superuser-status: + +Set Superuser Status +~~~~~~~~~~~~~~~~~~~~ + +Specify the superuser status of a user with a boolean value (``true`` or ``false``). + +.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of ``export`` below. + +.. code-block:: bash + + export SERVER_URL=http://localhost:8080 + export USERNAME=jdoe + export IS_SUPERUSER=true + curl -X PUT "$SERVER_URL/api/admin/superuser/$USERNAME" -d "$IS_SUPERUSER" + +The fully expanded example above (without environment variables) looks like this: + +.. code-block:: bash + + curl -X PUT "http://localhost:8080/api/admin/superuser/jdoe" -d true .. _delete-a-user: @@ -5549,6 +5720,17 @@ List permissions a user (based on API Token used) has on a Dataverse collection The ``$identifier`` can be a Dataverse collection alias or database id or a dataset persistent ID or database id. +.. note:: Datasets can be selected using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the dataset is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. + +Example: List permissions a user (based on API Token used) has on a dataset whose DOI is *10.5072/FK2/J8SJZB*: + +.. code-block:: bash + + export SERVER_URL=https://demo.dataverse.org + export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB + + curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/admin/permissions/:persistentId?persistentId=$PERSISTENT_IDENTIFIER" + Show Role Assignee ~~~~~~~~~~~~~~~~~~ @@ -5781,7 +5963,7 @@ Superusers can add a new license by posting a JSON file adapted from this exampl .. code-block:: bash export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - curl -X POST -H 'Content-Type: application/json' -H "X-Dataverse-key:$API_TOKEN" --data-binary @add-license.json "$SERVER_URL/api/licenses" + curl -X POST -H 'Content-Type: application/json' -H "X-Dataverse-key:$API_TOKEN" --upload-file add-license.json "$SERVER_URL/api/licenses" Superusers can change whether an existing license is active (usable for new dataset versions) or inactive (only allowed on already-published versions) specified by the license ``$ID``: diff --git a/doc/sphinx-guides/source/container/running/demo.rst b/doc/sphinx-guides/source/container/running/demo.rst index 2991c677618..0508639c616 100644 --- a/doc/sphinx-guides/source/container/running/demo.rst +++ b/doc/sphinx-guides/source/container/running/demo.rst @@ -220,4 +220,4 @@ Your feedback is extremely valuable to us! To let us know what you think, please Getting Help ------------ -Please do not be shy about reaching out for help. We very much want you to have a pleasant demo or evaluation experience. For ways to contact us, please see See :ref:`getting-help-containers`. +Please do not be shy about reaching out for help. We very much want you to have a pleasant demo or evaluation experience. For ways to contact us, please see :ref:`getting-help-containers`. diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 8d891e63317..5ea97029271 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -81,7 +81,12 @@ with the contents of the file cors.json as follows: Alternatively, you can enable CORS using the AWS S3 web interface, using json-encoded rules as in the example above. -Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 Tags to aid in identifying/removing such files. Upon upload, files are given a "dv-state":"temp" tag which is removed when the dataset changes are saved and the new file(s) are added in the Dataverse installation. Note that not all S3 implementations support Tags: Minio does not. WIth such stores, direct upload works, but Tags are not used. +.. _s3-tags-and-direct-upload: + +S3 Tags and Direct Upload +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 tags to aid in identifying/removing such files. Upon upload, files are given a "dv-state":"temp" tag which is removed when the dataset changes are saved and new files are added in the Dataverse installation. Note that not all S3 implementations support tags. Minio, for example, does not. With such stores, direct upload may not work and you might need to disable tagging. For details, see :ref:`s3-tagging` in the Installation Guide. Trusted Remote Storage with the ``remote`` Store Type ----------------------------------------------------- @@ -116,7 +121,7 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx export SERVER_URL=https://demo.dataverse.org export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV - export JSON_DATA="{'description':'My description.','directoryLabel':'data/subdir1','categories':['Data'], 'restrict':'false', 'storageIdentifier':'trs://images/dataverse_project_logo.svg', 'fileName':'dataverse_logo.svg', 'mimeType':'image/svg+xml', 'checksum': {'@type': 'SHA-1', '@value': '123456'}}" + export JSON_DATA='{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"trs://images/dataverse_project_logo.svg", "fileName":"dataverse_logo.svg", "mimeType":"image/svg+xml", "checksum": {"@type": "SHA-1", "@value": "123456"}}' curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA" diff --git a/doc/sphinx-guides/source/developers/coding-style.rst b/doc/sphinx-guides/source/developers/coding-style.rst index 9da7836bbf4..2a1c0d5d232 100755 --- a/doc/sphinx-guides/source/developers/coding-style.rst +++ b/doc/sphinx-guides/source/developers/coding-style.rst @@ -18,6 +18,11 @@ Tabs vs. Spaces Don't use tabs. Use 4 spaces. +Imports +^^^^^^^ + +Wildcard imports are neither encouraged nor discouraged. + Braces Placement ^^^^^^^^^^^^^^^^ diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst index 0040c1fd3f0..33b8e434e6e 100644 --- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst +++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst @@ -79,6 +79,12 @@ In the single part case, only one call to the supplied URL is required: curl -i -H 'x-amz-tagging:dv-state=temp' -X PUT -T "" +Or, if you have disabled S3 tagging (see :ref:`s3-tagging`), you should omit the header like this: + +.. code-block:: bash + + curl -i -X PUT -T "" + Note that without the ``-i`` flag, you should not expect any output from the command above. With the ``-i`` flag, you should expect to see a "200 OK" response. In the multipart case, the client must send each part and collect the 'eTag' responses from the server. The calls for this are the same as the one for the single part case except that each call should send a slice of the total file, with the last part containing the remaining bytes. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index ff786e900cc..907631e6236 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1189,12 +1189,31 @@ Larger installations may want to increase the number of open S3 connections allo ``./asadmin create-jvm-options "-Ddataverse.files..connection-pool-size=4096"`` +.. _s3-tagging: + +S3 Tagging +########## + +By default, when direct upload to an S3 store is configured, Dataverse will place a ``temp`` tag on the file being uploaded for an easier cleanup in case the file is not added to the dataset after upload (e.g., if the user cancels the operation). (See :ref:`s3-tags-and-direct-upload`.) +If your S3 store does not support tagging and gives an error when direct upload is configured, you can disable the tagging by using the ``dataverse.files..disable-tagging`` JVM option. For example: + +``./asadmin create-jvm-options "-Ddataverse.files..disable-tagging=true"`` + +Disabling the ``temp`` tag makes it harder to identify abandoned files that are not used by your Dataverse instance (i.e. one cannot search for the ``temp`` tag in a delete script). These should still be removed to avoid wasting storage space. To clean up these files and any other leftover files, regardless of whether the ``temp`` tag is applied, you can use the :ref:`cleanup-storage-api` API endpoint. + +Note that if you disable tagging, you should should omit the ``x-amz-tagging:dv-state=temp`` header when using the :doc:`/developers/s3-direct-upload-api`, as noted in that section. + +Finalizing S3 Configuration +########################### + In case you would like to configure Dataverse to use a custom S3 service instead of Amazon S3 services, please add the options for the custom URL and region as documented below. Please read above if your desired combination has been tested already and what other options have been set for a successful integration. Lastly, go ahead and restart your Payara server. With Dataverse deployed and the site online, you should be able to upload datasets and data files and see the corresponding files in your S3 bucket. Within a bucket, the folder structure emulates that found in local file storage. +.. _list-of-s3-storage-options: + List of S3 Storage Options ########################## @@ -1222,6 +1241,7 @@ List of S3 Storage Options dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` + dataverse.files..disable-tagging ``true``/``false`` Do not place the ``temp`` tag when redirecting the upload to the S3 server. ``false`` =========================================== ================== =================================================================================== ============= .. table:: @@ -1838,6 +1858,31 @@ JSON files for `Creative Commons licenses ` +- :download:`licenseApache-2.0.json <../../../../scripts/api/data/licenses/licenseApache-2.0.json>` + +Contributing to the Collection of Standard Licenses Above +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you do not find the license JSON you need above, you are encouraged to contribute it to this documentation. Following the Dataverse 6.2 release, we have standardized on the following procedure: + +- Look for the license at https://spdx.org/licenses/ +- ``cd scripts/api/data/licenses`` +- Copy an existing license as a starting point. +- Name your file using the SPDX identifier. For example, if the identifier is ``Apache-2.0``, you should name your file ``licenseApache-2.0.json``. +- For the ``name`` field, use the "short identifier" from the SPDX landing page (e.g. ``Apache-2.0``). +- For the ``description`` field, use the "full name" from the SPDX landing page (e.g. ``Apache License 2.0``). +- For the ``uri`` field, we encourage you to use the same resource that DataCite uses, which is often the same as the first "Other web pages for this license" on the SPDX page for the license. When these differ, or there are other concerns about the URI DataCite uses, please reach out to the community to see if a consensus can be reached. +- For the ``active`` field, put ``true``. +- For the ``sortOrder`` field, put the next sequential number after checking previous files with ``grep sortOrder scripts/api/data/licenses/*``. + +Note that prior to Dataverse 6.2, various license above have been added that do not adhere perfectly with this procedure. For example, the ``name`` for the CC0 license is ``CC0 1.0`` (no dash) rather than ``CC0-1.0`` (with a dash). We are keeping the existing names for backward compatibility. For more on standarizing license configuration, see https://github.com/IQSS/dataverse/issues/8512 + Adding Custom Licenses ^^^^^^^^^^^^^^^^^^^^^^ @@ -2131,26 +2176,51 @@ If you are not fronting Payara with Apache you'll need to prevent Payara from se Creating a Sitemap and Submitting it to Search Engines ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Search engines have an easier time indexing content when you provide them a sitemap. The Dataverse Software sitemap includes URLs to all published Dataverse collections and all published datasets that are not harvested or deaccessioned. +Creating a Sitemap +################## + +Search engines have an easier time indexing content when you provide them a sitemap. Dataverse can generate a sitemap that includes URLs to all published collections and all published datasets that are not harvested or deaccessioned. Create or update your sitemap by adding the following curl command to cron to run nightly or as you see fit: ``curl -X POST http://localhost:8080/api/admin/sitemap`` -This will create or update a file in the following location unless you have customized your installation directory for Payara: +On a Dataverse installation with many datasets, the creation or updating of the sitemap can take a while. You can check Payara's server.log file for "BEGIN updateSiteMap" and "END updateSiteMap" lines to know when the process started and stopped and any errors in between. + +For compliance with the `Sitemap protocol `_, the generated sitemap will be a single file with 50,000 items or fewer or it will be split into multiple files. + +Single Sitemap File +################### + +If you have 50,000 items or fewer, a single sitemap will be generated in the following location (unless you have customized your installation directory for Payara): ``/usr/local/payara6/glassfish/domains/domain1/docroot/sitemap/sitemap.xml`` -On Dataverse installation with many datasets, the creation or updating of the sitemap can take a while. You can check Payara's server.log file for "BEGIN updateSiteMap" and "END updateSiteMap" lines to know when the process started and stopped and any errors in between. +Once the sitemap has been generated in the location above, it will be served at ``/sitemap.xml`` like this: https://demo.dataverse.org/sitemap.xml -https://demo.dataverse.org/sitemap.xml is the sitemap URL for the Dataverse Project Demo site and yours should be similar. +Multiple Sitemap Files (Sitemap Index File) +########################################### -Once the sitemap has been generated and placed in the domain docroot directory, it will become available to the outside callers at /sitemap/sitemap.xml; it will also be accessible at /sitemap.xml (via a *pretty-faces* rewrite rule). Some search engines will be able to find it at this default location. Some, **including Google**, need to be **specifically instructed** to retrieve it. +According to the `Sitemaps.org protocol `_, a sitemap file must have no more than 50,000 URLs and must be no larger than 50MiB. In this case, the protocol instructs you to create a sitemap index file called ``sitemap_index.xml`` (instead of ``sitemap.xml``), which references multiple sitemap files named ``sitemap1.xml``, ``sitemap2.xml``, etc. These referenced files are also generated in the same place as other sitemap files (``domain1/docroot/sitemap``) and there will be as many files as necessary to contain the URLs of collections and datasets present in your installation, while respecting the limit of 50,000 URLs per file. -One way to submit your sitemap URL to Google is by using their "Search Console" (https://search.google.com/search-console). In order to use the console, you will need to authenticate yourself as the owner of your Dataverse site. Various authentication methods are provided; but if you are already using Google Analytics, the easiest way is to use that account. Make sure you are logged in on Google with the account that has the edit permission on your Google Analytics property; go to the search console and enter the root URL of your Dataverse installation, then choose Google Analytics as the authentication method. Once logged in, click on "Sitemaps" in the menu on the left. (todo: add a screenshot?) Consult `Google's "submit a sitemap" instructions`_ for more information; and/or similar instructions for other search engines. +If you have over 50,000 items, a sitemap index file will be generated in the following location (unless you have customized your installation directory for Payara): -.. _Google's "submit a sitemap" instructions: https://support.google.com/webmasters/answer/183668 +``/usr/local/payara6/glassfish/domains/domain1/docroot/sitemap/sitemap_index.xml`` + +Once the sitemap has been generated in the location above, it will be served at ``/sitemap_index.xml`` like this: https://demo.dataverse.org/sitemap_index.xml +Note that the sitemap is also available at (for example) https://demo.dataverse.org/sitemap/sitemap_index.xml and in that ``sitemap`` directory you will find the files it references such as ``sitemap1.xml``, ``sitemap2.xml``, etc. + +Submitting Your Sitemap to Search Engines +######################################### + +Some search engines will be able to find your sitemap file at ``/sitemap.xml`` or ``/sitemap_index.xml``, but others, **including Google**, need to be **specifically instructed** to retrieve it. + +As described above, Dataverse will automatically detect whether you need to create a single sitemap file or several files and generate them for you. However, when submitting your sitemap file to Google or other search engines, you must be careful to supply the correct file name (``sitemap.xml`` or ``sitemap_index.xml``) depending on your situation. + +One way to submit your sitemap URL to Google is by using their "Search Console" (https://search.google.com/search-console). In order to use the console, you will need to authenticate yourself as the owner of your Dataverse site. Various authentication methods are provided; but if you are already using Google Analytics, the easiest way is to use that account. Make sure you are logged in on Google with the account that has the edit permission on your Google Analytics property; go to the Search Console and enter the root URL of your Dataverse installation, then choose Google Analytics as the authentication method. Once logged in, click on "Sitemaps" in the menu on the left. Consult `Google's "submit a sitemap" instructions`_ for more information. + +.. _Google's "submit a sitemap" instructions: https://support.google.com/webmasters/answer/183668 Putting Your Dataverse Installation on the Map at dataverse.org +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -2875,6 +2945,24 @@ Defaults to ``false``. Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_API_ALLOW_INCOMPLETE_METADATA``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. +.. _dataverse.ui.show-validity-label-when-published: + +dataverse.ui.show-validity-label-when-published ++++++++++++++++++++++++++++++++++++++++++++++++ + +Even when you do not allow incomplete metadata to be saved in dataverse, some metadata may end up being incomplete, e.g., after making a metadata field mandatory. Datasets where that field is +not filled out, become incomplete, and therefore can be labeled with the ``incomplete metadata`` label. By default, this label is only shown for draft datasets and published datasets that the +user can edit. This option can be disabled by setting it to ``false`` where only draft datasets with incomplete metadata will have that label. When disabled, all published dataset will not have +that label. Note that you need to reindex the datasets after changing the metadata definitions. Reindexing will update the labels and other dataset information according to the new situation. + +When enabled (by default), published datasets with incomplete metadata will have an ``incomplete metadata`` label attached to them, but only for the datasets that the user can edit. +You can list these datasets, for example, with the validity of metadata filter shown in "My Data" page that can be turned on by enabling the :ref:`dataverse.ui.show-validity-filter` option. + +Defaults to ``true``. + +Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable +``DATAVERSE_API_SHOW_LABEL_FOR_INCOMPLETE_WHEN_PUBLISHED``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. + .. _dataverse.signposting.level1-author-limit: dataverse.signposting.level1-author-limit @@ -3072,6 +3160,8 @@ Defaults to ``false``. Can also be set via any `supported MicroProfile Config API source`_, e.g. the environment variable ``DATAVERSE_UI_ALLOW_REVIEW_FOR_INCOMPLETE``. Will accept ``[tT][rR][uU][eE]|1|[oO][nN]`` as "true" expressions. +.. _dataverse.ui.show-validity-filter: + dataverse.ui.show-validity-filter +++++++++++++++++++++++++++++++++ @@ -4458,9 +4548,13 @@ A boolean setting that, if true, will send an email and notification to users wh :CVocConf +++++++++ -A JSON-structured setting that configures Dataverse to associate specific metadatablock fields with external vocabulary services and specific vocabularies/sub-vocabularies managed by that service. More information about this capability is available at :doc:`/admin/metadatacustomization`. +The ``:CVocConf`` database setting is used to allow metadatablock fields to look up values in external vocabulary services. For example, you could configure the "Author Affiliation" field to look up organizations in the `Research Organization Registry (ROR) `_. For a high-level description of this feature, see :ref:`using-external-vocabulary-services` in the Admin Guide. -Scripts that implement this association for specific service protocols are maintained at https://github.com/gdcc/dataverse-external-vocab-support. That repository also includes a json-schema for validating the structure required by this setting along with an example metadatablock and sample :CVocConf setting values associating entries in the example block with ORCID and SKOSMOS based services. +The expected format for the ``:CVocConf`` database setting is JSON but the details are not documented here. Instead, please refer to `docs/readme.md `_ in the https://github.com/gdcc/dataverse-external-vocab-support repo. + +That repository also includes scripts that implement the lookup for specific service protocols, a JSON Schema for validating the structure required by this setting, and an example metadatablock with a sample ``:CVocConf`` config that associates fields in the example block with ORCID and SKOSMOS based services. + +The commands below should give you an idea of how to load the configuration, but you'll want to study the examples and make decisions about which configuration to use: ``wget https://gdcc.github.io/dataverse-external-vocab-support/examples/config/cvoc-conf.json`` @@ -4525,6 +4619,18 @@ can enter for an embargo end date. This limit will be enforced in the popup dial ``curl -X PUT -d 24 http://localhost:8080/api/admin/settings/:MaxEmbargoDurationInMonths`` +.. _:MinRetentionDurationInMonths: + +:MinRetentionDurationInMonths ++++++++++++++++++++++++++++++ + +This setting controls whether retention periods are allowed in a Dataverse instance and can limit the minimum duration users are allowed to specify. A value of 0 months or non-existent +setting indicates retention periods are not supported. A value of -1 allows retention periods of any length. Any other value indicates the minimum number of months (from the current date) a user +can enter for a retention period end date. This limit will be enforced in the popup dialog in which users enter the retention period end date. For example, to set a ten year minimum: + +``curl -X PUT -d 120 http://localhost:8080/api/admin/settings/:MinRetentionDurationInMonths`` + + :DataverseMetadataValidatorScript +++++++++++++++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index a56f4811ace..7f6e34789b8 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -276,17 +276,11 @@ jq Installing jq ============= -``jq`` is a command line tool for parsing JSON output that is used by the Dataverse Software installation script. It is available in the EPEL repository:: +``jq`` is a command line tool for parsing JSON output that is used by the Dataverse Software installation script. It is available in the ``appstream`` repository:: - # yum install epel-release - # yum install jq + # dnf install jq -or you may install it manually:: - - # cd /usr/bin - # wget https://stedolan.github.io/jq/download/linux64/jq - # chmod +x jq - # jq --version +or you may install the latest binary for your OS and platform, available from https://github.com/jqlang/jq/releases .. _install-imagemagick: diff --git a/doc/sphinx-guides/source/installation/shibboleth.rst b/doc/sphinx-guides/source/installation/shibboleth.rst index 9f7c04c1534..d5086d07a0e 100644 --- a/doc/sphinx-guides/source/installation/shibboleth.rst +++ b/doc/sphinx-guides/source/installation/shibboleth.rst @@ -148,6 +148,8 @@ When configuring the ``MetadataProvider`` section of ``shibboleth2.xml`` you sho Most Dataverse installations will probably only want to authenticate users via Shibboleth using their home institution's Identity Provider (IdP). The configuration above in ``shibboleth2.xml`` looks for the metadata for the Identity Providers (IdPs) in a file at ``/etc/shibboleth/dataverse-idp-metadata.xml``. You can download a :download:`sample dataverse-idp-metadata.xml file <../_static/installation/files/etc/shibboleth/dataverse-idp-metadata.xml>` and that includes the SAMLtest IdP from https://samltest.id but you will want to edit this file to include the metadata from the Identity Provider you care about. The identity people at your institution will be able to provide you with this metadata and they will very likely ask for a list of attributes that the Dataverse Software requires, which are listed at :ref:`shibboleth-attributes`. +.. _identity-federation: + Identity Federation ^^^^^^^^^^^^^^^^^^^ @@ -159,6 +161,10 @@ One of the benefits of using ``shibd`` is that it can be configured to periodica Once you've joined a federation the list of IdPs in the dropdown can be quite long! If you're curious how many are in the list you could try something like this: ``curl https://dataverse.example.edu/Shibboleth.sso/DiscoFeed | jq '.[].entityID' | wc -l`` +Joining the federation alone is not enough. For the InCommon Federation, one must `apply for Research and Scholarship entity category approval `_ and minimally your identity management group must release the attributes listed below to either the service provider (Dataverse instance) or optimally to all R&S service providers. See also https://refeds.org/category/research-and-scholarship + +When Dataverse does not receive :ref:`shibboleth-attributes` it needs, users see a confusing message. In the User Guide there is a section called :ref:`fix-shib-login` that attempts to explain the R&S situation as simply as possible and also links back here for more technical detail. + .. _shibboleth-attributes: Shibboleth Attributes diff --git a/doc/sphinx-guides/source/user/account.rst b/doc/sphinx-guides/source/user/account.rst index 81c416bafd1..bb73ff20dc7 100755 --- a/doc/sphinx-guides/source/user/account.rst +++ b/doc/sphinx-guides/source/user/account.rst @@ -83,7 +83,9 @@ Create a Dataverse installation account using Institutional Log In #. After you put in your institutional credentials successfully, you will be brought back to the Dataverse installation to confirm your account information, and click "Create Account". #. A username has been selected for you. You won't use this username to log in but it will appear next to your name when other users search for you to assign permissions within the system. To see what you username is, click on your name in the top right corner and click Account Information. -If you do not find your institution listed, you will need to request that it is added to the Research & Scholarship category of InCommon. Contact support for assistance on how to get this process started with the identity provider support team at your institution. +If you can't find your institution in a long list, you may need to request for it to be added to the "Research & Scholarship" category of an identity federation. See :ref:`fix-shib-login`. + +If your institution is listed but you get login error ("eppn was null" or similar), it may mean your institution has declared itself part of the "Research & Scholarship" category of an identity federation but it is not releasing required attributes (often email) as it should. To resolve this, see :ref:`fix-shib-login`. Convert your Dataverse installation account to use your Institutional Log In ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -106,6 +108,23 @@ Convert your Dataverse installation account away from your Institutional Log In If you are leaving your institution and need to convert your Dataverse installation account to the Dataverse Username/Email log in option, you will need to contact support for the Dataverse installation you are using. On your account page, there is a link that will open a popup form to contact support for assistance. +.. _fix-shib-login: + +Troubleshooting Federated Institutional Log In +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Dataverse can be configured to allow institutional log in from a worldwide federation (eduGAIN) but for a successful log in, the following Research & Scholarship (R&S) attributes must be released: + +- Shib-Identity-Provider +- eppn +- givenName +- sn +- email + +If you have attempted to log in but are seeing an error such as ``The SAML assertion for "eppn" was null``, you will need to contact the people who run the log in system (Identity Provider or IdP) for your organization and explain that the attributes above must be released. You can link them to this document, of course, as well as https://refeds.org/category/research-and-scholarship and :ref:`identity-federation` in the Installation Guide. + +Note that while Identity Providers (IdPs) who have joined R&S are required to release the attributes above to all Service Providers (SPs) who have joined R&S (Harvard Dataverse or UNC Dataverse, for example), for a successful login to a Dataverse installation, the IdP could decide to release attributes to just that individual installation. + ORCID Log In ~~~~~~~~~~~~~ diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index 9538be4a1ec..d803aae6d19 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -735,6 +735,14 @@ Once a dataset with embargoed files has been published, no further action is nee As the primary use case of embargoes is to make the existence of data known now, with a promise (to a journal, project team, etc.) that the data itself will become available at a given future date, users cannot change an embargo once a dataset version is published. Dataverse instance administrators do have the ability to correct mistakes and make changes if/when circumstances warrant. +Retention Periods +================= + +Support for file-level retention periods can also be configured in a Dataverse instance. Retention periods make file content inaccessible after the retention period end date. This means that file previews and the ability to download files will be blocked. The effect is similar to when a file is restricted except that the retention periods will end at the specified date without further action and after the retention periods expires, requests for file access cannot be made. + +Retention periods are intended to support use cases where files must be made unavailable - and in most cases destroyed, e.g. to meet legal requirements - after a certain period or date. +Actual destruction is not automatically handled, but would have to be done on the storage if needed. + Dataset Versions ================ diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index 91bf5a2c875..dae4a3aa272 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -21,7 +21,7 @@ ENV SCRIPT_DIR="/scripts" \ ENV PATH="${PATH}:${SCRIPT_DIR}" \ BOOTSTRAP_DIR="${SCRIPT_DIR}/bootstrap" -ARG APK_PACKAGES="curl bind-tools netcat-openbsd jq bash dumb-init wait4x ed" +ARG APK_PACKAGES="curl bind-tools netcat-openbsd jq bash dumb-init wait4x ed postgresql-client" RUN true && \ # Install necessary software and tools diff --git a/pom.xml b/pom.xml index 8f9d06b8744..091ea206bd2 100644 --- a/pom.xml +++ b/pom.xml @@ -311,7 +311,7 @@ org.apache.solr solr-solrj - 9.3.0 + 9.4.1 colt @@ -560,6 +560,12 @@ java-json-canonicalization 1.1 + + + io.gdcc + sitemapgen4j + 2.1.2 + edu.ucar cdm-core diff --git a/scripts/api/data/licenses/licenseApache-2.0.json b/scripts/api/data/licenses/licenseApache-2.0.json new file mode 100644 index 00000000000..5b7c3cf5c95 --- /dev/null +++ b/scripts/api/data/licenses/licenseApache-2.0.json @@ -0,0 +1,8 @@ +{ + "name": "Apache-2.0", + "uri": "http://www.apache.org/licenses/LICENSE-2.0", + "shortDescription": "Apache License 2.0", + "active": true, + "sortOrder": 9 + } + \ No newline at end of file diff --git a/scripts/api/data/licenses/licenseMIT.json b/scripts/api/data/licenses/licenseMIT.json new file mode 100644 index 00000000000..a879e8a5595 --- /dev/null +++ b/scripts/api/data/licenses/licenseMIT.json @@ -0,0 +1,7 @@ +{ + "name": "MIT", + "uri": "https://opensource.org/licenses/MIT", + "shortDescription": "MIT License", + "active": true, + "sortOrder": 8 +} diff --git a/scripts/api/data/metadatablocks/biomedical.tsv b/scripts/api/data/metadatablocks/biomedical.tsv index d70f754336a..06f1ebec1b4 100644 --- a/scripts/api/data/metadatablocks/biomedical.tsv +++ b/scripts/api/data/metadatablocks/biomedical.tsv @@ -45,6 +45,7 @@ studyFactorType Treatment Compound EFO_0000369 17 studyFactorType Treatment Type EFO_0000727 18 studyFactorType Other OTHER_FACTOR 19 + studyAssayMeasurementType cell counting ERO_0001899 0 studyAssayMeasurementType cell sorting CHMO_0001085 1 studyAssayMeasurementType clinical chemistry analysis OBI_0000520 2 studyAssayMeasurementType copy number variation profiling OBI_0000537 3 diff --git a/scripts/api/setup-all.sh b/scripts/api/setup-all.sh index 5ddd9a35fdc..b7f962209e4 100755 --- a/scripts/api/setup-all.sh +++ b/scripts/api/setup-all.sh @@ -65,7 +65,7 @@ echo echo "Setting up the admin user (and as superuser)" adminResp=$(curl -s -H "Content-type:application/json" -X POST -d @"$SCRIPT_PATH"/data/user-admin.json "${DATAVERSE_URL}/api/builtin-users?password=$DV_SU_PASSWORD&key=burrito") echo "$adminResp" -curl -X POST "${DATAVERSE_URL}/api/admin/superuser/dataverseAdmin" +curl -X PUT "${DATAVERSE_URL}/api/admin/superuser/dataverseAdmin" -d "true" echo echo "Setting up the root dataverse" diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFile.java b/src/main/java/edu/harvard/iq/dataverse/DataFile.java index 53cdff31cc2..29a4a14c021 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFile.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFile.java @@ -242,6 +242,18 @@ public void setEmbargo(Embargo embargo) { this.embargo = embargo; } + @ManyToOne + @JoinColumn(name="retention_id") + private Retention retention; + + public Retention getRetention() { + return retention; + } + + public void setRetention(Retention retention) { + this.retention = retention; + } + public DataFile() { this.fileMetadatas = new ArrayList<>(); initFileReplaceAttributes(); diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 8ceb529a5d4..41ea6ae39f0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -959,6 +959,7 @@ public boolean isThumbnailAvailable (DataFile file) { return true; } file.setPreviewImageFail(true); + file.setPreviewImageAvailable(false); this.save(file); return false; } @@ -1365,7 +1366,10 @@ public Embargo findEmbargo(Long id) { DataFile d = find(id); return d.getEmbargo(); } - + + public boolean isRetentionExpired(FileMetadata fm) { + return FileUtil.isRetentionExpired(fm); + } /** * Checks if the supplied DvObjectContainer (Dataset or Collection; although * only collection-level storage quotas are officially supported as of now) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index 6223cd83773..f6a566ae65f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -505,7 +505,14 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep HttpGet httpGet = new HttpGet(retrievalUri); //application/json+ld is for backward compatibility httpGet.addHeader("Accept", "application/ld+json, application/json+ld, application/json"); - + //Adding others custom HTTP request headers if exists + final JsonObject headers = cvocEntry.getJsonObject("headers"); + if (headers != null) { + final Set headerKeys = headers.keySet(); + for (final String hKey: headerKeys) { + httpGet.addHeader(hKey, headers.getString(hKey)); + } + } HttpResponse response = httpClient.execute(httpGet); String data = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); int statusCode = response.getStatusLine().getStatusCode(); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 4c436715f0d..d9cb10026a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -77,6 +77,7 @@ import java.lang.reflect.Method; import java.sql.Timestamp; import java.text.SimpleDateFormat; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -269,6 +270,8 @@ public enum DisplayMode { @Inject EmbargoServiceBean embargoService; @Inject + RetentionServiceBean retentionService; + @Inject LicenseServiceBean licenseServiceBean; @Inject DataFileCategoryServiceBean dataFileCategoryService; @@ -1234,8 +1237,17 @@ public boolean canDownloadFiles() { canDownloadFiles = false; for (FileMetadata fmd : workingVersion.getFileMetadatas()) { if (fileDownloadHelper.canDownloadFile(fmd)) { - canDownloadFiles = true; - break; + if (isVersionHasGlobus()) { + String driverId = DataAccess + .getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier()); + if (StorageIO.isDataverseAccessible(driverId)) { + canDownloadFiles = true; + break; + } + } else { + canDownloadFiles = true; + break; + } } } } @@ -2203,6 +2215,11 @@ private String init(boolean initFull) { } } + LocalDate minRetentiondate = settingsWrapper.getMinRetentionDate(); + if (minRetentiondate != null){ + selectionRetention.setDateUnavailable(minRetentiondate.plusDays(1L)); + } + displayLockInfo(dataset); displayPublishMessage(); @@ -2279,13 +2296,11 @@ private void displayPublishMessage(){ public boolean isValid() { if (valid == null) { - DatasetVersion version = dataset.getLatestVersion(); - if (!version.isDraft()) { + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(true))) { + valid = workingVersion.isValid(); + } else { valid = true; } - DatasetVersion newVersion = version.cloneDatasetVersion(); - newVersion.setDatasetFields(newVersion.initDatasetFields()); - valid = newVersion.isValid(); } return valid; } @@ -3260,7 +3275,7 @@ public void startDownloadSelectedOriginal() { private void startDownload(boolean downloadOriginal){ boolean guestbookRequired = isDownloadPopupRequired(); - boolean validate = validateFilesForDownload(downloadOriginal); + boolean validate = validateFilesForDownload(downloadOriginal, false); if (validate) { updateGuestbookResponse(guestbookRequired, downloadOriginal, false); if(!guestbookRequired && !getValidateFilesOutcome().equals("Mixed")){ @@ -3283,7 +3298,7 @@ public void setValidateFilesOutcome(String validateFilesOutcome) { this.validateFilesOutcome = validateFilesOutcome; } - public boolean validateFilesForDownload(boolean downloadOriginal){ + public boolean validateFilesForDownload(boolean downloadOriginal, boolean isGlobusTransfer){ if (this.selectedFiles.isEmpty()) { PrimeFaces.current().executeScript("PF('selectFilesForDownload').show()"); return false; @@ -3300,33 +3315,39 @@ public boolean validateFilesForDownload(boolean downloadOriginal){ return false; } - for (FileMetadata fmd : getSelectedDownloadableFiles()) { - DataFile dataFile = fmd.getDataFile(); - if (downloadOriginal && dataFile.isTabularData()) { - bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize(); - } else { - bytes += dataFile.getFilesize(); + if (!isGlobusTransfer) { + for (FileMetadata fmd : getSelectedDownloadableFiles()) { + DataFile dataFile = fmd.getDataFile(); + if (downloadOriginal && dataFile.isTabularData()) { + bytes += dataFile.getOriginalFileSize() == null ? 0 : dataFile.getOriginalFileSize(); + } else { + bytes += dataFile.getFilesize(); + } } - } - //if there are two or more files, with a total size - //over the zip limit, post a "too large" popup - if (bytes > settingsWrapper.getZipDownloadLimit() && selectedDownloadableFiles.size() > 1) { - setValidateFilesOutcome("FailSize"); - return false; + // if there are two or more files, with a total size + // over the zip limit, post a "too large" popup + if (bytes > settingsWrapper.getZipDownloadLimit() && selectedDownloadableFiles.size() > 1) { + setValidateFilesOutcome("FailSize"); + return false; + } } - + // If some of the files were restricted and we had to drop them off the // list, and NONE of the files are left on the downloadable list - // - we show them a "you're out of luck" popup: - if (getSelectedDownloadableFiles().isEmpty() && getSelectedGlobusTransferableFiles().isEmpty() && !getSelectedNonDownloadableFiles().isEmpty()) { + // - we show them a "you're out of luck" popup + // Same for globus transfer + if ((!isGlobusTransfer + && (getSelectedDownloadableFiles().isEmpty() && !getSelectedNonDownloadableFiles().isEmpty())) + || (isGlobusTransfer && (getSelectedGlobusTransferableFiles().isEmpty() + && !getSelectedNonGlobusTransferableFiles().isEmpty()))) { setValidateFilesOutcome("FailRestricted"); return false; } - //Some are selected and there are non-downloadable ones or there are both downloadable and globus transferable files - if ((!(getSelectedDownloadableFiles().isEmpty() && getSelectedGlobusTransferableFiles().isEmpty()) - && (!getSelectedNonDownloadableFiles().isEmpty()) || (!getSelectedDownloadableFiles().isEmpty() && !getSelectedGlobusTransferableFiles().isEmpty()))) { + //For download or transfer, there are some that can be downloaded/transferred and some that can't + if ((!isGlobusTransfer && (!getSelectedNonDownloadableFiles().isEmpty() && !getSelectedDownloadableFiles().isEmpty())) || + (isGlobusTransfer && (!getSelectedNonGlobusTransferableFiles().isEmpty() && !getSelectedGlobusTransferableFiles().isEmpty()))) { setValidateFilesOutcome("Mixed"); return true; } @@ -3684,6 +3705,25 @@ public String deleteFiles() throws CommandException{ } } + //Remove retentions that are no longer referenced + //Identify which ones are involved here + List orphanedRetentions = new ArrayList(); + if (selectedFiles != null && selectedFiles.size() > 0) { + for (FileMetadata fmd : workingVersion.getFileMetadatas()) { + for (FileMetadata fm : selectedFiles) { + if (fm.getDataFile().equals(fmd.getDataFile()) && !fmd.getDataFile().isReleased()) { + Retention ret = fmd.getDataFile().getRetention(); + if (ret != null) { + ret.getDataFiles().remove(fmd.getDataFile()); + if (ret.getDataFiles().isEmpty()) { + orphanedRetentions.add(ret); + } + } + } + } + } + } + deleteFiles(filesToDelete); String retVal; @@ -3693,12 +3733,14 @@ public String deleteFiles() throws CommandException{ } else { retVal = save(); } - - - //And delete them only after the dataset is updated + + // And delete them only after the dataset is updated for(Embargo emb: orphanedEmbargoes) { embargoService.deleteById(emb.getId(), ((AuthenticatedUser)session.getUser()).getUserIdentifier()); } + for(Retention ret: orphanedRetentions) { + retentionService.delete(ret, ((AuthenticatedUser)session.getUser()).getUserIdentifier()); + } return retVal; } @@ -5365,7 +5407,7 @@ public boolean isFileAccessRequestMultiSignUpButtonEnabled(){ return false; } for (FileMetadata fmd : this.selectedRestrictedFiles){ - if (!this.fileDownloadHelper.canDownloadFile(fmd)&& !FileUtil.isActivelyEmbargoed(fmd)){ + if (!this.fileDownloadHelper.canDownloadFile(fmd) && !FileUtil.isActivelyEmbargoed(fmd)){ return true; } } @@ -5726,7 +5768,10 @@ public boolean isShowPreviewButton(Long fileId) { public boolean isShowQueryButton(Long fileId) { DataFile dataFile = datafileService.find(fileId); - if(dataFile.isRestricted() || !dataFile.isReleased() || FileUtil.isActivelyEmbargoed(dataFile)){ + if(dataFile.isRestricted() + || !dataFile.isReleased() + || FileUtil.isActivelyEmbargoed(dataFile) + || FileUtil.isRetentionExpired(dataFile)){ return false; } @@ -6284,12 +6329,18 @@ public void clearSelectionEmbargo() { PrimeFaces.current().resetInputs("datasetForm:embargoInputs"); } - public boolean isCantDownloadDueToEmbargo() { + public boolean isCantDownloadDueToEmbargoOrDVAccess() { if (getSelectedNonDownloadableFiles() != null) { for (FileMetadata fmd : getSelectedNonDownloadableFiles()) { if (FileUtil.isActivelyEmbargoed(fmd)) { return true; } + if (isVersionHasGlobus()) { + if (StorageIO.isDataverseAccessible( + DataAccess.getStorageDriverFromIdentifier(fmd.getDataFile().getStorageIdentifier()))) { + return true; + } + } } } return false; @@ -6315,6 +6366,195 @@ private boolean containsOnlyActivelyEmbargoedFiles(List selectedFi return true; } + public Retention getSelectionRetention() { + return selectionRetention; + } + + public void setSelectionRetention(Retention selectionRetention) { + this.selectionRetention = selectionRetention; + } + + + private Retention selectionRetention = new Retention(); + + public boolean isValidRetentionSelection() { + //If fileMetadataForAction is set, someone is using the kebab/single file menu + if (fileMetadataForAction != null) { + if (!fileMetadataForAction.getDataFile().isReleased()) { + return true; + } else { + return false; + } + } + //Otherwise we check the selected files + for (FileMetadata fmd : selectedFiles) { + if (!fmd.getDataFile().isReleased()) { + return true; + } + } + return false; + } + + /* + * This method checks to see if the selected file/files have a retention that could be removed. It doesn't return true of a released file has a retention. + */ + public boolean isExistingRetention() { + if (fileMetadataForAction != null) { + if (!fileMetadataForAction.getDataFile().isReleased() + && (fileMetadataForAction.getDataFile().getRetention() != null)) { + return true; + } else { + return false; + } + } + for (FileMetadata fmd : selectedFiles) { + if (!fmd.getDataFile().isReleased() && (fmd.getDataFile().getRetention() != null)) { + return true; + } + } + + return false; + } + + public boolean isRetentionExpired(List fmdList) { + return FileUtil.isRetentionExpired(fmdList); + } + + public boolean isRetentionForWholeSelection() { + for (FileMetadata fmd : selectedFiles) { + if (fmd.getDataFile().isReleased()) { + return false; + } + } + return true; + } + + private boolean removeRetention=false; + + public boolean isRemoveRetention() { + return removeRetention; + } + + public void setRemoveRetention(boolean removeRetention) { + boolean existing = this.removeRetention; + this.removeRetention = removeRetention; + //If we flipped the state, update the selectedRetention. Otherwise (e.g. when save is hit) don't make changes + if(existing != this.removeRetention) { + logger.fine("State flip"); + selectionRetention= new Retention(); + if(removeRetention) { + logger.fine("Setting empty retention"); + selectionRetention= new Retention(null, null); + } + PrimeFaces.current().resetInputs("datasetForm:retentionInputs"); + } + } + + public String saveRetention() { + if (workingVersion.isReleased()) { + refreshSelectedFiles(selectedFiles); + } + + if(isRemoveRetention() || (selectionRetention.getDateUnavailable()==null && selectionRetention.getReason()==null)) { + selectionRetention=null; + } + + if(!(selectionRetention==null || (selectionRetention!=null && settingsWrapper.isValidRetentionDate(selectionRetention)))) { + logger.fine("Validation error: " + selectionRetention.getFormattedDateUnavailable()); + FacesContext.getCurrentInstance().validationFailed(); + return ""; + } + List orphanedRetentions = new ArrayList(); + List retentionFMs = null; + if (fileMetadataForAction != null) { + retentionFMs = new ArrayList(); + retentionFMs.add(fileMetadataForAction); + } else if (selectedFiles != null && selectedFiles.size() > 0) { + retentionFMs = selectedFiles; + } + + if(retentionFMs!=null && !retentionFMs.isEmpty()) { + if(selectionRetention!=null) { + selectionRetention = retentionService.merge(selectionRetention); + } + for (FileMetadata fmd : workingVersion.getFileMetadatas()) { + for (FileMetadata fm : retentionFMs) { + if (fm.getDataFile().equals(fmd.getDataFile()) && (isSuperUser()||!fmd.getDataFile().isReleased())) { + Retention ret = fmd.getDataFile().getRetention(); + if (ret != null) { + logger.fine("Before: " + ret.getDataFiles().size()); + ret.getDataFiles().remove(fmd.getDataFile()); + if (ret.getDataFiles().isEmpty()) { + orphanedRetentions.add(ret); + } + logger.fine("After: " + ret.getDataFiles().size()); + } + fmd.getDataFile().setRetention(selectionRetention); + } + } + } + } + if (selectionRetention != null) { + retentionService.save(selectionRetention, ((AuthenticatedUser) session.getUser()).getIdentifier()); + } + // success message: + String successMessage = BundleUtil.getStringFromBundle("file.assignedRetention.success"); + logger.fine(successMessage); + successMessage = successMessage.replace("{0}", "Selected Files"); + JsfHelper.addFlashMessage(successMessage); + selectionRetention = new Retention(); + + save(); + for(Retention ret: orphanedRetentions) { + retentionService.delete(ret, ((AuthenticatedUser)session.getUser()).getUserIdentifier()); + } + return returnToDraftVersion(); + } + + public void clearRetentionPopup() { + logger.fine("clearRetentionPopup called"); + selectionRetention= new Retention(); + setRemoveRetention(false); + PrimeFaces.current().resetInputs("datasetForm:retentionInputs"); + } + + public void clearSelectionRetention() { + logger.fine("clearSelectionRetention called"); + selectionRetention= new Retention(); + PrimeFaces.current().resetInputs("datasetForm:retentionInputs"); + } + + public boolean isCantDownloadDueToRetention() { + if (getSelectedNonDownloadableFiles() != null) { + for (FileMetadata fmd : getSelectedNonDownloadableFiles()) { + if (FileUtil.isRetentionExpired(fmd)) { + return true; + } + } + } + return false; + } + + public boolean isCantRequestDueToRetention() { + if (fileDownloadHelper.getFilesForRequestAccess() != null) { + for (DataFile df : fileDownloadHelper.getFilesForRequestAccess()) { + if (FileUtil.isRetentionExpired(df)) { + return true; + } + } + } + return false; + } + + private boolean containsOnlyRetentionExpiredFiles(List selectedFiles) { + for (FileMetadata fmd : selectedFiles) { + if (!FileUtil.isRetentionExpired(fmd)) { + return false; + } + } + return true; + } + public String getIngestMessage() { return BundleUtil.getStringFromBundle("file.ingestFailed.message", Arrays.asList(settingsWrapper.getGuidesBaseUrl(), settingsWrapper.getGuidesVersion())); } @@ -6381,7 +6621,8 @@ public void startGlobusTransfer(boolean transferAll, boolean popupShown) { } boolean guestbookRequired = isDownloadPopupRequired(); - boolean validated = validateFilesForDownload(true); + boolean validated = validateFilesForDownload(true, true); + if (validated) { globusTransferRequested = true; boolean mixed = "Mixed".equals(getValidateFilesOutcome()); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java index 9c182164d37..29f7570a645 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetServiceBean.java @@ -128,6 +128,7 @@ public Dataset findDeep(Object pk) { .setHint("eclipselink.left-join-fetch", "o.files.fileMetadatas.fileCategories") //.setHint("eclipselink.left-join-fetch", "o.files.guestbookResponses") .setHint("eclipselink.left-join-fetch", "o.files.embargo") + .setHint("eclipselink.left-join-fetch", "o.files.retention") .setHint("eclipselink.left-join-fetch", "o.files.fileAccessRequests") .setHint("eclipselink.left-join-fetch", "o.files.owner") .setHint("eclipselink.left-join-fetch", "o.files.releaseUser") @@ -860,18 +861,33 @@ public Dataset setDatasetFileAsThumbnail(Dataset dataset, DataFile datasetFileTh logger.fine("In setDatasetFileAsThumbnail but dataset is null! Returning null."); return null; } + // Just in case the previously designated thumbnail for the dataset was + // a "custom" kind, i.e. an uploaded "dataset_logo" file, the following method + // will try to delete it, and all the associated caches here (because there + // are no other uses for the file). This method is apparently called in all + // cases, without trying to check if the dataset was in fact using a custom + // logo; probably under the assumption that it can't hurt. DatasetUtil.deleteDatasetLogo(dataset); dataset.setThumbnailFile(datasetFileThumbnailToSwitchTo); dataset.setUseGenericThumbnail(false); return merge(dataset); } - public Dataset removeDatasetThumbnail(Dataset dataset) { + public Dataset clearDatasetLevelThumbnail(Dataset dataset) { if (dataset == null) { - logger.fine("In removeDatasetThumbnail but dataset is null! Returning null."); + logger.fine("In clearDatasetLevelThumbnail but dataset is null! Returning null."); return null; } + + // Just in case the thumbnail that was designated for the dataset was + // a "custom logo" kind, i.e. an uploaded "dataset_logo" file, the following method + // will try to delete it, and all the associated caches here (because there + // are no other uses for the file). This method is apparently called in all + // cases, without trying to check if the dataset was in fact using a custom + // logo; probably under the assumption that it can't hurt. DatasetUtil.deleteDatasetLogo(dataset); + + // Clear any designated thumbnails for the dataset: dataset.setThumbnailFile(null); dataset.setUseGenericThumbnail(true); return merge(dataset); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java index 5fd963f3931..943693355a3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersion.java @@ -1728,7 +1728,36 @@ public List> validateRequired() { } public boolean isValid() { - return validate().isEmpty(); + // first clone to leave the original untouched + final DatasetVersion newVersion = this.cloneDatasetVersion(); + // initDatasetFields + newVersion.setDatasetFields(newVersion.initDatasetFields()); + // remove special "N/A" values and empty values + newVersion.removeEmptyValues(); + // check validity of present fields and detect missing mandatory fields + return newVersion.validate().isEmpty(); + } + + private void removeEmptyValues() { + if (this.getDatasetFields() != null) { + for (DatasetField dsf : this.getDatasetFields()) { + removeEmptyValues(dsf); + } + } + } + + private void removeEmptyValues(DatasetField dsf) { + if (dsf.getDatasetFieldType().isPrimitive()) { // primitive + final Iterator i = dsf.getDatasetFieldValues().iterator(); + while (i.hasNext()) { + final String v = i.next().getValue(); + if (StringUtils.isBlank(v) || DatasetField.NA_VALUE.equals(v)) { + i.remove(); + } + } + } else { + dsf.getDatasetFieldCompoundValues().forEach(cv -> cv.getChildDatasetFields().forEach(v -> removeEmptyValues(v))); + } } public Set validate() { diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java index 99c3c65e3b8..afcfafe976c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionFilesServiceBean.java @@ -42,6 +42,16 @@ public enum FileDownloadSizeMode { All, Original, Archival } + /** + * Given a DatasetVersion, returns its total file metadata count + * + * @param datasetVersion the DatasetVersion to access + * @return long value of total file metadata count + */ + public long getFileMetadataCount(DatasetVersion datasetVersion) { + return getFileMetadataCount(datasetVersion, new FileSearchCriteria(null, null, null, null, null)); + } + /** * Given a DatasetVersion, returns its total file metadata count * @@ -189,6 +199,32 @@ public long getFilesDownloadSize(DatasetVersion datasetVersion, FileSearchCriter }; } + /** + * Determines whether or not a DataFile is present in a DatasetVersion + * + * @param datasetVersion the DatasetVersion to check + * @param dataFile the DataFile to check + * @return boolean value + */ + public boolean isDataFilePresentInDatasetVersion(DatasetVersion datasetVersion, DataFile dataFile) { + CriteriaBuilder criteriaBuilder = em.getCriteriaBuilder(); + CriteriaQuery criteriaQuery = criteriaBuilder.createQuery(Long.class); + Root dataFileRoot = criteriaQuery.from(DataFile.class); + Root fileMetadataRoot = criteriaQuery.from(FileMetadata.class); + Root datasetVersionRoot = criteriaQuery.from(DatasetVersion.class); + criteriaQuery + .select(criteriaBuilder.count(dataFileRoot)) + .where(criteriaBuilder.and( + criteriaBuilder.equal(dataFileRoot.get("id"), dataFile.getId()), + criteriaBuilder.equal(datasetVersionRoot.get("id"), datasetVersion.getId()), + fileMetadataRoot.in(dataFileRoot.get("fileMetadatas")), + fileMetadataRoot.in(datasetVersionRoot.get("fileMetadatas")) + ) + ); + Long count = em.createQuery(criteriaQuery).getSingleResult(); + return count != null && count > 0; + } + private void addAccessStatusCountToTotal(DatasetVersion datasetVersion, Map totalCounts, FileAccessStatus dataFileAccessStatus, FileSearchCriteria searchCriteria) { long fileMetadataCount = getFileMetadataCountByAccessStatus(datasetVersion, dataFileAccessStatus, searchCriteria); if (fileMetadataCount > 0) { @@ -210,6 +246,8 @@ private long getFileMetadataCountByAccessStatus(DatasetVersion datasetVersion, F private Predicate createSearchCriteriaAccessStatusPredicate(FileAccessStatus accessStatus, CriteriaBuilder criteriaBuilder, Root fileMetadataRoot) { Path dataFile = fileMetadataRoot.get("dataFile"); + Path retention = dataFile.get("retention"); + Predicate retentionExpiredPredicate = criteriaBuilder.lessThan(retention.get("dateUnavailable"), criteriaBuilder.currentDate()); Path embargo = dataFile.get("embargo"); Predicate activelyEmbargoedPredicate = criteriaBuilder.greaterThanOrEqualTo(embargo.get("dateAvailable"), criteriaBuilder.currentDate()); Predicate inactivelyEmbargoedPredicate = criteriaBuilder.isNull(embargo); @@ -217,6 +255,7 @@ private Predicate createSearchCriteriaAccessStatusPredicate(FileAccessStatus acc Predicate isRestrictedPredicate = criteriaBuilder.isTrue(isRestricted); Predicate isUnrestrictedPredicate = criteriaBuilder.isFalse(isRestricted); return switch (accessStatus) { + case RetentionPeriodExpired -> criteriaBuilder.and(retentionExpiredPredicate); case EmbargoedThenRestricted -> criteriaBuilder.and(activelyEmbargoedPredicate, isRestrictedPredicate); case EmbargoedThenPublic -> criteriaBuilder.and(activelyEmbargoedPredicate, isUnrestrictedPredicate); case Restricted -> criteriaBuilder.and(inactivelyEmbargoedPredicate, isRestrictedPredicate); diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java index 1ee517c9831..ab23fa779d5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java @@ -163,6 +163,7 @@ public DatasetVersion findDeep(Object pk) { .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.dataTables") .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.fileCategories") .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.embargo") + .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.retention") .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.datasetVersion") .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.releaseUser") .setHint("eclipselink.left-join-fetch", "o.fileMetadatas.dataFile.creator") @@ -802,6 +803,7 @@ public Long getThumbnailByVersionId(Long versionId) { + "AND fm.datafile_id = df.id " + "AND df.restricted = false " + "AND df.embargo_id is null " + + "AND df.retention_id is null " + "AND o.previewImageAvailable = true " + "ORDER BY df.id LIMIT 1;").getSingleResult(); } catch (Exception ex) { @@ -828,6 +830,7 @@ public Long getThumbnailByVersionId(Long versionId) { + "AND o.previewimagefail = false " + "AND df.restricted = false " + "AND df.embargo_id is null " + + "AND df.retention_id is null " + "AND df.contenttype LIKE 'image/%' " + "AND NOT df.contenttype = 'image/fits' " + "AND df.filesize < " + imageThumbnailSizeLimit + " " @@ -862,6 +865,7 @@ public Long getThumbnailByVersionId(Long versionId) { + "AND o.previewimagefail = false " + "AND df.restricted = false " + "AND df.embargo_id is null " + + "AND df.retention_id is null " + "AND df.contenttype = 'application/pdf' " + "AND df.filesize < " + imageThumbnailSizeLimit + " " + "ORDER BY df.filesize ASC LIMIT 1;").getSingleResult(); diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 42db9c1392a..78b1827c798 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -411,6 +411,14 @@ public List getDataverseFieldTypeInputLevels() { return dataverseFieldTypeInputLevels; } + public boolean isDatasetFieldTypeRequiredAsInputLevel(Long datasetFieldTypeId) { + for(DataverseFieldTypeInputLevel dataverseFieldTypeInputLevel : dataverseFieldTypeInputLevels) { + if (dataverseFieldTypeInputLevel.getDatasetFieldType().getId().equals(datasetFieldTypeId) && dataverseFieldTypeInputLevel.isRequired()) { + return true; + } + } + return false; + } public Template getDefaultTemplate() { return defaultTemplate; diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java index 33e708e7467..80cf3db8d53 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadHelper.java @@ -223,7 +223,10 @@ public boolean canDownloadFile(FileMetadata fileMetadata){ // Always allow download for PrivateUrlUser return true; } - + + // Retention expired files are always made unavailable, because they might be destroyed + if (FileUtil.isRetentionExpired(fileMetadata)) return false; + Long fid = fileMetadata.getId(); //logger.info("calling candownloadfile on filemetadata "+fid); // Note that `isRestricted` at the FileMetadata level is for expressing intent by version. Enforcement is done with `isRestricted` at the DataFile level. @@ -246,7 +249,9 @@ public boolean canDownloadFile(FileMetadata fileMetadata){ } } - if (!isRestrictedFile && !FileUtil.isActivelyEmbargoed(fileMetadata)){ + if (!isRestrictedFile + && !FileUtil.isActivelyEmbargoed(fileMetadata) + && !FileUtil.isRetentionExpired(fileMetadata)) { // Yes, save answer and return true this.fileDownloadPermissionMap.put(fid, true); return true; diff --git a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java index c5073693ab2..ab9e4f9be66 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileDownloadServiceBean.java @@ -354,7 +354,8 @@ public void explore(GuestbookResponse guestbookResponse, FileMetadata fmd, Exter ApiToken apiToken = null; User user = session.getUser(); DatasetVersion version = fmd.getDatasetVersion(); - if (version.isDraft() || fmd.getDatasetVersion().isDeaccessioned() || (fmd.getDataFile().isRestricted()) || (FileUtil.isActivelyEmbargoed(fmd))) { + if (version.isDraft() || fmd.getDatasetVersion().isDeaccessioned() || (fmd.getDataFile().isRestricted()) + || (FileUtil.isActivelyEmbargoed(fmd)) || (FileUtil.isRetentionExpired(fmd))) { apiToken = authService.getValidApiTokenForUser(user); } DataFile dataFile = null; diff --git a/src/main/java/edu/harvard/iq/dataverse/FilePage.java b/src/main/java/edu/harvard/iq/dataverse/FilePage.java index ca225dccb1c..9889d23cf55 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FilePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/FilePage.java @@ -34,6 +34,7 @@ import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean; import edu.harvard.iq.dataverse.makedatacount.MakeDataCountLoggingServiceBean.MakeDataCountEntry; import edu.harvard.iq.dataverse.privateurl.PrivateUrlServiceBean; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; @@ -44,6 +45,7 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.IOException; +import java.time.LocalDate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -152,6 +154,9 @@ public class FilePage implements java.io.Serializable { @Inject EmbargoServiceBean embargoService; + @Inject + RetentionServiceBean retentionService; + private static final Logger logger = Logger.getLogger(FilePage.class.getCanonicalName()); private boolean fileDeleteInProgress = false; @@ -277,7 +282,12 @@ public String init() { if(!hasValidTermsOfAccess && canUpdateDataset() ){ JsfHelper.addWarningMessage(BundleUtil.getStringFromBundle("dataset.message.editMetadata.invalid.TOUA.message")); } - + + LocalDate minRetentiondate = settingsWrapper.getMinRetentionDate(); + if (minRetentiondate != null){ + selectionRetention.setDateUnavailable(minRetentiondate.plusDays(1L)); + } + displayPublishMessage(); return null; } @@ -305,13 +315,18 @@ private void displayPublishMessage(){ } } + Boolean valid = null; + public boolean isValid() { - if (!fileMetadata.getDatasetVersion().isDraft()) { - return true; + if (valid == null) { + final DatasetVersion workingVersion = fileMetadata.getDatasetVersion(); + if (workingVersion.isDraft() || (canUpdateDataset() && JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(true))) { + valid = workingVersion.isValid(); + } else { + valid = true; + } } - DatasetVersion newVersion = fileMetadata.getDatasetVersion().cloneDatasetVersion(); - newVersion.setDatasetFields(newVersion.initDatasetFields()); - return newVersion.isValid(); + return valid; } private boolean canViewUnpublishedDataset() { @@ -1389,7 +1404,129 @@ public String getEmbargoPhrase() { return BundleUtil.getStringFromBundle("embargoed.willbeuntil"); } } - + + public boolean isValidRetentionSelection() { + if (!fileMetadata.getDataFile().isReleased()) { + return true; + } + return false; + } + + public boolean isExistingRetention() { + if (!fileMetadata.getDataFile().isReleased() && (fileMetadata.getDataFile().getRetention() != null)) { + return true; + } + return false; + } + + public boolean isRetentionForWholeSelection() { + return isValidRetentionSelection(); + } + + public Retention getSelectionRetention() { + return selectionRetention; + } + + public void setSelectionRetention(Retention selectionRetention) { + this.selectionRetention = selectionRetention; + } + + private Retention selectionRetention = new Retention(); + + private boolean removeRetention=false; + + public boolean isRemoveRetention() { + return removeRetention; + } + + public void setRemoveRetention(boolean removeRetention) { + boolean existing = this.removeRetention; + this.removeRetention = removeRetention; + if (existing != this.removeRetention) { + logger.info("State flip"); + selectionRetention = new Retention(); + if (removeRetention) { + selectionRetention = new Retention(null, null); + } + } + PrimeFaces.current().resetInputs("fileForm:retentionInputs"); + } + + public String saveRetention() { + + if(isRemoveRetention() || (selectionRetention.getDateUnavailable()==null && selectionRetention.getReason()==null)) { + selectionRetention=null; + } + + Retention ret = null; + // Note: this.fileMetadata.getDataFile() is not the same object as this.file. + // (Not sure there's a good reason for this other than that's the way it is.) + // So changes to this.fileMetadata.getDataFile() will not be saved with + // editDataset = this.file.getOwner() set as it is below. + if (!file.isReleased()) { + ret = file.getRetention(); + if (ret != null) { + logger.fine("Before: " + ret.getDataFiles().size()); + ret.getDataFiles().remove(fileMetadata.getDataFile()); + logger.fine("After: " + ret.getDataFiles().size()); + } + if (selectionRetention != null) { + retentionService.merge(selectionRetention); + } + file.setRetention(selectionRetention); + if (ret != null && !ret.getDataFiles().isEmpty()) { + ret = null; + } + } + if(selectionRetention!=null) { + retentionService.save(selectionRetention, ((AuthenticatedUser)session.getUser()).getIdentifier()); + } + // success message: + String successMessage = BundleUtil.getStringFromBundle("file.assignedRetention.success"); + logger.fine(successMessage); + successMessage = successMessage.replace("{0}", "Selected Files"); + JsfHelper.addFlashMessage(successMessage); + selectionRetention = new Retention(); + + //Caller has to set editDataset before calling save() + editDataset = this.file.getOwner(); + + save(); + init(); + if(ret!=null) { + retentionService.delete(ret,((AuthenticatedUser)session.getUser()).getIdentifier()); + } + return returnToDraftVersion(); + } + + public void clearRetentionPopup() { + setRemoveRetention(false); + selectionRetention = new Retention(); + PrimeFaces.current().resetInputs("fileForm:retentionInputs"); + } + + public void clearSelectionRetention() { + selectionRetention = new Retention(); + PrimeFaces.current().resetInputs("fileForm:retentionInputs"); + } + + public boolean isCantRequestDueToRetention() { + return FileUtil.isRetentionExpired(fileMetadata); + } + + public String getRetentionPhrase() { + //Should only be getting called when there is a retention + if(file.isReleased()) { + if(FileUtil.isRetentionExpired(file)) { + return BundleUtil.getStringFromBundle("retention.after"); + } else { + return BundleUtil.getStringFromBundle("retention.isfrom"); + } + } else { + return BundleUtil.getStringFromBundle("retention.willbeafter"); + } + } + public String getToolTabTitle(){ if (getAllAvailableTools().size() > 1) { return BundleUtil.getStringFromBundle("file.toolTab.header"); diff --git a/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java b/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java index 62f10c18bdf..e3ed507a9c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java +++ b/src/main/java/edu/harvard/iq/dataverse/FileSearchCriteria.java @@ -12,7 +12,7 @@ public class FileSearchCriteria { * Status of the particular DataFile based on active embargoes and restriction state */ public enum FileAccessStatus { - Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic + Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic, RetentionPeriodExpired } public FileSearchCriteria(String contentType, FileAccessStatus accessStatus, String categoryName, String tabularTagName, String searchText) { diff --git a/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java index c4c95fae551..1e2a34f5472 100644 --- a/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/MetadataBlockServiceBean.java @@ -58,10 +58,18 @@ public List listMetadataBlocksDisplayedOnCreate(Dataverse ownerDa if (ownerDataverse != null) { Root dataverseRoot = criteriaQuery.from(Dataverse.class); + Join datasetFieldTypeInputLevelJoin = dataverseRoot.join("dataverseFieldTypeInputLevels", JoinType.LEFT); + + Predicate requiredPredicate = criteriaBuilder.and( + datasetFieldTypeInputLevelJoin.get("datasetFieldType").in(metadataBlockRoot.get("datasetFieldTypes")), + criteriaBuilder.isTrue(datasetFieldTypeInputLevelJoin.get("required"))); + + Predicate unionPredicate = criteriaBuilder.or(displayOnCreatePredicate, requiredPredicate); + criteriaQuery.where(criteriaBuilder.and( criteriaBuilder.equal(dataverseRoot.get("id"), ownerDataverse.getId()), metadataBlockRoot.in(dataverseRoot.get("metadataBlocks")), - displayOnCreatePredicate + unionPredicate )); } else { criteriaQuery.where(displayOnCreatePredicate); diff --git a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java index 8fb762e3e5b..a389cbc735b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/PermissionServiceBean.java @@ -97,6 +97,9 @@ public class PermissionServiceBean { @Inject DataverseRequestServiceBean dvRequestService; + @Inject + DatasetVersionFilesServiceBean datasetVersionFilesServiceBean; + /** * A request-level permission query (e.g includes IP ras). */ @@ -442,23 +445,14 @@ private Set getInferredPermissions(DvObject dvo) { * download permission for everybody: */ private boolean isPublicallyDownloadable(DvObject dvo) { - if (dvo instanceof DataFile) { + if (dvo instanceof DataFile df) { // unrestricted files that are part of a release dataset // automatically get download permission for everybody: // -- L.A. 4.0 beta12 - - DataFile df = (DataFile) dvo; - if (!df.isRestricted()) { - if (df.getOwner().getReleasedVersion() != null) { - List fileMetadatas = df.getOwner().getReleasedVersion().getFileMetadatas(); - if (fileMetadatas != null) { - for (FileMetadata fm : fileMetadatas) { - if (df.equals(fm.getDataFile())) { - return true; - } - } - } + DatasetVersion releasedVersion = df.getOwner().getReleasedVersion(); + if (releasedVersion != null) { + return datasetVersionFilesServiceBean.isDataFilePresentInDatasetVersion(releasedVersion, df); } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/Retention.java b/src/main/java/edu/harvard/iq/dataverse/Retention.java new file mode 100644 index 00000000000..e1bd2231570 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/Retention.java @@ -0,0 +1,102 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.util.BundleUtil; +import jakarta.persistence.*; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Objects; + +@NamedQueries({ + @NamedQuery( name="Retention.findAll", + query = "SELECT r FROM Retention r"), + @NamedQuery( name="Retention.findById", + query = "SELECT r FROM Retention r WHERE r.id=:id"), + @NamedQuery( name="Retention.findByDateUnavailable", + query = "SELECT r FROM Retention r WHERE r.dateUnavailable=:dateUnavailable"), + @NamedQuery( name="Retention.deleteById", + query = "DELETE FROM Retention r WHERE r.id=:id") +}) +@Entity +public class Retention { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @Column(nullable = false) + private LocalDate dateUnavailable; + + @Column(columnDefinition="TEXT") + private String reason; + + @OneToMany(mappedBy="retention", cascade={ CascadeType.REMOVE, CascadeType.PERSIST}) + private List dataFiles; + + public Retention(){ + dateUnavailable = LocalDate.now().plusYears(1000); // Most likely valid with respect to configuration + } + + public Retention(LocalDate dateUnavailable, String reason) { + this.dateUnavailable = dateUnavailable; + this.reason = reason; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public LocalDate getDateUnavailable() { + return dateUnavailable; + } + + public void setDateUnavailable(LocalDate dateUnavailable) { + this.dateUnavailable = dateUnavailable; + } + + public String getFormattedDateUnavailable() { + return getDateUnavailable().format(DateTimeFormatter.ISO_LOCAL_DATE.withLocale(BundleUtil.getCurrentLocale())); + } + + public String getReason() { + return reason; + } + + public void setReason(String reason) { + this.reason = reason; + } + + public List getDataFiles() { + return dataFiles; + } + + public void setDataFiles(List dataFiles) { + this.dataFiles = dataFiles; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Retention retention = (Retention) o; + return id.equals(retention.id) && dateUnavailable.equals(retention.dateUnavailable) && Objects.equals(reason, retention.reason); + } + + @Override + public int hashCode() { + return Objects.hash(id, dateUnavailable, reason); + } + + @Override + public String toString() { + return "Retention{" + + "id=" + id + + ", dateUnavailable=" + dateUnavailable + + ", reason='" + reason + '\'' + + '}'; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/RetentionServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/RetentionServiceBean.java new file mode 100644 index 00000000000..1421ac61120 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/RetentionServiceBean.java @@ -0,0 +1,66 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; +import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; +import jakarta.ejb.EJB; +import jakarta.ejb.Stateless; +import jakarta.inject.Named; +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; +import jakarta.persistence.Query; + +import java.util.List; + + +@Stateless +@Named +public class RetentionServiceBean { + + @PersistenceContext + EntityManager em; + + @EJB + ActionLogServiceBean actionLogSvc; + + public List findAllRetentions() { + return em.createNamedQuery("Retention.findAll", Retention.class).getResultList(); + } + + public Retention findByRetentionId(Long id) { + Query query = em.createNamedQuery("Retention.findById", Retention.class); + query.setParameter("id", id); + try { + return (Retention) query.getSingleResult(); + } catch (Exception ex) { + return null; + } + } + + public Retention merge(Retention r) { + return em.merge(r); + } + + public Long save(Retention retention, String userIdentifier) { + if (retention.getId() == null) { + em.persist(retention); + em.flush(); + } + //Not quite from a command, but this action can be done by anyone, so command seems better than Admin or other alternatives + actionLogSvc.log(new ActionLogRecord(ActionLogRecord.ActionType.Command, "retentionCreate") + .setInfo("id: " + retention.getId() + " date unavailable: " + retention.getDateUnavailable() + " reason: " + retention.getReason()).setUserIdentifier(userIdentifier)); + return retention.getId(); + } + + private int deleteById(long id, String userIdentifier) { + //Not quite from a command, but this action can be done by anyone, so command seems better than Admin or other alternatives + actionLogSvc.log(new ActionLogRecord(ActionLogRecord.ActionType.Command, "retentionDelete") + .setInfo(Long.toString(id)) + .setUserIdentifier(userIdentifier)); + return em.createNamedQuery("Retention.deleteById") + .setParameter("id", id) + .executeUpdate(); + } + public int delete(Retention retention, String userIdentifier) { + return deleteById(retention.getId(), userIdentifier); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java index 91bcc508b78..7854f5adfd8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/SettingsWrapper.java @@ -78,6 +78,9 @@ public class SettingsWrapper implements java.io.Serializable { private boolean embargoDateChecked = false; private LocalDate maxEmbargoDate = null; + private boolean retentionDateChecked = false; + private LocalDate minRetentionDate = null; + private String siteUrl = null; private Dataverse rootDataverse = null; @@ -582,6 +585,89 @@ public void validateEmbargoDate(FacesContext context, UIComponent component, Obj } } + public LocalDate getMinRetentionDate() { + if (!retentionDateChecked) { + String months = getValueForKey(Key.MinRetentionDurationInMonths); + Long minMonths = null; + if (months != null) { + try { + minMonths = Long.parseLong(months); + } catch (NumberFormatException nfe) { + logger.warning("Cant interpret :MinRetentionDurationInMonths as a long"); + } + } + + if (minMonths != null && minMonths != 0) { + if (minMonths == -1) { + minMonths = 0l; // Absolute minimum is 0 + } + minRetentionDate = LocalDate.now().plusMonths(minMonths); + } + retentionDateChecked = true; + } + return minRetentionDate; + } + + public LocalDate getMaxRetentionDate() { + Long maxMonths = 12000l; // Arbitrary cutoff at 1000 years - needs to keep maxDate < year 999999999 and + // somehwere 1K> x >10K years the datepicker widget stops showing a popup + // calendar + return LocalDate.now().plusMonths(maxMonths); + } + + public boolean isValidRetentionDate(Retention r) { + + if (r.getDateUnavailable()==null || + isRetentionAllowed() && r.getDateUnavailable().isAfter(getMinRetentionDate())) { + return true; + } + + return false; + } + + public boolean isRetentionAllowed() { + //Need a valid :MinRetentionDurationInMonths setting to allow retentions + return getMinRetentionDate()!=null; + } + + public void validateRetentionDate(FacesContext context, UIComponent component, Object value) + throws ValidatorException { + if (isRetentionAllowed()) { + UIComponent cb = component.findComponent("retentionCheckbox"); + UIInput endComponent = (UIInput) cb; + boolean removedState = false; + if (endComponent != null) { + try { + removedState = (Boolean) endComponent.getSubmittedValue(); + } catch (NullPointerException npe) { + // Do nothing - checkbox is not being shown (and is therefore not checked) + } + } + if (!removedState && value == null) { + String msgString = BundleUtil.getStringFromBundle("retention.date.required"); + FacesMessage msg = new FacesMessage(msgString); + msg.setSeverity(FacesMessage.SEVERITY_ERROR); + throw new ValidatorException(msg); + } + Retention newR = new Retention(((LocalDate) value), null); + if (!isValidRetentionDate(newR)) { + String minDate = getMinRetentionDate().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + String maxDate = getMaxRetentionDate().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + String msgString = BundleUtil.getStringFromBundle("retention.date.invalid", + Arrays.asList(minDate, maxDate)); + // If we don't throw an exception here, the datePicker will use it's own + // vaidator and display a default message. The value for that can be set by + // adding validatorMessage="#{bundle['retention.date.invalid']}" (a version with + // no params) to the datepicker + // element in file-edit-popup-fragment.html, but it would be better to catch all + // problems here (so we can show a message with the min/max dates). + FacesMessage msg = new FacesMessage(msgString); + msg.setSeverity(FacesMessage.SEVERITY_ERROR); + throw new ValidatorException(msg); + } + } + } + Map languageMap = null; public Map getBaseMetadataLanguageMap(boolean refresh) { diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java index 24c0f9d7926..f9cf061e771 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Shib.java +++ b/src/main/java/edu/harvard/iq/dataverse/Shib.java @@ -59,6 +59,8 @@ public class Shib implements java.io.Serializable { SettingsServiceBean settingsService; @EJB SystemConfig systemConfig; + @EJB + UserServiceBean userService; HttpServletRequest request; @@ -259,6 +261,7 @@ else if (ShibAffiliationOrder.equals("firstAffiliation")) { state = State.REGULAR_LOGIN_INTO_EXISTING_SHIB_ACCOUNT; logger.fine("Found user based on " + userPersistentId + ". Logging in."); logger.fine("Updating display info for " + au.getName()); + userService.updateLastLogin(au); authSvc.updateAuthenticatedUser(au, displayInfo); logInUserAndSetShibAttributes(au); String prettyFacesHomePageString = getPrettyFacesHomePageString(false); diff --git a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java index b6ab23848e2..d31fdd4e380 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java +++ b/src/main/java/edu/harvard/iq/dataverse/ThumbnailServiceWrapper.java @@ -177,7 +177,7 @@ public String getDatasetCardImageAsUrl(Dataset dataset, Long versionId, boolean StorageIO storageIO = null; try { storageIO = DataAccess.getStorageIO(dataset); - if (storageIO.isAuxObjectCached(DatasetUtil.datasetLogoFilenameFinal)) { + if (storageIO != null && storageIO.isAuxObjectCached(DatasetUtil.datasetLogoFilenameFinal)) { // If not, return null/use the default, otherwise pass the logo URL hasDatasetLogo = true; } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java index b7305a24f69..b7e94464db9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/AbstractApiBean.java @@ -42,6 +42,7 @@ import jakarta.persistence.NoResultException; import jakarta.persistence.PersistenceContext; import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.constraints.NotNull; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; @@ -531,17 +532,21 @@ protected DvObject findDvo( Long id ) { * with that alias. If that fails, tries to get a {@link Dataset} with that global id. * @param id a value identifying the DvObject, either numeric of textual. * @return A DvObject, or {@code null} + * @throws WrappedResponse */ - protected DvObject findDvo( String id ) { - if ( isNumeric(id) ) { - return findDvo( Long.valueOf(id)) ; + @NotNull + protected DvObject findDvo(@NotNull final String id) throws WrappedResponse { + DvObject d = null; + if (isNumeric(id)) { + d = findDvo(Long.valueOf(id)); } else { - Dataverse d = dataverseSvc.findByAlias(id); - return ( d != null ) ? - d : datasetSvc.findByGlobalId(id); - + d = dataverseSvc.findByAlias(id); } - } + if (d == null) { + return findDatasetOrDie(id); + } + return d; + } protected T failIfNull( T t, String errorMessage ) throws WrappedResponse { if ( t != null ) return t; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 297ec2d3681..e95500426c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -466,7 +466,9 @@ public String tabularDatafileMetadataDDI(@Context ContainerRequestContext crc, @ if (!dataFile.isTabularData()) { throw new BadRequestException("tabular data required"); } - + if (FileUtil.isRetentionExpired(dataFile)) { + throw new BadRequestException("unable to download file with expired retention"); + } if (dataFile.isRestricted() || FileUtil.isActivelyEmbargoed(dataFile)) { boolean hasPermissionToDownloadFile = false; DataverseRequest dataverseRequest; @@ -921,14 +923,15 @@ public void write(OutputStream os) throws IOException, } } else { boolean embargoed = FileUtil.isActivelyEmbargoed(file); - if (file.isRestricted() || embargoed) { + boolean retentionExpired = FileUtil.isRetentionExpired(file); + if (file.isRestricted() || embargoed || retentionExpired) { if (zipper == null) { fileManifest = fileManifest + file.getFileMetadata().getLabel() + " IS " - + (embargoed ? "EMBARGOED" : "RESTRICTED") + + (embargoed ? "EMBARGOED" : retentionExpired ? "RETENTIONEXPIRED" : "RESTRICTED") + " AND CANNOT BE DOWNLOADED\r\n"; } else { zipper.addToManifest(file.getFileMetadata().getLabel() + " IS " - + (embargoed ? "EMBARGOED" : "RESTRICTED") + + (embargoed ? "EMBARGOED" : retentionExpired ? "RETENTIONEXPIRED" : "RESTRICTED") + " AND CANNOT BE DOWNLOADED\r\n"); } } else { @@ -1402,6 +1405,10 @@ public Response requestFileAccess(@Context ContainerRequestContext crc, @PathPar return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.fileNotFound", args)); } + if (FileUtil.isRetentionExpired(dataFile)) { + return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.failure.retentionExpired")); + } + if (!dataFile.getOwner().isFileAccessRequest()) { return error(BAD_REQUEST, BundleUtil.getStringFromBundle("access.api.requestAccess.requestsNotAccepted")); } @@ -1735,8 +1742,11 @@ private boolean isAccessAuthorized(User requestUser, DataFile df) { //True if there's an embargo that hasn't yet expired //In this state, we block access as though the file is restricted (even if it is not restricted) boolean embargoed = FileUtil.isActivelyEmbargoed(df); - - + // access is also blocked for retention expired files + boolean retentionExpired = FileUtil.isRetentionExpired(df); + // No access ever if retention is expired + if(retentionExpired) return false; + /* SEK 7/26/2018 for 3661 relying on the version state of the dataset versions to which this file is attached check to see if at least one is RELEASED @@ -1801,7 +1811,7 @@ private boolean isAccessAuthorized(User requestUser, DataFile df) { //The one case where we don't need to check permissions - if (!restricted && !embargoed && published) { + if (!restricted && !embargoed && !retentionExpired && published) { // If they are not published, they can still be downloaded, if the user // has the permission to view unpublished versions! (this case will // be handled below) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index d098c2fe16a..802904b5173 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -17,6 +17,7 @@ import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.StringUtil; import edu.harvard.iq.dataverse.validation.EMailValidator; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.Template; @@ -1029,29 +1030,49 @@ public Response deleteRole(@Context ContainerRequestContext crc, @PathParam("id" }, getRequestUser(crc)); } - @Path("superuser/{identifier}") - @POST - public Response toggleSuperuser(@PathParam("identifier") String identifier) { - ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "toggleSuperuser") - .setInfo(identifier); - try { - AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); - if (user.isDeactivated()) { - return error(Status.BAD_REQUEST, "You cannot make a deactivated user a superuser."); - } + @Path("superuser/{identifier}") + @Deprecated + @POST + public Response toggleSuperuser(@PathParam("identifier") String identifier) { + ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "toggleSuperuser") + .setInfo(identifier); + try { + final AuthenticatedUser user = authSvc.getAuthenticatedUser(identifier); + return setSuperuserStatus(user, !user.isSuperuser()); + } catch (Exception e) { + alr.setActionResult(ActionLogRecord.Result.InternalError); + alr.setInfo(alr.getInfo() + "// " + e.getMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + actionLogSvc.log(alr); + } + } - user.setSuperuser(!user.isSuperuser()); + private Response setSuperuserStatus(AuthenticatedUser user, Boolean isSuperuser) { + if (user.isDeactivated()) { + return error(Status.BAD_REQUEST, "You cannot make a deactivated user a superuser."); + } + user.setSuperuser(isSuperuser); + return ok("User " + user.getIdentifier() + " " + (user.isSuperuser() ? "set" : "removed") + + " as a superuser."); + } - return ok("User " + user.getIdentifier() + " " + (user.isSuperuser() ? "set" : "removed") - + " as a superuser."); - } catch (Exception e) { - alr.setActionResult(ActionLogRecord.Result.InternalError); - alr.setInfo(alr.getInfo() + "// " + e.getMessage()); - return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } finally { - actionLogSvc.log(alr); - } - } + @Path("superuser/{identifier}") + @PUT + // Using string instead of boolean so user doesn't need to add a Content-type header in their request + public Response setSuperuserStatus(@PathParam("identifier") String identifier, String isSuperuser) { + ActionLogRecord alr = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "setSuperuserStatus") + .setInfo(identifier + ":" + isSuperuser); + try { + return setSuperuserStatus(authSvc.getAuthenticatedUser(identifier), StringUtil.isTrue(isSuperuser)); + } catch (Exception e) { + alr.setActionResult(ActionLogRecord.Result.InternalError); + alr.setInfo(alr.getInfo() + "// " + e.getMessage()); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + actionLogSvc.log(alr); + } + } @GET @Path("validate/datasets") @@ -1332,26 +1353,24 @@ public Response convertUserFromBcryptToSha1(String json) { } - @Path("permissions/{dvo}") - @AuthRequired - @GET - public Response findPermissonsOn(@Context ContainerRequestContext crc, @PathParam("dvo") String dvo) { - try { - DvObject dvObj = findDvo(dvo); - if (dvObj == null) { - return notFound("DvObject " + dvo + " not found"); - } - User aUser = getRequestUser(crc); - JsonObjectBuilder bld = Json.createObjectBuilder(); - bld.add("user", aUser.getIdentifier()); - bld.add("permissions", json(permissionSvc.permissionsFor(createDataverseRequest(aUser), dvObj))); - return ok(bld); - - } catch (Exception e) { - logger.log(Level.SEVERE, "Error while testing permissions", e); - return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); - } - } + @Path("permissions/{dvo}") + @AuthRequired + @GET + public Response findPermissonsOn(@Context final ContainerRequestContext crc, @PathParam("dvo") final String dvo) { + try { + final DvObject dvObj = findDvo(dvo); + final User aUser = getRequestUser(crc); + final JsonObjectBuilder bld = Json.createObjectBuilder(); + bld.add("user", aUser.getIdentifier()); + bld.add("permissions", json(permissionSvc.permissionsFor(createDataverseRequest(aUser), dvObj))); + return ok(bld); + } catch (WrappedResponse r) { + return r.getResponse(); + } catch (Exception e) { + logger.log(Level.SEVERE, "Error while testing permissions", e); + return error(Response.Status.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } @Path("assignee/{idtf}") @GET diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 930776ffb0f..b9428129dc8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -80,6 +80,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ExecutionException; @@ -154,6 +155,9 @@ public class Datasets extends AbstractApiBean { @EJB EmbargoServiceBean embargoService; + @EJB + RetentionServiceBean retentionService; + @Inject MakeDataCountLoggingServiceBean mdcLogService; @@ -1690,6 +1694,306 @@ public Response removeFileEmbargo(@Context ContainerRequestContext crc, @PathPar } } + @POST + @AuthRequired + @Path("{id}/files/actions/:set-retention") + public Response createFileRetention(@Context ContainerRequestContext crc, @PathParam("id") String id, String jsonBody){ + + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + Dataset dataset; + try { + dataset = findDatasetOrDie(id); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + + boolean hasValidTerms = TermsOfUseAndAccessValidator.isTOUAValid(dataset.getLatestVersion().getTermsOfUseAndAccess(), null); + + if (!hasValidTerms){ + return error(Status.CONFLICT, BundleUtil.getStringFromBundle("dataset.message.toua.invalid")); + } + + // client is superadmin or (client has EditDataset permission on these files and files are unreleased) + // check if files are unreleased(DRAFT?) + if ((!authenticatedUser.isSuperuser() && (dataset.getLatestVersion().getVersionState() != DatasetVersion.VersionState.DRAFT) ) || !permissionService.userOn(authenticatedUser, dataset).has(Permission.EditDataset)) { + return error(Status.FORBIDDEN, "Either the files are released and user is not a superuser or user does not have EditDataset permissions"); + } + + // check if retentions are allowed(:MinRetentionDurationInMonths), gets the :MinRetentionDurationInMonths setting variable, if 0 or not set(null) return 400 + long minRetentionDurationInMonths = 0; + try { + minRetentionDurationInMonths = Long.parseLong(settingsService.get(SettingsServiceBean.Key.MinRetentionDurationInMonths.toString())); + } catch (NumberFormatException nfe){ + if (nfe.getMessage().contains("null")) { + return error(Status.BAD_REQUEST, "No Retention periods allowed"); + } + } + if (minRetentionDurationInMonths == 0){ + return error(Status.BAD_REQUEST, "No Retention periods allowed"); + } + + JsonObject json; + try { + json = JsonUtil.getJsonObject(jsonBody); + } catch (JsonException ex) { + return error(Status.BAD_REQUEST, "Invalid JSON; error message: " + ex.getMessage()); + } + + Retention retention = new Retention(); + + + LocalDate currentDateTime = LocalDate.now(); + + // Extract the dateUnavailable - check if specified and valid + String dateUnavailableStr = ""; + LocalDate dateUnavailable; + try { + dateUnavailableStr = json.getString("dateUnavailable"); + dateUnavailable = LocalDate.parse(dateUnavailableStr); + } catch (NullPointerException npex) { + return error(Status.BAD_REQUEST, "Invalid retention period; no dateUnavailable specified"); + } catch (ClassCastException ccex) { + return error(Status.BAD_REQUEST, "Invalid retention period; dateUnavailable must be a string"); + } catch (DateTimeParseException dtpex) { + return error(Status.BAD_REQUEST, "Invalid date format for dateUnavailable: " + dateUnavailableStr); + } + + // check :MinRetentionDurationInMonths if -1 + LocalDate minRetentionDateTime = minRetentionDurationInMonths != -1 ? LocalDate.now().plusMonths(minRetentionDurationInMonths) : null; + // dateUnavailable is not in the past + if (dateUnavailable.isAfter(currentDateTime)){ + retention.setDateUnavailable(dateUnavailable); + } else { + return error(Status.BAD_REQUEST, "Date unavailable can not be in the past"); + } + + // dateAvailable is within limits + if (minRetentionDateTime != null){ + if (dateUnavailable.isBefore(minRetentionDateTime)){ + return error(Status.BAD_REQUEST, "Date unavailable can not be earlier than MinRetentionDurationInMonths: "+minRetentionDurationInMonths + " from now"); + } + } + + try { + String reason = json.getString("reason"); + retention.setReason(reason); + } catch (NullPointerException npex) { + // ignoring; no reason specified is OK, it is optional + } catch (ClassCastException ccex) { + return error(Status.BAD_REQUEST, "Invalid retention period; reason must be a string"); + } + + + List datasetFiles = dataset.getFiles(); + List filesToRetention = new LinkedList<>(); + + // extract fileIds from json, find datafiles and add to list + if (json.containsKey("fileIds")){ + try { + JsonArray fileIds = json.getJsonArray("fileIds"); + for (JsonValue jsv : fileIds) { + try { + DataFile dataFile = findDataFileOrDie(jsv.toString()); + filesToRetention.add(dataFile); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + } catch (ClassCastException ccex) { + return error(Status.BAD_REQUEST, "Invalid retention period; fileIds must be an array of id strings"); + } catch (NullPointerException npex) { + return error(Status.BAD_REQUEST, "Invalid retention period; no fileIds specified"); + } + } else { + return error(Status.BAD_REQUEST, "No fileIds specified"); + } + + List orphanedRetentions = new ArrayList(); + // check if files belong to dataset + if (datasetFiles.containsAll(filesToRetention)) { + JsonArrayBuilder restrictedFiles = Json.createArrayBuilder(); + boolean badFiles = false; + for (DataFile datafile : filesToRetention) { + // superuser can overrule an existing retention, even on released files + if (datafile.isReleased() && !authenticatedUser.isSuperuser()) { + restrictedFiles.add(datafile.getId()); + badFiles = true; + } + } + if (badFiles) { + return Response.status(Status.FORBIDDEN) + .entity(NullSafeJsonBuilder.jsonObjectBuilder().add("status", ApiConstants.STATUS_ERROR) + .add("message", "You do not have permission to set a retention period for the following files") + .add("files", restrictedFiles).build()) + .type(MediaType.APPLICATION_JSON_TYPE).build(); + } + retention=retentionService.merge(retention); + // Good request, so add the retention. Track any existing retentions so we can + // delete them if there are no files left that reference them. + for (DataFile datafile : filesToRetention) { + Retention ret = datafile.getRetention(); + if (ret != null) { + ret.getDataFiles().remove(datafile); + if (ret.getDataFiles().isEmpty()) { + orphanedRetentions.add(ret); + } + } + // Save merges the datafile with an retention into the context + datafile.setRetention(retention); + fileService.save(datafile); + } + //Call service to get action logged + long retentionId = retentionService.save(retention, authenticatedUser.getIdentifier()); + if (orphanedRetentions.size() > 0) { + for (Retention ret : orphanedRetentions) { + retentionService.delete(ret, authenticatedUser.getIdentifier()); + } + } + //If superuser, report changes to any released files + if (authenticatedUser.isSuperuser()) { + String releasedFiles = filesToRetention.stream().filter(d -> d.isReleased()) + .map(d -> d.getId().toString()).collect(Collectors.joining(",")); + if (!releasedFiles.isBlank()) { + actionLogSvc + .log(new ActionLogRecord(ActionLogRecord.ActionType.Admin, "retentionAddedTo") + .setInfo("Retention id: " + retention.getId() + " added for released file(s), id(s) " + + releasedFiles + ".") + .setUserIdentifier(authenticatedUser.getIdentifier())); + } + } + return ok(Json.createObjectBuilder().add("message", "File(s) retention period has been set or updated")); + } else { + return error(BAD_REQUEST, "Not all files belong to dataset"); + } + } + + @POST + @AuthRequired + @Path("{id}/files/actions/:unset-retention") + public Response removeFileRetention(@Context ContainerRequestContext crc, @PathParam("id") String id, String jsonBody){ + + // user is authenticated + AuthenticatedUser authenticatedUser = null; + try { + authenticatedUser = getRequestAuthenticatedUserOrDie(crc); + } catch (WrappedResponse ex) { + return error(Status.UNAUTHORIZED, "Authentication is required."); + } + + Dataset dataset; + try { + dataset = findDatasetOrDie(id); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + + // client is superadmin or (client has EditDataset permission on these files and files are unreleased) + // check if files are unreleased(DRAFT?) + //ToDo - here and below - check the release status of files and not the dataset state (draft dataset version still can have released files) + if ((!authenticatedUser.isSuperuser() && (dataset.getLatestVersion().getVersionState() != DatasetVersion.VersionState.DRAFT) ) || !permissionService.userOn(authenticatedUser, dataset).has(Permission.EditDataset)) { + return error(Status.FORBIDDEN, "Either the files are released and user is not a superuser or user does not have EditDataset permissions"); + } + + // check if retentions are allowed(:MinRetentionDurationInMonths), gets the :MinRetentionDurationInMonths setting variable, if 0 or not set(null) return 400 + int minRetentionDurationInMonths = 0; + try { + minRetentionDurationInMonths = Integer.parseInt(settingsService.get(SettingsServiceBean.Key.MinRetentionDurationInMonths.toString())); + } catch (NumberFormatException nfe){ + if (nfe.getMessage().contains("null")) { + return error(Status.BAD_REQUEST, "No Retention periods allowed"); + } + } + if (minRetentionDurationInMonths == 0){ + return error(Status.BAD_REQUEST, "No Retention periods allowed"); + } + + JsonObject json; + try { + json = JsonUtil.getJsonObject(jsonBody); + } catch (JsonException ex) { + return error(Status.BAD_REQUEST, "Invalid JSON; error message: " + ex.getMessage()); + } + + List datasetFiles = dataset.getFiles(); + List retentionFilesToUnset = new LinkedList<>(); + + // extract fileIds from json, find datafiles and add to list + if (json.containsKey("fileIds")){ + try { + JsonArray fileIds = json.getJsonArray("fileIds"); + for (JsonValue jsv : fileIds) { + try { + DataFile dataFile = findDataFileOrDie(jsv.toString()); + retentionFilesToUnset.add(dataFile); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + } catch (ClassCastException ccex) { + return error(Status.BAD_REQUEST, "fileIds must be an array of id strings"); + } catch (NullPointerException npex) { + return error(Status.BAD_REQUEST, "No fileIds specified"); + } + } else { + return error(Status.BAD_REQUEST, "No fileIds specified"); + } + + List orphanedRetentions = new ArrayList(); + // check if files belong to dataset + if (datasetFiles.containsAll(retentionFilesToUnset)) { + JsonArrayBuilder restrictedFiles = Json.createArrayBuilder(); + boolean badFiles = false; + for (DataFile datafile : retentionFilesToUnset) { + // superuser can overrule an existing retention, even on released files + if (datafile.getRetention()==null || ((datafile.isReleased() && datafile.getRetention() != null) && !authenticatedUser.isSuperuser())) { + restrictedFiles.add(datafile.getId()); + badFiles = true; + } + } + if (badFiles) { + return Response.status(Status.FORBIDDEN) + .entity(NullSafeJsonBuilder.jsonObjectBuilder().add("status", ApiConstants.STATUS_ERROR) + .add("message", "The following files do not have retention periods or you do not have permission to remove their retention periods") + .add("files", restrictedFiles).build()) + .type(MediaType.APPLICATION_JSON_TYPE).build(); + } + // Good request, so remove the retention from the files. Track any existing retentions so we can + // delete them if there are no files left that reference them. + for (DataFile datafile : retentionFilesToUnset) { + Retention ret = datafile.getRetention(); + if (ret != null) { + ret.getDataFiles().remove(datafile); + if (ret.getDataFiles().isEmpty()) { + orphanedRetentions.add(ret); + } + } + // Save merges the datafile with an retention into the context + datafile.setRetention(null); + fileService.save(datafile); + } + if (orphanedRetentions.size() > 0) { + for (Retention ret : orphanedRetentions) { + retentionService.delete(ret, authenticatedUser.getIdentifier()); + } + } + String releasedFiles = retentionFilesToUnset.stream().filter(d -> d.isReleased()).map(d->d.getId().toString()).collect(Collectors.joining(",")); + if(!releasedFiles.isBlank()) { + ActionLogRecord removeRecord = new ActionLogRecord(ActionLogRecord.ActionType.Admin, "retentionRemovedFrom").setInfo("Retention removed from released file(s), id(s) " + releasedFiles + "."); + removeRecord.setUserIdentifier(authenticatedUser.getIdentifier()); + actionLogSvc.log(removeRecord); + } + return ok(Json.createObjectBuilder().add("message", "Retention periods were removed from file(s)")); + } else { + return error(BAD_REQUEST, "Not all files belong to dataset"); + } + } @PUT @AuthRequired @@ -3756,11 +4060,11 @@ public Response getGlobusDownloadParams(@Context ContainerRequestContext crc, @P // ------------------------------------- // (1) Get the user from the ContainerRequestContext // ------------------------------------- - AuthenticatedUser authUser; + AuthenticatedUser authUser = null; try { authUser = getRequestAuthenticatedUserOrDie(crc); } catch (WrappedResponse e) { - return e.getResponse(); + logger.fine("guest user globus download"); } // ------------------------------------- // (2) Get the Dataset Id diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 250257fc33b..02b60fdb32a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -1,24 +1,10 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DataverseFacet; -import edu.harvard.iq.dataverse.DataverseContact; -import edu.harvard.iq.dataverse.DataverseMetadataBlockFacet; -import edu.harvard.iq.dataverse.DataverseServiceBean; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.api.datadeposit.SwordServiceBean; import edu.harvard.iq.dataverse.api.dto.DataverseMetadataBlockFacetDTO; import edu.harvard.iq.dataverse.authorization.DataverseRole; -import edu.harvard.iq.dataverse.DvObject; -import edu.harvard.iq.dataverse.GlobalId; -import edu.harvard.iq.dataverse.GuestbookResponseServiceBean; -import edu.harvard.iq.dataverse.GuestbookServiceBean; -import edu.harvard.iq.dataverse.MetadataBlock; -import edu.harvard.iq.dataverse.RoleAssignment; import edu.harvard.iq.dataverse.api.dto.ExplicitGroupDTO; import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO; @@ -34,45 +20,7 @@ import edu.harvard.iq.dataverse.authorization.users.User; import edu.harvard.iq.dataverse.dataverse.DataverseUtil; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; -import edu.harvard.iq.dataverse.engine.command.impl.AddRoleAssigneesToExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.AssignRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.CreateDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.CreateExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand; -import edu.harvard.iq.dataverse.engine.command.impl.CreateRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.DeleteCollectionQuotaCommand; -import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataverseLinkingDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.DeleteExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDatasetSchemaCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetCollectionQuotaCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetCollectionStorageUseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateMetadataBlockFacetRootCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetDataverseStorageSizeCommand; -import edu.harvard.iq.dataverse.engine.command.impl.GetExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ImportDatasetCommand; -import edu.harvard.iq.dataverse.engine.command.impl.LinkDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListDataverseContentCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListExplicitGroupsCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListFacetsCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListMetadataBlockFacetsCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListMetadataBlocksCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ListRoleAssignments; -import edu.harvard.iq.dataverse.engine.command.impl.ListRolesCommand; -import edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetCommand; -import edu.harvard.iq.dataverse.engine.command.impl.PublishDatasetResult; -import edu.harvard.iq.dataverse.engine.command.impl.MoveDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.PublishDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RemoveRoleAssigneesFromExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.RevokeRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.SetCollectionQuotaCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseDefaultContributorRoleCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateDataverseMetadataBlocksCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateExplicitGroupCommand; -import edu.harvard.iq.dataverse.engine.command.impl.UpdateMetadataBlockFacetsCommand; -import edu.harvard.iq.dataverse.engine.command.impl.ValidateDatasetJsonCommand; +import edu.harvard.iq.dataverse.engine.command.impl.*; import edu.harvard.iq.dataverse.pidproviders.PidProvider; import edu.harvard.iq.dataverse.pidproviders.PidUtil; import edu.harvard.iq.dataverse.settings.JvmSettings; @@ -87,23 +35,14 @@ import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.JsonUtil; -import static edu.harvard.iq.dataverse.util.json.JsonPrinter.brief; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.TreeSet; +import java.io.StringReader; +import java.util.*; import java.util.logging.Level; import java.util.logging.Logger; import jakarta.ejb.EJB; import jakarta.ejb.EJBException; import jakarta.ejb.Stateless; -import jakarta.json.Json; -import jakarta.json.JsonArrayBuilder; -import jakarta.json.JsonNumber; -import jakarta.json.JsonObject; -import jakarta.json.JsonObjectBuilder; -import jakarta.json.JsonString; -import jakarta.json.JsonValue; +import jakarta.json.*; import jakarta.json.JsonValue.ValueType; import jakarta.json.stream.JsonParsingException; import jakarta.validation.ConstraintViolationException; @@ -127,10 +66,6 @@ import java.io.OutputStream; import java.text.MessageFormat; import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; import jakarta.servlet.http.HttpServletResponse; import jakarta.ws.rs.WebApplicationException; @@ -168,6 +103,12 @@ public class Dataverses extends AbstractApiBean { @EJB DataverseServiceBean dataverseService; + @EJB + DataverseLinkingServiceBean linkingService; + + @EJB + FeaturedDataverseServiceBean featuredDataverseService; + @EJB SwordServiceBean swordService; @@ -696,6 +637,43 @@ public Response updateAttribute(@Context ContainerRequestContext crc, @PathParam } } + @PUT + @AuthRequired + @Path("{identifier}/inputLevels") + public Response updateInputLevels(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier, String jsonBody) { + try { + Dataverse dataverse = findDataverseOrDie(identifier); + List newInputLevels = parseInputLevels(jsonBody, dataverse); + execCommand(new UpdateDataverseInputLevelsCommand(dataverse, createDataverseRequest(getRequestUser(crc)), newInputLevels)); + return ok(BundleUtil.getStringFromBundle("dataverse.update.success"), JsonPrinter.json(dataverse)); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } + + private List parseInputLevels(String jsonBody, Dataverse dataverse) throws WrappedResponse { + JsonArray inputLevelsArray = Json.createReader(new StringReader(jsonBody)).readArray(); + + List newInputLevels = new ArrayList<>(); + for (JsonValue value : inputLevelsArray) { + JsonObject inputLevel = (JsonObject) value; + String datasetFieldTypeName = inputLevel.getString("datasetFieldTypeName"); + DatasetFieldType datasetFieldType = datasetFieldSvc.findByName(datasetFieldTypeName); + + if (datasetFieldType == null) { + String errorMessage = MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.updateinputlevels.error.invalidfieldtypename"), datasetFieldTypeName); + throw new WrappedResponse(badRequest(errorMessage)); + } + + boolean required = inputLevel.getBoolean("required"); + boolean include = inputLevel.getBoolean("include"); + + newInputLevels.add(new DataverseFieldTypeInputLevel(datasetFieldType, dataverse, required, include)); + } + + return newInputLevels; + } + @DELETE @AuthRequired @Path("{linkingDataverseId}/deleteLink/{linkedDataverseId}") @@ -715,14 +693,15 @@ public Response listMetadataBlocks(@Context ContainerRequestContext crc, @QueryParam("onlyDisplayedOnCreate") boolean onlyDisplayedOnCreate, @QueryParam("returnDatasetFieldTypes") boolean returnDatasetFieldTypes) { try { + Dataverse dataverse = findDataverseOrDie(dvIdtf); final List metadataBlocks = execCommand( new ListMetadataBlocksCommand( createDataverseRequest(getRequestUser(crc)), - findDataverseOrDie(dvIdtf), + dataverse, onlyDisplayedOnCreate ) ); - return ok(json(metadataBlocks, returnDatasetFieldTypes, onlyDisplayedOnCreate)); + return ok(json(metadataBlocks, returnDatasetFieldTypes, onlyDisplayedOnCreate, dataverse)); } catch (WrappedResponse we) { return we.getResponse(); } @@ -826,6 +805,111 @@ public Response listFacets(@Context ContainerRequestContext crc, @PathParam("ide } } + + @GET + @AuthRequired + @Path("{identifier}/featured") + /* + Allows user to get the collections that are featured by a given collection + probably more for SPA than end user + */ + public Response getFeaturedDataverses(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, String dvAliases) { + + try { + User u = getRequestUser(crc); + DataverseRequest r = createDataverseRequest(u); + Dataverse dataverse = findDataverseOrDie(dvIdtf); + JsonArrayBuilder fs = Json.createArrayBuilder(); + for (Dataverse f : execCommand(new ListFeaturedCollectionsCommand(r, dataverse))) { + fs.add(f.getAlias()); + } + return ok(fs); + } catch (WrappedResponse e) { + return e.getResponse(); + } + } + + + @POST + @AuthRequired + @Path("{identifier}/featured") + /** + * Allows user to set featured dataverses - must have edit dataverse permission + * + */ + public Response setFeaturedDataverses(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf, String dvAliases) { + List dvsFromInput = new LinkedList<>(); + + + try { + + for (JsonString dvAlias : Util.asJsonArray(dvAliases).getValuesAs(JsonString.class)) { + Dataverse dvToBeFeatured = dataverseService.findByAlias(dvAlias.getString()); + if (dvToBeFeatured == null) { + return error(Response.Status.BAD_REQUEST, "Can't find dataverse collection with alias '" + dvAlias + "'"); + } + dvsFromInput.add(dvToBeFeatured); + } + + if (dvsFromInput.isEmpty()) { + return error(Response.Status.BAD_REQUEST, "Please provide a valid Json array of dataverse collection aliases to be featured."); + } + + Dataverse dataverse = findDataverseOrDie(dvIdtf); + List featuredSource = new ArrayList<>(); + List featuredTarget = new ArrayList<>(); + featuredSource.addAll(dataverseService.findAllPublishedByOwnerId(dataverse.getId())); + featuredSource.addAll(linkingService.findLinkedDataverses(dataverse.getId())); + List featuredList = featuredDataverseService.findByDataverseId(dataverse.getId()); + + if (featuredSource.isEmpty()) { + return error(Response.Status.BAD_REQUEST, "There are no collections avaialble to be featured in Dataverse collection '" + dataverse.getDisplayName() + "'."); + } + + for (DataverseFeaturedDataverse dfd : featuredList) { + Dataverse fd = dfd.getFeaturedDataverse(); + featuredTarget.add(fd); + featuredSource.remove(fd); + } + + for (Dataverse test : dvsFromInput) { + if (featuredTarget.contains(test)) { + return error(Response.Status.BAD_REQUEST, "Dataverse collection '" + test.getDisplayName() + "' is already featured in Dataverse collection '" + dataverse.getDisplayName() + "'."); + } + + if (featuredSource.contains(test)) { + featuredTarget.add(test); + } else { + return error(Response.Status.BAD_REQUEST, "Dataverse collection '" + test.getDisplayName() + "' may not be featured in Dataverse collection '" + dataverse.getDisplayName() + "'."); + } + + } + // by passing null for Facets and DataverseFieldTypeInputLevel, those are not changed + execCommand(new UpdateDataverseCommand(dataverse, null, featuredTarget, createDataverseRequest(getRequestUser(crc)), null)); + return ok("Featured Dataverses of dataverse " + dvIdtf + " updated."); + + } catch (WrappedResponse ex) { + return ex.getResponse(); + } catch (JsonParsingException jpe){ + return error(Response.Status.BAD_REQUEST, "Please provide a valid Json array of dataverse collection aliases to be featured."); + } + + } + + @DELETE + @AuthRequired + @Path("{identifier}/featured") + public Response deleteFeaturedCollections(@Context ContainerRequestContext crc, @PathParam("identifier") String dvIdtf) throws WrappedResponse { + try { + Dataverse dataverse = findDataverseOrDie(dvIdtf); + List featuredTarget = new ArrayList<>(); + execCommand(new UpdateDataverseCommand(dataverse, null, featuredTarget, createDataverseRequest(getRequestUser(crc)), null)); + return ok(BundleUtil.getStringFromBundle("dataverses.api.delete.featured.collections.successful")); + } catch (WrappedResponse ex) { + return ex.getResponse(); + } + } + @POST @AuthRequired @Path("{identifier}/facets") diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java index 89b22b76a7d..c815caa09eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/DownloadInstanceWriter.java @@ -283,7 +283,7 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] storageIO = ImageThumbConverter.getImageThumbnailAsInputStream(storageIO, ImageThumbConverter.DEFAULT_THUMBNAIL_SIZE); } else { try { - int size = new Integer(di.getConversionParamValue()); + int size = Integer.parseInt(di.getConversionParamValue()); if (size > 0) { storageIO = ImageThumbConverter.getImageThumbnailAsInputStream(storageIO, size); } @@ -294,8 +294,10 @@ public void writeTo(DownloadInstance di, Class clazz, Type type, Annotation[] // and, since we now have tabular data files that can // have thumbnail previews... obviously, we don't want to // add the variable header to the image stream! - storageIO.setNoVarHeader(Boolean.TRUE); - storageIO.setVarHeader(null); + if (storageIO != null) { // ImageThumbConverter returns null if thumbnail conversion fails + storageIO.setNoVarHeader(Boolean.TRUE); + storageIO.setVarHeader(null); + } } } else if (dataFile.isTabularData()) { logger.fine("request for tabular data download;"); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java index 71e2865ca4d..6b9fcb38305 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java @@ -1,10 +1,8 @@ package edu.harvard.iq.dataverse.api; -import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.api.auth.AuthRequired; import edu.harvard.iq.dataverse.search.SearchFields; -import edu.harvard.iq.dataverse.DataverseServiceBean; -import edu.harvard.iq.dataverse.DvObjectServiceBean; import edu.harvard.iq.dataverse.search.FacetCategory; import edu.harvard.iq.dataverse.search.FacetLabel; import edu.harvard.iq.dataverse.search.SolrSearchResult; @@ -16,7 +14,6 @@ import edu.harvard.iq.dataverse.search.SearchConstants; import edu.harvard.iq.dataverse.search.SearchException; import edu.harvard.iq.dataverse.search.SearchUtil; -import edu.harvard.iq.dataverse.search.SolrIndexServiceBean; import edu.harvard.iq.dataverse.search.SortBy; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import java.io.IOException; @@ -26,6 +23,7 @@ import java.util.Map; import java.util.logging.Logger; import jakarta.ejb.EJB; +import jakarta.inject.Inject; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObjectBuilder; @@ -51,10 +49,8 @@ public class Search extends AbstractApiBean { SearchServiceBean searchService; @EJB DataverseServiceBean dataverseService; - @EJB - DvObjectServiceBean dvObjectService; - @EJB - SolrIndexServiceBean SolrIndexService; + @Inject + DatasetVersionFilesServiceBean datasetVersionFilesServiceBean; @GET @AuthRequired @@ -179,7 +175,7 @@ public Response search( JsonArrayBuilder itemsArrayBuilder = Json.createArrayBuilder(); List solrSearchResults = solrQueryResponse.getSolrSearchResults(); for (SolrSearchResult solrSearchResult : solrSearchResults) { - itemsArrayBuilder.add(solrSearchResult.toJsonObject(showRelevance, showEntityIds, showApiUrls, metadataFields)); + itemsArrayBuilder.add(solrSearchResult.json(showRelevance, showEntityIds, showApiUrls, metadataFields, getDatasetFileCount(solrSearchResult))); } JsonObjectBuilder spelling_alternatives = Json.createObjectBuilder(); @@ -187,31 +183,32 @@ public Response search( spelling_alternatives.add(entry.getKey(), entry.getValue().toString()); } - JsonArrayBuilder facets = Json.createArrayBuilder(); - JsonObjectBuilder facetCategoryBuilder = Json.createObjectBuilder(); - for (FacetCategory facetCategory : solrQueryResponse.getFacetCategoryList()) { - JsonObjectBuilder facetCategoryBuilderFriendlyPlusData = Json.createObjectBuilder(); - JsonArrayBuilder facetLabelBuilderData = Json.createArrayBuilder(); - for (FacetLabel facetLabel : facetCategory.getFacetLabel()) { - JsonObjectBuilder countBuilder = Json.createObjectBuilder(); - countBuilder.add(facetLabel.getName(), facetLabel.getCount()); - facetLabelBuilderData.add(countBuilder); - } - facetCategoryBuilderFriendlyPlusData.add("friendly", facetCategory.getFriendlyName()); - facetCategoryBuilderFriendlyPlusData.add("labels", facetLabelBuilderData); - facetCategoryBuilder.add(facetCategory.getName(), facetCategoryBuilderFriendlyPlusData); - } - facets.add(facetCategoryBuilder); - JsonObjectBuilder value = Json.createObjectBuilder() .add("q", query) .add("total_count", solrQueryResponse.getNumResultsFound()) .add("start", solrQueryResponse.getResultsStart()) .add("spelling_alternatives", spelling_alternatives) .add("items", itemsArrayBuilder.build()); + if (showFacets) { + JsonArrayBuilder facets = Json.createArrayBuilder(); + JsonObjectBuilder facetCategoryBuilder = Json.createObjectBuilder(); + for (FacetCategory facetCategory : solrQueryResponse.getFacetCategoryList()) { + JsonObjectBuilder facetCategoryBuilderFriendlyPlusData = Json.createObjectBuilder(); + JsonArrayBuilder facetLabelBuilderData = Json.createArrayBuilder(); + for (FacetLabel facetLabel : facetCategory.getFacetLabel()) { + JsonObjectBuilder countBuilder = Json.createObjectBuilder(); + countBuilder.add(facetLabel.getName(), facetLabel.getCount()); + facetLabelBuilderData.add(countBuilder); + } + facetCategoryBuilderFriendlyPlusData.add("friendly", facetCategory.getFriendlyName()); + facetCategoryBuilderFriendlyPlusData.add("labels", facetLabelBuilderData); + facetCategoryBuilder.add(facetCategory.getName(), facetCategoryBuilderFriendlyPlusData); + } + facets.add(facetCategoryBuilder); value.add("facets", facets); } + value.add("count_in_response", solrSearchResults.size()); /** * @todo Returning the fq might be useful as a troubleshooting aid @@ -232,6 +229,15 @@ public Response search( } } + private Long getDatasetFileCount(SolrSearchResult solrSearchResult) { + DvObject dvObject = solrSearchResult.getEntity(); + if (dvObject.isInstanceofDataset()) { + DatasetVersion datasetVersion = ((Dataset) dvObject).getVersionFromId(solrSearchResult.getDatasetVersionId()); + return datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion); + } + return null; + } + private User getUser(ContainerRequestContext crc) throws WrappedResponse { User userToExecuteSearchAs = GuestUser.get(); try { diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java index 1c0f5010059..4a8fb123fd4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/AuthenticationServiceBean.java @@ -307,11 +307,9 @@ public AuthenticatedUser getUpdateAuthenticatedUser( String authenticationProvid if (user != null && !user.isDeactivated()) { user = userService.updateLastLogin(user); } - + if ( user == null ) { throw new IllegalStateException("Authenticated user does not exist. The functionality to support creating one at this point in authentication has been removed."); - //return createAuthenticatedUser( - // new UserRecordIdentifier(authenticationProviderId, resp.getUserId()), resp.getUserId(), resp.getUserDisplayInfo(), true ); } else { if (BuiltinAuthenticationProvider.PROVIDER_ID.equals(user.getAuthenticatedUserLookup().getAuthenticationProviderId())) { return user; diff --git a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java index 0fd0852b4df..8f3dc07fdea 100644 --- a/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/authorization/providers/oauth2/OAuth2LoginBackingBean.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.authorization.providers.oauth2; import edu.harvard.iq.dataverse.DataverseSession; +import edu.harvard.iq.dataverse.UserServiceBean; import edu.harvard.iq.dataverse.authorization.AuthenticationProvider; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.UserRecordIdentifier; @@ -65,6 +66,9 @@ public class OAuth2LoginBackingBean implements Serializable { @EJB SystemConfig systemConfig; + @EJB + UserServiceBean userService; + @Inject DataverseSession session; @@ -128,6 +132,7 @@ public void exchangeCodeForToken() throws IOException { } else { // login the user and redirect to HOME of intended page (if any). // setUser checks for deactivated users. + dvUser = userService.updateLastLogin(dvUser); session.setUser(dvUser); final OAuth2TokenData tokenData = oauthUser.getTokenData(); if (tokenData != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index a1bcbe49327..bc4c69390cf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -270,6 +270,10 @@ public static StorageIO createNewStorageIO(T dvObject, S logger.warning("Could not find storage driver for: " + storageTag); throw new IOException("createDataAccessObject: Unsupported storage method " + storageDriverId); } + if (storageIO == null) { + logger.warning("Could not find storage driver for: " + storageTag); + throw new IOException("createDataAccessObject: Unsupported storage method " + storageDriverId); + } // Note: All storageIO classes must assure that dvObject instances' storageIdentifiers are prepended with // the :// + any additional storageIO type information required (e.g. the bucketname for s3/swift) // This currently happens when the storageIO is opened for write access diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java index 1be2bb79e0f..2435e3f778a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java @@ -35,21 +35,21 @@ import javax.imageio.stream.ImageOutputStream; import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.DataFileServiceBean; import edu.harvard.iq.dataverse.util.FileUtil; import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.ByteArrayOutputStream; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.OutputStream; import java.nio.channels.Channel; import java.nio.channels.Channels; -import java.nio.channels.FileChannel; import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.logging.Level; import java.util.logging.Logger; + +import jakarta.enterprise.inject.spi.CDI; import org.apache.commons.io.IOUtils; //import org.primefaces.util.Base64; import java.util.Base64; @@ -110,6 +110,12 @@ private static boolean isThumbnailAvailable(StorageIO storageIO, int s return false; } + // check if thumbnail generation failed: + if (file.isPreviewImageFail()) { + logger.fine("Thumbnail failed to be generated for "+ file.getId()); + return false; + } + if (isThumbnailCached(storageIO, size)) { logger.fine("Found cached thumbnail for " + file.getId()); return true; @@ -119,22 +125,23 @@ private static boolean isThumbnailAvailable(StorageIO storageIO, int s } private static boolean generateThumbnail(DataFile file, StorageIO storageIO, int size) { - logger.log(Level.FINE, (file.isPreviewImageFail() ? "Not trying" : "Trying") + " to generate thumbnail, file id: " + file.getId()); + logger.fine((file.isPreviewImageFail() ? "Not trying" : "Trying") + " to generate thumbnail, file id: " + file.getId()); + boolean thumbnailGenerated = false; // Don't try to generate if there have been failures: if (!file.isPreviewImageFail()) { - boolean thumbnailGenerated = false; if (file.getContentType().substring(0, 6).equalsIgnoreCase("image/")) { thumbnailGenerated = generateImageThumbnail(storageIO, size); } else if (file.getContentType().equalsIgnoreCase("application/pdf")) { thumbnailGenerated = generatePDFThumbnail(storageIO, size); } if (!thumbnailGenerated) { + file.setPreviewImageFail(true); + file.setPreviewImageAvailable(false); logger.fine("No thumbnail generated for " + file.getId()); } - return thumbnailGenerated; } - return false; + return thumbnailGenerated; } // Note that this method works on ALL file types for which thumbnail @@ -165,15 +172,30 @@ public static InputStreamIO getImageThumbnailAsInputStream(StorageIO s return null; } int cachedThumbnailSize = (int) storageIO.getAuxObjectSize(THUMBNAIL_SUFFIX + size); + InputStreamIO inputStreamIO = cachedThumbnailSize > 0 ? new InputStreamIO(cachedThumbnailInputStream, cachedThumbnailSize) : null; - InputStreamIO inputStreamIO = new InputStreamIO(cachedThumbnailInputStream, cachedThumbnailSize); - - inputStreamIO.setMimeType(THUMBNAIL_MIME_TYPE); + if (inputStreamIO != null) { + inputStreamIO.setMimeType(THUMBNAIL_MIME_TYPE); - String fileName = storageIO.getFileName(); - if (fileName != null) { - fileName = fileName.replaceAll("\\.[^\\.]*$", THUMBNAIL_FILE_EXTENSION); - inputStreamIO.setFileName(fileName); + String fileName = storageIO.getFileName(); + if (fileName != null) { + fileName = fileName.replaceAll("\\.[^\\.]*$", THUMBNAIL_FILE_EXTENSION); + inputStreamIO.setFileName(fileName); + } + } else { + if (storageIO.getDataFile() != null && cachedThumbnailSize == 0) { + // We found an older 0 length thumbnail. Newer image uploads will not have this issue. + // Once cleaned up, this thumbnail will no longer have this issue + logger.warning("Cleaning up zero sized thumbnail ID: "+ storageIO.getDataFile().getId()); + storageIO.getDataFile().setPreviewImageFail(true); + storageIO.getDataFile().setPreviewImageAvailable(false); + DataFileServiceBean datafileService = CDI.current().select(DataFileServiceBean.class).get(); + datafileService.save(storageIO.getDataFile()); + + // Now that we have marked this File as a thumbnail failure, + // no reason not to try and delete this 0-size cache here: + storageIO.deleteAuxObject(THUMBNAIL_SUFFIX + size); + } } return inputStreamIO; } catch (Exception ioex) { @@ -307,6 +329,7 @@ private static boolean generateImageThumbnail(StorageIO storageIO, int private static boolean generateImageThumbnailFromInputStream(StorageIO storageIO, int size, InputStream inputStream) { BufferedImage fullSizeImage; + boolean thumbnailGenerated = false; try { logger.fine("attempting to read the image file with ImageIO.read(InputStream), " + storageIO.getDataFile().getStorageIdentifier()); @@ -359,23 +382,15 @@ private static boolean generateImageThumbnailFromInputStream(StorageIO try { rescaleImage(fullSizeImage, width, height, size, outputStream); - /* - // while we are at it, let's make sure other size thumbnails are - // generated too: - for (int s : (new int[]{DEFAULT_PREVIEW_SIZE, DEFAULT_THUMBNAIL_SIZE, DEFAULT_CARDIMAGE_SIZE})) { - if (size != s && !thumbnailFileExists(fileLocation, s)) { - rescaleImage(fullSizeImage, width, height, s, fileLocation); - } - } - */ if (tempFileRequired) { storageIO.savePathAsAux(Paths.get(tempFile.getAbsolutePath()), THUMBNAIL_SUFFIX + size); } + thumbnailGenerated = true; } catch (Exception ioex) { logger.warning("Failed to rescale and/or save the image: " + ioex.getMessage()); - return false; + thumbnailGenerated = false; } finally { if(tempFileRequired) { @@ -383,10 +398,19 @@ private static boolean generateImageThumbnailFromInputStream(StorageIO tempFile.delete(); } catch (Exception e) {} + } else if (!thumbnailGenerated) { + // if it was a local file - let's make sure we are not leaving + // behind a half-baked, broken image - such as a 0-size file - + // if this was a failure. + try { + storageIO.deleteAuxObject(THUMBNAIL_SUFFIX + size); + } catch (IOException ioex) { + logger.fine("Failed attempt to delete the result of a failed thumbnail rescaling; this is most likely ok - for ex., because it was never created in the first place."); + } } } - return true; + return thumbnailGenerated; } @@ -544,12 +568,10 @@ private static String getImageAsBase64FromInputStream(InputStream inputStream) { public static String getImageAsBase64FromFile(File imageFile) { InputStream imageInputStream = null; try { - - int imageSize = (int) imageFile.length(); - - imageInputStream = new FileInputStream(imageFile); - - return getImageAsBase64FromInputStream(imageInputStream); //, imageSize); + if (imageFile.length() > 0) { + imageInputStream = new FileInputStream(imageFile); + return getImageAsBase64FromInputStream(imageInputStream); + } } catch (IOException ex) { // too bad - but not fatal logger.warning("getImageAsBase64FromFile: Failed to read data from thumbnail file"); @@ -609,16 +631,12 @@ public static String generateImageThumbnailFromFile(String fileLocation, int siz logger.fine("image dimensions: " + width + "x" + height); - thumbFileLocation = rescaleImage(fullSizeImage, width, height, size, fileLocation); + return rescaleImage(fullSizeImage, width, height, size, fileLocation); - if (thumbFileLocation != null) { - return thumbFileLocation; - } } catch (Exception e) { logger.warning("Failed to read in an image from " + fileLocation + ": " + e.getMessage()); } return null; - } /* @@ -657,10 +675,14 @@ public static String rescaleImage(BufferedImage fullSizeImage, int width, int he try { rescaleImage(fullSizeImage, width, height, size, outputFileStream); } catch (Exception ioex) { - logger.warning("caught Exceptiopn trying to create rescaled image " + outputLocation); - return null; + logger.warning("caught Exception trying to create rescaled image " + outputLocation); + outputLocation = null; } finally { IOUtils.closeQuietly(outputFileStream); + // delete the file if the rescaleImage failed + if (outputLocation == null) { + outputFile.delete(); + } } return outputLocation; @@ -716,13 +738,19 @@ private static void rescaleImage(BufferedImage fullSizeImage, int width, int hei if (iter.hasNext()) { writer = (ImageWriter) iter.next(); } else { - throw new IOException("Failed to locatie ImageWriter plugin for image type PNG"); + throw new IOException("Failed to locate ImageWriter plugin for image type PNG"); } - BufferedImage lowRes = new BufferedImage(thumbWidth, thumbHeight, BufferedImage.TYPE_INT_ARGB); - Graphics2D g2 = lowRes.createGraphics(); - g2.drawImage(thumbImage, 0, 0, null); - g2.dispose(); + BufferedImage lowRes = null; + try { + lowRes = new BufferedImage(thumbWidth, thumbHeight, BufferedImage.TYPE_INT_ARGB); + Graphics2D g2 = lowRes.createGraphics(); + g2.drawImage(thumbImage, 0, 0, null); + g2.dispose(); + } catch (Exception ex) { + logger.warning("Failed to create LoRes Image: " + ex.getMessage()); + throw new IOException("Caught exception trying to generate thumbnail: " + ex.getMessage()); + } try (ImageOutputStream ios = ImageIO.createImageOutputStream(outputStream);) { @@ -838,6 +866,7 @@ public static String generatePDFThumbnailFromFile(String fileLocation, int size) // generate the thumbnail for the requested size, *using the already scaled-down // 400x400 png version, above*: + // (the "exists()" check below appears to be unnecessary - we've already checked early on - ?) if (!((new File(thumbFileLocation)).exists())) { thumbFileLocation = runImageMagick(imageMagickExec, previewFileLocation, thumbFileLocation, size, "png"); } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index c2143bd4789..d2fdec7b323 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -40,6 +40,7 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DvObject; import edu.harvard.iq.dataverse.datavariable.DataVariable; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.FileUtil; import opennlp.tools.util.StringUtil; @@ -991,7 +992,10 @@ private String generateTemporaryS3UploadUrl(String key, Date expiration) throws GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key).withMethod(HttpMethod.PUT).withExpiration(expiration); //Require user to add this header to indicate a temporary file - generatePresignedUrlRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + final boolean taggingDisabled = JvmSettings.DISABLE_S3_TAGGING.lookupOptional(Boolean.class, this.driverId).orElse(false); + if (!taggingDisabled) { + generatePresignedUrlRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + } URL presignedUrl; try { @@ -1040,7 +1044,10 @@ public JsonObjectBuilder generateTemporaryS3UploadUrls(String globalId, String s } else { JsonObjectBuilder urls = Json.createObjectBuilder(); InitiateMultipartUploadRequest initiationRequest = new InitiateMultipartUploadRequest(bucketName, key); - initiationRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + final boolean taggingDisabled = JvmSettings.DISABLE_S3_TAGGING.lookupOptional(Boolean.class, this.driverId).orElse(false); + if (!taggingDisabled) { + initiationRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + } InitiateMultipartUploadResult initiationResponse = s3.initiateMultipartUpload(initiationRequest); String uploadId = initiationResponse.getUploadId(); for (int i = 1; i <= (fileSize / minPartSize) + (fileSize % minPartSize > 0 ? 1 : 0); i++) { diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index 03a0044a987..98bd26b51d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -1,17 +1,14 @@ package edu.harvard.iq.dataverse.dataset; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.TermsOfUseAndAccess; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import static edu.harvard.iq.dataverse.api.ApiConstants.DS_VERSION_DRAFT; import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO; + +import edu.harvard.iq.dataverse.dataaccess.InputStreamIO; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.util.BundleUtil; @@ -31,15 +28,14 @@ import java.util.*; import java.util.logging.Logger; import javax.imageio.ImageIO; + +import jakarta.enterprise.inject.spi.CDI; import org.apache.commons.io.IOUtils; -import static edu.harvard.iq.dataverse.dataaccess.DataAccess.getStorageIO; import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.license.License; -import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.StringUtil; import static edu.harvard.iq.dataverse.util.json.JsonPrinter.json; -import static edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder.jsonObjectBuilder; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.EnumUtils; @@ -218,7 +214,8 @@ public static boolean deleteDatasetLogo(Dataset dataset) { storageIO.deleteAuxObject(datasetLogoThumbnail + thumbExtension + ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); } catch (IOException ex) { - logger.info("Failed to delete dataset logo: " + ex.getMessage()); + logger.fine("Failed to delete dataset logo: " + ex.getMessage() + + " (this is most likely harmless; this method is often called without checking if the custom dataset logo was in fact present)"); return false; } return true; @@ -293,7 +290,7 @@ public static Dataset persistDatasetLogoToStorageAndCreateThumbnails(Dataset dat dataAccess = DataAccess.getStorageIO(dataset); } catch(IOException ioex){ - //TODO: Add a suitable waing message + //TODO: Add a suitable warning message logger.warning("Failed to save the file, storage id " + dataset.getStorageIdentifier() + " (" + ioex.getMessage() + ")"); } @@ -355,30 +352,44 @@ public static Dataset persistDatasetLogoToStorageAndCreateThumbnails(Dataset dat // We'll try to pre-generate the rescaled versions in both the // DEFAULT_DATASET_LOGO (currently 140) and DEFAULT_CARDIMAGE_SIZE (48) String thumbFileLocation = ImageThumbConverter.rescaleImage(fullSizeImage, width, height, ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE, tmpFileForResize.toPath().toString()); - logger.fine("thumbFileLocation = " + thumbFileLocation); - logger.fine("tmpFileLocation=" + tmpFileForResize.toPath().toString()); - //now we must save the updated thumbnail - try { - dataAccess.savePathAsAux(Paths.get(thumbFileLocation), datasetLogoThumbnail+thumbExtension+ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE); - } catch (IOException ex) { - logger.severe("Failed to move updated thumbnail file from " + tmpFile.getAbsolutePath() + " to its DataAccess location" + ": " + ex); + if (thumbFileLocation == null) { + logger.warning("Rescale Thumbnail Image to logo failed"); + dataset.setPreviewImageAvailable(false); + dataset.setUseGenericThumbnail(true); + } else { + logger.fine("thumbFileLocation = " + thumbFileLocation); + logger.fine("tmpFileLocation=" + tmpFileForResize.toPath().toString()); + //now we must save the updated thumbnail + try { + dataAccess.savePathAsAux(Paths.get(thumbFileLocation), datasetLogoThumbnail + thumbExtension + ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE); + } catch (IOException ex) { + logger.severe("Failed to move updated thumbnail file from " + tmpFile.getAbsolutePath() + " to its DataAccess location" + ": " + ex); + } } thumbFileLocation = ImageThumbConverter.rescaleImage(fullSizeImage, width, height, ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE, tmpFileForResize.toPath().toString()); - logger.fine("thumbFileLocation = " + thumbFileLocation); - logger.fine("tmpFileLocation=" + tmpFileForResize.toPath().toString()); - //now we must save the updated thumbnail - try { - dataAccess.savePathAsAux(Paths.get(thumbFileLocation), datasetLogoThumbnail+thumbExtension+ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); - } catch (IOException ex) { - logger.severe("Failed to move updated thumbnail file from " + tmpFile.getAbsolutePath() + " to its DataAccess location" + ": " + ex); + if (thumbFileLocation == null) { + logger.warning("Rescale Thumbnail Image to card failed"); + dataset.setPreviewImageAvailable(false); + dataset.setUseGenericThumbnail(true); + } else { + logger.fine("thumbFileLocation = " + thumbFileLocation); + logger.fine("tmpFileLocation=" + tmpFileForResize.toPath().toString()); + //now we must save the updated thumbnail + try { + dataAccess.savePathAsAux(Paths.get(thumbFileLocation), datasetLogoThumbnail + thumbExtension + ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); + } catch (IOException ex) { + logger.severe("Failed to move updated thumbnail file from " + tmpFile.getAbsolutePath() + " to its DataAccess location" + ": " + ex); + } } //This deletes the tempfiles created for rescaling and encoding boolean tmpFileWasDeleted = tmpFile.delete(); boolean originalTempFileWasDeleted = tmpFileForResize.delete(); try { - Files.delete(Paths.get(thumbFileLocation)); + if (thumbFileLocation != null) { + Files.delete(Paths.get(thumbFileLocation)); + } } catch (IOException ioex) { logger.fine("Failed to delete temporary thumbnail file"); } @@ -463,8 +474,19 @@ public static InputStream getLogoAsInputStream(Dataset dataset) { } try { - in = ImageThumbConverter.getImageThumbnailAsInputStream(thumbnailFile.getStorageIO(), - ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE).getInputStream(); + + boolean origImageFailed = thumbnailFile.isPreviewImageFail(); + InputStreamIO isIO = ImageThumbConverter.getImageThumbnailAsInputStream(thumbnailFile.getStorageIO(), + ImageThumbConverter.DEFAULT_DATASETLOGO_SIZE); + if (!origImageFailed && thumbnailFile.isPreviewImageFail()) { + // We found an older 0 length thumbnail. Newer image uploads will not have this issue. + // Once cleaned up, this thumbnail will no longer have this issue + // ImageThumbConverter fixed the DataFile + // Now we need to update dataset since this is a bad logo + DatasetServiceBean datasetService = CDI.current().select(DatasetServiceBean.class).get(); + datasetService.clearDatasetLevelThumbnail(dataset); + } + in = isIO != null ? isIO.getInputStream() : null; } catch (IOException ioex) { logger.warning("getLogo(): Failed to get logo from DataFile for " + dataset.getStorageIdentifier() + " (" + ioex.getMessage() + ")"); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListFeaturedCollectionsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListFeaturedCollectionsCommand.java new file mode 100644 index 00000000000..4dca522e499 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/ListFeaturedCollectionsCommand.java @@ -0,0 +1,50 @@ + +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFeaturedDataverse; +import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * + * @author stephenkraffmiller + */ +public class ListFeaturedCollectionsCommand extends AbstractCommand> { + + private final Dataverse dv; + + public ListFeaturedCollectionsCommand(DataverseRequest aRequest, Dataverse aDataverse) { + super(aRequest, aDataverse); + dv = aDataverse; + } + + @Override + public List execute(CommandContext ctxt) throws CommandException { + List featuredTarget = new ArrayList<>(); + List featuredList = ctxt.featuredDataverses().findByDataverseId(dv.getId()); + for (DataverseFeaturedDataverse dfd : featuredList) { + Dataverse fd = dfd.getFeaturedDataverse(); + featuredTarget.add(fd); + } + return featuredTarget; + + } + + @Override + public Map> getRequiredPermissions() { + return Collections.singletonMap("", + dv.isReleased() ? Collections.emptySet() + : Collections.singleton(Permission.ViewUnpublishedDataverse)); + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetThumbnailCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetThumbnailCommand.java index 3f4b3c36b70..b8c70ec6c46 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetThumbnailCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetThumbnailCommand.java @@ -11,12 +11,14 @@ import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; +import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.FileUtil; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.io.IOUtils; @@ -59,17 +61,18 @@ public DatasetThumbnail execute(CommandContext ctxt) throws CommandException { // throw new CommandException("Just testing what an error would look like in the GUI.", this); // } if (userIntent == null) { - throw new IllegalCommandException("No changes to save.", this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.noChange"), this); } switch (userIntent) { case setDatasetFileAsThumbnail: if (dataFileIdSupplied == null) { - throw new CommandException("A file was not selected to be the new dataset thumbnail.", this); + throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.fileNotSupplied"), this); } DataFile datasetFileThumbnailToSwitchTo = ctxt.files().find(dataFileIdSupplied); if (datasetFileThumbnailToSwitchTo == null) { - throw new CommandException("Could not find file based on id supplied: " + dataFileIdSupplied + ".", this); + throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.fileNotFound", + List.of(dataFileIdSupplied.toString())), this); } Dataset ds1 = ctxt.datasets().setDatasetFileAsThumbnail(dataset, datasetFileThumbnailToSwitchTo); DatasetThumbnail datasetThumbnail = ds1.getDatasetThumbnail(ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); @@ -79,11 +82,12 @@ public DatasetThumbnail execute(CommandContext ctxt) throws CommandException { if (dataFile.getId().equals(dataFileIdSupplied)) { return datasetThumbnail; } else { - throw new CommandException("Dataset thumbnail is should be based on file id " + dataFile.getId() + " but instead it is " + dataFileIdSupplied + ".", this); + throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.basedOnWrongFileId", + List.of(String.valueOf(dataFile.getId()),String.valueOf(dataFileIdSupplied))), this); } } } else { - throw new CommandException("Dataset thumbnail is unexpectedly absent.", this); + throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.missing"), this); } case setNonDatasetFileAsThumbnail: @@ -91,14 +95,14 @@ public DatasetThumbnail execute(CommandContext ctxt) throws CommandException { try { uploadedFile = FileUtil.inputStreamToFile(inputStream); } catch (IOException ex) { - throw new CommandException("In setNonDatasetFileAsThumbnail caught exception calling inputStreamToFile: " + ex, this); + throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.inputStreamToFile.exception", List.of(ex.getMessage())), this); } if (uploadedFile == null) { - throw new CommandException("In setNonDatasetFileAsThumbnail uploadedFile was null.", this); + throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.nonDatasetsFileIsNull"), this); } long uploadLogoSizeLimit = ctxt.systemConfig().getUploadLogoSizeLimit(); if (uploadedFile.length() > uploadLogoSizeLimit) { - throw new IllegalCommandException("File is larger than maximum size: " + uploadLogoSizeLimit + ".", this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.fileToLarge", List.of(String.valueOf(uploadLogoSizeLimit))), this); } FileInputStream fileAsStream = null; try { @@ -107,23 +111,25 @@ public DatasetThumbnail execute(CommandContext ctxt) throws CommandException { Logger.getLogger(UpdateDatasetThumbnailCommand.class.getName()).log(Level.SEVERE, null, ex); } Dataset datasetWithNewThumbnail = ctxt.datasets().setNonDatasetFileAsThumbnail(dataset, fileAsStream); - IOUtils.closeQuietly(fileAsStream); + IOUtils.closeQuietly(fileAsStream); if (datasetWithNewThumbnail != null) { - return datasetWithNewThumbnail.getDatasetThumbnail(ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); - } else { - return null; + DatasetThumbnail thumbnail = datasetWithNewThumbnail.getDatasetThumbnail(ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); + if (thumbnail != null) { + return thumbnail; + } } + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.nonDatasetFailed"), this); case removeThumbnail: - Dataset ds2 = ctxt.datasets().removeDatasetThumbnail(dataset); + Dataset ds2 = ctxt.datasets().clearDatasetLevelThumbnail(dataset); DatasetThumbnail datasetThumbnail2 = ds2.getDatasetThumbnail(ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE); if (datasetThumbnail2 == null) { return null; } else { - throw new CommandException("User wanted to remove the thumbnail it still has one!", this); + throw new CommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.notDeleted"), this); } default: - throw new IllegalCommandException("Whatever you are trying to do to the dataset thumbnail is not supported.", this); + throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasets.api.thumbnail.actionNotSupported"), this); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java index fe9415f39f9..bdb69dc918f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java @@ -18,7 +18,6 @@ import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; -import jakarta.persistence.TypedQuery; /** * Update an existing dataverse. @@ -30,10 +29,10 @@ public class UpdateDataverseCommand extends AbstractCommand { private final Dataverse editedDv; private final List facetList; - private final List featuredDataverseList; - private final List inputLevelList; - - private boolean datasetsReindexRequired = false; + private final List featuredDataverseList; + private final List inputLevelList; + + private boolean datasetsReindexRequired = false; public UpdateDataverseCommand(Dataverse editedDv, List facetList, List featuredDataverseList, DataverseRequest aRequest, List inputLevelList ) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseInputLevelsCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseInputLevelsCommand.java new file mode 100644 index 00000000000..cf7b4a6f69c --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseInputLevelsCommand.java @@ -0,0 +1,51 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevel; +import edu.harvard.iq.dataverse.MetadataBlock; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.engine.command.AbstractCommand; +import edu.harvard.iq.dataverse.engine.command.CommandContext; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.engine.command.exception.CommandException; + +import java.util.ArrayList; +import java.util.List; + +@RequiredPermissions(Permission.EditDataverse) +public class UpdateDataverseInputLevelsCommand extends AbstractCommand { + private final Dataverse dataverse; + private final List inputLevelList; + + public UpdateDataverseInputLevelsCommand(Dataverse dataverse, DataverseRequest request, List inputLevelList) { + super(request, dataverse); + this.dataverse = dataverse; + this.inputLevelList = new ArrayList<>(inputLevelList); + } + + @Override + public Dataverse execute(CommandContext ctxt) throws CommandException { + if (inputLevelList == null || inputLevelList.isEmpty()) { + throw new CommandException("Error while updating dataverse input levels: Input level list cannot be null or empty", this); + } + addInputLevelMetadataBlocks(); + dataverse.setMetadataBlockRoot(true); + return ctxt.engine().submit(new UpdateDataverseCommand(dataverse, null, null, getRequest(), inputLevelList)); + } + + private void addInputLevelMetadataBlocks() { + List dataverseMetadataBlocks = dataverse.getMetadataBlocks(); + for (DataverseFieldTypeInputLevel inputLevel : inputLevelList) { + MetadataBlock inputLevelMetadataBlock = inputLevel.getDatasetFieldType().getMetadataBlock(); + if (!dataverseHasMetadataBlock(dataverseMetadataBlocks, inputLevelMetadataBlock)) { + dataverseMetadataBlocks.add(inputLevelMetadataBlock); + } + } + dataverse.setMetadataBlocks(dataverseMetadataBlocks); + } + + private boolean dataverseHasMetadataBlock(List dataverseMetadataBlocks, MetadataBlock metadataBlock) { + return dataverseMetadataBlocks.stream().anyMatch(block -> block.getId().equals(metadataBlock.getId())); + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java index 9fda7a0f7f1..fb50214c259 100644 --- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java @@ -985,10 +985,14 @@ public void globusDownload(String jsonData, Dataset dataset, User authUser) thro if (taskStatus.startsWith("FAILED") || taskStatus.startsWith("INACTIVE")) { String comment = "Reason : " + taskStatus.split("#")[1] + "
Short Description : " + taskStatus.split("#")[2]; - userNotificationService.sendNotification((AuthenticatedUser) authUser, new Timestamp(new Date().getTime()), - UserNotification.Type.GLOBUSDOWNLOADCOMPLETEDWITHERRORS, dataset.getId(), comment, true); - globusLogger.info("Globus task failed during download process"); - } else { + if (authUser != null && authUser instanceof AuthenticatedUser) { + userNotificationService.sendNotification((AuthenticatedUser) authUser, new Timestamp(new Date().getTime()), + UserNotification.Type.GLOBUSDOWNLOADCOMPLETEDWITHERRORS, dataset.getId(), comment, true); + } + + globusLogger.info("Globus task failed during download process: "+comment); + } else if (authUser != null && authUser instanceof AuthenticatedUser) { + boolean taskSkippedFiles = (task.getSkip_source_errors() == null) ? false : task.getSkip_source_errors(); if (!taskSkippedFiles) { userNotificationService.sendNotification((AuthenticatedUser) authUser, @@ -1257,11 +1261,11 @@ public void writeGuestbookAndStartTransfer(GuestbookResponse guestbookResponse, Long fileId = Long.parseLong(idAsString); // If we need to create a GuestBookResponse record, we have to // look up the DataFile object for this file: + df = dataFileService.findCheapAndEasy(fileId); + selectedFiles.add(df); if (!doNotSaveGuestbookResponse) { - df = dataFileService.findCheapAndEasy(fileId); guestbookResponse.setDataFile(df); fileDownloadService.writeGuestbookResponseRecord(guestbookResponse); - selectedFiles.add(df); } } catch (NumberFormatException nfe) { logger.warning( diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java index 0a64f42d840..6fccbe35e44 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/DataRetrieverAPI.java @@ -3,6 +3,7 @@ */ package edu.harvard.iq.dataverse.mydata; +import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DataverseRoleServiceBean; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.DataverseSession; @@ -63,7 +64,7 @@ public class DataRetrieverAPI extends AbstractApiBean { private static final String retrieveDataPartialAPIPath = "retrieve"; @Inject - DataverseSession session; + DataverseSession session; @EJB DataverseRoleServiceBean dataverseRoleService; @@ -81,6 +82,8 @@ public class DataRetrieverAPI extends AbstractApiBean { //MyDataQueryHelperServiceBean myDataQueryHelperServiceBean; @EJB GroupServiceBean groupService; + @EJB + DatasetServiceBean datasetService; private List roleList; private DataverseRolePermissionHelper rolePermissionHelper; @@ -491,9 +494,10 @@ private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagR // ------------------------------------------- // (a) Get core card data from solr // ------------------------------------------- - myDataCardInfo = doc.getJsonForMyData(); - if (!doc.getEntity().isInstanceofDataFile()){ + myDataCardInfo = doc.getJsonForMyData(isValid(doc)); + + if (doc.getEntity() != null && !doc.getEntity().isInstanceofDataFile()){ String parentAlias = dataverseService.getParentAliasString(doc); myDataCardInfo.add("parent_alias",parentAlias); } @@ -514,4 +518,8 @@ private JsonArrayBuilder formatSolrDocs(SolrQueryResponse solrResponse, RoleTagR return jsonSolrDocsArrayBuilder; } + + private boolean isValid(SolrSearchResult result) { + return result.isValid(x -> true); + } } \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java index 2ab248fcc0b..277fa9ee12f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/mydata/MyDataFilterParams.java @@ -292,7 +292,7 @@ public String getSolrFragmentForPublicationStatus(){ } public String getSolrFragmentForDatasetValidity(){ - if ((this.datasetValidities == null) || (this.datasetValidities.isEmpty())){ + if ((this.datasetValidities == null) || (this.datasetValidities.isEmpty()) || (this.datasetValidities.size() > 1)){ return ""; } diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java index c68554db180..e61b93a741f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java @@ -835,16 +835,7 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set accessObject = null; InputStream instream = null; ContentHandler textHandler = null; @@ -1335,11 +1337,13 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set dataverses = new ArrayList<>(); dataverses.add(dataverse); solrQueryResponse = searchService.search(dataverseRequest, dataverses, queryToPassToSolr, filterQueriesFinal, sortField, sortOrder.toString(), paginationStart, onlyDataRelatedToMe, numRows, false, null, null, !isFacetsDisabled(), true); @@ -1480,9 +1479,23 @@ public boolean isActivelyEmbargoed(SolrSearchResult result) { return false; } } + + public boolean isRetentionExpired(SolrSearchResult result) { + Long retentionEndDate = result.getRetentionEndDate(); + if(retentionEndDate != null) { + return LocalDate.now().toEpochDay() > retentionEndDate; + } else { + return false; + } + } + private DataverseRequest getDataverseRequest() { + final HttpServletRequest httpServletRequest = (HttpServletRequest) FacesContext.getCurrentInstance().getExternalContext().getRequest(); + return new DataverseRequest(session.getUser(), httpServletRequest); + } + public boolean isValid(SolrSearchResult result) { - return result.isValid(); + return result.isValid(x -> permissionsWrapper.canUpdateDataset(getDataverseRequest(), datasetService.find(x.getEntityId()))); } public enum SortOrder { diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java index c6f08151050..42d61231f93 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java @@ -499,6 +499,8 @@ public SolrQueryResponse search( String identifierOfDataverse = (String) solrDocument.getFieldValue(SearchFields.IDENTIFIER_OF_DATAVERSE); String nameOfDataverse = (String) solrDocument.getFieldValue(SearchFields.DATAVERSE_NAME); Long embargoEndDate = (Long) solrDocument.getFieldValue(SearchFields.EMBARGO_END_DATE); + Long retentionEndDate = (Long) solrDocument.getFieldValue(SearchFields.RETENTION_END_DATE); + // Boolean datasetValid = (Boolean) solrDocument.getFieldValue(SearchFields.DATASET_VALID); List matchedFields = new ArrayList<>(); @@ -580,7 +582,8 @@ public SolrQueryResponse search( } solrSearchResult.setEmbargoEndDate(embargoEndDate); - + solrSearchResult.setRetentionEndDate(retentionEndDate); + /** * @todo start using SearchConstants class here */ diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java index 6ad7f9dbbf6..e84c8f133da 100644 --- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java +++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java @@ -7,8 +7,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.logging.Logger; +import edu.harvard.iq.dataverse.*; import jakarta.json.Json; import jakarta.json.JsonArrayBuilder; import jakarta.json.JsonObject; @@ -16,22 +18,14 @@ import org.apache.commons.collections4.CollectionUtils; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetRelPublication; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.DvObject; -import edu.harvard.iq.dataverse.GlobalId; -import edu.harvard.iq.dataverse.MetadataBlock; import edu.harvard.iq.dataverse.api.Util; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; +import edu.harvard.iq.dataverse.settings.JvmSettings; import edu.harvard.iq.dataverse.util.DateUtil; import edu.harvard.iq.dataverse.util.json.JsonPrinter; import edu.harvard.iq.dataverse.util.json.NullSafeJsonBuilder; public class SolrSearchResult { - private static final Logger logger = Logger.getLogger(SolrSearchResult.class.getCanonicalName()); private String id; @@ -123,6 +117,8 @@ public class SolrSearchResult { private Long embargoEndDate; + private Long retentionEndDate; + private boolean datasetValid; public String getDvTree() { @@ -403,21 +399,12 @@ public JsonArrayBuilder getRelevance() { return matchedFieldsArray; } - public JsonObject toJsonObject(boolean showRelevance, boolean showEntityIds, boolean showApiUrls) { - return toJsonObject(showRelevance, showEntityIds, showApiUrls, null); - } - - public JsonObject toJsonObject(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, - List metadataFields) { - return json(showRelevance, showEntityIds, showApiUrls, metadataFields).build(); - } - /** * Add additional fields for the MyData page * * @return */ - public JsonObjectBuilder getJsonForMyData() { + public JsonObjectBuilder getJsonForMyData(boolean isValid) { JsonObjectBuilder myDataJson = json(true, true, true);// boolean showRelevance, boolean showEntityIds, boolean showApiUrls) @@ -425,7 +412,7 @@ public JsonObjectBuilder getJsonForMyData() { .add("is_draft_state", this.isDraftState()).add("is_in_review_state", this.isInReviewState()) .add("is_unpublished_state", this.isUnpublishedState()).add("is_published", this.isPublishedState()) .add("is_deaccesioned", this.isDeaccessionedState()) - .add("is_valid", this.isValid()) + .add("is_valid", isValid) .add("date_to_display_on_card", getDateToDisplayOnCard()); // Add is_deaccessioned attribute, even though MyData currently screens any deaccessioned info out @@ -436,7 +423,7 @@ public JsonObjectBuilder getJsonForMyData() { if ((this.getParent() != null) && (!this.getParent().isEmpty())) { // System.out.println("keys:" + parent.keySet().toString()); - if (this.entity.isInstanceofDataFile()) { + if (this.entity != null && this.entity.isInstanceofDataFile()) { myDataJson.add("parentIdentifier", this.getParent().get(SolrSearchResult.PARENT_IDENTIFIER)) .add("parentName", this.getParent().get("name")); @@ -450,12 +437,10 @@ public JsonObjectBuilder getJsonForMyData() { } // getJsonForMydata public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls) { - return json(showRelevance, showEntityIds, showApiUrls, null); + return json(showRelevance, showEntityIds, showApiUrls, null, null); } - public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, - List metadataFields) { - + public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, List metadataFields, Long datasetFileCount) { if (this.type == null) { return jsonObjectBuilder(); } @@ -571,7 +556,7 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool subjects.add(subject); } nullSafeJsonBuilder.add("subjects", subjects); - nullSafeJsonBuilder.add("fileCount", dv.getFileMetadatas().size()); + nullSafeJsonBuilder.add("fileCount", datasetFileCount); nullSafeJsonBuilder.add("versionId", dv.getId()); nullSafeJsonBuilder.add("versionState", dv.getVersionState().toString()); if (this.isPublishedState()) { @@ -1261,11 +1246,31 @@ public void setEmbargoEndDate(Long embargoEndDate) { this.embargoEndDate = embargoEndDate; } + public Long getRetentionEndDate() { + return retentionEndDate; + } + + public void setRetentionEndDate(Long retentionEndDate) { + this.retentionEndDate = retentionEndDate; + } + public void setDatasetValid(Boolean datasetValid) { this.datasetValid = datasetValid == null || Boolean.valueOf(datasetValid); } - public boolean isValid() { - return datasetValid; + public boolean isValid(Predicate canUpdateDataset) { + if (this.datasetValid) { + return true; + } + if (!this.getType().equals("datasets")) { + return true; + } + if (this.isDraftState()) { + return false; + } + if (!JvmSettings.UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED.lookupOptional(Boolean.class).orElse(true)) { + return true; + } + return !canUpdateDataset.test(this); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java index 8f749ba64cd..9d13be005c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/JvmSettings.java @@ -51,6 +51,10 @@ public enum JvmSettings { DOCROOT_DIRECTORY(SCOPE_FILES, "docroot"), GUESTBOOK_AT_REQUEST(SCOPE_FILES, "guestbook-at-request"), GLOBUS_CACHE_MAXAGE(SCOPE_FILES, "globus-cache-maxage"), + + //STORAGE DRIVER SETTINGS + SCOPE_DRIVER(SCOPE_FILES), + DISABLE_S3_TAGGING(SCOPE_DRIVER, "disable-tagging"), // SOLR INDEX SETTINGS SCOPE_SOLR(PREFIX, "solr"), @@ -226,6 +230,7 @@ public enum JvmSettings { SCOPE_UI(PREFIX, "ui"), UI_ALLOW_REVIEW_INCOMPLETE(SCOPE_UI, "allow-review-for-incomplete"), UI_SHOW_VALIDITY_FILTER(SCOPE_UI, "show-validity-filter"), + UI_SHOW_VALIDITY_LABEL_WHEN_PUBLISHED(SCOPE_UI, "show-validity-label-when-published"), // NetCDF SETTINGS SCOPE_NETCDF(PREFIX, "netcdf"), diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 9888db84696..35d70498c3f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -586,6 +586,12 @@ Whether Harvesting (OAI) service is enabled * n: embargo enabled with n months the maximum allowed duration */ MaxEmbargoDurationInMonths, + /** This setting enables Retention capabilities in Dataverse and sets the minimum Retention duration allowed. + * 0 or not set: new retentions disabled + * -1: retention enabled, no time limit + * n: retention enabled with n months the minimum allowed duration + */ + MinRetentionDurationInMonths, /* * Include "Custom Terms" as an item in the license drop-down or not. */ diff --git a/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java b/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java index 86ae697f771..8408e7d91f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/sitemap/SiteMapUtil.java @@ -1,194 +1,140 @@ package edu.harvard.iq.dataverse.sitemap; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DvObjectContainer; -import edu.harvard.iq.dataverse.settings.ConfigCheckService; -import edu.harvard.iq.dataverse.settings.JvmSettings; -import edu.harvard.iq.dataverse.util.SystemConfig; -import edu.harvard.iq.dataverse.util.xml.XmlValidator; import java.io.File; import java.io.IOException; import java.net.MalformedURLException; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.text.SimpleDateFormat; +import java.text.ParseException; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.logging.Logger; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.xml.sax.SAXException; + +import com.redfin.sitemapgenerator.W3CDateFormat; +import com.redfin.sitemapgenerator.W3CDateFormat.Pattern; +import com.redfin.sitemapgenerator.WebSitemapGenerator; +import com.redfin.sitemapgenerator.WebSitemapUrl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.Dataverse; +import edu.harvard.iq.dataverse.DvObjectContainer; +import edu.harvard.iq.dataverse.settings.ConfigCheckService; +import edu.harvard.iq.dataverse.settings.JvmSettings; +import edu.harvard.iq.dataverse.util.SystemConfig; public class SiteMapUtil { + static final String DATE_PATTERN = "yyyy-MM-dd"; + static final String SITEMAP_FILENAME_STAGED = "sitemap.xml.staged"; + /** @see https://www.sitemaps.org/protocol.html#index */ + static final int SITEMAP_LIMIT = 50000; + private static final Logger logger = Logger.getLogger(SiteMapUtil.class.getCanonicalName()); + private static DateTimeFormatter formatter = DateTimeFormatter.ofPattern(DATE_PATTERN); - static final String SITEMAP_FILENAME_FINAL = "sitemap.xml"; - static final String SITEMAP_FILENAME_STAGED = "sitemap.xml.staged"; - /** - * TODO: Handle more than 50,000 entries in the sitemap. - * - * (As of this writing Harvard Dataverse only has ~3000 dataverses and - * ~30,000 datasets.) - * - * "each Sitemap file that you provide must have no more than 50,000 URLs" - * https://www.sitemaps.org/protocol.html - * - * Consider using a third party library: "One sitemap can contain a maximum - * of 50,000 URLs. (Some sitemaps, like Google News sitemaps, can contain - * only 1,000 URLs.) If you need to put more URLs than that in a sitemap, - * you'll have to use a sitemap index file. Fortunately, WebSitemapGenerator - * can manage the whole thing for you." - * https://github.com/dfabulich/sitemapgen4j - */ public static void updateSiteMap(List dataverses, List datasets) { logger.info("BEGIN updateSiteMap"); - String sitemapPathString = getSitemapPathString(); - String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; - String finalSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_FINAL; - - Path stagedPath = Paths.get(stagedSitemapPathAndFileString); - if (Files.exists(stagedPath)) { - logger.warning("Unable to update sitemap! The staged file from a previous run already existed. Delete " + stagedSitemapPathAndFileString + " and try again."); + final String dataverseSiteUrl = SystemConfig.getDataverseSiteUrlStatic(); + final String msgErrorFormat = "Problem with %s : %s. The exception is %s"; + final String msgErrorW3CFormat = "%s isn't a valid W3C date time for %s. The exception is %s"; + final String sitemapPathString = getSitemapPathString(); + final String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; + final Path stagedSitemapPath = Paths.get(stagedSitemapPathAndFileString); + + if (Files.exists(stagedSitemapPath)) { + logger.warning(String.format( + "Unable to update sitemap! The staged file from a previous run already existed. Delete %s and try again.", + stagedSitemapPathAndFileString)); return; } - DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder documentBuilder = null; + final File directory = new File(sitemapPathString); + if (!directory.exists()) { + directory.mkdir(); + } + + // Use DAY pattern (YYYY-MM-DD), local machine timezone + final W3CDateFormat dateFormat = new W3CDateFormat(Pattern.DAY); + WebSitemapGenerator wsg = null; try { - documentBuilder = documentBuilderFactory.newDocumentBuilder(); - } catch (ParserConfigurationException ex) { - logger.warning("Unable to update sitemap! ParserConfigurationException: " + ex.getLocalizedMessage()); + // All sitemap files are in "sitemap" folder, see "getSitemapPathString" method. + // But with pretty-faces configuration, "sitemap.xml" and "sitemap_index.xml" are accessible directly, + // like "https://demo.dataverse.org/sitemap.xml". So "/sitemap/" need to be added on "WebSitemapGenerator" + // in order to have valid URL for sitemap location. + wsg = WebSitemapGenerator.builder(dataverseSiteUrl + "/sitemap/", directory).autoValidate(true).dateFormat(dateFormat) + .build(); + } catch (MalformedURLException e) { + logger.warning(String.format(msgErrorFormat, "Dataverse site URL", dataverseSiteUrl, e.getLocalizedMessage())); return; } - Document document = documentBuilder.newDocument(); - - Element urlSet = document.createElement("urlset"); - urlSet.setAttribute("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9"); - urlSet.setAttribute("xmlns:xhtml", "http://www.w3.org/1999/xhtml"); - document.appendChild(urlSet); for (Dataverse dataverse : dataverses) { if (!dataverse.isReleased()) { continue; } - Element url = document.createElement("url"); - urlSet.appendChild(url); - - Element loc = document.createElement("loc"); - String dataverseAlias = dataverse.getAlias(); - loc.appendChild(document.createTextNode(SystemConfig.getDataverseSiteUrlStatic() + "/dataverse/" + dataverseAlias)); - url.appendChild(loc); - - Element lastmod = document.createElement("lastmod"); - lastmod.appendChild(document.createTextNode(getLastModDate(dataverse))); - url.appendChild(lastmod); + final String dvAlias = dataverse.getAlias(); + final String dataverseUrl = dataverseSiteUrl + "/dataverse/" + dvAlias; + final String lastModDate = getLastModDate(dataverse); + try { + final WebSitemapUrl url = new WebSitemapUrl.Options(dataverseUrl).lastMod(lastModDate).build(); + wsg.addUrl(url); + } catch (MalformedURLException e) { + logger.fine(String.format(msgErrorFormat, "dataverse URL", dataverseUrl, e.getLocalizedMessage())); + } catch (ParseException e) { + logger.fine(String.format(msgErrorW3CFormat, lastModDate, "dataverse alias " + dvAlias, e.getLocalizedMessage())); + } } for (Dataset dataset : datasets) { - if (!dataset.isReleased()) { - continue; - } - if (dataset.isHarvested()) { - continue; - } // The deaccessioned check is last because it has to iterate through dataset versions. - if (dataset.isDeaccessioned()) { + if (!dataset.isReleased() || dataset.isHarvested() || dataset.isDeaccessioned()) { continue; } - Element url = document.createElement("url"); - urlSet.appendChild(url); - - Element loc = document.createElement("loc"); - String datasetPid = dataset.getGlobalId().asString(); - loc.appendChild(document.createTextNode(SystemConfig.getDataverseSiteUrlStatic() + "/dataset.xhtml?persistentId=" + datasetPid)); - url.appendChild(loc); - - Element lastmod = document.createElement("lastmod"); - lastmod.appendChild(document.createTextNode(getLastModDate(dataset))); - url.appendChild(lastmod); - } - - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = null; - try { - transformer = transformerFactory.newTransformer(); - } catch (TransformerConfigurationException ex) { - logger.warning("Unable to update sitemap! TransformerConfigurationException: " + ex.getLocalizedMessage()); - return; - } - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); - DOMSource source = new DOMSource(document); - File directory = new File(sitemapPathString); - if (!directory.exists()) { - directory.mkdir(); - } - - boolean debug = false; - if (debug) { - logger.info("Writing sitemap to console/logs"); - StreamResult consoleResult = new StreamResult(System.out); + final String datasetPid = dataset.getGlobalId().asString(); + final String datasetUrl = dataverseSiteUrl + "/dataset.xhtml?persistentId=" + datasetPid; + final String lastModDate = getLastModDate(dataset); try { - transformer.transform(source, consoleResult); - } catch (TransformerException ex) { - logger.warning("Unable to print sitemap to the console: " + ex.getLocalizedMessage()); + final WebSitemapUrl url = new WebSitemapUrl.Options(datasetUrl).lastMod(lastModDate).build(); + wsg.addUrl(url); + } catch (MalformedURLException e) { + logger.fine(String.format(msgErrorFormat, "dataset URL", datasetUrl, e.getLocalizedMessage())); + } catch (ParseException e) { + logger.fine(String.format(msgErrorW3CFormat, lastModDate, "dataset " + datasetPid, e.getLocalizedMessage())); } } - logger.info("Writing staged sitemap to " + stagedSitemapPathAndFileString); - StreamResult result = new StreamResult(new File(stagedSitemapPathAndFileString)); - try { - transformer.transform(source, result); - } catch (TransformerException ex) { - logger.warning("Unable to update sitemap! Unable to write staged sitemap to " + stagedSitemapPathAndFileString + ". TransformerException: " + ex.getLocalizedMessage()); - return; - } - - logger.info("Checking staged sitemap for well-formedness. The staged file is " + stagedSitemapPathAndFileString); + logger.info(String.format("Writing and checking sitemap file into %s", sitemapPathString)); try { - XmlValidator.validateXmlWellFormed(stagedSitemapPathAndFileString); + wsg.write(); + if (dataverses.size() + datasets.size() > SITEMAP_LIMIT) { + wsg.writeSitemapsWithIndex(); + } } catch (Exception ex) { - logger.warning("Unable to update sitemap! Staged sitemap file is not well-formed XML! The exception for " + stagedSitemapPathAndFileString + " is " + ex.getLocalizedMessage()); - return; - } - - logger.info("Checking staged sitemap against XML schema. The staged file is " + stagedSitemapPathAndFileString); - URL schemaUrl = null; - try { - schemaUrl = new URL("https://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd"); - } catch (MalformedURLException ex) { - // This URL is hard coded and it's fine. We should never get MalformedURLException so we just swallow the exception and carry on. - } - try { - XmlValidator.validateXmlSchema(stagedSitemapPathAndFileString, schemaUrl); - } catch (SAXException | IOException ex) { - logger.warning("Unable to update sitemap! Exception caught while checking XML staged file (" + stagedSitemapPathAndFileString + " ) against XML schema: " + ex.getLocalizedMessage()); + final StringBuffer errorMsg = new StringBuffer("Unable to write or validate sitemap ! The exception is "); + errorMsg.append(ex.getLocalizedMessage()); + // Add causes messages exception + Throwable cause = ex.getCause(); + // Fix limit to 5 causes + final int causeLimit = 5; + int cpt = 0; + while (cause != null && cpt < causeLimit) { + errorMsg.append(" with cause ").append(cause.getLocalizedMessage()); + cause = ex.getCause(); + cpt = cpt + 1; + } + logger.warning(errorMsg.toString()); return; } - Path finalPath = Paths.get(finalSitemapPathAndFileString); - logger.info("Copying staged sitemap from " + stagedSitemapPathAndFileString + " to " + finalSitemapPathAndFileString); + logger.info(String.format("Remove staged sitemap %s", stagedSitemapPathAndFileString)); try { - Files.move(stagedPath, finalPath, StandardCopyOption.REPLACE_EXISTING); + Files.deleteIfExists(stagedSitemapPath); } catch (IOException ex) { - logger.warning("Unable to update sitemap! Unable to copy staged sitemap from " + stagedSitemapPathAndFileString + " to " + finalSitemapPathAndFileString + ". IOException: " + ex.getLocalizedMessage()); + logger.warning("Unable to delete sitemap staged file! IOException: " + ex.getLocalizedMessage()); return; } @@ -199,12 +145,11 @@ private static String getLastModDate(DvObjectContainer dvObjectContainer) { // TODO: Decide if YYYY-MM-DD is enough. https://www.sitemaps.org/protocol.html // says "The date of last modification of the file. This date should be in W3C Datetime format. // This format allows you to omit the time portion, if desired, and use YYYY-MM-DD." - return new SimpleDateFormat("yyyy-MM-dd").format(dvObjectContainer.getModificationTime()); + return dvObjectContainer.getModificationTime().toLocalDateTime().format(formatter); } public static boolean stageFileExists() { - String sitemapPathString = getSitemapPathString(); - String stagedSitemapPathAndFileString = sitemapPathString + File.separator + SITEMAP_FILENAME_STAGED; + String stagedSitemapPathAndFileString = getSitemapPathString() + File.separator + SITEMAP_FILENAME_STAGED; Path stagedPath = Paths.get(stagedSitemapPathAndFileString); if (Files.exists(stagedPath)) { logger.warning("Unable to update sitemap! The staged file from a previous run already existed. Delete " + stagedSitemapPathAndFileString + " and try again."); @@ -212,7 +157,7 @@ public static boolean stageFileExists() { } return false; } - + /** * Lookup the location where to generate the sitemap. * @@ -223,6 +168,6 @@ public static boolean stageFileExists() { */ private static String getSitemapPathString() { return JvmSettings.DOCROOT_DIRECTORY.lookup() + File.separator + "sitemap"; - } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 8decf74fe13..6c427672e6d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -21,14 +21,8 @@ package edu.harvard.iq.dataverse.util; -import edu.harvard.iq.dataverse.DataFile; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DataFile.ChecksumType; -import edu.harvard.iq.dataverse.DataFileServiceBean; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetVersion; -import edu.harvard.iq.dataverse.Embargo; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; @@ -86,6 +80,7 @@ import java.util.HashMap; import java.util.List; import java.util.Optional; +import java.util.ResourceBundle; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; @@ -182,6 +177,7 @@ public class FileUtil implements java.io.Serializable { public static final String MIME_TYPE_NETCDF = "application/netcdf"; public static final String MIME_TYPE_XNETCDF = "application/x-netcdf"; public static final String MIME_TYPE_HDF5 = "application/x-hdf5"; + public static final String MIME_TYPE_RO_CRATE = "application/ld+json; profile=\"http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate\""; // File type "thumbnail classes" tags: @@ -278,6 +274,11 @@ public static String getUserFriendlyFileType(DataFile dataFile) { if (fileType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE)){ return ShapefileHandler.SHAPEFILE_FILE_TYPE_FRIENDLY_NAME; } + try { + return BundleUtil.getStringFromPropertyFile(fileType,"MimeTypeDisplay" ); + } catch (MissingResourceException e) { + //NOOP: we will try again after trimming ";" + } if (fileType.contains(";")) { fileType = fileType.substring(0, fileType.indexOf(";")); } @@ -292,6 +293,11 @@ public static String getUserFriendlyFileType(DataFile dataFile) { } public static String getIndexableFacetFileType(DataFile dataFile) { + try { + return BundleUtil.getStringFromDefaultPropertyFile(dataFile.getContentType(),"MimeTypeFacets" ); + } catch (MissingResourceException e) { + //NOOP: we will try again after trimming ";" + } String fileType = getFileType(dataFile); try { return BundleUtil.getStringFromDefaultPropertyFile(fileType,"MimeTypeFacets" ); @@ -421,7 +427,10 @@ public static String retestIngestableFileType(File file, String fileType) { } public static String determineFileType(File f, String fileName) throws IOException{ - String fileType = null; + String fileType = lookupFileTypeByFileName(fileName); + if (fileType != null) { + return fileType; + } String fileExtension = getFileExtension(fileName); @@ -480,17 +489,17 @@ public static String determineFileType(File f, String fileName) throws IOExcepti if (fileType != null && fileType.startsWith("text/plain") && STATISTICAL_FILE_EXTENSION.containsKey(fileExtension)) { fileType = STATISTICAL_FILE_EXTENSION.get(fileExtension); } else { - fileType = determineFileTypeByNameAndExtension(fileName); + fileType = lookupFileTypeByExtension(fileName); } logger.fine("mime type recognized by extension: "+fileType); } } else { logger.fine("fileExtension is null"); - String fileTypeByName = lookupFileTypeFromPropertiesFile(fileName); - if(!StringUtil.isEmpty(fileTypeByName)) { - logger.fine(String.format("mime type: %s recognized by filename: %s", fileTypeByName, fileName)); - fileType = fileTypeByName; + final String fileTypeByExtension = lookupFileTypeByExtensionFromPropertiesFile(fileName); + if(!StringUtil.isEmpty(fileTypeByExtension)) { + logger.fine(String.format("mime type: %s recognized by extension: %s", fileTypeByExtension, fileName)); + fileType = fileTypeByExtension; } } @@ -501,24 +510,15 @@ public static String determineFileType(File f, String fileName) throws IOExcepti if ("application/x-gzip".equals(fileType)) { logger.fine("we'll run additional checks on this gzipped file."); - // We want to be able to support gzipped FITS files, same way as - // if they were just regular FITS files: - FileInputStream gzippedIn = new FileInputStream(f); - // (new FileInputStream() can throw a "filen not found" exception; - // however, if we've made it this far, it really means that the - // file does exist and can be opened) - InputStream uncompressedIn = null; - try { - uncompressedIn = new GZIPInputStream(gzippedIn); + try (FileInputStream gzippedIn = new FileInputStream(f); + InputStream uncompressedIn = new GZIPInputStream(gzippedIn)) { if (isFITSFile(uncompressedIn)) { fileType = "application/fits-gzipped"; } } catch (IOException ioex) { - if (uncompressedIn != null) { - try {uncompressedIn.close();} catch (IOException e) {} - } + logger.warning("IOException while processing gzipped FITS file: " + ioex.getMessage()); } - } + } if ("application/zip".equals(fileType)) { // Is this a zipped Shapefile? @@ -544,33 +544,41 @@ public static String determineFileType(File f, String fileName) throws IOExcepti return fileType; } - public static String determineFileTypeByNameAndExtension(String fileName) { - String mimetypesFileTypeMapResult = MIME_TYPE_MAP.getContentType(fileName); + public static String determineFileTypeByNameAndExtension(final String fileName) { + final String fileType = lookupFileTypeByFileName(fileName); + if (fileType != null) { + return fileType; + } + return lookupFileTypeByExtension(fileName); + } + + private static String lookupFileTypeByExtension(final String fileName) { + final String mimetypesFileTypeMapResult = MIME_TYPE_MAP.getContentType(fileName); logger.fine("MimetypesFileTypeMap type by extension, for " + fileName + ": " + mimetypesFileTypeMapResult); - if (mimetypesFileTypeMapResult != null) { - if ("application/octet-stream".equals(mimetypesFileTypeMapResult)) { - return lookupFileTypeFromPropertiesFile(fileName); - } else { - return mimetypesFileTypeMapResult; - } - } else { + if (mimetypesFileTypeMapResult == null) { return null; } + if ("application/octet-stream".equals(mimetypesFileTypeMapResult)) { + return lookupFileTypeByExtensionFromPropertiesFile(fileName); + } + return mimetypesFileTypeMapResult; } - public static String lookupFileTypeFromPropertiesFile(String fileName) { - String fileKey = FilenameUtils.getExtension(fileName); - String propertyFileName = "MimeTypeDetectionByFileExtension"; - if(fileKey == null || fileKey.isEmpty()) { - fileKey = fileName; - propertyFileName = "MimeTypeDetectionByFileName"; + private static String lookupFileTypeByFileName(final String fileName) { + return lookupFileTypeFromPropertiesFile("MimeTypeDetectionByFileName", fileName); + } - } - String propertyFileNameOnDisk = propertyFileName + ".properties"; + private static String lookupFileTypeByExtensionFromPropertiesFile(final String fileName) { + final String fileKey = FilenameUtils.getExtension(fileName); + return lookupFileTypeFromPropertiesFile("MimeTypeDetectionByFileExtension", fileKey); + } + + private static String lookupFileTypeFromPropertiesFile(final String propertyFileName, final String fileKey) { + final String propertyFileNameOnDisk = propertyFileName + ".properties"; try { logger.fine("checking " + propertyFileNameOnDisk + " for file key " + fileKey); return BundleUtil.getStringFromPropertyFile(fileKey, propertyFileName); - } catch (MissingResourceException ex) { + } catch (final MissingResourceException ex) { logger.info(fileKey + " is a filename/extension Dataverse doesn't know about. Consider adding it to the " + propertyFileNameOnDisk + " file."); return null; } @@ -825,7 +833,8 @@ public static boolean useRecognizedType(String suppliedContentType, String recog || canIngestAsTabular(recognizedType) || recognizedType.equals("application/fits-gzipped") || recognizedType.equalsIgnoreCase(ShapefileHandler.SHAPEFILE_FILE_TYPE) || recognizedType.equalsIgnoreCase(BagItFileHandler.FILE_TYPE) - || recognizedType.equals(MIME_TYPE_ZIP)) { + || recognizedType.equals(MIME_TYPE_ZIP) + || recognizedType.equals(MIME_TYPE_RO_CRATE)) { return true; } return false; @@ -1223,6 +1232,9 @@ public static boolean isPubliclyDownloadable(FileMetadata fileMetadata) { if (isActivelyEmbargoed(fileMetadata)) { return false; } + if (isRetentionExpired(fileMetadata)) { + return false; + } boolean popupReasons = isDownloadPopupRequired(fileMetadata.getDatasetVersion()); if (popupReasons == true) { /** @@ -1776,6 +1788,29 @@ public static boolean isActivelyEmbargoed(List fmdList) { return false; } + public static boolean isRetentionExpired(DataFile df) { + Retention e = df.getRetention(); + if (e != null) { + LocalDate endDate = e.getDateUnavailable(); + if (endDate != null && endDate.isBefore(LocalDate.now())) { + return true; + } + } + return false; + } + + public static boolean isRetentionExpired(FileMetadata fileMetadata) { + return isRetentionExpired(fileMetadata.getDataFile()); + } + + public static boolean isRetentionExpired(List fmdList) { + for (FileMetadata fmd : fmdList) { + if (isRetentionExpired(fmd)) { + return true; + } + } + return false; + } public static String getStorageDriver(DataFile dataFile) { String storageIdentifier = dataFile.getStorageIdentifier(); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java index a3293e0cd28..90557a530c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/URLTokenUtil.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.util; import java.util.Arrays; +import java.util.Random; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -261,7 +262,9 @@ public JsonObject getParams(JsonObject toolParameters) { public static String getScriptForUrl(String url) { String msg = BundleUtil.getStringFromBundle("externaltools.enable.browser.popups"); - String script = "const newWin = window.open('" + url + "', target='_blank'); if (!newWin || newWin.closed || typeof newWin.closed == \"undefined\") {alert(\"" + msg + "\");}"; + String newWin = "newWin" + (new Random()).nextInt(1000000000); + //Always use a unique identifier so that more than one script can run (or one can be rerun) without conflicts + String script = String.format("const %1$s = window.open('" + url + "', target='_blank'); if (!%1$s || %1$s.closed || typeof %1$s.closed == \"undefined\") {alert(\"" + msg + "\");}", newWin); return script; } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java index aa653a6e360..84bc7834ab9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/bagit/OREMap.java @@ -1,19 +1,7 @@ package edu.harvard.iq.dataverse.util.bagit; -import edu.harvard.iq.dataverse.DataFile; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetField; -import edu.harvard.iq.dataverse.DatasetFieldCompoundValue; -import edu.harvard.iq.dataverse.DatasetFieldConstant; -import edu.harvard.iq.dataverse.DatasetFieldServiceBean; -import edu.harvard.iq.dataverse.DatasetFieldType; -import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.*; import edu.harvard.iq.dataverse.DatasetVersion.VersionState; -import edu.harvard.iq.dataverse.Dataverse; -import edu.harvard.iq.dataverse.DvObjectContainer; -import edu.harvard.iq.dataverse.Embargo; -import edu.harvard.iq.dataverse.FileMetadata; -import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.export.OAI_OREExporter; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; @@ -236,6 +224,17 @@ public JsonObjectBuilder getOREMapBuilder(boolean aggregationOnly) { } aggRes.add(JsonLDTerm.DVCore("embargoed").getLabel(), embargoObject); } + Retention retention = df.getRetention(); + if(retention!=null) { + String date = retention.getFormattedDateUnavailable(); + String reason= retention.getReason(); + JsonObjectBuilder retentionObject = Json.createObjectBuilder(); + retentionObject.add(JsonLDTerm.DVCore("dateUnavailable").getLabel(), date); + if(reason!=null) { + retentionObject.add(JsonLDTerm.DVCore("reason").getLabel(), reason); + } + aggRes.add(JsonLDTerm.DVCore("retained").getLabel(), retentionObject); + } addIfNotNull(aggRes, JsonLDTerm.directoryLabel, fmd.getDirectoryLabel()); addIfNotNull(aggRes, JsonLDTerm.schemaOrg("version"), fmd.getVersion()); addIfNotNull(aggRes, JsonLDTerm.datasetVersionId, fmd.getDatasetVersion().getId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java index a0278d1a60f..95f14b79ece 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java @@ -273,7 +273,7 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re } if (returnOwners){ bld.add("isPartOf", getOwnersFromDvObject(dv)); - } + } bld.add("permissionRoot", dv.isPermissionRoot()) .add("description", dv.getDescription()) .add("dataverseType", dv.getDataverseType().name()); @@ -292,6 +292,12 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re if (dv.getFilePIDsEnabled() != null) { bld.add("filePIDsEnabled", dv.getFilePIDsEnabled()); } + bld.add("isReleased", dv.isReleased()); + + List inputLevels = dv.getDataverseFieldTypeInputLevels(); + if(!inputLevels.isEmpty()) { + bld.add("inputLevels", JsonPrinter.jsonDataverseFieldTypeInputLevels(inputLevels)); + } return bld; } @@ -588,9 +594,13 @@ public static JsonObjectBuilder json(MetadataBlock block, List fie } public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes) { + return json(metadataBlocks, returnDatasetFieldTypes, printOnlyDisplayedOnCreateDatasetFieldTypes, null); + } + + public static JsonArrayBuilder json(List metadataBlocks, boolean returnDatasetFieldTypes, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse) { JsonArrayBuilder arrayBuilder = Json.createArrayBuilder(); for (MetadataBlock metadataBlock : metadataBlocks) { - arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes) : brief.json(metadataBlock)); + arrayBuilder.add(returnDatasetFieldTypes ? json(metadataBlock, printOnlyDisplayedOnCreateDatasetFieldTypes, ownerDataverse) : brief.json(metadataBlock)); } return arrayBuilder; } @@ -618,20 +628,25 @@ public static JsonObject json(DatasetField dfv) { } public static JsonObjectBuilder json(MetadataBlock metadataBlock) { - return json(metadataBlock, false); + return json(metadataBlock, false, null); } - public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes) { + public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printOnlyDisplayedOnCreateDatasetFieldTypes, Dataverse ownerDataverse) { JsonObjectBuilder jsonObjectBuilder = jsonObjectBuilder(); jsonObjectBuilder.add("id", metadataBlock.getId()); jsonObjectBuilder.add("name", metadataBlock.getName()); jsonObjectBuilder.add("displayName", metadataBlock.getDisplayName()); jsonObjectBuilder.add("displayOnCreate", metadataBlock.isDisplayOnCreate()); - JsonObjectBuilder fieldsBuilder = jsonObjectBuilder(); - for (DatasetFieldType datasetFieldType : new TreeSet<>(metadataBlock.getDatasetFieldTypes())) { - if (!printOnlyDisplayedOnCreateDatasetFieldTypes || datasetFieldType.isDisplayOnCreate()) { - fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType)); + JsonObjectBuilder fieldsBuilder = Json.createObjectBuilder(); + Set datasetFieldTypes = new TreeSet<>(metadataBlock.getDatasetFieldTypes()); + for (DatasetFieldType datasetFieldType : datasetFieldTypes) { + boolean requiredInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(datasetFieldType.getId()); + boolean displayCondition = !printOnlyDisplayedOnCreateDatasetFieldTypes || + datasetFieldType.isDisplayOnCreate() || + requiredInOwnerDataverse; + if (displayCondition) { + fieldsBuilder.add(datasetFieldType.getName(), json(datasetFieldType, ownerDataverse)); } } @@ -641,6 +656,10 @@ public static JsonObjectBuilder json(MetadataBlock metadataBlock, boolean printO } public static JsonObjectBuilder json(DatasetFieldType fld) { + return json(fld, null); + } + + public static JsonObjectBuilder json(DatasetFieldType fld, Dataverse ownerDataverse) { JsonObjectBuilder fieldsBld = jsonObjectBuilder(); fieldsBld.add("name", fld.getName()); fieldsBld.add("displayName", fld.getDisplayName()); @@ -653,8 +672,11 @@ public static JsonObjectBuilder json(DatasetFieldType fld) { fieldsBld.add("multiple", fld.isAllowMultiples()); fieldsBld.add("isControlledVocabulary", fld.isControlledVocabulary()); fieldsBld.add("displayFormat", fld.getDisplayFormat()); - fieldsBld.add("isRequired", fld.isRequired()); fieldsBld.add("displayOrder", fld.getDisplayOrder()); + + boolean requiredInOwnerDataverse = ownerDataverse != null && ownerDataverse.isDatasetFieldTypeRequiredAsInputLevel(fld.getId()); + fieldsBld.add("isRequired", requiredInOwnerDataverse || fld.isRequired()); + if (fld.isControlledVocabulary()) { // If the field has a controlled vocabulary, // add all values to the resulting JSON @@ -664,10 +686,11 @@ public static JsonObjectBuilder json(DatasetFieldType fld) { } fieldsBld.add("controlledVocabularyValues", jab); } + if (!fld.getChildDatasetFieldTypes().isEmpty()) { JsonObjectBuilder subFieldsBld = jsonObjectBuilder(); for (DatasetFieldType subFld : fld.getChildDatasetFieldTypes()) { - subFieldsBld.add(subFld.getName(), JsonPrinter.json(subFld)); + subFieldsBld.add(subFld.getName(), JsonPrinter.json(subFld, ownerDataverse)); } fieldsBld.add("childFields", subFieldsBld); } @@ -749,6 +772,7 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo String pidString = (filePid!=null)? filePid.asString(): ""; JsonObjectBuilder embargo = df.getEmbargo() != null ? JsonPrinter.json(df.getEmbargo()) : null; + JsonObjectBuilder retention = df.getRetention() != null ? JsonPrinter.json(df.getRetention()) : null; NullSafeJsonBuilder builder = jsonObjectBuilder() .add("id", df.getId()) @@ -761,6 +785,7 @@ public static JsonObjectBuilder json(DataFile df, FileMetadata fileMetadata, boo .add("description", fileMetadata.getDescription()) .add("categories", getFileCategories(fileMetadata)) .add("embargo", embargo) + .add("retention", retention) //.add("released", df.isReleased()) .add("storageIdentifier", df.getStorageIdentifier()) .add("originalFileFormat", df.getOriginalFileFormat()) @@ -1166,6 +1191,11 @@ public static JsonObjectBuilder json(Embargo embargo) { embargo.getReason()); } + public static JsonObjectBuilder json(Retention retention) { + return jsonObjectBuilder().add("dateUnavailable", retention.getDateUnavailable().toString()).add("reason", + retention.getReason()); + } + public static JsonObjectBuilder json(License license) { return jsonObjectBuilder() .add("id", license.getId()) @@ -1334,4 +1364,16 @@ private static JsonObjectBuilder jsonLicense(DatasetVersion dsv) { } return licenseJsonObjectBuilder; } + + public static JsonArrayBuilder jsonDataverseFieldTypeInputLevels(List inputLevels) { + JsonArrayBuilder jsonArrayOfInputLevels = Json.createArrayBuilder(); + for (DataverseFieldTypeInputLevel inputLevel : inputLevels) { + NullSafeJsonBuilder inputLevelJsonObject = NullSafeJsonBuilder.jsonObjectBuilder(); + inputLevelJsonObject.add("datasetFieldTypeName", inputLevel.getDatasetFieldType().getName()); + inputLevelJsonObject.add("required", inputLevel.isRequired()); + inputLevelJsonObject.add("include", inputLevel.isInclude()); + jsonArrayOfInputLevels.add(inputLevelJsonObject); + } + return jsonArrayOfInputLevels; + } } diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 692ab9e0686..1e847d3eeb3 100644 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -15,6 +15,7 @@ embargoed=Embargoed embargoedaccess=Embargoed with Access embargoedandrestricted=Embargoed and then Restricted embargoedandrestrictedaccess=Embargoed and then Restricted with Access +retentionExpired=Retention Period Expired incomplete=Incomplete metadata valid=Valid find=Find @@ -30,6 +31,12 @@ embargoed.wasthrough=Was embargoed until embargoed.willbeuntil=Draft: will be embargoed until embargo.date.invalid=Date is outside the allowed range: ({0} to {1}) embargo.date.required=An embargo date is required +retention.after=Was retained until +retention.isfrom=Is retained until +retention.willbeafter=Draft: will be retained until +retention.enddateinfo=after which it will no longer be accessible +retention.date.invalid=Date is outside the allowed range: ({0} to {1}) +retention.date.required=A retention period end date is required cancel=Cancel ok=OK saveChanges=Save Changes @@ -940,6 +947,7 @@ dataverse.default=(Default) dataverse.metadatalanguage.setatdatasetcreation=Chosen at Dataset Creation dataverse.guestbookentry.atdownload=Guestbook Entry At Download dataverse.guestbookentry.atrequest=Guestbook Entry At Access Request +dataverse.updateinputlevels.error.invalidfieldtypename=Invalid dataset field type name: {0} # rolesAndPermissionsFragment.xhtml # advanced.xhtml @@ -1027,7 +1035,7 @@ dataverse.theme.inheritCustomization.title=For this dataverse, use the same them dataverse.theme.inheritCustomization.label=Inherit Theme dataverse.theme.inheritCustomization.checkbox=Inherit theme from {0} dataverse.theme.logo=Logo -dataverse.theme.logo.tip=Supported image types are JPG, TIF, or PNG and should be no larger than 500 KB. The maximum display size for an image file in a dataverse's theme is 940 pixels wide by 120 pixels high. +dataverse.theme.logo.tip=Supported image types are JPG and PNG, must be no larger than 500 KB. The maximum display size for an image file in a dataverse's theme is 940 pixels wide by 120 pixels high. dataverse.theme.logo.format=Logo Format dataverse.theme.logo.format.selectTab.square=Square dataverse.theme.logo.format.selectTab.rectangle=Rectangle @@ -1665,17 +1673,19 @@ dataset.noSelectedFiles=Please select one or more files. dataset.noSelectedFilesForDownload=Please select a file or files to be downloaded. dataset.noSelectedFilesForRequestAccess=Please select a file or files for access request. dataset.embargoedSelectedFilesForRequestAccess=Embargoed files cannot be accessed. Please select an unembargoed file or files for your access request. -dataset.inValidSelectedFilesForDownload=Restricted Files Selected -dataset.inValidSelectedFilesForDownloadWithEmbargo=Embargoed and/or Restricted Files Selected -dataset.noValidSelectedFilesForDownload=The selected file(s) may not be downloaded because you have not been granted access. -dataset.mixedSelectedFilesForDownload=The restricted file(s) selected may not be downloaded because you have not been granted access. -dataset.mixedSelectedFilesForDownloadWithEmbargo=The embargoed and/or restricted file(s) selected may not be downloaded because you have not been granted access. -dataset.mixedSelectedFilesForTransfer=Some file(s) cannot be transferred. (They are restricted, embargoed, or not Globus accessible.) +dataset.inValidSelectedFilesForDownload=Inaccessible Files Selected +dataset.inValidSelectedFilesForDownloadWithEmbargo=Inaccessible Files Selected +dataset.inValidSelectedFilesForTransferWithEmbargo=Inaccessible Files Selected +dataset.noValidSelectedFilesForDownload=The selected file(s) may not be downloaded because you have not been granted access or the file(s) have a retention period that has expired or the files can only be transferred via Globus. +dataset.noValidSelectedFilesForTransfer=The selected file(s) may not be transferred because you have not been granted access or the file(s) have a retention period that has expired or the files are not Globus accessible. +dataset.mixedSelectedFilesForDownload=The selected file(s) may not be downloaded because you have not been granted access or the file(s) have a retention period that has expired. +dataset.mixedSelectedFilesForDownloadWithEmbargo=Any embargoed and/or restricted file(s) selected may not be downloaded because you have not been granted access. Some files may have a retention period that has expired. Some files may only be accessible via Globus. +dataset.mixedSelectedFilesForTransfer=Some file(s) cannot be transferred. (They are restricted, embargoed, with an expired retention period, or not Globus accessible.) dataset.inValidSelectedFilesForTransfer=Ineligible Files Selected dataset.downloadUnrestricted=Click Continue to download the files you have access to download. dataset.transferUnrestricted=Click Continue to transfer the elligible files. -dataset.requestAccessToRestrictedFiles=You may request access to the restricted file(s) by clicking the Request Access button. +dataset.requestAccessToRestrictedFiles=You may request access to any restricted file(s) by clicking the Request Access button. dataset.requestAccessToRestrictedFilesWithEmbargo=Embargoed files cannot be accessed during the embargo period. If your selection contains restricted files, you may request access to them by clicking the Request Access button. dataset.privateurl.infoMessageAuthor=Privately share this dataset before it is published: {0} dataset.privateurl.infoMessageReviewer=This unpublished dataset is being privately shared. @@ -1849,6 +1859,18 @@ file.editEmbargoDialog.newReason=Add a reason... file.editEmbargoDialog.newDate=Select the embargo end-date file.editEmbargoDialog.remove=Remove existing embargo(es) on selected files +file.retention=Retention Period +file.editRetention=Edit Retention Period +file.editRetention.add=Add or Change +file.editRetention.delete=Remove +file.editRetentionDialog.tip=Edit the planned retention period for the selected file or files. Once this dataset version is published, you will need to contact an administrator to change the retention period end date or reason of the file or files. \n After the retention period expires the files become unavailable for download. +file.editRetentionDialog.some.tip=One or more of the selected files have already been published. Contact an administrator to change the retention period date or reason of the file or files. +file.editRetentionDialog.none.tip=The selected file or files have already been published. Contact an administrator to change the retention period date or reason of the file or files. +file.editRetentionDialog.partial.tip=Any changes you make here will not be made to these files. +file.editRetentionDialog.reason.tip=Enter a short reason why this retention period exists +file.editRetentionDialog.newReason=Add a reason... +file.editRetentionDialog.newDate=Select the retention period end date +file.editRetentionDialog.remove=Remove existing retention period(s) on selected files file.setThumbnail=Set Thumbnail file.setThumbnail.header=Set Dataset Thumbnail @@ -1861,6 +1883,7 @@ file.advancedIngestOptions=Advanced Ingest Options file.assignedDataverseImage.success={0} has been saved as the thumbnail for this dataset. file.assignedTabFileTags.success=The tags were successfully added for {0}. file.assignedEmbargo.success=An Embargo was successfully added for {0}. +file.assignedRetention.success=A Retention Period was successfully added for {0}. file.tabularDataTags=Tabular Data Tags file.tabularDataTags.tip=Select a tag to describe the type(s) of data this is (survey, time series, geospatial, etc). file.spss-savEncoding=Language Encoding @@ -2119,7 +2142,7 @@ dataset.thumbnailsAndWidget.thumbnails.title=Thumbnail dataset.thumbnailsAndWidget.widgets.title=Widgets dataset.thumbnailsAndWidget.thumbnailImage=Thumbnail Image dataset.thumbnailsAndWidget.thumbnailImage.title=The logo or image file you wish to display as the thumbnail of this dataset. -dataset.thumbnailsAndWidget.thumbnailImage.tip=Supported image types are JPG, TIF, or PNG and should be no larger than {0} KB. The maximum display size for an image file as a dataset thumbnail is 48 pixels wide by 48 pixels high. +dataset.thumbnailsAndWidget.thumbnailImage.tip=Supported image types are JPG and PNG, must be no larger than {0} KB. The maximum display size for an image file as a dataset thumbnail is 140 pixels wide by 140 pixels high. dataset.thumbnailsAndWidget.thumbnailImage.default=Default Icon dataset.thumbnailsAndWidget.thumbnailImage.selectAvailable=Select Available File dataset.thumbnailsAndWidget.thumbnailImage.selectThumbnail=Select Thumbnail @@ -2180,6 +2203,8 @@ file.metadataTab.fileMetadata.type.label=Type file.metadataTab.fileMetadata.description.label=Description file.metadataTab.fileMetadata.publicationDate.label=Publication Date file.metadataTab.fileMetadata.embargoReason.label=Embargo Reason +file.metadataTab.fileMetadata.retentionDate.label=Retention End Date +file.metadataTab.fileMetadata.retentionReason.label=Retention Reason file.metadataTab.fileMetadata.metadataReleaseDate.label=Metadata Release Date file.metadataTab.fileMetadata.depositDate.label=Deposit Date file.metadataTab.fileMetadata.hierarchy.label=File Path @@ -2686,6 +2711,17 @@ datasets.api.globusdownloaddisabled=File transfer from Dataverse via Globus is n datasets.api.globusdownloadnotfound=List of files to transfer not found. datasets.api.globusuploaddisabled=File transfer to Dataverse via Globus is not available for this dataset. datasets.api.pidgenerator.notfound=No PID Generator configured for the give id. +datasets.api.thumbnail.fileToLarge=File is larger than maximum size: {0} +datasets.api.thumbnail.nonDatasetFailed=In setNonDatasetFileAsThumbnail could not generate thumbnail from uploaded file. +datasets.api.thumbnail.notDeleted=User wanted to remove the thumbnail it still has one! +datasets.api.thumbnail.actionNotSupported=Whatever you are trying to do to the dataset thumbnail is not supported. +datasets.api.thumbnail.nonDatasetsFileIsNull=In setNonDatasetFileAsThumbnail uploadedFile was null. +datasets.api.thumbnail.inputStreamToFile.exception=In setNonDatasetFileAsThumbnail caught exception calling inputStreamToFile: {0} +datasets.api.thumbnail.missing=Dataset thumbnail is unexpectedly absent. +datasets.api.thumbnail.basedOnWrongFileId=Dataset thumbnail should be based on file id {0} but instead it is {1} +datasets.api.thumbnail.fileNotFound=Could not find file based on id supplied: {0} +datasets.api.thumbnail.fileNotSupplied=A file was not selected to be the new dataset thumbnail. +datasets.api.thumbnail.noChange=No changes to save. #Dataverses.java dataverses.api.update.default.contributor.role.failure.role.not.found=Role {0} not found. @@ -2698,6 +2734,7 @@ dataverses.api.move.dataverse.failure.not.published=Published dataverse may not dataverses.api.move.dataverse.error.guestbook=Dataset guestbook is not in target dataverse. dataverses.api.move.dataverse.error.template=Dataverse template is not in target dataverse. dataverses.api.move.dataverse.error.featured=Dataverse is featured in current dataverse. +dataverses.api.delete.featured.collections.successful=Featured dataverses have been removed dataverses.api.move.dataverse.error.metadataBlock=Dataverse metadata block is not in target dataverse. dataverses.api.move.dataverse.error.dataverseLink=Dataverse is linked to target dataverse or one of its parents. dataverses.api.move.dataverse.error.datasetLink=Dataset is linked to target dataverse or one of its parents. @@ -2720,6 +2757,8 @@ access.api.fileAccess.failure.noUser=Could not find user to execute command: {0} access.api.requestAccess.failure.commandError=Problem trying request access on {0} : {1} access.api.requestAccess.failure.requestExists=An access request for this file on your behalf already exists. access.api.requestAccess.failure.invalidRequest=You may not request access to this file. It may already be available to you. +access.api.requestAccess.failure.retentionExpired=You may not request access to this file. It is not available because its retention period has ended. + access.api.requestAccess.noKey=You must provide a key to request access to a file. access.api.requestAccess.fileNotFound=Could not find datafile with id {0}. access.api.requestAccess.invalidRequest=This file is already available to you for download or you have a pending request @@ -2916,6 +2955,7 @@ Public=Public Restricted=Restricted EmbargoedThenPublic=Embargoed then Public EmbargoedThenRestricted=Embargoed then Restricted +RetentionPeriodExpired=Retention Period Expired #metadata source - Facet Label Harvested=Harvested diff --git a/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties b/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties index 70b0c4e371e..5c1a22bfd5f 100644 --- a/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties +++ b/src/main/java/propertyFiles/MimeTypeDetectionByFileName.properties @@ -2,3 +2,5 @@ Makefile=text/x-makefile Snakemake=text/x-snakemake Dockerfile=application/x-docker-file Vagrantfile=application/x-vagrant-file +ro-crate-metadata.json=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" +ro-crate-metadata.jsonld=application/ld+json; profile="http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate" diff --git a/src/main/java/propertyFiles/MimeTypeDisplay.properties b/src/main/java/propertyFiles/MimeTypeDisplay.properties index 295ac226fa1..8e5a251abbf 100644 --- a/src/main/java/propertyFiles/MimeTypeDisplay.properties +++ b/src/main/java/propertyFiles/MimeTypeDisplay.properties @@ -207,6 +207,7 @@ audio/ogg=OGG Audio audio/wav=Waveform Audio audio/x-wav=Waveform Audio audio/x-wave=Waveform Audio +audio/vnd.wave=Waveform Audio # Video video/avi=AVI Video video/x-msvideo=AVI Video @@ -222,5 +223,6 @@ text/xml-graphml=GraphML Network Data application/octet-stream=Unknown application/x-docker-file=Docker Image File application/x-vagrant-file=Vagrant Image File +application/ld+json;\u0020profile\u003d\u0022http\u003a//www.w3.org/ns/json-ld#flattened\u0020http\u003a//www.w3.org/ns/json-ld#compacted\u0020https\u003a//w3id.org/ro/crate\u0022=RO-Crate metadata # Dataverse-specific application/vnd.dataverse.file-package=Dataverse Package diff --git a/src/main/java/propertyFiles/MimeTypeFacets.properties b/src/main/java/propertyFiles/MimeTypeFacets.properties index aaab66f20ae..0dad8daff4c 100644 --- a/src/main/java/propertyFiles/MimeTypeFacets.properties +++ b/src/main/java/propertyFiles/MimeTypeFacets.properties @@ -209,6 +209,7 @@ audio/ogg=Audio audio/wav=Audio audio/x-wav=Audio audio/x-wave=Audio +audio/vnd.wave=Audio # (anything else that looks like audio/* will also be indexed as facet type "Audio") # Video video/avi=Video @@ -224,5 +225,6 @@ video/webm=Video text/xml-graphml=Network Data # Other application/octet-stream=Unknown +application/ld+json;\u0020profile\u003d\u0022http\u003a//www.w3.org/ns/json-ld#flattened\u0020http\u003a//www.w3.org/ns/json-ld#compacted\u0020https\u003a//w3id.org/ro/crate\u0022=Metadata # Dataverse-specific application/vnd.dataverse.file-package=Data diff --git a/src/main/resources/db/migration/V6.2.0.1.sql b/src/main/resources/db/migration/V6.2.0.1.sql new file mode 100644 index 00000000000..cb23d589542 --- /dev/null +++ b/src/main/resources/db/migration/V6.2.0.1.sql @@ -0,0 +1 @@ +ALTER TABLE datafile ADD COLUMN IF NOT EXISTS retention_id BIGINT; \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/pretty-config.xml b/src/main/webapp/WEB-INF/pretty-config.xml index ab5f37a1051..5f8f4877af8 100644 --- a/src/main/webapp/WEB-INF/pretty-config.xml +++ b/src/main/webapp/WEB-INF/pretty-config.xml @@ -27,4 +27,9 @@ + + + + + \ No newline at end of file diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index 3947c4415f9..527b829960f 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -32,7 +32,7 @@ and !permissionsWrapper.canIssuePublishDatasetCommand(DatasetPage.dataset)}"/> - + @@ -235,20 +236,21 @@ - -
  • - - #{bundle.transfer} - - - - - -
    • -     - + +
  • + + #{bundle.transfer} + + + + + +
    • +     + + @@ -1003,8 +1005,10 @@ - + + +

    #{bundle['dataset.share.datasetShare.tip']}

    @@ -1053,10 +1057,10 @@     - +

    #{bundle['dataset.noValidSelectedFilesForDownload']}

    -

    #{DatasetPage.cantDownloadDueToEmbargo ? bundle['dataset.requestAccessToRestrictedFilesWithEmbargo'] : bundle['dataset.requestAccessToRestrictedFiles']}

    +

    #{DatasetPage.cantDownloadDueToEmbargoOrDVAccess ? bundle['dataset.requestAccessToRestrictedFilesWithEmbargo'] : bundle['dataset.requestAccessToRestrictedFiles']}
    + +

    #{bundle['dataset.noValidSelectedFilesForTransfer']}

    + +

    #{DatasetPage.cantDownloadDueToEmbargoOrDVAccess ? bundle['dataset.requestAccessToRestrictedFilesWithEmbargo'] : bundle['dataset.requestAccessToRestrictedFiles']}

    +     +
    + +
    +

    #{bundle['file.zip.download.exceeds.limit.info']}

    @@ -1085,8 +1100,8 @@     - -

    #{DatasetPage.cantDownloadDueToEmbargo ? bundle['dataset.mixedSelectedFilesForDownloadWithEmbargo'] : bundle['dataset.mixedSelectedFilesForDownload']}

    + +

    #{DatasetPage.cantDownloadDueToEmbargoOrDVAccess ? bundle['dataset.mixedSelectedFilesForDownloadWithEmbargo'] : bundle['dataset.mixedSelectedFilesForDownload']}

    @@ -1970,7 +1985,11 @@ PF('downloadTooLarge').show(); } if (outcome ==='FailRestricted'){ - PF('downloadInvalid').show(); + if(isTransfer) { + PF('transferInvalid').show(); + } else { + PF('downloadInvalid').show(); + } } if (outcome ==='GuestbookRequired'){ PF('guestbookAndTermsPopup').show(); diff --git a/src/main/webapp/datasetFieldForEditFragment.xhtml b/src/main/webapp/datasetFieldForEditFragment.xhtml index e72ee351ea0..d8c005366cb 100644 --- a/src/main/webapp/datasetFieldForEditFragment.xhtml +++ b/src/main/webapp/datasetFieldForEditFragment.xhtml @@ -42,6 +42,7 @@ + diff --git a/src/main/webapp/editdatafiles.xhtml b/src/main/webapp/editdatafiles.xhtml index 02acb224827..be78359e02b 100644 --- a/src/main/webapp/editdatafiles.xhtml +++ b/src/main/webapp/editdatafiles.xhtml @@ -75,8 +75,10 @@ - + + + diff --git a/src/main/webapp/file-download-button-fragment.xhtml b/src/main/webapp/file-download-button-fragment.xhtml index 9c29fd777a1..cd6a6b06523 100644 --- a/src/main/webapp/file-download-button-fragment.xhtml +++ b/src/main/webapp/file-download-button-fragment.xhtml @@ -14,16 +14,18 @@
  • #{bundle['file.accessBtn.header.access']}
  • - + value=" #{dataFileServiceBean.isRetentionExpired(fileMetadata) ? bundle['retentionExpired'] : !fileDownloadHelper.isRestrictedOrEmbargoed(fileMetadata) ? bundle['public'] : (!fileDownloadHelper.canDownloadFile(fileMetadata) ? (!dataFileServiceBean.isActivelyEmbargoed(fileMetadata) ? bundle['restricted'] : bundle['embargoed']) : (!dataFileServiceBean.isActivelyEmbargoed(fileMetadata) ? bundle['restrictedaccess'] : bundle['embargoed']) )}" + styleClass="#{dataFileServiceBean.isRetentionExpired(fileMetadata) ? 'text-danger' : !fileDownloadHelper.isRestrictedOrEmbargoed(fileMetadata) ? 'text-success' : (!fileDownloadHelper.canDownloadFile(fileMetadata) ? 'text-danger' : 'text-success')}"/>
    • + and fileMetadata.dataFile.owner.fileAccessRequest + and !dataFileServiceBean.isActivelyEmbargoed(fileMetadata) + and !dataFileServiceBean.isRetentionExpired(fileMetadata)}"> + +
  • + + + + +
    • +     +
  • diff --git a/src/main/webapp/file-edit-popup-fragment.xhtml b/src/main/webapp/file-edit-popup-fragment.xhtml index ffc4a1fcef7..3b1141816c8 100644 --- a/src/main/webapp/file-edit-popup-fragment.xhtml +++ b/src/main/webapp/file-edit-popup-fragment.xhtml @@ -168,7 +168,83 @@ PF('blockDatasetForm').hide();" action="#{bean.clearEmbargoPopup()}" update="#{updateElements}" immediate="true"/> - + + + + +

    #{bundle['file.editRetentionDialog.tip']}

    +

    #{bundle['file.editRetentionDialog.some.tip']} #{bundle['file.editRetentionDialog.partial.tip']}

    +

    #{bundle['file.editRetentionDialog.none.tip']}

    + + +
    + +
    +
    + +
    +
    +
    + + + + +
    + +
    +
    +
    +

    #{bundle['file.editRetentionDialog.reason.tip']}

    + +
    +
    +
    +
    +
    +
    +
    + +
    +
    + + + +
    +
    +
    +     +
    + + +
    +     +

    #{bundle['file.deleteFileDialog.immediate']}

    diff --git a/src/main/webapp/file-info-fragment.xhtml b/src/main/webapp/file-info-fragment.xhtml index 72fe279fbf8..dca5c4a8cec 100644 --- a/src/main/webapp/file-info-fragment.xhtml +++ b/src/main/webapp/file-info-fragment.xhtml @@ -64,6 +64,7 @@
    +
    diff --git a/src/main/webapp/file.xhtml b/src/main/webapp/file.xhtml index ea7b51f9640..835764d9cf5 100644 --- a/src/main/webapp/file.xhtml +++ b/src/main/webapp/file.xhtml @@ -43,7 +43,7 @@
    #{FilePage.fileMetadata.label} - +
    @@ -64,22 +64,23 @@

    - + - + + - + - + + value="#{bundle['file.DatasetVersion']} #{FilePage.fileMetadata.datasetVersion.versionNumber}.#{FilePage.fileMetadata.datasetVersion.minorVersionNumber}"/>
    @@ -98,9 +99,9 @@ - -
    @@ -145,11 +146,11 @@ - - -
    @@ -192,7 +193,7 @@
    -
      - @@ -227,12 +228,12 @@
      #{bundle['file.editBtn']} - +
      #{bundle['metrics.file.title']} - -
      @@ -317,7 +318,7 @@
      -
      @@ -351,7 +352,7 @@ - @@ -376,7 +377,7 @@ - +
        @@ -393,8 +394,8 @@
        - -
        @@ -443,7 +444,7 @@ #{bundle['file.dataFilesTab.metadata.addBtn']}
      @@ -483,7 +484,7 @@
      -
      @@ -495,7 +496,7 @@ #{bundle['file.metadata.filetags']}
    • + - + - @@ -566,6 +574,13 @@ + + + +
    -
    +
    @@ -504,7 +505,7 @@
    -
    @@ -552,11 +553,18 @@ #{FilePage.file.publicationDateFormattedYYYYMMDD}
    #{bundle['file.metadataTab.fileMetadata.publicationDate.label']} #{!(empty FilePage.file.embargo) ? FilePage.embargoPhrase: ''} #{!(empty FilePage.file.embargo) ? FilePage.file.embargo.dateAvailable : FilePage.file.publicationDateFormattedYYYYMMDD} + #{(empty FilePage.file.embargo) and !(empty FilePage.file.released) and !(empty FilePage.file.retention) ? ''.concat(FilePage.file.publicationDateFormattedYYYYMMDD).concat('; ') : ''} + #{!(empty FilePage.file.embargo) ? ''.concat(FilePage.embargoPhrase).concat(' ').concat(FilePage.file.embargo.dateAvailable) : ''} + #{!(empty FilePage.file.embargo) and !(empty FilePage.file.retention) ? '; ': ''} + #{!(empty FilePage.file.retention) ? ''.concat(FilePage.retentionPhrase) + .concat(' ').concat(FilePage.file.retention.dateUnavailable) + .concat(', ').concat(bundle['retention.enddateinfo']) + : ''} + #{(empty FilePage.file.embargo and empty FilePage.file.retention)? FilePage.file.publicationDateFormattedYYYYMMDD : ''}
    #{FilePage.file.embargo.reason}
    + #{bundle['file.metadataTab.fileMetadata.retentionReason.label']} + #{FilePage.file.retention.reason} +
    #{bundle['file.metadataTab.fileMetadata.size.label']} @@ -623,9 +638,11 @@ + - - + + +

    #{bundle['file.share.tip']}

    @@ -634,7 +651,7 @@ #{bundle.close} -     + @@ -645,7 +662,7 @@ - + @@ -657,7 +674,7 @@ - + @@ -678,7 +695,7 @@ #{bundle.close} - +

    #{bundle['file.compute.fileAccessDenied']}

    @@ -690,7 +707,7 @@