From 61ecb32e1e9836a613cea2f71e8376dee400ac4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Thu, 15 Feb 2024 15:03:33 +0100 Subject: [PATCH 1/5] CVOC : add HTTP request headers --- .../harvard/iq/dataverse/DatasetFieldServiceBean.java | 9 ++++++++- src/main/webapp/datasetFieldForEditFragment.xhtml | 1 + src/main/webapp/metadataFragment.xhtml | 4 ++++ src/main/webapp/search-include-fragment.xhtml | 4 ++++ src/main/webapp/search/advanced.xhtml | 1 + 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java index ce2b00086ec..b05dd6b40c0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldServiceBean.java @@ -502,7 +502,14 @@ public void process(HttpResponse response, HttpContext context) throws HttpExcep HttpGet httpGet = new HttpGet(retrievalUri); //application/json+ld is for backward compatibility httpGet.addHeader("Accept", "application/ld+json, application/json+ld, application/json"); - + //Adding others custom HTTP request headers if exists + final JsonObject headers = cvocEntry.getJsonObject("headers"); + if (headers != null) { + final Set headerKeys = headers.keySet(); + for (final String hKey: headerKeys) { + httpGet.addHeader(hKey, headers.getString(hKey)); + } + } HttpResponse response = httpClient.execute(httpGet); String data = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); int statusCode = response.getStatusLine().getStatusCode(); diff --git a/src/main/webapp/datasetFieldForEditFragment.xhtml b/src/main/webapp/datasetFieldForEditFragment.xhtml index e72ee351ea0..d8c005366cb 100644 --- a/src/main/webapp/datasetFieldForEditFragment.xhtml +++ b/src/main/webapp/datasetFieldForEditFragment.xhtml @@ -42,6 +42,7 @@ + diff --git a/src/main/webapp/metadataFragment.xhtml b/src/main/webapp/metadataFragment.xhtml index 200d2917b9a..1cf4ac0bc71 100755 --- a/src/main/webapp/metadataFragment.xhtml +++ b/src/main/webapp/metadataFragment.xhtml @@ -88,6 +88,7 @@ escape="#{dsf.datasetFieldType.isEscapeOutputText()}"> + @@ -100,6 +101,7 @@ escape="#{dsf.datasetFieldType.isEscapeOutputText()}"> + @@ -146,6 +148,7 @@ rendered="${cvocOnCvPart and cvPart.key.datasetFieldType.name.equals(cvocConf.get(cvPart.key.datasetFieldType.id).getString('term-uri-field'))}"> + @@ -156,6 +159,7 @@ rendered="${cvocOnDsf and cvPart.key.datasetFieldType.name.equals(cvocConf.get(dsf.datasetFieldType.id).getString('term-uri-field'))}"> + diff --git a/src/main/webapp/search-include-fragment.xhtml b/src/main/webapp/search-include-fragment.xhtml index af568170157..2eed454800d 100644 --- a/src/main/webapp/search-include-fragment.xhtml +++ b/src/main/webapp/search-include-fragment.xhtml @@ -213,6 +213,7 @@ + @@ -234,6 +235,7 @@ + @@ -384,6 +386,7 @@ + @@ -621,6 +624,7 @@ + diff --git a/src/main/webapp/search/advanced.xhtml b/src/main/webapp/search/advanced.xhtml index 06e662064e7..24b78ba0dc9 100644 --- a/src/main/webapp/search/advanced.xhtml +++ b/src/main/webapp/search/advanced.xhtml @@ -121,6 +121,7 @@ + From 1c0915a1a98d0be30944424a15846d0bc90e04f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Wed, 17 Apr 2024 11:40:44 +0200 Subject: [PATCH 2/5] Add minimal release note snippet --- doc/release-notes/10316_cvoc_http_headers.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes/10316_cvoc_http_headers.md diff --git a/doc/release-notes/10316_cvoc_http_headers.md b/doc/release-notes/10316_cvoc_http_headers.md new file mode 100644 index 00000000000..4b557383a2e --- /dev/null +++ b/doc/release-notes/10316_cvoc_http_headers.md @@ -0,0 +1,5 @@ +You are now able to add HTTP request headers required by the External Vocabulary Services you are implementing. + +A combined documentation can be found on pull request [#10404](https://github.com/IQSS/dataverse/pull/10404). + +For more information, see issue [#10316](https://github.com/IQSS/dataverse/issues/10316) and pull request [gddc/dataverse-external-vocab-support#19](https://github.com/gdcc/dataverse-external-vocab-support/pull/19). From f4da07d15df95d2a9e903dd63c30c6f35a830870 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 17 Apr 2024 09:46:15 -0400 Subject: [PATCH 3/5] address failing test - add sleep for reindexing --- src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java index be8d5effbfe..b62b050b949 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/SearchIT.java @@ -862,7 +862,7 @@ public void testNestedSubtree() { Response publishDataset = UtilIT.publishDatasetViaNativeApi(datasetPid, "major", apiToken); publishDataset.then().assertThat() .statusCode(OK.getStatusCode()); - + UtilIT.sleepForReindex(datasetPid, apiToken, 5); Response searchPublishedSubtreeWDS = UtilIT.search(searchPart, apiToken, "&subtree="+dataverseAlias); searchPublishedSubtreeWDS.prettyPrint(); searchPublishedSubtreeWDS.then().assertThat() From fa4a03cb023bbebe2464c02d15d5d312c4306bcd Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 17 Apr 2024 09:49:55 -0400 Subject: [PATCH 4/5] doc changes --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 66911aa0ad1..c1ea375d614 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -577,9 +577,9 @@ In general, the external vocabulary support mechanism may be a better choice for The specifics of the user interface for entering/selecting a vocabulary term and how that term is then displayed are managed by third-party Javascripts. The initial Javascripts that have been created provide auto-completion, displaying a list of choices that match what the user has typed so far, but other interfaces, such as displaying a tree of options for a hierarchical vocabulary, are possible. Similarly, existing scripts do relatively simple things for displaying a term - showing the term's name in the appropriate language and providing a link to an external URL with more information, but more sophisticated displays are possible. -Scripts supporting use of vocabularies from services supporting the SKOMOS protocol (see https://skosmos.org) and retrieving ORCIDs (from https://orcid.org) are available https://github.com/gdcc/dataverse-external-vocab-support. (Custom scripts can also be used and community members are encouraged to share new scripts through the dataverse-external-vocab-support repository.) +Scripts supporting use of vocabularies from services supporting the SKOMOS protocol (see https://skosmos.org), retrieving ORCIDs (from https://orcid.org), and using ROR (https://ror.org/) are available https://github.com/gdcc/dataverse-external-vocab-support. (Custom scripts can also be used and community members are encouraged to share new scripts through the dataverse-external-vocab-support repository.) -Configuration involves specifying which fields are to be mapped, whether free-text entries are allowed, which vocabulary(ies) should be used, what languages those vocabulary(ies) are available in, and several service protocol and service instance specific parameters. +Configuration involves specifying which fields are to be mapped, whether free-text entries are allowed, which vocabulary(ies) should be used, what languages those vocabulary(ies) are available in, and several service protocol and service instance specific parameters, including the ability to send HTTP headers on calls to the service. These are all defined in the :ref:`:CVocConf <:CVocConf>` setting as a JSON array. Details about the required elements as well as example JSON arrays are available at https://github.com/gdcc/dataverse-external-vocab-support, along with an example metadata block that can be used for testing. The scripts required can be hosted locally or retrieved dynamically from https://gdcc.github.io/ (similar to how dataverse-previewers work). From ccf34c9ed9e3e86a4a29bc8811ce04a6d8029bf8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 17 Apr 2024 10:24:15 -0400 Subject: [PATCH 5/5] rewrite :CVocConf docs, explain where to find readme.me #10316 --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 2 ++ doc/sphinx-guides/source/installation/config.rst | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 66911aa0ad1..7c31743b454 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -552,6 +552,8 @@ Great care must be taken when reloading a metadata block. Matching is done on fi The ability to reload metadata blocks means that SQL update scripts don't need to be written for these changes. See also the :doc:`/developers/sql-upgrade-scripts` section of the Developer Guide. +.. _using-external-vocabulary-services: + Using External Vocabulary Services ---------------------------------- diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index ff786e900cc..d79a1c4bfe6 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -4458,9 +4458,13 @@ A boolean setting that, if true, will send an email and notification to users wh :CVocConf +++++++++ -A JSON-structured setting that configures Dataverse to associate specific metadatablock fields with external vocabulary services and specific vocabularies/sub-vocabularies managed by that service. More information about this capability is available at :doc:`/admin/metadatacustomization`. +The ``:CVocConf`` database setting is used to allow metadatablock fields to look up values in external vocabulary services. For example, you could configure the "Author Affiliation" field to look up organizations in the `Research Organization Registry (ROR) `_. For a high-level description of this feature, see :ref:`using-external-vocabulary-services` in the Admin Guide. -Scripts that implement this association for specific service protocols are maintained at https://github.com/gdcc/dataverse-external-vocab-support. That repository also includes a json-schema for validating the structure required by this setting along with an example metadatablock and sample :CVocConf setting values associating entries in the example block with ORCID and SKOSMOS based services. +The expected format for the ``:CVocConf`` database setting is JSON but the details are not documented here. Instead, please refer to `docs/readme.md `_ in the https://github.com/gdcc/dataverse-external-vocab-support repo. + +That repository also includes scripts that implement the lookup for specific service protocols, a JSON Schema for validating the structure required by this setting, and an example metadatablock with a sample ``:CVocConf`` config that associates fields in the example block with ORCID and SKOSMOS based services. + +The commands below should give you an idea of how to load the configuration, but you'll want to study the examples and make decisions about which configuration to use: ``wget https://gdcc.github.io/dataverse-external-vocab-support/examples/config/cvoc-conf.json``