-
-
-
-
-
-
-
-
-
+
+
@@ -163,7 +156,7 @@
-
+
@@ -200,7 +193,7 @@
-
+
@@ -208,9 +201,9 @@
-
+
-
+
@@ -218,10 +211,10 @@
-
+
-
+
@@ -253,7 +246,7 @@
-
+
@@ -658,7 +651,8 @@
+ RESTRICTION: the glob-like pattern in the name attribute must have a "*"
+ only at the start or the end. -->
@@ -666,19 +660,23 @@
-
+
+
+
+
+
+
+
-
-
-
-
+
+
@@ -724,43 +722,6 @@
field first in an ascending sort and last in a descending sort.
-->
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
-
-
-
-
-
- 7.3.0
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ${solr.data.dir:}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ${solr.lock.type:native}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ${solr.ulog.dir:}
- ${solr.ulog.numVersionBuckets:65536}
-
-
-
-
- ${solr.autoCommit.maxTime:15000}
- false
-
-
-
-
-
- ${solr.autoSoftCommit.maxTime:-1}
-
-
-
-
-
-
-
-
-
-
-
-
-
- 1024
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- true
-
-
-
-
-
- 20
-
-
- 200
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- false
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- explicit
- 10
- edismax
- 0.075
-
- dvName^400
- authorName^180
- dvSubject^190
- dvDescription^180
- dvAffiliation^170
- title^130
- subject^120
- keyword^110
- topicClassValue^100
- dsDescriptionValue^90
- authorAffiliation^80
- publicationCitation^60
- producerName^50
- fileName^30
- fileDescription^30
- variableLabel^20
- variableName^10
- _text_^1.0
-
-
- dvName^200
- authorName^100
- dvSubject^100
- dvDescription^100
- dvAffiliation^100
- title^75
- subject^75
- keyword^75
- topicClassValue^75
- dsDescriptionValue^75
- authorAffiliation^75
- publicationCitation^75
- producerName^75
-
-
-
- isHarvested:false^25000
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- explicit
- json
- true
-
-
-
-
-
-
-
- explicit
-
-
-
-
-
- _text_
-
-
-
-
-
-
- true
- ignored_
- _text_
-
-
-
-
-
-
-
-
- text_general
-
-
-
-
-
- default
- _text_
- solr.DirectSolrSpellChecker
-
- internal
-
- 0.5
-
- 2
-
- 1
-
- 5
-
- 4
-
- 0.01
-
-
-
-
-
-
-
-
-
-
-
- default
- on
- true
- 10
- 5
- 5
- true
- true
- 10
- 5
-
-
- spellcheck
-
-
-
-
-
-
-
-
-
- true
-
-
- tvComponent
-
-
-
-
-
-
-
-
-
-
-
- true
- false
-
-
- terms
-
-
-
-
-
-
-
- string
-
-
-
-
-
- explicit
-
-
- elevator
-
-
-
-
-
-
-
-
-
-
- 100
-
-
-
-
-
-
-
- 70
-
- 0.5
-
- [-\w ,/\n\"']{20,200}
-
-
-
-
-
-
- ]]>
- ]]>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- ,,
- ,,
- ,,
- ,,
- ,]]>
- ]]>
-
-
-
-
-
- 10
- .,!?
-
-
-
-
-
-
- WORD
-
-
- en
- US
-
-
-
-
-
-
-
-
-
-
-
-
-
- [^\w-\.]
- _
-
-
-
-
-
-
- yyyy-MM-dd'T'HH:mm:ss.SSSZ
- yyyy-MM-dd'T'HH:mm:ss,SSSZ
- yyyy-MM-dd'T'HH:mm:ss.SSS
- yyyy-MM-dd'T'HH:mm:ss,SSS
- yyyy-MM-dd'T'HH:mm:ssZ
- yyyy-MM-dd'T'HH:mm:ss
- yyyy-MM-dd'T'HH:mmZ
- yyyy-MM-dd'T'HH:mm
- yyyy-MM-dd HH:mm:ss.SSSZ
- yyyy-MM-dd HH:mm:ss,SSSZ
- yyyy-MM-dd HH:mm:ss.SSS
- yyyy-MM-dd HH:mm:ss,SSS
- yyyy-MM-dd HH:mm:ssZ
- yyyy-MM-dd HH:mm:ss
- yyyy-MM-dd HH:mmZ
- yyyy-MM-dd HH:mm
- yyyy-MM-dd
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- text/plain; charset=UTF-8
-
-
-
-
- ${velocity.template.base.dir:}
- ${velocity.solr.resource.loader.enabled:true}
- ${velocity.params.resource.loader.enabled:false}
-
-
-
-
- 5
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+ 9.7
+
+
+
+
+
+
+
+
+
+
+ ${solr.data.dir:}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${solr.lock.type:native}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${solr.ulog.dir:}
+ ${solr.ulog.numVersionBuckets:65536}
+
+
+
+
+ ${solr.autoCommit.maxTime:15000}
+ false
+
+
+
+
+
+ ${solr.autoSoftCommit.maxTime:-1}
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ${solr.max.booleanClauses:1024}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ true
+
+
+
+
+
+ 20
+
+
+ 200
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ false
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ explicit
+ 10
+
+ edismax
+ 0.075
+
+ dvName^400
+ authorName^180
+ dvSubject^190
+ dvDescription^180
+ dvAffiliation^170
+ title^130
+ subject^120
+ keyword^110
+ topicClassValue^100
+ dsDescriptionValue^90
+ authorAffiliation^80
+ publicationCitation^60
+ producerName^50
+ fileName^30
+ fileDescription^30
+ variableLabel^20
+ variableName^10
+ _text_^1.0
+
+
+ dvName^200
+ authorName^100
+ dvSubject^100
+ dvDescription^100
+ dvAffiliation^100
+ title^75
+ subject^75
+ keyword^75
+ topicClassValue^75
+ dsDescriptionValue^75
+ authorAffiliation^75
+ publicationCitation^75
+ producerName^75
+
+
+
+ isHarvested:false^25000
+
+
+
+
+
+
+
+ explicit
+ json
+ true
+
+
+
+
+
+
+ _text_
+
+
+
+
+
+
+ text_general
+
+
+
+
+
+ default
+ _text_
+ solr.DirectSolrSpellChecker
+
+ internal
+
+ 0.5
+
+ 2
+
+ 1
+
+ 5
+
+ 4
+
+ 0.01
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 100
+
+
+
+
+
+
+
+ 70
+
+ 0.5
+
+ [-\w ,/\n\"']{20,200}
+
+
+
+
+
+
+ ]]>
+ ]]>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ,,
+ ,,
+ ,,
+ ,,
+ ,]]>
+ ]]>
+
+
+
+
+
+ 10
+ .,!?
+
+
+
+
+
+
+ WORD
+
+
+ en
+ US
+
+
+
+
+
+
+
+
+
+
+
+
+ [^\w-\.]
+ _
+
+
+
+
+
+
+ yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z
+ yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z
+ yyyy-MM-dd HH:mm[:ss[.SSS]][z
+ yyyy-MM-dd HH:mm[:ss[,SSS]][z
+ [EEE, ]dd MMM yyyy HH:mm[:ss] z
+ EEEE, dd-MMM-yy HH:mm:ss z
+ EEE MMM ppd HH:mm:ss [z ]yyyy
+
+
+
+
+ java.lang.String
+ text_general
+
+ *_str
+ 256
+
+
+ true
+
+
+ java.lang.Boolean
+ booleans
+
+
+ java.util.Date
+ pdates
+
+
+ java.lang.Long
+ java.lang.Integer
+ plongs
+
+
+ java.lang.Number
+ pdoubles
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/conf/solr/8.11.1/update-fields.sh b/conf/solr/9.3.0/update-fields.sh
similarity index 100%
rename from conf/solr/8.11.1/update-fields.sh
rename to conf/solr/9.3.0/update-fields.sh
diff --git a/conf/vagrant/etc/shibboleth/attribute-map.xml b/conf/vagrant/etc/shibboleth/attribute-map.xml
deleted file mode 100644
index f6386b620f5..00000000000
--- a/conf/vagrant/etc/shibboleth/attribute-map.xml
+++ /dev/null
@@ -1,141 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/conf/vagrant/etc/shibboleth/dataverse-idp-metadata.xml b/conf/vagrant/etc/shibboleth/dataverse-idp-metadata.xml
deleted file mode 100644
index 67225b5e670..00000000000
--- a/conf/vagrant/etc/shibboleth/dataverse-idp-metadata.xml
+++ /dev/null
@@ -1,298 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- testshib.org
-
- TestShib Test IdP
- TestShib IdP. Use this as a source of attributes
- for your test SP.
- https://www.testshib.org/images/testshib-transp.png
-
-
-
-
-
-
-
- MIIEDjCCAvagAwIBAgIBADANBgkqhkiG9w0BAQUFADBnMQswCQYDVQQGEwJVUzEV
- MBMGA1UECBMMUGVubnN5bHZhbmlhMRMwEQYDVQQHEwpQaXR0c2J1cmdoMREwDwYD
- VQQKEwhUZXN0U2hpYjEZMBcGA1UEAxMQaWRwLnRlc3RzaGliLm9yZzAeFw0wNjA4
- MzAyMTEyMjVaFw0xNjA4MjcyMTEyMjVaMGcxCzAJBgNVBAYTAlVTMRUwEwYDVQQI
- EwxQZW5uc3lsdmFuaWExEzARBgNVBAcTClBpdHRzYnVyZ2gxETAPBgNVBAoTCFRl
- c3RTaGliMRkwFwYDVQQDExBpZHAudGVzdHNoaWIub3JnMIIBIjANBgkqhkiG9w0B
- AQEFAAOCAQ8AMIIBCgKCAQEArYkCGuTmJp9eAOSGHwRJo1SNatB5ZOKqDM9ysg7C
- yVTDClcpu93gSP10nH4gkCZOlnESNgttg0r+MqL8tfJC6ybddEFB3YBo8PZajKSe
- 3OQ01Ow3yT4I+Wdg1tsTpSge9gEz7SrC07EkYmHuPtd71CHiUaCWDv+xVfUQX0aT
- NPFmDixzUjoYzbGDrtAyCqA8f9CN2txIfJnpHE6q6CmKcoLADS4UrNPlhHSzd614
- kR/JYiks0K4kbRqCQF0Dv0P5Di+rEfefC6glV8ysC8dB5/9nb0yh/ojRuJGmgMWH
- gWk6h0ihjihqiu4jACovUZ7vVOCgSE5Ipn7OIwqd93zp2wIDAQABo4HEMIHBMB0G
- A1UdDgQWBBSsBQ869nh83KqZr5jArr4/7b+QazCBkQYDVR0jBIGJMIGGgBSsBQ86
- 9nh83KqZr5jArr4/7b+Qa6FrpGkwZzELMAkGA1UEBhMCVVMxFTATBgNVBAgTDFBl
- bm5zeWx2YW5pYTETMBEGA1UEBxMKUGl0dHNidXJnaDERMA8GA1UEChMIVGVzdFNo
- aWIxGTAXBgNVBAMTEGlkcC50ZXN0c2hpYi5vcmeCAQAwDAYDVR0TBAUwAwEB/zAN
- BgkqhkiG9w0BAQUFAAOCAQEAjR29PhrCbk8qLN5MFfSVk98t3CT9jHZoYxd8QMRL
- I4j7iYQxXiGJTT1FXs1nd4Rha9un+LqTfeMMYqISdDDI6tv8iNpkOAvZZUosVkUo
- 93pv1T0RPz35hcHHYq2yee59HJOco2bFlcsH8JBXRSRrJ3Q7Eut+z9uo80JdGNJ4
- /SJy5UorZ8KazGj16lfJhOBXldgrhppQBb0Nq6HKHguqmwRfJ+WkxemZXzhediAj
- Geka8nz8JjwxpUjAiSWYKLtJhGEaTqCYxCCX2Dw+dOTqUzHOZ7WKv4JXPK5G/Uhr
- 8K/qhmFT2nIQi538n6rVYLeWj8Bbnl+ev0peYzxFyF5sQA==
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- urn:mace:shibboleth:1.0:nameIdentifier
- urn:oasis:names:tc:SAML:2.0:nameid-format:transient
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- MIIEDjCCAvagAwIBAgIBADANBgkqhkiG9w0BAQUFADBnMQswCQYDVQQGEwJVUzEV
- MBMGA1UECBMMUGVubnN5bHZhbmlhMRMwEQYDVQQHEwpQaXR0c2J1cmdoMREwDwYD
- VQQKEwhUZXN0U2hpYjEZMBcGA1UEAxMQaWRwLnRlc3RzaGliLm9yZzAeFw0wNjA4
- MzAyMTEyMjVaFw0xNjA4MjcyMTEyMjVaMGcxCzAJBgNVBAYTAlVTMRUwEwYDVQQI
- EwxQZW5uc3lsdmFuaWExEzARBgNVBAcTClBpdHRzYnVyZ2gxETAPBgNVBAoTCFRl
- c3RTaGliMRkwFwYDVQQDExBpZHAudGVzdHNoaWIub3JnMIIBIjANBgkqhkiG9w0B
- AQEFAAOCAQ8AMIIBCgKCAQEArYkCGuTmJp9eAOSGHwRJo1SNatB5ZOKqDM9ysg7C
- yVTDClcpu93gSP10nH4gkCZOlnESNgttg0r+MqL8tfJC6ybddEFB3YBo8PZajKSe
- 3OQ01Ow3yT4I+Wdg1tsTpSge9gEz7SrC07EkYmHuPtd71CHiUaCWDv+xVfUQX0aT
- NPFmDixzUjoYzbGDrtAyCqA8f9CN2txIfJnpHE6q6CmKcoLADS4UrNPlhHSzd614
- kR/JYiks0K4kbRqCQF0Dv0P5Di+rEfefC6glV8ysC8dB5/9nb0yh/ojRuJGmgMWH
- gWk6h0ihjihqiu4jACovUZ7vVOCgSE5Ipn7OIwqd93zp2wIDAQABo4HEMIHBMB0G
- A1UdDgQWBBSsBQ869nh83KqZr5jArr4/7b+QazCBkQYDVR0jBIGJMIGGgBSsBQ86
- 9nh83KqZr5jArr4/7b+Qa6FrpGkwZzELMAkGA1UEBhMCVVMxFTATBgNVBAgTDFBl
- bm5zeWx2YW5pYTETMBEGA1UEBxMKUGl0dHNidXJnaDERMA8GA1UEChMIVGVzdFNo
- aWIxGTAXBgNVBAMTEGlkcC50ZXN0c2hpYi5vcmeCAQAwDAYDVR0TBAUwAwEB/zAN
- BgkqhkiG9w0BAQUFAAOCAQEAjR29PhrCbk8qLN5MFfSVk98t3CT9jHZoYxd8QMRL
- I4j7iYQxXiGJTT1FXs1nd4Rha9un+LqTfeMMYqISdDDI6tv8iNpkOAvZZUosVkUo
- 93pv1T0RPz35hcHHYq2yee59HJOco2bFlcsH8JBXRSRrJ3Q7Eut+z9uo80JdGNJ4
- /SJy5UorZ8KazGj16lfJhOBXldgrhppQBb0Nq6HKHguqmwRfJ+WkxemZXzhediAj
- Geka8nz8JjwxpUjAiSWYKLtJhGEaTqCYxCCX2Dw+dOTqUzHOZ7WKv4JXPK5G/Uhr
- 8K/qhmFT2nIQi538n6rVYLeWj8Bbnl+ev0peYzxFyF5sQA==
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- urn:mace:shibboleth:1.0:nameIdentifier
- urn:oasis:names:tc:SAML:2.0:nameid-format:transient
-
-
-
-
- TestShib Two Identity Provider
- TestShib Two
- http://www.testshib.org/testshib-two/
-
-
- Nate
- Klingenstein
- ndk@internet2.edu
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- TestShib Test SP
- TestShib SP. Log into this to test your machine.
- Once logged in check that all attributes that you expected have been
- released.
- https://www.testshib.org/images/testshib-transp.png
-
-
-
-
-
-
-
- MIIEPjCCAyagAwIBAgIBADANBgkqhkiG9w0BAQUFADB3MQswCQYDVQQGEwJVUzEV
- MBMGA1UECBMMUGVubnN5bHZhbmlhMRMwEQYDVQQHEwpQaXR0c2J1cmdoMSIwIAYD
- VQQKExlUZXN0U2hpYiBTZXJ2aWNlIFByb3ZpZGVyMRgwFgYDVQQDEw9zcC50ZXN0
- c2hpYi5vcmcwHhcNMDYwODMwMjEyNDM5WhcNMTYwODI3MjEyNDM5WjB3MQswCQYD
- VQQGEwJVUzEVMBMGA1UECBMMUGVubnN5bHZhbmlhMRMwEQYDVQQHEwpQaXR0c2J1
- cmdoMSIwIAYDVQQKExlUZXN0U2hpYiBTZXJ2aWNlIFByb3ZpZGVyMRgwFgYDVQQD
- Ew9zcC50ZXN0c2hpYi5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
- AQDJyR6ZP6MXkQ9z6RRziT0AuCabDd3x1m7nLO9ZRPbr0v1LsU+nnC363jO8nGEq
- sqkgiZ/bSsO5lvjEt4ehff57ERio2Qk9cYw8XCgmYccVXKH9M+QVO1MQwErNobWb
- AjiVkuhWcwLWQwTDBowfKXI87SA7KR7sFUymNx5z1aoRvk3GM++tiPY6u4shy8c7
- vpWbVfisfTfvef/y+galxjPUQYHmegu7vCbjYP3On0V7/Ivzr+r2aPhp8egxt00Q
- XpilNai12LBYV3Nv/lMsUzBeB7+CdXRVjZOHGuQ8mGqEbsj8MBXvcxIKbcpeK5Zi
- JCVXPfarzuriM1G5y5QkKW+LAgMBAAGjgdQwgdEwHQYDVR0OBBYEFKB6wPDxwYrY
- StNjU5P4b4AjBVQVMIGhBgNVHSMEgZkwgZaAFKB6wPDxwYrYStNjU5P4b4AjBVQV
- oXukeTB3MQswCQYDVQQGEwJVUzEVMBMGA1UECBMMUGVubnN5bHZhbmlhMRMwEQYD
- VQQHEwpQaXR0c2J1cmdoMSIwIAYDVQQKExlUZXN0U2hpYiBTZXJ2aWNlIFByb3Zp
- ZGVyMRgwFgYDVQQDEw9zcC50ZXN0c2hpYi5vcmeCAQAwDAYDVR0TBAUwAwEB/zAN
- BgkqhkiG9w0BAQUFAAOCAQEAc06Kgt7ZP6g2TIZgMbFxg6vKwvDL0+2dzF11Onpl
- 5sbtkPaNIcj24lQ4vajCrrGKdzHXo9m54BzrdRJ7xDYtw0dbu37l1IZVmiZr12eE
- Iay/5YMU+aWP1z70h867ZQ7/7Y4HW345rdiS6EW663oH732wSYNt9kr7/0Uer3KD
- 9CuPuOidBacospDaFyfsaJruE99Kd6Eu/w5KLAGG+m0iqENCziDGzVA47TngKz2v
- PVA+aokoOyoz3b53qeti77ijatSEoKjxheBWpO+eoJeGq/e49Um3M2ogIX/JAlMa
- Inh+vYSYngQB2sx9LGkR9KHaMKNIGCDehk93Xla4pWJx1w==
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- urn:oasis:names:tc:SAML:2.0:nameid-format:transient
- urn:mace:shibboleth:1.0:nameIdentifier
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- TestShib Two Service Provider
- TestShib Two
- http://www.testshib.org/testshib-two/
-
-
- Nate
- Klingenstein
- ndk@internet2.edu
-
-
-
-
-
-
-
diff --git a/conf/vagrant/etc/shibboleth/shibboleth2.xml b/conf/vagrant/etc/shibboleth/shibboleth2.xml
deleted file mode 100644
index 946e73bdf6a..00000000000
--- a/conf/vagrant/etc/shibboleth/shibboleth2.xml
+++ /dev/null
@@ -1,85 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- SAML2 SAML1
-
-
-
- SAML2 Local
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/conf/vagrant/etc/yum.repos.d/epel-apache-maven.repo b/conf/vagrant/etc/yum.repos.d/epel-apache-maven.repo
deleted file mode 100644
index 1e0f8200040..00000000000
--- a/conf/vagrant/etc/yum.repos.d/epel-apache-maven.repo
+++ /dev/null
@@ -1,15 +0,0 @@
-# Place this file in your /etc/yum.repos.d/ directory
-
-[epel-apache-maven]
-name=maven from apache foundation.
-baseurl=http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-$releasever/$basearch/
-enabled=1
-skip_if_unavailable=1
-gpgcheck=0
-
-[epel-apache-maven-source]
-name=maven from apache foundation. - Source
-baseurl=http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-$releasever/SRPMS
-enabled=0
-skip_if_unavailable=1
-gpgcheck=0
diff --git a/conf/vagrant/etc/yum.repos.d/shibboleth.repo b/conf/vagrant/etc/yum.repos.d/shibboleth.repo
deleted file mode 100644
index adf42185d8a..00000000000
--- a/conf/vagrant/etc/yum.repos.d/shibboleth.repo
+++ /dev/null
@@ -1,9 +0,0 @@
-[shibboleth]
-name=Shibboleth (rockylinux8)
-# Please report any problems to https://shibboleth.atlassian.net/jira
-type=rpm-md
-mirrorlist=https://shibboleth.net/cgi-bin/mirrorlist.cgi/rockylinux8
-gpgcheck=1
-gpgkey=https://shibboleth.net/downloads/service-provider/RPMS/repomd.xml.key
- https://shibboleth.net/downloads/service-provider/RPMS/cantor.repomd.xml.key
-enabled=1
diff --git a/conf/vagrant/var/lib/pgsql/data/pg_hba.conf b/conf/vagrant/var/lib/pgsql/data/pg_hba.conf
deleted file mode 100644
index e3244686066..00000000000
--- a/conf/vagrant/var/lib/pgsql/data/pg_hba.conf
+++ /dev/null
@@ -1,74 +0,0 @@
-# PostgreSQL Client Authentication Configuration File
-# ===================================================
-#
-# Refer to the "Client Authentication" section in the
-# PostgreSQL documentation for a complete description
-# of this file. A short synopsis follows.
-#
-# This file controls: which hosts are allowed to connect, how clients
-# are authenticated, which PostgreSQL user names they can use, which
-# databases they can access. Records take one of these forms:
-#
-# local DATABASE USER METHOD [OPTIONS]
-# host DATABASE USER CIDR-ADDRESS METHOD [OPTIONS]
-# hostssl DATABASE USER CIDR-ADDRESS METHOD [OPTIONS]
-# hostnossl DATABASE USER CIDR-ADDRESS METHOD [OPTIONS]
-#
-# (The uppercase items must be replaced by actual values.)
-#
-# The first field is the connection type: "local" is a Unix-domain socket,
-# "host" is either a plain or SSL-encrypted TCP/IP socket, "hostssl" is an
-# SSL-encrypted TCP/IP socket, and "hostnossl" is a plain TCP/IP socket.
-#
-# DATABASE can be "all", "sameuser", "samerole", a database name, or
-# a comma-separated list thereof.
-#
-# USER can be "all", a user name, a group name prefixed with "+", or
-# a comma-separated list thereof. In both the DATABASE and USER fields
-# you can also write a file name prefixed with "@" to include names from
-# a separate file.
-#
-# CIDR-ADDRESS specifies the set of hosts the record matches.
-# It is made up of an IP address and a CIDR mask that is an integer
-# (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that specifies
-# the number of significant bits in the mask. Alternatively, you can write
-# an IP address and netmask in separate columns to specify the set of hosts.
-#
-# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi", "krb5",
-# "ident", "pam", "ldap" or "cert". Note that "password" sends passwords
-# in clear text; "md5" is preferred since it sends encrypted passwords.
-#
-# OPTIONS are a set of options for the authentication in the format
-# NAME=VALUE. The available options depend on the different authentication
-# methods - refer to the "Client Authentication" section in the documentation
-# for a list of which options are available for which authentication methods.
-#
-# Database and user names containing spaces, commas, quotes and other special
-# characters must be quoted. Quoting one of the keywords "all", "sameuser" or
-# "samerole" makes the name lose its special character, and just match a
-# database or username with that name.
-#
-# This file is read on server startup and when the postmaster receives
-# a SIGHUP signal. If you edit the file on a running system, you have
-# to SIGHUP the postmaster for the changes to take effect. You can use
-# "pg_ctl reload" to do that.
-
-# Put your actual configuration here
-# ----------------------------------
-#
-# If you want to allow non-local connections, you need to add more
-# "host" records. In that case you will also need to make PostgreSQL listen
-# on a non-local interface via the listen_addresses configuration parameter,
-# or via the -i or -h command line switches.
-#
-
-
-
-# TYPE DATABASE USER CIDR-ADDRESS METHOD
-
-# "local" is for Unix domain socket connections only
-local all all trust
-# IPv4 local connections:
-host all all 127.0.0.1/32 trust
-# IPv6 local connections:
-host all all ::1/128 trust
diff --git a/conf/vagrant/var/www/dataverse/error-documents/503.html b/conf/vagrant/var/www/dataverse/error-documents/503.html
deleted file mode 100644
index 95a7dea4107..00000000000
--- a/conf/vagrant/var/www/dataverse/error-documents/503.html
+++ /dev/null
@@ -1 +0,0 @@
-Custom "site is unavailable" 503 page.
diff --git a/doc/release-notes/10155-datasets-can-download-at-least-one-file.md b/doc/release-notes/10155-datasets-can-download-at-least-one-file.md
new file mode 100644
index 00000000000..a0b0d02310a
--- /dev/null
+++ b/doc/release-notes/10155-datasets-can-download-at-least-one-file.md
@@ -0,0 +1,3 @@
+The getCanDownloadAtLeastOneFile (/api/datasets/{id}/versions/{versionId}/canDownloadAtLeastOneFile) endpoint has been created.
+
+This API endpoint indicates if the calling user can download at least one file from a dataset version. Note that Shibboleth group permissions are not considered.
diff --git a/doc/release-notes/1249-collapse_dataverse_description.md b/doc/release-notes/1249-collapse_dataverse_description.md
deleted file mode 100644
index 8fe933005de..00000000000
--- a/doc/release-notes/1249-collapse_dataverse_description.md
+++ /dev/null
@@ -1 +0,0 @@
-Long descriptions for collections are now truncated but can be expanded to read the full description.
diff --git a/doc/release-notes/215-handles-upgrade-reminder.md b/doc/release-notes/215-handles-upgrade-reminder.md
deleted file mode 100644
index e1cbb8b0d8c..00000000000
--- a/doc/release-notes/215-handles-upgrade-reminder.md
+++ /dev/null
@@ -1,8 +0,0 @@
-### A reminder for the installations using Handles persistent identifiers (instead of DOIs): remember to upgrade your Handles service installation to a currently supported version.
-
-Generally, Handles is known to be working reliably even when running older versions that haven't been officially supported in years. We still recommend to check on your service and make sure to upgrade to a supported version (the latest version is 9.3.1, https://www.handle.net/hnr-source/handle-9.3.1-distribution.tar.gz, as of writing this). An older version may be running for you seemingly just fine, but do keep in mind that it may just stop working unexpectedly at any moment, because of some incompatibility introduced in a Java rpm upgrade, or anything similarly unpredictable.
-
-Handles is also very good about backward incompatibility. Meaning, in most cases you can simply stop the old version, unpack the new version from the distribution and start it on the existing config and database files, and it'll just keep working. However, it is a good idea to keep up with the recommended format upgrades, for the sake of efficiency and to avoid any unexpected surprises, should they finally decide to drop the old database format, for example. The two specific things we recommend: 1) Make sure your service is using a json version of the `siteinfo` bundle (i.e., if you are still using `siteinfo.bin`, convert it to `siteinfo.json` and remove the binary file from the service directory) and 2) Make sure you are using the newer bdbje database format for your handles catalog (i.e., if you still have the files `handles.jdb` and `nas.jdb` in your server directory, convert them to the new format). Follow the simple conversion instructions in the file README.txt in the Handles software distribution. Make sure to stop the service before converting the files and make sure to have a full backup of the existing server directory, just in case). Do not hesitate to contact the Handles support with any questions you may have, as they are very responsive and helpful.
-
-
-
diff --git a/doc/release-notes/3913-delete-file-endpoint b/doc/release-notes/3913-delete-file-endpoint
deleted file mode 100644
index b8b0c2f1ec5..00000000000
--- a/doc/release-notes/3913-delete-file-endpoint
+++ /dev/null
@@ -1 +0,0 @@
-Support for deleting files using native API: http://preview.guides.gdcc.io/en/develop/api/native-api.html#deleting-files
diff --git a/doc/release-notes/5.14-release-notes.md b/doc/release-notes/5.14-release-notes.md
new file mode 100644
index 00000000000..ef2a3b59659
--- /dev/null
+++ b/doc/release-notes/5.14-release-notes.md
@@ -0,0 +1,404 @@
+# Dataverse Software 5.14
+
+(If this note appears truncated on the GitHub Releases page, you can view it in full in the source tree: https://github.com/IQSS/dataverse/blob/master/doc/release-notes/5.14-release-notes.md)
+
+This release brings new features, enhancements, and bug fixes to the Dataverse software. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project.
+
+Please note that, as an experiment, the sections of this release note are organized in a different order. The Upgrade and Installation sections are at the top, with the detailed sections highlighting new features and fixes further down.
+
+## Installation
+
+If this is a new installation, please see our [Installation Guide](https://guides.dataverse.org/en/5.14/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/5.14/installation/intro.html#getting-help) if you need it!
+
+After your installation has gone into production, you are welcome to add it to our [map of installations](https://dataverse.org/installations) by opening an issue in the [dataverse-installations](https://github.com/IQSS/dataverse-installations) repo.
+
+## Upgrade Instructions
+
+0\. These instructions assume that you are upgrading from 5.13. If you are running an earlier version, the only safe way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to 5.14.
+
+If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user.
+
+In the following commands we assume that Payara 5 is installed in `/usr/local/payara5`. If not, adjust as needed.
+
+`export PAYARA=/usr/local/payara5`
+
+(or `setenv PAYARA /usr/local/payara5` if you are using a `csh`-like shell)
+
+1\. Undeploy the previous version.
+
+- `$PAYARA/bin/asadmin undeploy dataverse-5.13`
+
+2\. Stop Payara and remove the generated directory
+
+- `service payara stop`
+- `rm -rf $PAYARA/glassfish/domains/domain1/generated`
+
+3\. Start Payara
+
+- `service payara start`
+
+4\. Deploy this version.
+
+- `$PAYARA/bin/asadmin deploy dataverse-5.14.war`
+
+5\. Restart Payara
+
+- `service payara stop`
+- `service payara start`
+
+6\. Update the Citation metadata block: (the update makes the field Series repeatable)
+
+- `wget https://github.com/IQSS/dataverse/releases/download/v5.14/citation.tsv`
+- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"`
+
+If you are running an English-only installation, you are finished with the citation block. Otherwise, download the updated citation.properties file and place it in the [`dataverse.lang.directory`](https://guides.dataverse.org/en/5.14/installation/config.html#configuring-the-lang-directory); `/home/dataverse/langBundles` used in the example below.
+
+- `wget https://github.com/IQSS/dataverse/releases/download/v5.14/citation.properties`
+- `cp citation.properties /home/dataverse/langBundles`
+
+7\. Upate Solr schema.xml to allow multiple series to be used. See specific instructions below for those installations without custom metadata blocks (7a) and those with custom metadata blocks (7b).
+
+7a\. For installations without custom or experimental metadata blocks:
+
+- Stop Solr instance (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/5.14/installation/prerequisites.html#solr-init-script))
+
+- Replace schema.xml
+
+ - `cp /tmp/dvinstall/schema.xml /usr/local/solr/solr-8.11.1/server/solr/collection1/conf`
+
+- Start Solr instance (usually `service solr start`, depending on Solr/OS)
+
+7b\. For installations with custom or experimental metadata blocks:
+
+- Stop Solr instance (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/5.14/installation/prerequisites.html#solr-init-script))
+
+- There are 2 ways to regenerate the schema: Either by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed):
+```
+ wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/8.11.1/update-fields.sh
+ chmod +x update-fields.sh
+ curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-8.8.1/server/solr/collection1/conf/schema.xml
+```
+OR, alternatively, you can edit the following lines in your schema.xml by hand as follows (to indicate that series and its components are now `multiValued="true"`):
+```
+
+
+
+```
+
+- Restart Solr instance (usually `service solr restart` depending on solr/OS)
+
+8\. Run ReExportAll to update dataset metadata exports. Follow the directions in the [Admin Guide](http://guides.dataverse.org/en/5.14/admin/metadataexport.html#batch-exports-through-the-api).
+
+9\. If your installation did not have :FilePIDsEnabled set, you will need to set it to true to keep file PIDs enabled:
+
+ curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:FilePIDsEnabled
+
+10\. If your installation uses Handles as persistent identifiers (instead of DOIs): remember to upgrade your Handles service installation to a currently supported version.
+
+Generally, Handles is known to be working reliably even when running older versions that haven't been officially supported in years. We still recommend to check on your service and make sure to upgrade to a supported version (the latest version is 9.3.1, https://www.handle.net/hnr-source/handle-9.3.1-distribution.tar.gz, as of writing this). An older version may be running for you seemingly just fine, but do keep in mind that it may just stop working unexpectedly at any moment, because of some incompatibility introduced in a Java rpm upgrade, or anything similarly unpredictable.
+
+Handles is also very good about backward incompatibility. Meaning, in most cases you can simply stop the old version, unpack the new version from the distribution and start it on the existing config and database files, and it'll just keep working. However, it is a good idea to keep up with the recommended format upgrades, for the sake of efficiency and to avoid any unexpected surprises, should they finally decide to drop the old database format, for example. The two specific things we recommend: 1) Make sure your service is using a json version of the `siteinfo` bundle (i.e., if you are still using `siteinfo.bin`, convert it to `siteinfo.json` and remove the binary file from the service directory) and 2) Make sure you are using the newer bdbje database format for your handles catalog (i.e., if you still have the files `handles.jdb` and `nas.jdb` in your server directory, convert them to the new format). Follow the simple conversion instructions in the file README.txt in the Handles software distribution. Make sure to stop the service before converting the files and make sure to have a full backup of the existing server directory, just in case. Do not hesitate to contact the Handles support with any questions you may have, as they are very responsive and helpful.
+
+## New JVM Options and MicroProfile Config Options
+
+The following PID provider options are now available. See the section "Changes to PID Provider JVM Settings" below for more information.
+
+- `dataverse.pid.datacite.mds-api-url`
+- `dataverse.pid.datacite.rest-api-url`
+- `dataverse.pid.datacite.username`
+- `dataverse.pid.datacite.password`
+- `dataverse.pid.handlenet.key.path`
+- `dataverse.pid.handlenet.key.passphrase`
+- `dataverse.pid.handlenet.index`
+- `dataverse.pid.permalink.base-url`
+- `dataverse.pid.ezid.api-url`
+- `dataverse.pid.ezid.username`
+- `dataverse.pid.ezid.password`
+
+The following MicroProfile Config options have been added as part of [Signposting](https://signposting.org/) support. See the section "Signposting for Dataverse" below for details.
+
+- `dataverse.signposting.level1-author-limit`
+- `dataverse.signposting.level1-item-limit`
+
+The following JVM options are described in the "Creating datasets with incomplete metadata through API" section below.
+
+- `dataverse.api.allow-incomplete-metadata`
+- `dataverse.ui.show-validity-filter`
+- `dataverse.ui.allow-review-for-incomplete`
+
+The following JVM/MicroProfile setting is for External Exporters. See "Mechanism Added for Adding External Exporters" below.
+
+- `dataverse.spi.export.directory`
+
+The following JVM/MicroProfile settings are for handling of support emails. See "Contact Email Improvements" below.
+
+- `dataverse.mail.support-email`
+- `dataverse.mail.cc-support-on-contact-emails`
+
+The following JVM/MicroProfile setting is for extracting a geospatial bounding box even if S3 direct upload is enabled.
+
+- `dataverse.netcdf.geo-extract-s3-direct-upload`
+
+## Backward Incompatibilities
+
+The following list of potential backward incompatibilities references the sections of the "Detailed Release Highlights..." portion of the document further below where the corresponding changes are explained in detail.
+
+### Using the new External Exporters framework
+
+Care should be taken when replacing Dataverse's internal metadata export formats as third party code, including other third party Exporters, may depend on the contents of those export formats. When replacing an existing format, one must also remember to delete the cached metadata export files or run the reExport command for the metadata exports of existing datasets to be updated.
+
+See "Mechanism Added for Adding External Exporters".
+
+### Publishing via API
+
+When publishing a dataset via API, it now mirrors the UI behavior by requiring that the dataset has either a standard license configured, or has valid Custom Terms of Use (if allowed by the instance). Attempting to publish a dataset without such **will fail with an error message**.
+
+See "Handling of license information fixed in the API" for guidance on how to ensure that datasets created or updated via native API have a license configured.
+
+
+
+## Detailed Release Highlights, New Features and Use Case Scenarios
+
+### For Dataverse developers, support for running Dataverse in Docker (experimental)
+
+Developers can experiment with running Dataverse in Docker: (PR #9439)
+
+This is an image developers build locally (or can pull from Docker Hub). It is not meant for production use!
+
+To provide a complete container-based local development environment, developers can deploy a Dataverse container from
+the new image in addition to other containers for necessary dependencies:
+https://guides.dataverse.org/en/5.14/container/dev-usage.html
+
+Please note that with this emerging solution we will sunset older tooling like `docker-aio` and `docker-dcm`.
+We envision more testing possibilities in the future, to be discussed as part of the
+[Dataverse Containerization Working Group](https://ct.gdcc.io). There is no sunsetting roadmap yet, but you have been warned.
+If there is some specific feature of these tools you would like to be kept, please [reach out](https://ct.gdcc.io).
+
+### Indexing performance improved
+
+Noticeable improvements in performance, especially for large datasets containing thousands of files.
+Uploading files one by one to the dataset is much faster now, allowing uploading thousands of files in an acceptable timeframe. Not only uploading a file, but all edit operations on datasets containing many files, got faster.
+Performance tweaks include indexing of the datasets in the background and optimizations in the amount of the indexing operations needed. Furthermore, updates to the dateset no longer wait for ingesting to finish. Ingesting was already running in the background, but it took a lock, preventing updating the dataset and degrading performance for datasets containing many files. (PR #9558)
+
+### For installations using MDC (Make Data Count), it is now possible to display both the MDC metrics and the legacy access counts, generated before MDC was enabled.
+
+This is enabled via the new setting `:MDCStartDate` that specifies the cutoff date. If a dataset has any legacy access counts collected prior to that date, those numbers will be displayed in addition to any MDC numbers recorded since then. (PR #6543)
+
+### Changes to PID Provider JVM Settings
+
+In preparation for a future feature to use multiple PID providers at the same time, all JVM settings for PID providers
+have been enabled to be configured using MicroProfile Config. In the same go, they were renamed to match the name
+of the provider to be configured.
+
+Please watch your log files for deprecation warnings. Your old settings will be picked up, but you should migrate
+to the new names to avoid unnecessary log clutter and get prepared for more future changes. An example message
+looks like this:
+
+```
+[#|2023-03-31T16:55:27.992+0000|WARNING|Payara 5.2022.5|edu.harvard.iq.dataverse.settings.source.AliasConfigSource|_ThreadID=30;_ThreadName=RunLevelControllerThread-1680281704925;_TimeMillis=1680281727992;_LevelValue=900;|
+ Detected deprecated config option doi.username in use. Please update your config to use dataverse.pid.datacite.username.|#]
+```
+
+Here is a list of the new settings:
+
+- dataverse.pid.datacite.mds-api-url
+- dataverse.pid.datacite.rest-api-url
+- dataverse.pid.datacite.username
+- dataverse.pid.datacite.password
+- dataverse.pid.handlenet.key.path
+- dataverse.pid.handlenet.key.passphrase
+- dataverse.pid.handlenet.index
+- dataverse.pid.permalink.base-url
+- dataverse.pid.ezid.api-url
+- dataverse.pid.ezid.username
+- dataverse.pid.ezid.password
+
+See also https://guides.dataverse.org/en/5.14/installation/config.html#persistent-identifiers-and-publishing-datasets (multiple PRs: #8823 #8828)
+
+### Signposting for Dataverse
+
+This release adds [Signposting](https://signposting.org) support to Dataverse to improve machine discoverability of datasets and files. (PR #8424)
+
+The following MicroProfile Config options are now available (these can be treated as JVM options):
+
+- dataverse.signposting.level1-author-limit
+- dataverse.signposting.level1-item-limit
+
+Signposting is described in more detail in a new page in the Admin Guide on discoverability: https://guides.dataverse.org/en/5.14/admin/discoverability.html
+
+### Permalinks support
+
+Dataverse now optionally supports PermaLinks, a type of persistent identifier that does not involve a global registry service. PermaLinks are appropriate for Intranet deployment and catalog use cases. (PR #8674)
+
+
+### Creating datasets with incomplete metadata through API
+
+It is now possible to create a dataset with some nominally mandatory metadata fields left unpopulated. For details on the use case that lead to this feature see issue #8822 and PR #8940.
+
+The create dataset API call (POST to /api/dataverses/#dataverseId/datasets) is extended with the "doNotValidate" parameter. However, in order to be able to create a dataset with incomplete metadata, the Solr configuration must be updated first with the new "schema.xml" file (do not forget to run the metadata fields update script when you use custom metadata). Reindexing is optional, but recommended. Also, even when this feature is not used, it is recommended to update the Solr configuration and reindex the metadata. Finally, this new feature can be activated with the "dataverse.api.allow-incomplete-metadata" JVM option.
+
+You can also enable a valid/incomplete metadata filter in the "My Data" page using the "dataverse.ui.show-validity-filter" JVM option. By default, this filter is not shown. When you wish to use this filter, you must reindex the datasets first, otherwise datasets with valid metadata will not be shown in the results.
+
+It is not possible to publish datasets with incomplete or incomplete metadata. By default, you also cannot send such datasets for review. If you wish to enable sending for review of datasets with incomplete metadata, turn on the "dataverse.ui.allow-review-for-incomplete" JVM option.
+
+In order to customize the wording and add translations to the UI sections extended by this feature, you can edit the "Bundle.properties" file and the localized versions of that file. The property keys used by this feature are:
+- incomplete
+- valid
+- dataset.message.incomplete.warning
+- mydataFragment.validity
+- dataverses.api.create.dataset.error.mustIncludeAuthorName
+
+### Registering PIDs (DOIs or Handles) for files in select collections
+
+It is now possible to configure registering PIDs for files in individual collections.
+
+For example, registration of PIDs for files can be enabled in a specific collection when it is disabled instance-wide. Or it can be disabled in specific collections where it is enabled by default. See the [:FilePIDsEnabled](https://guides.dataverse.org/en/5.14/installation/config.html#filepidsenabled) section of the Configuration guide for details. (PR #9614)
+
+### Mechanism Added for Adding External Exporters
+
+It is now possible for third parties to develop and share code to provide new metadata export formats for Dataverse. Export formats can be made available via the Dataverse UI and API or configured for use in Harvesting. Dataverse now provides developers with a separate dataverse-spi JAR file that contains the Java interfaces and classes required to create a new metadata Exporter. Once a new Exporter has been created and packaged as a JAR file, administrators can use it by specifying a local directory for third party Exporters, dropping then Exporter JAR there, and restarting Payara. This mechanism also allows new Exporters to replace any of Dataverse's existing metadata export formats. (PR #9175). See also https://guides.dataverse.org/en/5.14/developers/metadataexport.html
+
+#### Backward Incompatibilities
+
+Care should be taken when replacing Dataverse's internal metadata export formats as third party code, including other third party Exporters may depend on the contents of those export formats. When replacing an existing format, one must also remember to delete the cached metadata export files or run the reExport command for the metadata exports of existing datasets to be updated.
+
+#### New JVM/MicroProfile Settings
+
+dataverse.spi.export.directory - specifies a directory, readable by the Dataverse server. Any Exporter JAR files placed in this directory will be read by Dataverse and used to add/replace the specified metadata format.
+
+### Contact Email Improvements
+
+Email sent from the contact forms to the contact(s) for a collection, dataset, or datafile can now optionally be cc'd to a support email address. The support email address can be changed from the default :SystemEmail address to a separate :SupportEmail address. When multiple contacts are listed, the system will now send one email to all contacts (with the optional cc if configured) instead of separate emails to each contact. Contact names with a comma that refer to Organizations will no longer have the name parts reversed in the email greeting. A new protected/admin feedback API has been added. (PR #9186) See https://guides.dataverse.org/en/5.14/api/native-api.html#send-feedback-to-contact-s
+
+#### New JVM/MicroProfile Settings
+
+dataverse.mail.support-email - allows a separate email, distinct from the :SystemEmail to be used as the to address in emails from the contact form/ feedback api.
+dataverse.mail.cc-support-on-contact-emails - include the support email address as a CC: entry when contact/feedback emails are sent to the contacts for a collection, dataset, or datafile.
+
+### Support for Grouping Dataset Files by Folder and Category Tag
+
+Dataverse now supports grouping dataset files by folder and/or optionally by Tag/Category. The default for whether to order by folder can be changed via :OrderByFolder. Ordering by category must be enabled by an administrator via the :CategoryOrder parameter which is used to specify which tags appear first (e.g. to put Documentation files before Data or Code files, etc.) These Group-By options work with the existing sort options, i.e. sorting alphabetically means that files within each folder or tag group will be sorted alphabetically. :AllowUsersToManageOrdering can be set to true to allow users to turn folder ordering and category ordering (if enabled) on or off in the current dataset view. (PR #9204)
+
+#### New Settings
+
+:CategoryOrder - a comma separated list of Category/Tag names defining the order in which files with those tags should be displayed. The setting can include custom tag names along with the pre-defined defaults ( Documentation, Data, and Code, which can be overridden by the ::FileCategories setting.)
+:OrderByFolder - defaults to true - whether to group files in the same folder together
+:AllowUserManagementOfOrder - default false - allow users to toggle ordering on/off in the dataset display
+
+### Metadata field Series now repeatable
+
+This enhancement allows depositors to define multiple instances of the metadata field Series in the Citation Metadata block.
+
+Data contained in a dataset may belong to multiple series. Making the field repeatable makes it possible to reflect this fact in the dataset metadata. (PR #9256)
+
+### Guides in PDF Format
+
+An experimental version of the guides in PDF format is available at (PR #9474)
+
+Advice for anyone who wants to help improve the PDF is available at https://guides.dataverse.org/en/5.14/developers/documentation.html#pdf-version-of-the-guides
+
+### Datasets API extended
+
+The following APIs have been added: (PR #9592)
+
+- `/api/datasets/summaryFieldNames`
+- `/api/datasets/privateUrlDatasetVersion/{privateUrlToken}`
+- `/api/datasets/privateUrlDatasetVersion/{privateUrlToken}/citation`
+- `/api/datasets/{datasetId}/versions/{version}/citation`
+
+### Extra fields included in the JSON metadata
+
+The following fields are now available in the native JSON output:
+
+- `alternativePersistentId`
+- `publicationDate`
+- `citationDate`
+
+(PR #9657)
+
+
+### Files downloaded from Binder are now in their original format.
+
+For example, data.dta (a Stata file) will be downloaded instead of data.tab (the archival version Dataverse creates as part of a successful ingest). (PR #9483)
+
+This should make it easier to write code to reproduce results as the dataset authors and subsequent researchers are likely operating on the original file format rather that the format that Dataverse creates.
+
+For details, see #9374, , and .
+
+### Handling of license information fixed in the API
+
+(PR #9568)
+
+When publishing a dataset via API, it now requires the dataset to either have a standard license configured, or have valid Custom Terms of Use (if allowed by the instance). Attempting to publish a dataset without such **will fail with an error message**. This introduces a backward incompatibility, and if you have scripts that automatically create, update and publish datasets, this last step may start failing. Because, unfortunately, there were some problems with the datasets APIs that made it difficult to manage licenses, so an API user was likely to end up with a dataset missing either of the above. In this release we have addressed it by making the following fixes:
+
+We fixed the incompatibility between the format in which license information was *exported* in json, and the format the create and update APIs were expecting it for *import* (https://github.com/IQSS/dataverse/issues/9155). This means that the following json format can now be imported:
+```
+"license": {
+ "name": "CC0 1.0",
+ "uri": "http://creativecommons.org/publicdomain/zero/1.0"
+}
+```
+However, for the sake of backward compatibility the old format
+```
+"license" : "CC0 1.0"
+```
+will be accepted as well.
+
+We have added the default license (CC0) to the model json file that we provide and recommend to use as the model in the Native API Guide (https://github.com/IQSS/dataverse/issues/9364).
+
+And we have corrected the misleading language in the same guide where we used to recommend to users that they select, edit and re-import only the `.metadataBlocks` fragment of the json metadata representing the latest version. There are in fact other useful pieces of information that need to be preserved in the update (such as the `"license"` section above). So the recommended way of creating base json for updates via the API is to select *everything but* the `"files"` section, with (for example) the following `jq` command:
+
+```
+jq '.data | del(.files)'
+```
+
+Please see the [Update Metadata For a Dataset](https://guides.dataverse.org/en/5.14/api/native-api.html#update-metadata-for-a-dataset) section of our Native Api guide for more information.
+
+
+### New External Tool Type and Implementation
+
+With this release a new experimental external tool type has been added to the Dataverse Software. The tool type is "query" and its first implementation is an experimental tool named [Ask the Data](https://github.com/IQSS/askdataverse) which allows users to ask natural language queries of tabular files in Dataverse. More information is available in the External Tools section of the guides. (PR #9737) See https://guides.dataverse.org/en/5.14/admin/external-tools.html#file-level-query-tools
+
+### Default Value for File PIDs registration has changed
+
+The default for whether PIDs are registered for files or not is now false.
+
+Installations where file PIDs were enabled by default will have to add the :FilePIDsEnabled = true setting to maintain the existing functionality.
+
+See Step 9 of the upgrade instructions:
+
+ If your installation did not have :FilePIDsEnabled set, you will need to set it to true to keep file PIDs enabled:
+
+ curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:FilePIDsEnabled
+
+
+It is now possible to allow File PIDs to be enabled/disabled per collection. See the [:AllowEnablingFilePIDsPerCollection](https://guides.dataverse.org/en/latest/installation/config.html#allowenablingfilepidspercollection) section of the Configuration guide for details.
+
+For example, registration of PIDs for files can now be enabled in a specific collection when it is disabled instance-wide. Or it can be disabled in specific collections where it is enabled by default.
+
+
+### Changes and fixes in this release not already mentioned above include:
+
+- An endpoint for deleting a file has been added to the native API: https://guides.dataverse.org/en/5.14/api/native-api.html#deleting-files (PR #9383)
+- A date column has been added to the restricted file access request overview, indicating when the earliest request by that user was made. An issue was fixed where where the request list was not updated when a request was approved or rejected. (PR #9257)
+- Changes made in v5.13 and v5.14 in multiple PRs to improve the embedded Schema.org metadata in dataset pages will only be propagated to the Schema.Org JSON-LD metadata export if a reExportAll() is done. (PR #9102)
+- It is now possible to write external vocabulary scripts that target a single child field in a metadata block. Example scripts are now available at https://github.com/gdcc/dataverse-external-vocab-support that can be configured to support lookup from the Research Orgnaization Registry (ROR) for the Author Affiliation Field and for the CrossRef Funding Registry (Fundreg) in the Funding Information/Agency field, both in the standard Citation metadata block. Application if these scripts to other fields, and the development of other scripts targetting child fields are now possible (PR #9402)
+- Dataverse now supports requiring a secret key to add or edit metadata in specified "system" metadata blocks. Changing the metadata in such system metadata blocks is not allowed without the key and is currently only allowed via API. (PR #9388)
+- An attempt will be made to extract a geospatial bounding box (west, south, east, north) from NetCDF and HDF5 files and then insert these values into the geospatial metadata block, if enabled. (#9541) See https://guides.dataverse.org/en/5.14/user/dataset-management.html#geospatial-bounding-box
+- A file previewer called H5Web is now available for exploring and visualizing NetCDF and HDF5 files. (PR #9600) See https://guides.dataverse.org/en/5.14/user/dataset-management.html#h5web-previewer
+- Two file previewers for GeoTIFF and Shapefiles are now available for visualizing geotiff image files and zipped Shapefiles on a map. See https://github.com/gdcc/dataverse-previewers
+- New alternative to setup the Dataverse dependencies for the development environment through Docker Compose. (PR #9417)
+- New alternative, explained in the documentation, to build the Sphinx guides through a Docker container. (PR #9417)
+- A container has been added called "configbaker" that configures Dataverse while running in containers. This allows developers to spin up Dataverse with a single command. (PR #9574)
+- Direct upload via the Dataverse UI will now support any algorithm configured via the `:FileFixityChecksumAlgorithm` setting. External apps using the direct upload API can now query Dataverse to discover which algorithm should be used. Sites that have been using an algorithm other than MD5 and direct upload and/or dvwebloader may want to use the `/api/admin/updateHashValues` call (see https://guides.dataverse.org/en/5.14/installation/config.html?highlight=updatehashvalues#filefixitychecksumalgorithm) to replace any MD5 hashes on existing files. (PR #9482)
+- The OAI_ORE metadata export (and hence the archival Bag for a dataset) now includes information about file embargoes. (PR #9698)
+- DatasetFieldType attribute "displayFormat", is now returned by the API. (PR #9668)
+- An API named "MyData" has been available for years but is newly documented. It is used to get a list of the objects (datasets, collections or datafiles) that an authenticated user can modify. (PR #9596)
+- A Go client library for Dataverse APIs is now available. See https://guides.dataverse.org/en/5.14/api/client-libraries.html
+- A feature flag called "api-session-auth" has been added temporarily to aid in the development of the new frontend (#9063) but will be removed once bearer tokens (#9229) have been implemented. There is a security risk (CSRF) in enabling this flag! Do not use it in production! For more information, see https://guides.dataverse.org/en/5.14/installation/config.html#feature-flags
+- A feature flag called "api-bearer-auth" has been added. This allows OIDC useraccounts to send authenticated API requests using Bearer Tokens. Note: This feature is limited to OIDC! For more information, see https://guides.dataverse.org/en/5.14/installation/config.html#feature-flags (PR #9591)
+
+
+## Complete List of Changes
+
+For the complete list of code changes in this release, see the [5.14 milestone](https://github.com/IQSS/dataverse/milestone/108?closed=1) on GitHub.
diff --git a/doc/release-notes/6.0-release-notes.md b/doc/release-notes/6.0-release-notes.md
new file mode 100644
index 00000000000..858cd604bda
--- /dev/null
+++ b/doc/release-notes/6.0-release-notes.md
@@ -0,0 +1,316 @@
+# Dataverse 6.0
+
+This is a platform upgrade release. Payara, Solr, and Java have been upgraded. No features have been added to the Dataverse software itself. Only a handful of bugs were fixed.
+
+Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project!
+
+## Release Highlights (Major Upgrades, Breaking Changes)
+
+This release contains major upgrades to core components. Detailed upgrade instructions can be found below.
+
+### Runtime
+
+- The required Java version has been increased from version 11 to 17.
+ - See PR #9764 for details.
+- Payara application server has been upgraded to version 6.2023.8.
+ - This is a required update.
+ - Please note that Payara Community 5 has reached [end of life](https://www.payara.fish/products/payara-platform-product-lifecycle/)
+ - See PR #9685 and PR #9795 for details.
+- Solr has been upgraded to version 9.3.0.
+ - See PR #9787 for details.
+- PostgreSQL 13 remains the tested and supported version.
+ - That said, the installer and Flyway have been upgraded to support PostgreSQL 14 and 15. See the [PostgreSQL](https://guides.dataverse.org/en/6.0/installation/prerequisites.html#postgresql) section of the Installation Guide and PR #9877 for details.
+
+### Development
+
+- Removal of Vagrant and Docker All In One (docker-aio), deprecated in Dataverse v5.14. See PR #9838 and PR #9685 for details.
+- All tests have been migrated to use JUnit 5 exclusively from now on. See PR #9796 for details.
+
+## Installation
+
+If this is a new installation, please follow our [Installation Guide](https://guides.dataverse.org/en/latest/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/latest/installation/intro.html#getting-help) if you need it!
+
+Once you are in production, we would be delighted to update our [map of Dataverse installations](https://dataverse.org/installations) around the world to include yours! Please [create an issue](https://github.com/IQSS/dataverse-installations/issues) or email us at support@dataverse.org to join the club!
+
+You are also very welcome to join the [Global Dataverse Community Consortium](https://dataversecommunity.global) (GDCC).
+
+## Upgrade Instructions
+
+Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc.
+
+These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 5.14.
+
+### Upgrade from Java 11 to Java 17
+
+Java 17 is now required for Dataverse. Solr can run under Java 11 or Java 17 but the latter is recommended. In preparation for the Java upgrade, stop both Dataverse/Payara and Solr.
+
+1. Undeploy Dataverse, if deployed, using the unprivileged service account.
+
+ `sudo -u dataverse /usr/local/payara5/bin/asadmin list-applications`
+
+ `sudo -u dataverse /usr/local/payara5/bin/asadmin undeploy dataverse-5.14`
+
+1. Stop Payara 5.
+
+ `sudo -u dataverse /usr/local/payara5/bin/asadmin stop-domain`
+
+1. Stop Solr 8.
+
+ `sudo systemctl stop solr.service`
+
+1. Install Java 17.
+
+ Assuming you are using RHEL or a derivative such as Rocky Linux:
+
+ `sudo yum install java-17-openjdk`
+
+1. Set Java 17 as the default.
+
+ Assuming you are using RHEL or a derivative such as Rocky Linux:
+
+ `sudo alternatives --config java`
+
+1. Test that Java 17 is the default.
+
+ `java -version`
+
+### Upgrade from Payara 5 to Payara 6
+
+If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user.
+
+1. Download Payara 6.2023.8.
+
+ `curl -L -O https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip`
+
+1. Unzip it to /usr/local (or your preferred location).
+
+ `sudo unzip payara-6.2023.8.zip -d /usr/local/`
+
+1. Change ownership of the unzipped Payara to your "service" user ("dataverse" by default).
+
+ `sudo chown -R dataverse /usr/local/payara6`
+
+1. Undeploy Dataverse, if deployed, using the unprivileged service account.
+
+ `sudo -u dataverse /usr/local/payara5/bin/asadmin list-applications`
+
+ `sudo -u dataverse /usr/local/payara5/bin/asadmin undeploy dataverse-5.14`
+
+1. Stop Payara 5, if running.
+
+ `sudo -u dataverse /usr/local/payara5/bin/asadmin stop-domain`
+
+1. Copy Dataverse-related lines from Payara 5 to Payara 6 domain.xml.
+
+ `sudo -u dataverse cp /usr/local/payara6/glassfish/domains/domain1/config/domain.xml /usr/local/payara6/glassfish/domains/domain1/config/domain.xml.orig`
+
+ `sudo egrep 'dataverse|doi' /usr/local/payara5/glassfish/domains/domain1/config/domain.xml > lines.txt`
+
+ `sudo vi /usr/local/payara6/glassfish/domains/domain1/config/domain.xml`
+
+ The lines will appear in two sections, examples shown below (but your content will vary).
+
+ Section 1: system properties (under ``)
+
+ ```
+
+
+
+
+
+ ```
+
+ Note: if you used the Dataverse installer, you won't have a `dataverse.db.password` property. See "Create password aliases" below.
+
+ Section 2: JVM options (under ``, the one under ``, not under ``)
+
+ ```
+ -Ddataverse.files.directory=/usr/local/dvn/data
+ -Ddataverse.files.file.type=file
+ -Ddataverse.files.file.label=file
+ -Ddataverse.files.file.directory=/usr/local/dvn/data
+ -Ddataverse.rserve.host=localhost
+ -Ddataverse.rserve.port=6311
+ -Ddataverse.rserve.user=rserve
+ -Ddataverse.rserve.password=rserve
+ -Ddataverse.auth.password-reset-timeout-in-minutes=60
+ -Ddataverse.timerServer=true
+ -Ddataverse.fqdn=dev1.dataverse.org
+ -Ddataverse.siteUrl=https://dev1.dataverse.org
+ -Ddataverse.files.storage-driver-id=file
+ -Ddoi.username=testaccount
+ -Ddoi.password=notmypassword
+ -Ddoi.baseurlstring=https://mds.test.datacite.org/
+ -Ddoi.dataciterestapiurlstring=https://api.test.datacite.org
+ ```
+
+1. Check the `Xmx` setting in `domain.xml`.
+
+ Under `/usr/local/payara6/glassfish/domains/domain1/config/domain.xml`, check the `Xmx` setting under ``, where you put the JVM options, not the one under ``. Note that there are two such settings, and you want to adjust the one in the stanza with Dataverse options. This sets the JVM heap size; a good rule of thumb is half of your system's total RAM. You may specify the value in MB (`8192m`) or GB (`8g`).
+
+1. Copy `jhove.conf` and `jhoveConfig.xsd` from Payara 5, edit and change `payara5` to `payara6`.
+
+ `sudo cp /usr/local/payara5/glassfish/domains/domain1/config/jhove* /usr/local/payara6/glassfish/domains/domain1/config/`
+
+ `sudo chown dataverse /usr/local/payara6/glassfish/domains/domain1/config/jhove*`
+
+ `sudo -u dataverse vi /usr/local/payara6/glassfish/domains/domain1/config/jhove.conf`
+
+1. Copy logos from Payara 5 to Payara 6.
+
+ These logos are for collections (dataverses).
+
+ `sudo -u dataverse cp -r /usr/local/payara5/glassfish/domains/domain1/docroot/logos /usr/local/payara6/glassfish/domains/domain1/docroot`
+
+1. If you are using Make Data Count (MDC), edit :MDCLogPath.
+
+ Your `:MDCLogPath` database setting might be pointing to a Payara 5 directory such as `/usr/local/payara5/glassfish/domains/domain1/logs`. If so, edit this to be Payara 6. You'll probably want to copy your logs over as well.
+
+1. If you've enabled access logging or any other site-specific configuration, be sure to preserve them. For instance, the default domain.xml includes
+
+ ```
+
+
+ ```
+
+ but you may wish to include
+
+ ```
+
+
+ ```
+
+ Be sure to keep a previous copy of your domain.xml for reference.
+
+1. Update systemd unit file (or other init system) from `/usr/local/payara5` to `/usr/local/payara6`, if applicable.
+
+1. Start Payara.
+
+ `sudo -u dataverse /usr/local/payara6/bin/asadmin start-domain`
+
+1. Create a Java mail resource, replacing "localhost" for mailhost with your mail relay server, and replacing "localhost" for fromaddress with the FQDN of your Dataverse server.
+
+ `sudo -u dataverse /usr/local/payara6/bin/asadmin create-javamail-resource --mailhost "localhost" --mailuser "dataversenotify" --fromaddress "do-not-reply@localhost" mail/notifyMailSession`
+
+1. Create password aliases for your database, rserve and datacite jvm-options, if you're using them.
+
+ `echo "AS_ADMIN_ALIASPASSWORD=yourDBpassword" > /tmp/dataverse.db.password.txt`
+
+ `sudo -u dataverse /usr/local/payara6/bin/asadmin create-password-alias --passwordfile /tmp/dataverse.db.password.txt`
+
+ When you are prompted "Enter the value for the aliasname operand", enter `dataverse.db.password`
+
+ You should see "Command create-password-alias executed successfully."
+
+ You'll want to perform similar commands for `rserve_password_alias` and `doi_password_alias` if you're using Rserve and/or DataCite.
+
+1. Enable workaround for FISH-7722.
+
+ The following workaround is for https://github.com/payara/Payara/issues/6337
+
+ `sudo -u dataverse /usr/local/payara6/bin/asadmin create-jvm-options --add-opens=java.base/java.io=ALL-UNNAMED`
+
+1. Create the network listener on port 8009.
+
+ `sudo -u dataverse /usr/local/payara6/bin/asadmin create-network-listener --protocol http-listener-1 --listenerport 8009 --jkenabled true jk-connector`
+
+1. Deploy the Dataverse 6.0 war file.
+
+ `sudo -u dataverse /usr/local/payara6/bin/asadmin deploy /path/to/dataverse-6.0.war`
+
+1. Check that you get a version number from Dataverse.
+
+ This is just a sanity check that Dataverse has been deployed properly.
+
+ `curl http://localhost:8080/api/info/version`
+
+1. Perform one final Payara restart to ensure that timers are initialized properly.
+
+ `sudo -u dataverse /usr/local/payara6/bin/asadmin stop-domain`
+
+ `sudo -u dataverse /usr/local/payara6/bin/asadmin start-domain`
+
+### Upgrade from Solr 8 to 9
+
+Solr has been upgraded to Solr 9. You must install Solr fresh and reindex. You cannot use your old `schema.xml` because the format has changed.
+
+The instructions below are copied from https://guides.dataverse.org/en/6.0/installation/prerequisites.html#installing-solr and tweaked a bit for an upgrade scenario.
+
+We assume that you already have a user called "solr" (from the instructions above), added during your initial installation of Solr. We also assume that you have already stopped Solr 8 as explained in the instructions above about upgrading Java.
+
+1. Become the "solr" user and then download and configure Solr.
+
+ `su - solr`
+
+ `cd /usr/local/solr`
+
+ `wget https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz`
+
+ `tar xvzf solr-9.3.0.tgz`
+
+ `cd solr-9.3.0`
+
+ `cp -r server/solr/configsets/_default server/solr/collection1`
+
+1. Unzip "dvinstall.zip" from this release. Unzip it into /tmp. Then copy the following files into place.
+
+ `cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf`
+
+ `cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf`
+
+1. A Dataverse installation requires a change to the jetty.xml file that ships with Solr.
+
+ Edit `/usr/local/solr/solr-9.3.0/server/etc/jetty.xml`, increasing `requestHeaderSize` from `8192` to `102400`
+
+1. Tell Solr to create the core "collection1" on startup.
+
+ `echo "name=collection1" > /usr/local/solr/solr-9.3.0/server/solr/collection1/core.properties`
+
+1. Update your init script.
+
+ Your init script may be located at `/etc/systemd/system/solr.service`, for example. Update the path to Solr to be `/usr/local/solr/solr-9.3.0`.
+
+1. Start Solr using your init script and check collection1.
+
+ The collection1 check below should print out fields Dataverse uses like "dsDescription".
+
+ `systemctl start solr.service`
+
+ `curl http://localhost:8983/solr/collection1/schema/fields`
+
+1. Reindex Solr.
+
+ For details, see https://guides.dataverse.org/en/6.0/admin/solr-search-index.html but here is the reindex command:
+
+ `curl http://localhost:8080/api/admin/index`
+
+1. If you have custom metadata blocks installed, you must update your Solr `schema.xml` to include your custom fields.
+
+ For details, please see https://guides.dataverse.org/en/6.0/admin/metadatacustomization.html#updating-the-solr-schema
+
+ At a high level you will be copying custom fields from the output of http://localhost:8080/api/admin/index/solr/schema or using a script to automate this.
+
+## Potential Archiver Incompatibilities with Payara 6
+
+The [Google Cloud and DuraCloud archivers](https://guides.dataverse.org/en/5.14/installation/config.html#bagit-export) may not work in Dataverse 6.0.
+
+This is due to the archivers' dependence on libraries that include classes in `javax.* packages` that are no longer available. If these classes are actually used when the archivers run, the archivers would fail. As these two archivers require additional setup, they have not been tested in 6.0. Community members using these archivers or considering their use are encouraged to test them with 6.0 and report any errors and/or provide fixes for them that can be included in future releases.
+
+## Bug Fix for Dataset Templates with Custom Terms of Use
+
+A bug was fixed for the following scenario:
+
+- Create a template with custom terms.
+- Set that template as the default.
+- Try to create a dataset.
+- A 500 error appears before the form to create dataset is even shown.
+
+For more details, see issue #9825 and PR #9892
+
+## Complete List of Changes
+
+For the complete list of code changes in this release, see the [6.0 Milestone](https://github.com/IQSS/dataverse/milestone/109?closed=1) in GitHub.
+
+## Getting Help
+
+For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org.
diff --git a/doc/release-notes/6.1-release-notes.md b/doc/release-notes/6.1-release-notes.md
new file mode 100644
index 00000000000..1279d09a023
--- /dev/null
+++ b/doc/release-notes/6.1-release-notes.md
@@ -0,0 +1,322 @@
+# Dataverse 6.1
+
+Please note: To read these instructions in full, please go to https://github.com/IQSS/dataverse/releases/tag/v6.1 rather than the list of releases, which will cut them off.
+
+This release brings new features, enhancements, and bug fixes to the Dataverse software.
+Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project.
+
+## Release highlights
+
+### Guestbook at request
+
+Dataverse can now be configured (via the `dataverse.files.guestbook-at-request` option) to display any configured guestbook to users when they request restricted files (new functionality) or when they download files (previous behavior).
+
+The global default defined by this setting can be overridden at the collection level on the collection page and at the individual dataset level by a superuser using the API. The default, showing guestbooks when files are downloaded, remains as it was in prior Dataverse versions.
+
+For details, see [dataverse.files.guestbook-at-request](https://guides.dataverse.org/en/6.1/installation/config.html#dataverse-files-guestbook-at-request) and PR #9599.
+
+### Collection-level storage quotas
+
+This release adds support for defining storage size quotas for collections. Please see the API guide for details. This is an experimental feature that has not yet been used in production on any real life Dataverse instance, but we are planning to try it out at Harvard/IQSS.
+
+Please note that this release includes a database update (via a Flyway script) that will calculate the storage sizes of all the existing datasets and collections on the first deployment. On a large production database with tens of thousands of datasets this may add a couple of extra minutes to the first, initial deployment of Dataverse 6.1.
+
+For details, see [Storage Quotas for Collections](https://guides.dataverse.org/en/6.1/admin/collectionquotas.html) in the Admin Guide.
+
+### Globus support (experimental), continued
+
+Globus support in Dataverse has been expanded to include support for using file-based Globus endpoints, including the case where files are stored on tape and are not immediately accessible and for the case of referencing files stored on remote Globus endpoints. Support for using the Globus S3 Connector with an S3 store has been retained but requires changes to the Dataverse configuration. Please note:
+
+- Globus functionality remains experimental/advanced in that it requires significant setup, differs in multiple ways from other file storage mechanisms, and may continue to evolve with the potential for backward incompatibilities.
+- The functionality is configured per store and replaces the previous single-S3-Connector-per-Dataverse-instance model.
+- Adding files to a dataset, and accessing files is supported via the Dataverse user interface through a separate [dataverse-globus app](https://github.com/scholarsportal/dataverse-globus).
+- The functionality is also accessible via APIs (combining calls to the Dataverse and Globus APIs)
+
+Backward incompatibilities:
+- The configuration for use of a Globus S3 Connector has changed and is aligned with the standard store configuration mechanism
+- The new functionality is incompatible with older versions of the globus-dataverse app and the Globus-related functionality in the UI will only function correctly if a Dataverse 6.1 compatible version of the dataverse-globus app is configured.
+
+New JVM options:
+- A new "globus" store type and associated store-related options have been added. These are described in the [File Storage](https://guides.dataverse.org/en/6.1/installation/config.html#file-storage) section of the Installation Guide.
+- dataverse.files.globus-cache-maxage - specifies the number of minutes Dataverse will wait between an initial request for a file transfer occurs and when that transfer must begin.
+
+Obsolete Settings: the :GlobusBasicToken, :GlobusEndpoint, and :GlobusStores settings are no longer used
+
+Further details can be found in the [Big Data Support](https://guides.dataverse.org/en/6.1/developers/big-data-support.html#big-data-support) section of the Developer Guide.
+
+### Alternative Title now allows multiple values
+
+Alternative Title now allows multiples. Note that JSON used to create a dataset with an Alternate Title must be changed. See "Backward incompatibilities" below and PR #9440 for details.
+
+### External tools: configure tools now available at the dataset level
+
+Read/write "configure" tools (a type of external tool) are now available at the dataset level. They appear under the "Edit Dataset" menu. See [External Tools](https://guides.dataverse.org/en/6.1/admin/external-tools.html#dataset-level-configure-tools) in the Admin Guide and PR #9925.
+
+### S3 out-of-band upload
+
+In some situations, direct upload might not work from the UI, e.g., when s3 storage is not accessible from the internet. This pull request adds an option to [allow direct uploads via API only](https://github.com/IQSS/dataverse/pull/9003). This way, a third party application can use direct upload from within the internal network, while there is no direct download available to the users via UI.
+By default, Dataverse supports uploading files via the [add a file to a dataset](https://guides.dataverse.org/en/6.1/api/native-api.html#add-a-file-to-a-dataset) API. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
+With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the [Adding the Uploaded file to the Dataset](https://guides.dataverse.org/en/6.1/developers/s3-direct-upload-api.html#adding-the-uploaded-file-to-the-dataset) API call (described in the [Direct DataFile Upload/Replace API](https://guides.dataverse.org/en/6.1/developers/s3-direct-upload-api.html) page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
+
+### JSON Schema for datasets
+
+Functionality has been added to help validate dataset JSON prior to dataset creation. There are two new API endpoints in this release. The first takes in a collection alias and returns a custom dataset schema based on the required fields of the collection. The second takes in a collection alias and a dataset JSON file and does an automated validation of the JSON file against the custom schema for the collection. In this release functionality is limited to JSON format validation and validating required elements. Future releases will address field types, controlled vocabulary, etc. See [Retrieve a Dataset JSON Schema for a Collection](https://guides.dataverse.org/en/6.1/api/native-api.html#retrieve-a-dataset-json-schema-for-a-collection) in the API Guide and PR #10109.
+
+### OpenID Connect (OIDC) improvements
+
+#### Using MicroProfile Config for provisioning
+
+With this release it is possible to provision a single OIDC-based authentication provider by using MicroProfile Config instead of or in addition to the classic Admin API provisioning.
+
+If you are using an external OIDC provider component as an identity management system and/or broker to other authentication providers such as Google, eduGain SAML and so on, this might make your life easier during instance setups and reconfiguration. You no longer need to generate the necessary JSON file.
+
+#### Adding PKCE Support
+
+Some OIDC providers require using PKCE as additional security layer. As of this version, you can enable support for this on any OIDC provider you configure. (Note that OAuth2 providers have not been upgraded.)
+
+For both features, see the [OIDC](https://guides.dataverse.org/en/6.0/installation/oidc.html) section of the Installation Guide and PR #9273.
+
+### Solr improvements
+
+As of this release, application-side support has been added for the "circuit breaker" mechanism in Solr that makes it drop requests more gracefully when the search engine is experiencing load issues.
+
+Please see the [Installing Solr](https://guides.dataverse.org/en/6.1/installation/prerequisites.html#installing-solr) section of the Installation Guide.
+
+### New release of Dataverse Previewers (including a Markdown previewer)
+
+Version 1.4 of the standard Dataverse Previewers from https://github/com/gdcc/dataverse-previewers is available. The new version supports the use of signedUrls rather than API keys when previewing restricted files (including files in draft dataset versions). Upgrading is highly recommended. Please note:
+
+- SignedUrls can now be used with PrivateUrl access tokens, which allows PrivateUrl users to view previewers that are configured to use SignedUrls. See #10093.
+- Launching a dataset-level configuration tool will automatically generate an API token when needed. This is consistent with how other types of tools work. See #10045.
+- There is now a [Markdown (.md)](https://guides.dataverse.org/en/6.1/user/dataset-management.html#file-previews) previewer.
+
+### New or improved APIs
+
+The development of a [new UI for Dataverse](https://github.com/IQSS/dataverse-frontend) is driving the addition or improvement of many APIs.
+
+#### New API endpoints
+
+- deaccessionDataset (/api/datasets/{id}/versions/{versionId}/deaccession): version deaccessioning through API (Given a dataset and a version).
+- /api/files/{id}/downloadCount
+- /api/files/{id}/dataTables
+- /api/files/{id}/metadata/tabularTags New endpoint to set tabular file tags.
+- canManageFilePermissions (/access/datafile/{id}/userPermissions) Added for getting user permissions on a file.
+- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts): Given a dataset and its version, retrieves file counts based on different criteria (Total count, per content type, per access status and per category name).
+- setFileCategories (/api/files/{id}/metadata/categories): Updates the categories (by name) for an existing file. If the specified categories do not exist, they will be created.
+- userFileAccessRequested (/api/access/datafile/{id}/userFileAccessRequested): Returns true or false depending on whether or not the calling user has requested access to a particular file.
+- hasBeenDeleted (/api/files/{id}/hasBeenDeleted): Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
+- getZipDownloadLimit (/api/info/zipDownloadLimit): Get the configured zip file download limit. The response contains the long value of the limit in bytes.
+- getMaxEmbargoDurationInMonths (/api/info/settings/:MaxEmbargoDurationInMonths): Get the maximum embargo duration in months, if available, configured through the database setting :MaxEmbargoDurationInMonths.
+- getDatasetJsonSchema (/api/dataverses/{id}/datasetSchema): Get a dataset schema with the fields required by a given dataverse collection.
+- validateDatasetJsonSchema (/api/dataverses/{id}/validateDatasetJson): Validate that a dataset JSON file is in proper format and contains the required elements and fields for a given dataverse collection.
+- downloadTmpFile (/api/admin/downloadTmpFile): For testing purposes, allows files to be downloaded from /tmp.
+
+#### Pagination of files in dataset versions
+
+- optional pagination has been added to `/api/datasets/{id}/versions` that may be useful in datasets with a large number of versions
+- a new flag `includeFiles` is added to both `/api/datasets/{id}/versions` and `/api/datasets/{id}/versions/{vid}` (true by default), providing an option to drop the file information from the output
+- when files are requested to be included, some database lookup optimizations have been added to improve the performance on datasets with large numbers of files.
+
+This is reflected in the [Dataset Versions API](https://guides.dataverse.org/en/6.1/api/native-api.html#dataset-versions-api) section of the Guide.
+
+
+#### DataFile API payload has been extended to include the following fields
+
+- tabularData: Boolean field to know if the DataFile is of tabular type
+- fileAccessRequest: Boolean field to know if the file access requests are enabled on the Dataset (DataFile owner)
+- friendlyType: String
+
+#### The getVersionFiles endpoint (/api/datasets/{id}/versions/{versionId}/files) has been extended to support pagination, ordering, and optional filtering
+
+- Access status: through the `accessStatus` query parameter, which supports the following values:
+ - Public
+ - Restricted
+ - EmbargoedThenRestricted
+ - EmbargoedThenPublic
+- Category name: through the `categoryName` query parameter. To return files to which the particular category has been added.
+- Content type: through the `contentType` query parameter. To return files matching the requested content type. For example: "image/png".
+
+#### Additional improvements to existing API endpoints
+
+- getVersionFiles (/api/datasets/{id}/versions/{versionId}/files): Extended to support optional filtering by search text through the `searchText` query parameter. The search will be applied to the labels and descriptions of the dataset files. Added `tabularTagName` to return files to which the particular tabular tag has been added. Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain files.
+- getVersionFileCounts (/api/datasets/{id}/versions/{versionId}/files/counts): Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain file counts. Added support for filtering by optional criteria query parameter:
+ - contentType
+ - accessStatus
+ - categoryName
+ - tabularTagName
+ - searchText
+- getDownloadSize ("api/datasets/{identifier}/versions/{versionId}/downloadsize"): Added optional boolean query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain files. Added a new optional query parameter "mode"
+This parameter applies a filter criteria to the operation and supports the following values:
+ - All (Default): Includes both archival and original sizes for tabular files
+ - Archival: Includes only the archival size for tabular files
+ - Original: Includes only the original size for tabular files.
+- /api/datasets/{id}/versions/{versionId} New query parameter `includeDeaccessioned` added to consider deaccessioned versions when searching for versions.
+- /api/datasets/{id}/userPermissions Get user permissions on a dataset, in particular, the user permissions that this API call checks, returned as booleans, are the following:
+ - Can view the unpublished dataset
+ - Can edit the dataset
+ - Can publish the dataset
+ - Can manage the dataset permissions
+ - Can delete the dataset draft
+- getDatasetVersionCitation (/api/datasets/{id}/versions/{versionId}/citation) endpoint now accepts a new boolean optional query parameter "includeDeaccessioned", which, if enabled, causes the endpoint to consider deaccessioned versions when searching for versions to obtain the citation.
+
+### Improvements for developers
+
+- Developers can enjoy a dramatically faster feedback loop when iterating on code if they are using Netbeans or IntelliJ IDEA Ultimate (with the Payara Platform Tools plugin). For details, see https://guides.dataverse.org/en/6.1/container/dev-usage.html#intellij-idea-ultimate-and-payara-platform-tools and [the thread](https://groups.google.com/g/dataverse-community/c/zNBDzSMF2Q0/m/Z-xS6fA2BgAJ) on the mailing list.
+- Developers can now test S3 locally by using the Dockerized development environment, which now includes both LocalStack and MinIO. API (end to end) tests are in S3AccessIT.
+- In addition, a new integration test class (not an API test, the new Testcontainers-based test launched with `mvn verify`) has been added at S3AccessIOLocalstackIT. It uses Testcontainers to spin up Localstack for S3 testing and does not require Dataverse to be running.
+- With this release, we add a new type of testing to Dataverse: integration tests which are not end-to-end tests (like our API tests). Starting with OIDC authentication support, we test regularly on CI for working condition of both OIDC login options in UI and API.
+- The testing and development Keycloak realm has been updated with more users and compatibility with Keycloak 21.
+- The support for setting JVM options during testing has been improved for developers. You now may add the `@JvmSetting` annotation to classes (also inner classes) and reference factory methods for values. This improvement is also paving the way to enable manipulating JVM options during end-to-end tests on remote ends.
+- As part of these testing improvements, the code coverage report file for unit tests has moved from `target/jacoco.exec` to `target/coverage-reports/jacoco-unit.exec`.
+
+## Major use cases and infrastructure enhancements
+
+Changes and fixes in this release not already mentioned above include:
+
+- Validation has been added for the Geographic Bounding Box values in the Geospatial metadata block. This will prevent improperly defined bounding boxes from being created via the edit page or metadata imports. This also fixes the issue where existing datasets with invalid geoboxes were quietly failing to get reindexed. See PR #10142.
+- Dataverse's OAI_ORE Metadata Export format and archival BagIT exports
+(which include the OAI-ORE metadata export file) have been updated to include
+information about the dataset version state, e.g. RELEASED or DEACCESSIONED
+and to indicate which version of Dataverse was used to create the archival Bag.
+As part of the latter, the current OAI_ORE Metadata format has been given a 1.0.0
+version designation and it is expected that any future changes to the OAI_ORE export
+format will result in a version change and that tools such as DVUploader that can
+recreate datasets from archival Bags will start indicating which version(s) of the
+OAI_ORE format they can read.
+Dataverse installations that have been using archival Bags may wish to update any
+existing archival Bags they have, e.g. by deleting existing Bags and using the Dataverse
+[archival Bag export API](https://guides.dataverse.org/en/6.1/installation/config.html#bagit-export-api-calls)
+to generate updated versions.
+- For BagIT export, it is now possible to configure the following information in bag-info.txt. (Previously, customization was possible by editing `Bundle.properties` but this is no longer supported.) For details, see https://guides.dataverse.org/en/6.1/installation/config.html#bag-info-txt
+ - Source-Organization from `dataverse.bagit.sourceorg.name`.
+ - Organization-Address from `dataverse.bagit.sourceorg.address`.
+ - Organization-Email from `dataverse.bagit.sourceorg.address`.
+- This release fixes several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification. These changes introduce backward-incompatibility, but since Signposting support was added recently (in Dataverse 5.14 in PR #8981), we feel it's best to do this clean up and not support the old implementation that was not fully compliant with the spec.
+ - To fix #9952, we surround the license info with `<` and `>`.
+ - To fix #9953, we no longer wrap the response in a `{"status":"OK","data":{` JSON object. This has also been noted in the guides at https://dataverse-guide--9955.org.readthedocs.build/en/9955/api/native-api.html#retrieve-signposting-information
+ - To fix #9957, we corrected the mime/content type, changing it from `json+ld` to `ld+json`. For backward compatibility, we are still supporting the old one, for now.
+- It's now possible to configure the docroot, which holds collection logos and more. See [dataverse.files.docroot](https://guides.dataverse.org/en/6.1/installation/config.html#dataverse-files-docroot) in the Installation Guide and PR #9819.
+- We have started maintaining an API changelog of breaking changes: https://guides.dataverse.org/en/6.1/api/changelog.html
+See also #10060.
+
+## New configuration options
+
+- dataverse.auth.oidc.auth-server-url
+- dataverse.auth.oidc.client-id
+- dataverse.auth.oidc.client-secret
+- dataverse.auth.oidc.enabled
+- dataverse.auth.oidc.pkce.enabled
+- dataverse.auth.oidc.pkce.max-cache-age
+- dataverse.auth.oidc.pkce.max-cache-size
+- dataverse.auth.oidc.pkce.method
+- dataverse.auth.oidc.subtitle
+- dataverse.auth.oidc.title
+- dataverse.bagit.sourceorg.address
+- dataverse.bagit.sourceorg.address
+- dataverse.bagit.sourceorg.name
+- dataverse.files.docroot
+- dataverse.files.globus-cache-maxage
+- dataverse.files.guestbook-at-request
+- dataverse.files.{driverId}.upload-out-of-band
+
+## Backward incompatibilities
+
+- Since Alternative Title is now repeatable, the JSON you send to create or edit a dataset must be an array rather than a simple string. For example, instead of "value": "Alternative Title", you must send "value": ["Alternative Title1", "Alternative Title2"]
+- Several issues (#9952, #9953, #9957) where the Signposting output did not match the Signposting specification introduce backward-incompatibility. See above for details.
+- For BagIT export, if you were configuring values in bag-info.txt using `Bundle.properties`, you must switch to the new `dataverse.bagit` JVM options mentioned above. For details, see https://guides.dataverse.org/en/6.1/installation/config.html#bag-info-txt
+- See "Globus support" above for backward incompatibilies specific to Globus.
+
+## Complete list of changes
+
+For the complete list of code changes in this release, see the [6.1 Milestone](https://github.com/IQSS/dataverse/milestone/110?closed=1) in GitHub.
+
+## Getting help
+
+For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/forum/#!forum/dataverse-community) or email support@dataverse.org.
+
+## Installation
+
+If this is a new installation, please follow our [Installation Guide](https://guides.dataverse.org/en/latest/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/latest/installation/intro.html#getting-help) if you need it!
+
+Once you are in production, we would be delighted to update our [map of Dataverse installations](https://dataverse.org/installations) around the world to include yours! Please [create an issue](https://github.com/IQSS/dataverse-installations/issues) or email us at support@dataverse.org to join the club!
+
+You are also very welcome to join the [Global Dataverse Community Consortium](https://www.gdcc.io/) (GDCC).
+
+## Upgrade instructions
+Upgrading requires a maintenance window and downtime. Please plan ahead, create backups of your database, etc.
+
+These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.0.
+
+0\. These instructions assume that you are upgrading from 6.0. If you are running an earlier version, the only safe way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to 5.14.
+
+If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user.
+
+In the following commands we assume that Payara 6 is installed in `/usr/local/payara6`. If not, adjust as needed.
+
+`export PAYARA=/usr/local/payara6`
+
+(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell)
+
+1\. Undeploy the previous version.
+
+- `$PAYARA/bin/asadmin undeploy dataverse-6.0`
+
+2\. Stop Payara and remove the generated directory
+
+- `service payara stop`
+- `rm -rf $PAYARA/glassfish/domains/domain1/generated`
+
+3\. Start Payara
+
+- `service payara start`
+
+4\. Deploy this version.
+
+- `$PAYARA/bin/asadmin deploy dataverse-6.1.war`
+
+As noted above, deployment of the war file might take several minutes due a database migration script required for the new storage quotas feature.
+
+5\. Restart Payara
+
+- `service payara stop`
+- `service payara start`
+
+6\. Update Geospatial Metadata Block (to improve validation of bounding box values)
+
+- `wget https://github.com/IQSS/dataverse/releases/download/v6.1/geospatial.tsv`
+- `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file @geospatial.tsv`
+
+6a\. Update Citation Metadata Block (to make Alternative Title repeatable)
+
+- `curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file scripts/api/data/metadatablocks/citation.tsv`
+
+7\. Upate Solr schema.xml to allow multiple Alternative Titles to be used. See specific instructions below for those installations without custom metadata blocks (7a) and those with custom metadata blocks (7b).
+
+7a\. For installations without custom or experimental metadata blocks:
+
+- Stop Solr instance (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/5.14/installation/prerequisites.html#solr-init-script))
+
+- Replace schema.xml
+
+ - `cp /tmp/dvinstall/schema.xml /usr/local/solr/solr-9.3.0/server/solr/collection1/conf`
+
+- Start Solr instance (usually `service solr start`, depending on Solr/OS)
+
+7b\. For installations with custom or experimental metadata blocks:
+
+- Stop Solr instance (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/5.14/installation/prerequisites.html#solr-init-script))
+
+- There are 2 ways to regenerate the schema: Either by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed):
+```
+ wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.3.0/update-fields.sh
+ chmod +x update-fields.sh
+ curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.3.0/server/solr/collection1/conf/schema.xml
+```
+OR, alternatively, you can edit the following line in your schema.xml by hand as follows (to indicate that alternative title is now `multiValued="true"`):
+```
+
+```
+
+- Restart Solr instance (usually `service solr restart` depending on solr/OS)
+
+8\. Run ReExportAll to update dataset metadata exports. Follow the directions in the [Admin Guide](http://guides.dataverse.org/en/5.14/admin/metadataexport.html#batch-exports-through-the-api).
diff --git a/doc/release-notes/6542-mdc-legacy-counts.md b/doc/release-notes/6542-mdc-legacy-counts.md
deleted file mode 100644
index 1f439747a08..00000000000
--- a/doc/release-notes/6542-mdc-legacy-counts.md
+++ /dev/null
@@ -1,3 +0,0 @@
-###For installations using MDC (Make Data Count), it is now possible to display both the MDC metrics and the legacy access counts, generated before MDC was enabled.
-
-This is enabled via the new setting `:MDCStartDate` that specifies the cutoff date. If a dataset has any legacy access counts collected prior to that date, those numbers will be displayed in addition to the any MDC numbers recorded since then.
diff --git a/doc/release-notes/7000-pidproviders.md b/doc/release-notes/7000-pidproviders.md
deleted file mode 100644
index 04f7c25078d..00000000000
--- a/doc/release-notes/7000-pidproviders.md
+++ /dev/null
@@ -1,30 +0,0 @@
-# Changes to PID Provider JVM Settings
-
-In prepration for a future feature to use multiple PID providers at the same time, all JVM settings for PID providers
-have been enabled to be configured using MicroProfile Config. In the same go, they were renamed to match the name
-of the provider to be configured.
-
-Please watch your log files for deprecation warnings. Your old settings will be picked up, but you should migrate
-to the new names to avoid unnecessary log clutter and get prepared for more future changes. An example message
-looks like this:
-
-```
-[#|2023-03-31T16:55:27.992+0000|WARNING|Payara 5.2022.5|edu.harvard.iq.dataverse.settings.source.AliasConfigSource|_ThreadID=30;_ThreadName=RunLevelControllerThread-1680281704925;_TimeMillis=1680281727992;_LevelValue=900;|
- Detected deprecated config option doi.username in use. Please update your config to use dataverse.pid.datacite.username.|#]
-```
-
-Here is a list of the new settings:
-
-- dataverse.pid.datacite.mds-api-url
-- dataverse.pid.datacite.rest-api-url
-- dataverse.pid.datacite.username
-- dataverse.pid.datacite.password
-- dataverse.pid.handlenet.key.path
-- dataverse.pid.handlenet.key.passphrase
-- dataverse.pid.handlenet.index
-- dataverse.pid.permalink.base-url
-- dataverse.pid.ezid.api-url
-- dataverse.pid.ezid.username
-- dataverse.pid.ezid.password
-
-See also http://preview.guides.gdcc.io/en/develop/installation/config.html#persistent-identifiers-and-publishing-datasets
diff --git a/doc/release-notes/8092-timestamp-of-data-access-request.md b/doc/release-notes/8092-timestamp-of-data-access-request.md
deleted file mode 100644
index 506b78018a9..00000000000
--- a/doc/release-notes/8092-timestamp-of-data-access-request.md
+++ /dev/null
@@ -1,3 +0,0 @@
-A date column has been added to the restricted file access request overview, indicating when the earliest request by that user was made.
-
-An issue was fixed where where the request list was not updated when a request was approved or rejected.
diff --git a/doc/release-notes/8424-signposting.md b/doc/release-notes/8424-signposting.md
deleted file mode 100644
index 6994ba92cb7..00000000000
--- a/doc/release-notes/8424-signposting.md
+++ /dev/null
@@ -1,8 +0,0 @@
-# Signposting for Dataverse
-
-This release adds [Signposting](https://signposting.org/) support to Dataverse to improve machine discoverability of datasets and files.
-
-The following MicroProfile Config options are now available (these can be treated as JVM options):
-
-- dataverse.signposting.level1-author-limit
-- dataverse.signposting.level1-item-limit
diff --git a/doc/release-notes/8674-permalinks.md b/doc/release-notes/8674-permalinks.md
deleted file mode 100644
index 5d47b378c91..00000000000
--- a/doc/release-notes/8674-permalinks.md
+++ /dev/null
@@ -1 +0,0 @@
-Dataverse now optionally supports PermaLinks, a type of persistent identifier that does not involve a global registry service. PermaLinks are appropriate for Intranet deployment and catalog use cases.
diff --git a/doc/release-notes/8822-incomplete-datasets-via-api.md b/doc/release-notes/8822-incomplete-datasets-via-api.md
deleted file mode 100644
index dd0d98e2d3f..00000000000
--- a/doc/release-notes/8822-incomplete-datasets-via-api.md
+++ /dev/null
@@ -1,14 +0,0 @@
-### Creating datasets with incomplete metadata through API
-
-The create dataset API call (POST to /api/dataverses/#dataverseId/datasets) is extended with the "doNotValidate" parameter. However, in order to be able to create a dataset with incomplete metadata, the solr configuration must be updated first with the new "schema.xml" file (do not forget to run the metadata fields update script when you use custom metadata). Reindexing is optional, but recommended. Also, even when this feature is not used, it is recommended to update the solar configuration and reindex the metadata. Finally, this new feature can be activated with the "dataverse.api.allow-incomplete-metadata" JVM option.
-
-You can also enable a valid/incomplete metadata filter in the "My Data" page using the "dataverse.ui.show-validity-filter" JVM option. By default, this filter is not shown. When you wish to use this filter, you must reindex the datasets first, otherwise datasets with valid metadata will not be shown in the results.
-
-It is not possible to publish datasets with incomplete or incomplete metadata. By default, you also cannot send such datasets for review. If you wish to enable sending for review of datasets with incomplete metadata, turn on the "dataverse.ui.allow-review-for-incomplete" JVM option.
-
-In order to customize the wording and add translations to the UI sections extended by this feature, you can edit the "Bundle.properties" file and the localized versions of that file. The property keys used by this feature are:
-- incomplete
-- valid
-- dataset.message.incomplete.warning
-- mydataFragment.validity
-- dataverses.api.create.dataset.error.mustIncludeAuthorName
diff --git a/doc/release-notes/9063-session-api-auth.md b/doc/release-notes/9063-session-api-auth.md
deleted file mode 100644
index a972486e481..00000000000
--- a/doc/release-notes/9063-session-api-auth.md
+++ /dev/null
@@ -1 +0,0 @@
-A feature flag called "api-session-auth" has been added temporarily to aid in the development of the new frontend (#9063) but will be removed once bearer tokens (#9229) have been implemented. There is a security risk (CSRF) in enabling this flag! Do not use it in production! For more information, see http://preview.guides.gdcc.io/en/develop/installation/config.html#feature-flags
diff --git a/doc/release-notes/9100-schema.org-updates.md b/doc/release-notes/9100-schema.org-updates.md
deleted file mode 100644
index c76a0cc38e5..00000000000
--- a/doc/release-notes/9100-schema.org-updates.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Changes made in v5.13 and v5.14 in multiple PRs to improve the embedded Schema.org metadata in dataset pages will only be propagated to the Schema.Org JSON-LD metadata export if a reExportAll() is done.
-
-The 5.14 release notes should include the standard instructions for doing a reExportAll after updating the code.
\ No newline at end of file
diff --git a/doc/release-notes/9148-license-via-api.md b/doc/release-notes/9148-license-via-api.md
deleted file mode 100644
index 4c27af941e3..00000000000
--- a/doc/release-notes/9148-license-via-api.md
+++ /dev/null
@@ -1,28 +0,0 @@
-# License management via API
-
-See https://github.com/IQSS/dataverse/issues/9148.
-
-When publishing a dataset via API, it now requires the dataset to either have a standard license configured, or have valid Custom Terms of Use (if allowed by the instance). Attempting to publish a dataset without such **will fail with an error message**. This introduces a backward incompatibility, and if you have scripts that automatically create, update and publish datasets, this last step may start failing. Because, unfortunately, there were some problems with the datasets APIs that made it difficult to manage licenses, so an API user was likely to end up with a dataset missing either of the above. In this release we have addressed it by making the following fixes:
-
-We fixed the incompatibility between the format in which license information was *exported* in json, and the format the create and update APIs were expecting it for *import* (https://github.com/IQSS/dataverse/issues/9155). This means that the following json format can now be imported:
-```
-"license": {
- "name": "CC0 1.0",
- "uri": "http://creativecommons.org/publicdomain/zero/1.0"
-}
-```
-However, for the sake of backward compatibility the old format
-```
-"license" : "CC0 1.0"
-```
-will be accepted as well.
-
-We have added the default license (CC0) to the model json file that we provide and recommend to use as the model in the Native API Guide (https://github.com/IQSS/dataverse/issues/9364).
-
-And we have corrected the misleading language in the same guide where we used to recommend to users that they select, edit and re-import only the `.metadataBlocks` fragment of the json metadata representing the latest version. There are in fact other useful pieces of information that need to be preserved in the update (such as the `"license"` section above). So the recommended way of creating base json for updates via the API is to select *everything but* the `"files"` section, with (for example) the following `jq` command:
-
-```
-jq '.data | del(.files)'
-```
-
-Please see the [Update Metadata For a Dataset](https://guides.dataverse.org/en/latest/api/native-api.html#update-metadata-for-a-dataset) section of our Native Api guide for more information.
diff --git a/doc/release-notes/9150-improved-external-vocab-supprt.md b/doc/release-notes/9150-improved-external-vocab-supprt.md
deleted file mode 100644
index 5ae678e185b..00000000000
--- a/doc/release-notes/9150-improved-external-vocab-supprt.md
+++ /dev/null
@@ -1 +0,0 @@
-It is now possible to write external vocabulary scripts that target a single child field in a metadata block. Example scripts are now available at https://github.com/gdcc/dataverse-external-vocab-support that can be configured to support lookup from the Research Orgnaization Registry (ROR) for the Author Affiliation Field and for the CrossRef Funding Registry (Fundreg) in the Funding Information/Agency field, both in the standard Citation metadata block. Application if these scripts to other fields, and the development of other scripts targetting child fields are now possible.
\ No newline at end of file
diff --git a/doc/release-notes/9175-external-exporters.md b/doc/release-notes/9175-external-exporters.md
deleted file mode 100644
index 79b6358dac5..00000000000
--- a/doc/release-notes/9175-external-exporters.md
+++ /dev/null
@@ -1,11 +0,0 @@
-## Ability to Create New Exporters
-
-It is now possible for third parties to develop and share code to provide new metadata export formats for Dataverse. Export formats can be made available via the Dataverse UI and API or configured for use in Harvesting. Dataverse now provides developers with a separate dataverse-spi JAR file that contains the Java interfaces and classes required to create a new metadata Exporter. Once a new Exporter has been created and packaged as a JAR file, administrators can use it by specifying a local directory for third party Exporters, dropping then Exporter JAR there, and restarting Payara. This mechanism also allows new Exporters to replace any of Dataverse's existing metadata export formats.
-
-## Backward Incompatibilities
-
-Care should be taken when replacing Dataverse's internal metadata export formats as third party code, including other third party Exporters may depend on the contents of those export formats. When replacing an existing format, one must also remember to delete the cached metadata export files or run the reExport command for the metadata exports of existing datasets to be updated.
-
-## New JVM/MicroProfile Settings
-
-dataverse.spi.export.directory - specifies a directory, readable by the Dataverse server. Any Exporter JAR files placed in this directory will be read by Dataverse and used to add/replace the specified metadata format.
\ No newline at end of file
diff --git a/doc/release-notes/9185-contact-email-updates.md b/doc/release-notes/9185-contact-email-updates.md
deleted file mode 100644
index 3e03461a383..00000000000
--- a/doc/release-notes/9185-contact-email-updates.md
+++ /dev/null
@@ -1,12 +0,0 @@
-## Contact Email Improvements
-
-Email sent from the contact forms to the contact(s) for a collection, dataset, or datafile can now optionally be cc'd to a support email address. The support email address can be changed from the default :SystemEmail address to a separate :SupportEmail address. When multiple contacts are listed, the system will now send one email to all contacts (with the optional cc if configured) instead of separate emails to each contact. Contact names with a comma that refer to Organizations will no longer have the name parts reversed in the email greeting. A new protected feedback API has been added.
-
-## Backward Incompatibilities
-
-When there are multiple contacts, the system will now send one email with all of the contacts in the To: header instead of sending one email to each contact (with no indication that others have been notified).
-
-## New JVM/MicroProfile Settings
-
-dataverse.mail.support-email - allows a separate email, distinct from the :SystemEmail to be used as the to address in emails from the contact form/ feedback api.
-dataverse.mail.cc-support-on-contact-emails - include the support email address as a CC: entry when contact/feedback emails are sent to the contacts for a collection, dataset, or datafile.
\ No newline at end of file
diff --git a/doc/release-notes/9204-group-by-file-ordering.md b/doc/release-notes/9204-group-by-file-ordering.md
deleted file mode 100644
index 97f6b0750fb..00000000000
--- a/doc/release-notes/9204-group-by-file-ordering.md
+++ /dev/null
@@ -1,9 +0,0 @@
-### Support for Grouping Dataset Files by Folder and Category Tag
-
-Dataverse now supports grouping dataset files by folder and/or optionally by Tag/Category. The default for whether to order by folder can be changed via :OrderByFolder. Ordering by category must be enabled by an administrator via the :CategoryOrder parameter which is used to specify which tags appear first (e.g. to put Documentation files before Data or Code files, etc.) These Group-By options work with the existing sort options, i.e. sorting alphabetically means that files within each folder or tag group will be sorted alphabetically. :AllowUsersToManageOrdering can be set to true to allow users to turn folder ordering and category ordering (if enabled) on or off in the current dataset view.
-
-### New Setting
-
-:CategoryOrder - a comma separated list of Category/Tag names defining the order in which files with those tags should be displayed. The setting can include custom tag names along with the pre-defined defaults ( Documentation, Data, and Code, which can be overridden by the ::FileCategories setting.)
-:OrderByFolder - defaults to true - whether to group files in the same folder together
-:AllowUserManagementOfOrder - default false - allow users to toggle ordering on/off in the dataset display
\ No newline at end of file
diff --git a/doc/release-notes/9229-bearer-api-auth.md b/doc/release-notes/9229-bearer-api-auth.md
deleted file mode 100644
index 543b803469b..00000000000
--- a/doc/release-notes/9229-bearer-api-auth.md
+++ /dev/null
@@ -1 +0,0 @@
-A feature flag called "api-bearer-auth" has been added. This allows OIDC useraccounts to send authenticated API requests using Bearer Tokens. Note: This feature is limited to OIDC! For more information, see http://preview.guides.gdcc.io/en/develop/installation/config.html#feature-flags
diff --git a/doc/release-notes/9256-series.md b/doc/release-notes/9256-series.md
deleted file mode 100644
index 97419af4616..00000000000
--- a/doc/release-notes/9256-series.md
+++ /dev/null
@@ -1,42 +0,0 @@
-## Metadata field Series now repeatable
-This enhancement allows depositors to define multiple instances of the metadata field Series in the Citation Metadata block.
-
-## Major Use Cases and Infrastructure Enhancements
-* Data contained in a dataset may belong to multiple series. Making the field Series repeatable will make it possible to reflect this fact in the dataset metadata. (Issue #9255, PR #9256)
-
-### Additional Upgrade Steps
-
-Update the Citation metadata block:
-
-wget https://github.com/IQSS/dataverse/releases/download/v5.14/citation.tsv
-curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"
-
-## Additional Release Steps
-
-1. Replace Solr schema.xml to allow multiple series to be used. See specific instructions below for those installations without custom metadata blocks (1a) and those with custom metadata blocks (1b).
-
-1a.
-
-For installations without Custom Metadata Blocks:
-
--stop solr instance (usually service solr stop, depending on solr installation/OS, see the Installation Guide
-
--replace schema.xml
-
-cp /tmp/dvinstall/schema.xml /usr/local/solr/solr-8.11.1/server/solr/collection1/conf
-
--start solr instance (usually service solr start, depending on solr/OS)
-
-1b.
-
-For installations with Custom Metadata Blocks:
-
--stop solr instance (usually service solr stop, depending on solr installation/OS, see the Installation Guide
-
-edit the following lines to your schema.xml (to indicate that series and its components are now multiValued='true"):
-
-
-
-
-
--restart solr instance (usually service solr start, depending on solr/OS)
\ No newline at end of file
diff --git a/doc/release-notes/9277-nonstopmode-pdf-guides.md b/doc/release-notes/9277-nonstopmode-pdf-guides.md
deleted file mode 100644
index eab456416ba..00000000000
--- a/doc/release-notes/9277-nonstopmode-pdf-guides.md
+++ /dev/null
@@ -1,3 +0,0 @@
-An experimental version of the guides in PDF format is available at
-
-Advice for contributors to documentation who want to help improve the PDF is available at http://preview.guides.gdcc.io/en/develop/developers/documentation.html#pdf-version-of-the-guides
diff --git a/doc/release-notes/9331-extract-bounding-box.md b/doc/release-notes/9331-extract-bounding-box.md
deleted file mode 100644
index c4ff83e40c0..00000000000
--- a/doc/release-notes/9331-extract-bounding-box.md
+++ /dev/null
@@ -1 +0,0 @@
-An attempt will be made to extract a geospatial bounding box (west, south, east, north) from NetCDF and HDF5 files and then insert these values into the geospatial metadata block, if enabled.
diff --git a/doc/release-notes/9374-binder-orig.md b/doc/release-notes/9374-binder-orig.md
deleted file mode 100644
index 5e6ff0e5c4d..00000000000
--- a/doc/release-notes/9374-binder-orig.md
+++ /dev/null
@@ -1,7 +0,0 @@
-Files downloaded from Binder are now in their original format.
-
-For example, data.dta (a Stata file) will be downloaded instead of data.tab (the archival version Dataverse creates as part of a successful ingest).
-
-This should make it easier to write code to reproduce results as the dataset authors and subsequent researchers are likely operating on the original file format rather that the format that Dataverse creates.
-
-For details, see #9374, , and .
diff --git a/doc/release-notes/9414-containerized_dev_dependencies.md b/doc/release-notes/9414-containerized_dev_dependencies.md
deleted file mode 100644
index 4e3680573f9..00000000000
--- a/doc/release-notes/9414-containerized_dev_dependencies.md
+++ /dev/null
@@ -1,2 +0,0 @@
-New alternative to setup the Dataverse dependencies for the development environment through Docker Compose.
-New alternative, explained in the documentation, to build the Sphinx guides through a Docker container.
diff --git a/doc/release-notes/9434-app-container.md b/doc/release-notes/9434-app-container.md
deleted file mode 100644
index b5088bee328..00000000000
--- a/doc/release-notes/9434-app-container.md
+++ /dev/null
@@ -1,12 +0,0 @@
-Developers can experiment with running Dataverse in Docker: https://preview.guides.gdcc.io/en/develop/container/app-image.html
-
-This is an image developers build locally. It is not meant for production use!
-
-To provide a complete container-based local development environment, developers can deploy a Dataverse container from
-the new image in addition to other containers for the necessary dependencies:
-https://preview.guides.gdcc.io/en/develop/container/dev-usage.html
-
-Please note that with this emerging solution we will sunset older tooling like `docker-aio` and `docker-dcm`.
-We envision more testing possibilities in the future, to be discussed as part of the
-[Dataverse Containerization Working Group](https://dc.wgs.gdcc.io). There is no roadmap yet, but you have been warned.
-If there is some specific feature of these tools you would like to be kept, please [reach out](https://dc.wgs.gdcc.io).
diff --git a/doc/release-notes/9573-configbaker.md b/doc/release-notes/9573-configbaker.md
deleted file mode 100644
index bb68134794c..00000000000
--- a/doc/release-notes/9573-configbaker.md
+++ /dev/null
@@ -1 +0,0 @@
-A container has been added called "configbaker" that configures Dataverse while running in containers. This allows developers to spin up Dataverse with a single command.
diff --git a/doc/release-notes/9920-postgres16.md b/doc/release-notes/9920-postgres16.md
new file mode 100644
index 00000000000..8aab76e98b9
--- /dev/null
+++ b/doc/release-notes/9920-postgres16.md
@@ -0,0 +1,3 @@
+This release adds install script support for the new permissions model in Postgres versions 15+, and bumps FlyWay to support Postgres 16.
+
+Postgres 13 remains the version used with automated testing.
diff --git a/doc/shib/shib.md b/doc/shib/shib.md
index 2c178a93f35..9cff6d827e7 100644
--- a/doc/shib/shib.md
+++ b/doc/shib/shib.md
@@ -82,11 +82,7 @@ Run `service httpd restart`.
## Update/verify files under /etc/shibboleth
-For /etc/shibboleth/shibboleth2.xml use the version from https://github.com/IQSS/dataverse/blob/master/conf/vagrant/etc/shibboleth/shibboleth2.xml but replace "pdurbin.pagekite.me" with the "shibtest.dataverse.org".
-
-Put https://github.com/IQSS/dataverse/blob/master/conf/vagrant/etc/shibboleth/dataverse-idp-metadata.xml at /etc/shibboleth/dataverse-idp-metadata.xml
-
-Put https://github.com/IQSS/dataverse/blob/master/conf/vagrant/etc/shibboleth/attribute-map.xml at
+Get files from the Installation Guide.
After making these changes, run `service shibd restart` and `service httpd restart`.
diff --git a/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml b/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml
index 4f338905751..26144544d9e 100644
--- a/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml
+++ b/doc/sphinx-guides/source/_static/admin/counter-processor-config.yaml
@@ -1,8 +1,8 @@
# currently no other option but to have daily logs and have year-month-day format in the name with
# 4-digit year and 2-digit month and day
-# /usr/local/payara5/glassfish/domains/domain1/logs/counter_2019-01-11.log
+# /usr/local/payara6/glassfish/domains/domain1/logs/counter_2019-01-11.log
#log_name_pattern: sample_logs/counter_(yyyy-mm-dd).log
-log_name_pattern: /usr/local/payara5/glassfish/domains/domain1/logs/mdc/counter_(yyyy-mm-dd).log
+log_name_pattern: /usr/local/payara6/glassfish/domains/domain1/logs/mdc/counter_(yyyy-mm-dd).log
# path_types regular expressions allow matching to classify page urls as either an investigation or request
# based on specific URL structure for your system.
diff --git a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
index 4fc9849f7a5..4f4c29d0670 100644
--- a/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
+++ b/doc/sphinx-guides/source/_static/admin/dataverse-external-tools.tsv
@@ -1,6 +1,7 @@
-Tool Type Scope Description
-Data Explorer explore file A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse.
-Whole Tale explore dataset A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide `_.
-Binder explore dataset Binder allows you to spin up custom computing environments in the cloud (including Jupyter notebooks) with the files from your dataset. `Installation instructions `_ are in the Data Exploration Lab girder_ythub project. See also :ref:`binder`.
-File Previewers explore file A set of tools that display the content of files - including audio, html, `Hypothes.is `_ annotations, images, PDF, text, video, tabular data, spreadsheets, GeoJSON, zip, and NcML files - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers
-Data Curation Tool configure file A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions.
+Tool Type Scope Description
+Data Explorer explore file "A GUI which lists the variables in a tabular data file allowing searching, charting and cross tabulation analysis. See the README.md file at https://github.com/scholarsportal/dataverse-data-explorer-v2 for the instructions on adding Data Explorer to your Dataverse."
+Whole Tale explore dataset "A platform for the creation of reproducible research packages that allows users to launch containerized interactive analysis environments based on popular tools such as Jupyter and RStudio. Using this integration, Dataverse users can launch Jupyter and RStudio environments to analyze published datasets. For more information, see the `Whole Tale User Guide `_."
+Binder explore dataset Binder allows you to spin up custom computing environments in the cloud (including Jupyter notebooks) with the files from your dataset. `Installation instructions `_ are in the Data Exploration Lab girder_ythub project.
+File Previewers explore file "A set of tools that display the content of files - including audio, html, `Hypothes.is `_ annotations, images, PDF, Markdown, text, video, tabular data, spreadsheets, GeoJSON, zip, and NcML files - allowing them to be viewed without downloading the file. The previewers can be run directly from github.io, so the only required step is using the Dataverse API to register the ones you want to use. Documentation, including how to optionally brand the previewers, and an invitation to contribute through github are in the README.md file. Initial development was led by the Qualitative Data Repository and the spreasdheet previewer was added by the Social Sciences and Humanities Open Cloud (SSHOC) project. https://github.com/gdcc/dataverse-previewers"
+Data Curation Tool configure file "A GUI for curating data by adding labels, groups, weights and other details to assist with informed reuse. See the README.md file at https://github.com/scholarsportal/Dataverse-Data-Curation-Tool for the installation instructions."
+Ask the Data query file Ask the Data is an experimental tool that allows you ask natural language questions about the data contained in Dataverse tables (tabular data). See the README.md file at https://github.com/IQSS/askdataverse/tree/main/askthedata for the instructions on adding Ask the Data to your Dataverse installation.
diff --git a/doc/sphinx-guides/source/_static/api/dataset-schema.json b/doc/sphinx-guides/source/_static/api/dataset-schema.json
new file mode 100644
index 00000000000..34b8a1eeedb
--- /dev/null
+++ b/doc/sphinx-guides/source/_static/api/dataset-schema.json
@@ -0,0 +1,122 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "$defs": {
+ "field": {
+ "type": "object",
+ "required": ["typeClass", "multiple", "typeName"],
+ "properties": {
+ "value": {
+ "anyOf": [
+ {
+ "type": "array"
+ },
+ {
+ "type": "string"
+ },
+ {
+ "$ref": "#/$defs/field"
+ }
+ ]
+ },
+ "typeClass": {
+ "type": "string"
+ },
+ "multiple": {
+ "type": "boolean"
+ },
+ "typeName": {
+ "type": "string"
+ }
+ }
+ }
+},
+"type": "object",
+"properties": {
+ "datasetVersion": {
+ "type": "object",
+ "properties": {
+ "license": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "uri": {
+ "type": "string",
+ "format": "uri"
+ }
+ },
+ "required": ["name", "uri"]
+ },
+ "metadataBlocks": {
+ "type": "object",
+ "properties": {
+ "citation": {
+ "type": "object",
+ "properties": {
+ "fields": {
+ "type": "array",
+ "items": {
+ "$ref": "#/$defs/field"
+ },
+ "minItems": 5,
+ "allOf": [
+ {
+ "contains": {
+ "properties": {
+ "typeName": {
+ "const": "title"
+ }
+ }
+ }
+ },
+ {
+ "contains": {
+ "properties": {
+ "typeName": {
+ "const": "author"
+ }
+ }
+ }
+ },
+ {
+ "contains": {
+ "properties": {
+ "typeName": {
+ "const": "datasetContact"
+ }
+ }
+ }
+ },
+ {
+ "contains": {
+ "properties": {
+ "typeName": {
+ "const": "dsDescription"
+ }
+ }
+ }
+ },
+ {
+ "contains": {
+ "properties": {
+ "typeName": {
+ "const": "subject"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "required": ["fields"]
+ }
+ },
+ "required": ["citation"]
+ }
+ },
+ "required": ["metadataBlocks"]
+ }
+ },
+ "required": ["datasetVersion"]
+}
diff --git a/scripts/vagrant/counter-processor-config.yaml b/doc/sphinx-guides/source/_static/developers/counter-processor-config.yaml
similarity index 100%
rename from scripts/vagrant/counter-processor-config.yaml
rename to doc/sphinx-guides/source/_static/developers/counter-processor-config.yaml
diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.root b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.root
index 1de94331523..b9ef9960318 100755
--- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.root
+++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.root
@@ -4,7 +4,7 @@
set -e
-ASADMIN=/usr/local/payara5/bin/asadmin
+ASADMIN=/usr/local/payara6/bin/asadmin
case "$1" in
start)
diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.service b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.service
index 7c457e615d8..19bb190e740 100755
--- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.service
+++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/payara.init.service
@@ -3,7 +3,7 @@
# description: Payara App Server
set -e
-ASADMIN=/usr/local/payara5/bin/asadmin
+ASADMIN=/usr/local/payara6/bin/asadmin
APP_SERVER_USER=dataverse
case "$1" in
diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr
index 7ca04cdff3f..f7dba504e70 100755
--- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr
+++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr
@@ -5,9 +5,9 @@
# chkconfig: 35 92 08
# description: Starts and stops Apache Solr
-SOLR_DIR="/usr/local/solr/solr-8.11.1"
+SOLR_DIR="/usr/local/solr/solr-9.3.0"
SOLR_COMMAND="bin/solr"
-SOLR_ARGS="-m 1g -j jetty.host=127.0.0.1"
+SOLR_ARGS="-m 1g"
SOLR_USER=solr
case $1 in
diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/payara.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/payara.service
index c8c82f6d6b2..c8efcb9c6f9 100644
--- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/payara.service
+++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/payara.service
@@ -4,9 +4,9 @@ After = syslog.target network.target
[Service]
Type = forking
-ExecStart = /usr/bin/java -jar /usr/local/payara5/glassfish/lib/client/appserver-cli.jar start-domain
-ExecStop = /usr/bin/java -jar /usr/local/payara5/glassfish/lib/client/appserver-cli.jar stop-domain
-ExecReload = /usr/bin/java -jar /usr/local/payara5/glassfish/lib/client/appserver-cli.jar restart-domain
+ExecStart = /usr/bin/java -jar /usr/local/payara6/glassfish/lib/client/appserver-cli.jar start-domain
+ExecStop = /usr/bin/java -jar /usr/local/payara6/glassfish/lib/client/appserver-cli.jar stop-domain
+ExecReload = /usr/bin/java -jar /usr/local/payara6/glassfish/lib/client/appserver-cli.jar restart-domain
User=dataverse
LimitNOFILE=32768
Environment="LANG=en_US.UTF-8"
diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service
index d89ee108377..2ceeb0016d6 100644
--- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service
+++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service
@@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target
[Service]
User = solr
Type = forking
-WorkingDirectory = /usr/local/solr/solr-8.11.1
-ExecStart = /usr/local/solr/solr-8.11.1/bin/solr start -m 1g -j "jetty.host=127.0.0.1"
-ExecStop = /usr/local/solr/solr-8.11.1/bin/solr stop
+WorkingDirectory = /usr/local/solr/solr-9.3.0
+ExecStart = /usr/local/solr/solr-9.3.0/bin/solr start -m 1g
+ExecStop = /usr/local/solr/solr-9.3.0/bin/solr stop
LimitNOFILE=65000
LimitNPROC=65000
Restart=on-failure
diff --git a/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/oidc.json b/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/oidc.json
new file mode 100644
index 00000000000..9df38988a25
--- /dev/null
+++ b/doc/sphinx-guides/source/_static/installation/files/root/auth-providers/oidc.json
@@ -0,0 +1,8 @@
+{
+ "id":"",
+ "factoryAlias":"oidc",
+ "title":"",
+ "subtitle":"",
+ "factoryData":"type: oidc | issuer: | clientId: | clientSecret: | pkceEnabled: | pkceMethod: ",
+ "enabled":true
+}
\ No newline at end of file
diff --git a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json
index 47413c8a625..22dd6477cb4 100644
--- a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json
+++ b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/dynamicDatasetTool.json
@@ -14,14 +14,14 @@
{
"locale":"{localeCode}"
}
- ],
- "allowedApiCalls": [
- {
- "name":"retrieveDatasetJson",
- "httpMethod":"GET",
- "urlTemplate":"/api/v1/datasets/{datasetId}",
- "timeOut":10
- }
- ]
- }
+ ]
+ },
+ "allowedApiCalls": [
+ {
+ "name":"retrieveDatasetJson",
+ "httpMethod":"GET",
+ "urlTemplate":"/api/v1/datasets/{datasetId}",
+ "timeOut":10
+ }
+ ]
}
diff --git a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json
index 1c132576099..2b6a0b8e092 100644
--- a/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json
+++ b/doc/sphinx-guides/source/_static/installation/files/root/external-tools/fabulousFileTool.json
@@ -21,14 +21,14 @@
{
"locale":"{localeCode}"
}
- ],
- "allowedApiCalls": [
- {
- "name":"retrieveDataFile",
- "httpMethod":"GET",
- "urlTemplate":"/api/v1/access/datafile/{fileId}",
- "timeOut":270
- }
]
- }
+ },
+ "allowedApiCalls": [
+ {
+ "name":"retrieveDataFile",
+ "httpMethod":"GET",
+ "urlTemplate":"/api/v1/access/datafile/{fileId}",
+ "timeOut":270
+ }
+ ]
}
diff --git a/doc/sphinx-guides/source/_static/installation/files/usr/local/payara5/glassfish/domains/domain1/config/logging.properties b/doc/sphinx-guides/source/_static/installation/files/usr/local/payara5/glassfish/domains/domain1/config/logging.properties
deleted file mode 100644
index 4054c794452..00000000000
--- a/doc/sphinx-guides/source/_static/installation/files/usr/local/payara5/glassfish/domains/domain1/config/logging.properties
+++ /dev/null
@@ -1,166 +0,0 @@
-#
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
-#
-# Copyright (c) 2013 Oracle and/or its affiliates. All rights reserved.
-#
-# The contents of this file are subject to the terms of either the GNU
-# General Public License Version 2 only ("GPL") or the Common Development
-# and Distribution License("CDDL") (collectively, the "License"). You
-# may not use this file except in compliance with the License. You can
-# obtain a copy of the License at
-# https://glassfish.dev.java.net/public/CDDL+GPL_1_1.html
-# or packager/legal/LICENSE.txt. See the License for the specific
-# language governing permissions and limitations under the License.
-#
-# When distributing the software, include this License Header Notice in each
-# file and include the License file at packager/legal/LICENSE.txt.
-#
-# GPL Classpath Exception:
-# Oracle designates this particular file as subject to the "Classpath"
-# exception as provided by Oracle in the GPL Version 2 section of the License
-# file that accompanied this code.
-#
-# Modifications:
-# If applicable, add the following below the License Header, with the fields
-# enclosed by brackets [] replaced by your own identifying information:
-# "Portions Copyright [year] [name of copyright owner]"
-#
-# Contributor(s):
-# If you wish your version of this file to be governed by only the CDDL or
-# only the GPL Version 2, indicate your decision by adding "[Contributor]
-# elects to include this software in this distribution under the [CDDL or GPL
-# Version 2] license." If you don't indicate a single choice of license, a
-# recipient has the option to distribute your version of this file under
-# either the CDDL, the GPL Version 2 or to extend the choice of license to
-# its licensees as provided above. However, if you add GPL Version 2 code
-# and therefore, elected the GPL Version 2 license, then the option applies
-# only if the new code is made subject to such option by the copyright
-# holder.
-#
-# Portions Copyright [2016-2021] [Payara Foundation and/or its affiliates]
-
-#GlassFish logging.properties list
-#Update June 13 2012
-
-#All attributes details
-handlers=java.util.logging.ConsoleHandler
-handlerServices=com.sun.enterprise.server.logging.GFFileHandler,com.sun.enterprise.server.logging.SyslogHandler
-java.util.logging.ConsoleHandler.formatter=com.sun.enterprise.server.logging.UniformLogFormatter
-java.util.logging.FileHandler.count=1
-java.util.logging.FileHandler.formatter=java.util.logging.XMLFormatter
-java.util.logging.FileHandler.limit=50000
-java.util.logging.FileHandler.pattern=%h/java%u.log
-com.sun.enterprise.server.logging.GFFileHandler.compressOnRotation=false
-com.sun.enterprise.server.logging.GFFileHandler.excludeFields=
-com.sun.enterprise.server.logging.GFFileHandler.file=${com.sun.aas.instanceRoot}/logs/server.log
-com.sun.enterprise.server.logging.GFFileHandler.flushFrequency=1
-com.sun.enterprise.server.logging.GFFileHandler.formatter=com.sun.enterprise.server.logging.ODLLogFormatter
-com.sun.enterprise.server.logging.GFFileHandler.level=ALL
-com.sun.enterprise.server.logging.GFFileHandler.logStandardStreams=true
-com.sun.enterprise.server.logging.GFFileHandler.logtoConsole=false
-com.sun.enterprise.server.logging.GFFileHandler.logtoFile=true
-com.sun.enterprise.server.logging.GFFileHandler.maxHistoryFiles=0
-com.sun.enterprise.server.logging.GFFileHandler.multiLineMode=true
-com.sun.enterprise.server.logging.GFFileHandler.retainErrorsStasticsForHours=0
-com.sun.enterprise.server.logging.GFFileHandler.rotationLimitInBytes=2000000
-com.sun.enterprise.server.logging.GFFileHandler.rotationOnDateChange=false
-com.sun.enterprise.server.logging.GFFileHandler.rotationTimelimitInMinutes=0
-com.sun.enterprise.server.logging.SyslogHandler.level=ALL
-com.sun.enterprise.server.logging.SyslogHandler.useSystemLogging=false
-log4j.logger.org.hibernate.validator.util.Version=warn
-com.sun.enterprise.server.logging.UniformLogFormatter.ansiColor=true
-
-#Payara Notification logging properties
-fish.payara.enterprise.server.logging.PayaraNotificationFileHandler.compressOnRotation=false
-fish.payara.enterprise.server.logging.PayaraNotificationFileHandler.file=${com.sun.aas.instanceRoot}/logs/notification.log
-fish.payara.enterprise.server.logging.PayaraNotificationFileHandler.formatter=com.sun.enterprise.server.logging.ODLLogFormatter
-fish.payara.enterprise.server.logging.PayaraNotificationFileHandler.logtoFile=true
-fish.payara.enterprise.server.logging.PayaraNotificationFileHandler.maxHistoryFiles=0
-fish.payara.enterprise.server.logging.PayaraNotificationFileHandler.rotationLimitInBytes=2000000
-fish.payara.enterprise.server.logging.PayaraNotificationFileHandler.rotationOnDateChange=false
-fish.payara.enterprise.server.logging.PayaraNotificationFileHandler.rotationTimelimitInMinutes=0
-fish.payara.deprecated.jsonlogformatter.underscoreprefix=false
-
-#All log level details
-
-.level=INFO
-ShoalLogger.level=CONFIG
-com.hazelcast.level=WARNING
-java.util.logging.ConsoleHandler.level=FINEST
-javax.enterprise.resource.corba.level=INFO
-javax.enterprise.resource.javamail.level=INFO
-javax.enterprise.resource.jdo.level=INFO
-javax.enterprise.resource.jms.level=INFO
-javax.enterprise.resource.jta.level=INFO
-javax.enterprise.resource.resourceadapter.level=INFO
-javax.enterprise.resource.sqltrace.level=FINE
-javax.enterprise.resource.webcontainer.jsf.application.level=INFO
-javax.enterprise.resource.webcontainer.jsf.config.level=INFO
-javax.enterprise.resource.webcontainer.jsf.context.level=INFO
-javax.enterprise.resource.webcontainer.jsf.facelets.level=INFO
-javax.enterprise.resource.webcontainer.jsf.lifecycle.level=INFO
-javax.enterprise.resource.webcontainer.jsf.managedbean.level=INFO
-javax.enterprise.resource.webcontainer.jsf.renderkit.level=INFO
-javax.enterprise.resource.webcontainer.jsf.resource.level=INFO
-javax.enterprise.resource.webcontainer.jsf.taglib.level=INFO
-javax.enterprise.resource.webcontainer.jsf.timing.level=INFO
-javax.enterprise.system.container.cmp.level=INFO
-javax.enterprise.system.container.ejb.level=INFO
-javax.enterprise.system.container.ejb.mdb.level=INFO
-javax.enterprise.system.container.web.level=INFO
-javax.enterprise.system.core.classloading.level=INFO
-javax.enterprise.system.core.config.level=INFO
-javax.enterprise.system.core.level=INFO
-javax.enterprise.system.core.security.level=INFO
-javax.enterprise.system.core.selfmanagement.level=INFO
-javax.enterprise.system.core.transaction.level=INFO
-javax.enterprise.system.level=INFO
-javax.enterprise.system.ssl.security.level=INFO
-javax.enterprise.system.tools.admin.level=INFO
-javax.enterprise.system.tools.backup.level=INFO
-javax.enterprise.system.tools.deployment.common.level=WARNING
-javax.enterprise.system.tools.deployment.dol.level=WARNING
-javax.enterprise.system.tools.deployment.level=INFO
-javax.enterprise.system.util.level=INFO
-javax.enterprise.system.webservices.registry.level=INFO
-javax.enterprise.system.webservices.rpc.level=INFO
-javax.enterprise.system.webservices.saaj.level=INFO
-javax.level=INFO
-javax.mail.level=INFO
-javax.org.glassfish.persistence.level=INFO
-org.apache.catalina.level=INFO
-org.apache.coyote.level=INFO
-org.apache.jasper.level=INFO
-org.eclipse.persistence.session.level=INFO
-org.glassfish.admingui.level=INFO
-org.glassfish.naming.level=INFO
-org.jvnet.hk2.osgiadapter.level=INFO
-
-javax.enterprise.resource.corba.level=INFO
-javax.enterprise.resource.jta.level=INFO
-javax.enterprise.system.webservices.saaj.level=INFO
-javax.enterprise.system.container.ejb.level=INFO
-javax.enterprise.system.container.ejb.mdb.level=INFO
-javax.enterprise.resource.javamail.level=INFO
-javax.enterprise.system.webservices.rpc.level=INFO
-javax.enterprise.system.container.web.level=INFO
-javax.enterprise.resource.jms.level=INFO
-javax.enterprise.system.webservices.registry.level=INFO
-javax.enterprise.resource.webcontainer.jsf.application.level=INFO
-javax.enterprise.resource.webcontainer.jsf.resource.level=INFO
-javax.enterprise.resource.webcontainer.jsf.config.level=INFO
-javax.enterprise.resource.webcontainer.jsf.context.level=INFO
-javax.enterprise.resource.webcontainer.jsf.facelets.level=INFO
-javax.enterprise.resource.webcontainer.jsf.lifecycle.level=INFO
-javax.enterprise.resource.webcontainer.jsf.managedbean.level=INFO
-javax.enterprise.resource.webcontainer.jsf.renderkit.level=INFO
-javax.enterprise.resource.webcontainer.jsf.taglib.level=INFO
-javax.enterprise.resource.webcontainer.jsf.timing.level=INFO
-javax.org.glassfish.persistence.level=INFO
-javax.enterprise.system.tools.backup.level=INFO
-javax.mail.level=INFO
-org.glassfish.admingui.level=INFO
-org.glassfish.naming.level=INFO
-org.eclipse.persistence.session.level=INFO
-javax.enterprise.system.tools.deployment.dol.level=WARNING
-javax.enterprise.system.tools.deployment.common.level=WARNING
diff --git a/doc/sphinx-guides/source/_static/navbarscroll.js b/doc/sphinx-guides/source/_static/navbarscroll.js
index 66c9d4d7995..735f80870cd 100644
--- a/doc/sphinx-guides/source/_static/navbarscroll.js
+++ b/doc/sphinx-guides/source/_static/navbarscroll.js
@@ -1,6 +1,6 @@
/*
Use to fix hidden section headers behind the navbar when using links with targets
- See: http://stackoverflow.com/questions/10732690/offsetting-an-html-anchor-to-adjust-for-fixed-header
+ See: https://stackoverflow.com/questions/10732690/offsetting-an-html-anchor-to-adjust-for-fixed-header
*/
$jqTheme(document).ready(function() {
$jqTheme('a[href*="#"]:not([href="#"])').on('click', function() {
diff --git a/doc/sphinx-guides/source/_static/util/clear_timer.sh b/doc/sphinx-guides/source/_static/util/clear_timer.sh
index 1d9966e4e07..641b2695084 100755
--- a/doc/sphinx-guides/source/_static/util/clear_timer.sh
+++ b/doc/sphinx-guides/source/_static/util/clear_timer.sh
@@ -8,7 +8,7 @@
# if you'd like to avoid that.
# directory where Payara is installed
-PAYARA_DIR=/usr/local/payara5
+PAYARA_DIR=/usr/local/payara6
# directory within Payara (defaults)
DV_DIR=${PAYARA_DIR}/glassfish/domains/domain1
diff --git a/doc/sphinx-guides/source/_static/util/counter_daily.sh b/doc/sphinx-guides/source/_static/util/counter_daily.sh
index a12439d9cf8..674972b18f2 100644
--- a/doc/sphinx-guides/source/_static/util/counter_daily.sh
+++ b/doc/sphinx-guides/source/_static/util/counter_daily.sh
@@ -1,7 +1,7 @@
#! /bin/bash
COUNTER_PROCESSOR_DIRECTORY="/usr/local/counter-processor-0.1.04"
-MDC_LOG_DIRECTORY="/usr/local/payara5/glassfish/domains/domain1/logs/mdc"
+MDC_LOG_DIRECTORY="/usr/local/payara6/glassfish/domains/domain1/logs/mdc"
# counter_daily.sh
diff --git a/doc/sphinx-guides/source/_templates/navbar.html b/doc/sphinx-guides/source/_templates/navbar.html
index 538cccf74d7..c7b81dcb937 100644
--- a/doc/sphinx-guides/source/_templates/navbar.html
+++ b/doc/sphinx-guides/source/_templates/navbar.html
@@ -15,7 +15,7 @@
- Dataverse Project
+ Dataverse Project
@@ -24,15 +24,15 @@
About
-
+
Community
@@ -49,18 +49,18 @@
Software
-
+
Contact
diff --git a/doc/sphinx-guides/source/admin/collectionquotas.rst b/doc/sphinx-guides/source/admin/collectionquotas.rst
new file mode 100644
index 00000000000..2ce3132e2ba
--- /dev/null
+++ b/doc/sphinx-guides/source/admin/collectionquotas.rst
@@ -0,0 +1,19 @@
+
+Storage Quotas for Collections
+==============================
+
+Please note that this is a new and still experimental feature (as of Dataverse v6.1 release).
+
+Instance admins can now define storage quota limits for specific collections. These limits can be set, changed and/or deleted via the provided APIs (please see the :ref:`collection-storage-quotas` section of the :doc:`/api/native-api` guide). The Read version of the API is available to the individual collection admins (i.e., a collection owner can check on the quota configured for their collection), but only superusers can set, change or disable storage quotas.
+
+Storage quotas are *inherited* by subcollections. In other words, when storage use limit is set for a specific collection, it applies to all the datasets immediately under it and in its sub-collections, unless different quotas are defined there and so on. Each file added to any dataset in that hierarchy counts for the purposes of the quota limit defined for the top collection. A storage quota defined on a child sub-collection overrides whatever quota that may be defined on the parent, or inherited from an ancestor.
+
+For example, a collection ``A`` has the storage quota set to 10GB. It has 3 sub-collections, ``B``, ``C`` and ``D``. Users can keep uploading files into the datasets anywhere in this hierarchy until the combined size of 10GB is reached between them. However, if an admin has reasons to limit one of the sub-collections, ``B`` to 3GB only, that quota can be explicitly set there. This both limits the growth of ``B`` to 3GB, and also *guarantees* that allocation to it. I.e. the contributors to collection ``B`` will be able to keep adding data until the 3GB limit is reached, even after the parent collection ``A`` reaches the combined 10GB limit (at which point ``A`` and all its subcollections except for ``B`` will become read-only).
+
+We do not yet know whether this is going to be a popular, or needed use case - a child collection quota that is different from the quota it inherits from a parent. It is likely that for many instances it will be sufficient to be able to define quotas for collections and have them apply to all the child objects underneath. We will examine the response to this feature and consider making adjustments to this scheme based on it. We are already considering introducing other types of quotas, such as limits by users or specific storage volumes.
+
+Please note that only the sizes of the main datafiles and the archival tab-delimited format versions, as produced by the ingest process are counted for the purposes of enforcing the limits. Automatically generated "auxiliary" files, such as rescaled image thumbnails and metadata exports for datasets are not.
+
+When quotas are set and enforced, the users will be informed of the remaining storage allocation on the file upload page together with other upload and processing limits.
+
+Part of the new and experimental nature of this feature is that we don't know for the fact yet how well it will function in real life on a very busy production system, despite our best efforts to test it prior to the release. One specific issue is having to update the recorded storage use for every parent collection of the given dataset whenever new files are added. This includes updating the combined size of the root, top collection - which will need to be updated after *every* file upload. In an unlikely case that this will start causing problems with race conditions and database update conflicts, it is possible to disable these updates (and thus disable the storage quotas feature), by setting the :ref:`dataverse.storageuse.disable-storageuse-increments` JVM setting to true.
diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst
index 7f32e8c2514..37494c57fa1 100644
--- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst
+++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst
@@ -53,11 +53,15 @@ Configure a Dataverse Collection to Store All New Files in a Specific File Store
To direct new files (uploaded when datasets are created or edited) for all datasets in a given Dataverse collection, the store can be specified via the API as shown below, or by editing the 'General Information' for a Dataverse collection on the Dataverse collection page. Only accessible to superusers. ::
curl -H "X-Dataverse-key: $API_TOKEN" -X PUT -d $storageDriverLabel http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver
+
+(Note that for ``dataverse.files.store1.label=MyLabel``, you should pass ``MyLabel``.)
The current driver can be seen using::
curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver
+(Note that for ``dataverse.files.store1.label=MyLabel``, ``store1`` will be returned.)
+
and can be reset to the default store with::
curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE http://$SERVER/api/admin/dataverse/$dataverse-alias/storageDriver
@@ -118,6 +122,28 @@ Creates a link between a dataset and a Dataverse collection (see the :ref:`datas
curl -H "X-Dataverse-key: $API_TOKEN" -X PUT http://$SERVER/api/datasets/$linked-dataset-id/link/$linking-dataverse-alias
+List Collections that are Linked from a Dataset
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Lists the link(s) created between a dataset and a Dataverse collection (see the :ref:`dataset-linking` section of the User Guide for more information). ::
+
+ curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/datasets/$linked-dataset-id/links
+
+It returns a list in the following format:
+
+.. code-block:: json
+
+ {
+ "status": "OK",
+ "data": {
+ "dataverses that link to dataset id 56782": [
+ "crc990 (id 18802)"
+ ]
+ }
+ }
+
+.. _unlink-a-dataset:
+
Unlink a Dataset
^^^^^^^^^^^^^^^^
@@ -131,15 +157,35 @@ Mint a PID for a File That Does Not Have One
In the following example, the database id of the file is 42::
export FILE_ID=42
- curl http://localhost:8080/api/admin/$FILE_ID/registerDataFile
+ curl "http://localhost:8080/api/admin/$FILE_ID/registerDataFile"
+
+This method will return a FORBIDDEN response if minting of file PIDs is not enabled for the collection the file is in. (Note that it is possible to have file PIDs enabled for a specific collection, even when it is disabled for the Dataverse installation as a whole. See :ref:`collection-attributes-api` in the Native API Guide.)
-Mint PIDs for Files That Do Not Have Them
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Mint PIDs for all unregistered published files in the specified collection
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-If you have a large number of files, you might want to consider miniting PIDs for files individually using the ``registerDataFile`` endpoint above in a for loop, sleeping between each registration::
+The following API will register the PIDs for all the yet unregistered published files in the datasets **directly within the collection** specified by its alias::
+
+ curl "http://localhost:8080/api/admin/registerDataFiles/{collection_alias}"
+
+It will not attempt to register the datafiles in its sub-collections, so this call will need to be repeated on any sub-collections where files need to be registered as well.
+File-level PID registration must be enabled on the collection. (Note that it is possible to have it enabled for a specific collection, even when it is disabled for the Dataverse installation as a whole. See :ref:`collection-attributes-api` in the Native API Guide.)
+
+This API will sleep for 1 second between registration calls by default. A longer sleep interval can be specified with an optional ``sleep=`` parameter::
+
+ curl "http://localhost:8080/api/admin/registerDataFiles/{collection_alias}?sleep=5"
+
+Mint PIDs for ALL unregistered files in the database
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The following API will attempt to register the PIDs for all the published files in your instance, in collections that allow file PIDs, that do not yet have them::
curl http://localhost:8080/api/admin/registerDataFileAll
+The application will attempt to sleep for 1 second between registration attempts as not to overload your persistent identifier service provider. Note that if you have a large number of files that need to be registered in your Dataverse, you may want to consider minting file PIDs within indivdual collections, or even for individual files using the ``registerDataFiles`` and/or ``registerDataFile`` endpoints above in a loop, with a longer sleep interval between calls.
+
+
+
Mint a New DOI for a Dataset with a Handle
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/doc/sphinx-guides/source/admin/external-tools.rst b/doc/sphinx-guides/source/admin/external-tools.rst
index ad6181a867a..346ca0b15ee 100644
--- a/doc/sphinx-guides/source/admin/external-tools.rst
+++ b/doc/sphinx-guides/source/admin/external-tools.rst
@@ -92,7 +92,15 @@ File Level Preview Tools
File level preview tools allow the user to see a preview of the file contents without having to download it.
-When a file has a preview available, a preview icon will appear next to that file in the file listing on the dataset page. On the file page itself, the preview will appear in a Preview tab either immediately or once a guestbook has been filled in or terms, if any, have been agreed to.
+When a file has a preview available, a preview icon will appear next to that file in the file listing on the dataset page. On the file page itself, the preview will appear in a Preview tab (renamed File Tools, if multiple tools are available) either immediately or once a guestbook has been filled in or terms, if any, have been agreed to.
+
+File Level Query Tools
+++++++++++++++++++++++++
+
+File level query tools allow the user to ask questions (e.g. natural language queries) of a data table's contents without having to download it.
+
+When a file has a query tool available, a query icon will appear next to that file in the file listing on the dataset page. On the file page itself, the query tool will appear in a Query tab (renamed File Tools, if multiple tools are available) either immediately or once a guestbook has been filled in or terms, if any, have been agreed to.
+
File Level Configure Tools
++++++++++++++++++++++++++
@@ -107,7 +115,7 @@ Dataset level explore tools allow the user to explore all the files in a dataset
Dataset Level Configure Tools
+++++++++++++++++++++++++++++
-Configure tools at the dataset level are not currently supported.
+Dataset level configure tools can be launched by users who have edit access to the dataset. These tools are found under the "Edit Dataset" menu.
Writing Your Own External Tool
------------------------------
diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst
index da922459f46..59fc4dc2c64 100644
--- a/doc/sphinx-guides/source/admin/harvestclients.rst
+++ b/doc/sphinx-guides/source/admin/harvestclients.rst
@@ -35,8 +35,8 @@ For example:
.. code-block:: bash
- sudo touch /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916
- sudo chown dataverse /usr/local/payara5/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916
+ sudo touch /usr/local/payara6/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916
+ sudo chown dataverse /usr/local/payara6/glassfish/domains/domain1/logs/stopharvest_bigarchive.70916
Note: If the application server is stopped and restarted, any running harvesting jobs will be killed but may remain marked as in progress in the database. We thus recommend using the mechanism here to stop ongoing harvests prior to a server restart.
@@ -44,6 +44,6 @@ Note: If the application server is stopped and restarted, any running harvesting
What if a Run Fails?
~~~~~~~~~~~~~~~~~~~~
-Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara5/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong.
+Each harvesting client run logs a separate file per run to the app server's default logging directory (``/usr/local/payara6/glassfish/domains/domain1/logs/`` unless you've changed it). Look for filenames in the format ``harvest_TARGET_YYYY_MM_DD_timestamp.log`` to get a better idea of what's going wrong.
Note that you'll want to run a minimum of Dataverse Software 4.6, optimally 4.18 or beyond, for the best OAI-PMH interoperability.
diff --git a/doc/sphinx-guides/source/admin/index.rst b/doc/sphinx-guides/source/admin/index.rst
index ac81aa737a7..633842044b4 100755
--- a/doc/sphinx-guides/source/admin/index.rst
+++ b/doc/sphinx-guides/source/admin/index.rst
@@ -27,6 +27,7 @@ This guide documents the functionality only available to superusers (such as "da
solr-search-index
ip-groups
mail-groups
+ collectionquotas
monitoring
reporting-tools-and-queries
maintenance
diff --git a/doc/sphinx-guides/source/admin/integrations.rst b/doc/sphinx-guides/source/admin/integrations.rst
index 21adf8338d9..db566106b49 100644
--- a/doc/sphinx-guides/source/admin/integrations.rst
+++ b/doc/sphinx-guides/source/admin/integrations.rst
@@ -121,6 +121,18 @@ Its goal is to make the dashboard adjustable for a Dataverse installation's need
The integrations dashboard is currently in development. A preview and more information can be found at: `rdm-integration GitHub repository `_
+Globus
+++++++
+
+Globus transfer uses an efficient transfer mechanism and has additional features that make it suitable for large files and large numbers of files:
+
+* robust file transfer capable of restarting after network or endpoint failures
+* third-party transfer, which enables a user accessing a Dataverse installation in their desktop browser to initiate transfer of their files from a remote endpoint (i.e. on a local high-performance computing cluster), directly to an S3 store managed by the Dataverse installation
+
+Users can transfer files via `Globus `_ into and out of datasets, or reference files on a remote Globus endpoint, when their Dataverse installation is configured to use a Globus accessible store(s)
+and a community-developed `dataverse-globus `_ app has been properly installed and configured.
+
+
Embedding Data on Websites
--------------------------
@@ -217,7 +229,14 @@ Sponsored by the `Ontario Council of University Libraries (OCUL) `_ zipped `BagIt `_ bags to the `Chronopolis `_ via `DuraCloud `_, to a local file system, or to `Google Cloud Storage `_.
+A Dataverse installation can be configured to submit a copy of published Dataset versions, packaged as `Research Data Alliance conformant `_ zipped `BagIt `_ bags to `Chronopolis `_ via `DuraCloud `_, a local file system, any S3 store, or to `Google Cloud Storage `_.
+Submission can be automated to occur upon publication, or can be done periodically (via external scripting).
+The archival status of each Dataset version can be seen in the Dataset page version table and queried via API.
+
+The archival Bags include all of the files and metadata in a given dataset version and are sufficient to recreate the dataset, e.g. in a new Dataverse instance, or potentially in another RDA-conformant repository.
+Specifically, the archival Bags include an OAI-ORE Map serialized as JSON-LD that describe the dataset and it's files, as well as information about the version of Dataverse used to export the archival Bag.
+
+The `DVUploader `_ includes functionality to recreate a Dataset from an archival Bag produced by Dataverse (using the Dataverse API to do so).
For details on how to configure this integration, see :ref:`BagIt Export` in the :doc:`/installation/config` section of the Installation Guide.
diff --git a/doc/sphinx-guides/source/admin/make-data-count.rst b/doc/sphinx-guides/source/admin/make-data-count.rst
index 521ee4f97ca..fe32af6649a 100644
--- a/doc/sphinx-guides/source/admin/make-data-count.rst
+++ b/doc/sphinx-guides/source/admin/make-data-count.rst
@@ -72,7 +72,8 @@ Enable or Disable Display of Make Data Count Metrics
By default, when MDC logging is enabled (when ``:MDCLogPath`` is set), your Dataverse installation will display MDC metrics instead of it's internal (legacy) metrics. You can avoid this (e.g. to collect MDC metrics for some period of time before starting to display them) by setting ``:DisplayMDCMetrics`` to false.
-The following discussion assumes ``:MDCLogPath`` has been set to ``/usr/local/payara5/glassfish/domains/domain1/logs/mdc``
+The following discussion assumes ``:MDCLogPath`` has been set to ``/usr/local/payara6/glassfish/domains/domain1/logs/mdc``
+You can also decide to display MDC metrics along with Dataverse's traditional download counts from the time before MDC was enabled. To do this, set the :ref:`:MDCStartDate` to when you started MDC logging.
Configure Counter Processor
~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -102,7 +103,7 @@ Soon we will be setting up a cron job to run nightly but we start with a single
* If you are running Counter Processor for the first time in the middle of a month, you will need create blank log files for the previous days. e.g.:
- * ``cd /usr/local/payara5/glassfish/domains/domain1/logs/mdc``
+ * ``cd /usr/local/payara6/glassfish/domains/domain1/logs/mdc``
* ``touch counter_2019-02-01.log``
diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst
index 1480c66f8b6..4f737bd730b 100644
--- a/doc/sphinx-guides/source/admin/metadatacustomization.rst
+++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst
@@ -95,6 +95,11 @@ Each of the three main sections own sets of properties:
| displayName | Acts as a brief label for display related to this | Should be relatively brief. The limit is 256 character, |
| | #metadataBlock. | but very long names might cause display problems. |
+----------------+---------------------------------------------------------+---------------------------------------------------------+
+| displayFacet | Label displayed in the search area when this | Should be brief. Long names will cause display problems |
+| | #metadataBlock is configured as a search facet | in the search area. |
+| | for a collection. See | |
+| | :ref:`the API `. | |
++----------------+---------------------------------------------------------+---------------------------------------------------------+
| blockURI | Associates the properties in a block with an external | The citation #metadataBlock has the blockURI |
| | URI. | https://dataverse.org/schema/citation/ which assigns a |
| | Properties will be assigned the | default global URI to terms such as |
@@ -408,13 +413,10 @@ Setting Up a Dev Environment for Testing
You have several options for setting up a dev environment for testing metadata block changes:
-- Vagrant: See the :doc:`/developers/tools` section of the Developer Guide.
-- docker-aio: See https://github.com/IQSS/dataverse/tree/develop/conf/docker-aio
+- Docker: See :doc:`/container/index`.
- AWS deployment: See the :doc:`/developers/deployment` section of the Developer Guide.
- Full dev environment: See the :doc:`/developers/dev-environment` section of the Developer Guide.
-To get a clean environment in Vagrant, you'll be running ``vagrant destroy``. In Docker, you'll use ``docker rm``. For a full dev environment or AWS installation, you might find ``rebuild`` and related scripts at ``scripts/deploy/phoenix.dataverse.org`` useful.
-
Editing TSV files
~~~~~~~~~~~~~~~~~
@@ -452,12 +454,16 @@ metadatablock.name=(the value of **name** property from #metadatablock)
metadatablock.displayName=(the value of **displayName** property from #metadatablock)
+metadatablock.displayFacet=(the value of **displayFacet** property from #metadatablock)
+
example:
metadatablock.name=citation
metadatablock.displayName=Citation Metadata
+metadatablock.displayFacet=Citation
+
#datasetField (field) properties
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
datasetfieldtype.(the value of **name** property from #datasetField).title=(the value of **title** property from #datasetField)
@@ -507,7 +513,7 @@ the Solr schema configuration, including any enabled metadata schemas:
``curl "http://localhost:8080/api/admin/index/solr/schema"``
-You can use :download:`update-fields.sh <../../../../conf/solr/8.11.1/update-fields.sh>` to easily add these to the
+You can use :download:`update-fields.sh <../../../../conf/solr/9.3.0/update-fields.sh>` to easily add these to the
Solr schema you installed for your Dataverse installation.
The script needs a target XML file containing your Solr schema. (See the :doc:`/installation/prerequisites/` section of
@@ -531,7 +537,7 @@ from some place else than your Dataverse installation).
Please note that reconfigurations of your Solr index might require a re-index. Usually release notes indicate
a necessary re-index, but for your custom metadata you will need to keep track on your own.
-Please note also that if you are going to make a pull request updating ``conf/solr/8.11.1/schema.xml`` with fields you have
+Please note also that if you are going to make a pull request updating ``conf/solr/9.3.0/schema.xml`` with fields you have
added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you
don't care about) to create a complete list of fields. (This might change in the future.)
@@ -579,6 +585,58 @@ The scripts required can be hosted locally or retrieved dynamically from https:/
Please note that in addition to the :ref:`:CVocConf` described above, an alternative is the :ref:`:ControlledVocabularyCustomJavaScript` setting.
+Protecting MetadataBlocks
+-------------------------
+
+Dataverse can be configured to only allow entries for a metadata block to be changed (created, edited, deleted) by entities that know a defined secret key.
+Metadata blocks protected by such a key are referred to as "System" metadata blocks.
+A primary use case for system metadata blocks is to handle metadata created by third-party tools interacting with Dataverse where unintended changes to the metadata could cause a failure. Examples might include archiving systems or workflow engines.
+To protect an existing metadatablock, one must set a key (recommended to be long and un-guessable) for that block:
+
+dataverse.metadata.block-system-metadata-keys.=
+
+This can be done using system properties (see :ref:`jvm-options`), environment variables or other MicroProfile Config mechanisms supported by the app server.
+ `See Payara docs for supported sources `_. Note that a Payara restart may be required to enable the new option.
+
+For these secret keys, Payara password aliases are recommended.
+
+ Alias creation example using the codemeta metadata block (actual name: codeMeta20):
+
+ .. code-block:: shell
+
+ echo "AS_ADMIN_ALIASPASSWORD=1234ChangeMeToSomethingLong" > /tmp/key.txt
+ asadmin create-password-alias --passwordfile /tmp/key.txt dataverse.metadata.block-system-metadata-keys.codeMeta20
+ rm /tmp/key.txt
+
+ Alias deletion example for the codemeta metadata block (removes protected status)
+
+ .. code-block:: shell
+
+ asadmin delete-password-alias dataverse.metadata.block-system-metadata-keys.codeMeta20
+
+A Payara restart is required after these example commands.
+
+When protected via a key, a metadata block will not be shown in the user interface when a dataset is being created or when metadata is being edited. Entries in such a system metadata block will be shown to users, consistent with Dataverse's design in which all metadata in published datasets is publicly visible.
+
+Note that protecting a block with required fields, or using a template with an entry in a protected block, will make it impossible to create a new dataset via the user interface. Also note that for this reason protecting the citation metadatablock is not recommended. (Creating a dataset also automatically sets the date of deposit field in the citation block, which would be prohibited if the citation block is protected.)
+
+To remove proted status and return a block to working normally, remove the associated key.
+
+To add metadata to a system metadata block via API, one must include an additional key of the form
+
+mdkey.=
+
+as an HTTP Header or query parameter (case sensitive) for each system metadata block to any API call in which metadata values are changed in that block. Multiple keys are allowed if more than one system metadatablock is being changed in a given API call.
+
+For example, following the :ref:`Add Dataset Metadata ` example from the :doc:`/developers/dataset-semantic-metadata-api`:
+
+.. code-block:: bash
+
+ curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -H 'mdkey.codeMeta20:1234ChangeMeToSomethingLong' -d '{"codeVersion": "1.0.0", "@context":{"codeVersion": "https://schema.org/softwareVersion"}}' "$SERVER_URL/api/datasets/$DATASET_ID/metadata"
+
+ curl -X PUT -H X-Dataverse-key:$API_TOKEN -H 'Content-Type: application/ld+json' -d '{"codeVersion": "1.0.1", "@context":{"codeVersion": "https://schema.org/softwareVersion"}}' "$SERVER_URL/api/datasets/$DATASET_ID/metadata?mdkey.codeMeta20=1234ChangeMeToSomethingLong&replace=true"
+
+
Tips from the Dataverse Community
---------------------------------
diff --git a/doc/sphinx-guides/source/admin/monitoring.rst b/doc/sphinx-guides/source/admin/monitoring.rst
index a4affda1302..04fba23a3e8 100644
--- a/doc/sphinx-guides/source/admin/monitoring.rst
+++ b/doc/sphinx-guides/source/admin/monitoring.rst
@@ -1,7 +1,7 @@
Monitoring
===========
-Once you're in production, you'll want to set up some monitoring. This page may serve as a starting point for you but you are encouraged to share your ideas with the Dataverse community!
+Once you're in production, you'll want to set up some monitoring. This page may serve as a starting point for you but you are encouraged to share your ideas with the Dataverse community! You may also be interested in the :doc:`/developers/performance` section of the Developer Guide.
.. contents:: Contents:
:local:
@@ -14,7 +14,7 @@ In production you'll want to monitor the usual suspects such as CPU, memory, fre
Munin
+++++
-http://munin-monitoring.org says, "A default installation provides a lot of graphs with almost no work." From RHEL or CentOS 7, you can try the following steps.
+https://munin-monitoring.org says, "A default installation provides a lot of graphs with almost no work." From RHEL or CentOS 7, you can try the following steps.
Enable the EPEL yum repo (if you haven't already):
diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst
index 9f085ba90cd..acbdcaae17e 100644
--- a/doc/sphinx-guides/source/admin/troubleshooting.rst
+++ b/doc/sphinx-guides/source/admin/troubleshooting.rst
@@ -53,15 +53,13 @@ Long-Running Ingest Jobs Have Exhausted System Resources
Ingest is both CPU- and memory-intensive, and depending on your system resources and the size and format of tabular data files uploaded, may render your Dataverse installation unresponsive or nearly inoperable. It is possible to cancel these jobs by purging the ingest queue.
-``/usr/local/payara5/mq/bin/imqcmd -u admin query dst -t q -n DataverseIngest`` will query the DataverseIngest destination. The password, unless you have changed it, matches the username.
+``/usr/local/payara6/mq/bin/imqcmd -u admin query dst -t q -n DataverseIngest`` will query the DataverseIngest destination. The password, unless you have changed it, matches the username.
-``/usr/local/payara5/mq/bin/imqcmd -u admin purge dst -t q -n DataverseIngest`` will purge the DataverseIngest queue, and prompt for your confirmation.
+``/usr/local/payara6/mq/bin/imqcmd -u admin purge dst -t q -n DataverseIngest`` will purge the DataverseIngest queue, and prompt for your confirmation.
Finally, list destinations to verify that the purge was successful:
-``/usr/local/payara5/mq/bin/imqcmd -u admin list dst``
-
-If you are still running Glassfish, substitute glassfish4 for payara5 above. If you have installed your Dataverse installation in some other location, adjust the above paths accordingly.
+``/usr/local/payara6/mq/bin/imqcmd -u admin list dst``
.. _troubleshooting-payara:
@@ -73,7 +71,7 @@ Payara
Finding the Payara Log File
~~~~~~~~~~~~~~~~~~~~~~~~~~~
-``/usr/local/payara5/glassfish/domains/domain1/logs/server.log`` is the main place to look when you encounter problems (assuming you installed Payara in the default directory). Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version or commit. Send more of the stack trace (the entire file if possible) to developers who can help (see "Getting Help", below) and be sure to say which version of the Dataverse Software you have installed.
+``/usr/local/payara6/glassfish/domains/domain1/logs/server.log`` is the main place to look when you encounter problems (assuming you installed Payara in the default directory). Hopefully an error message has been logged. If there's a stack trace, it may be of interest to developers, especially they can trace line numbers back to a tagged version or commit. Send more of the stack trace (the entire file if possible) to developers who can help (see "Getting Help", below) and be sure to say which version of the Dataverse Software you have installed.
.. _increase-payara-logging:
diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst
index a498c62d3d4..44db666736c 100755
--- a/doc/sphinx-guides/source/api/apps.rst
+++ b/doc/sphinx-guides/source/api/apps.rst
@@ -94,6 +94,13 @@ This series of Python scripts offers a starting point for migrating datasets fro
https://github.com/scholarsportal/dataverse-migration-scripts
+idsc.dataverse
+~~~~~~~~~~~~~~
+
+This module can, among others, help you migrate one dataverse to another. (see `migrate.md `_)
+
+https://github.com/iza-institute-of-labor-economics/idsc.dataverse
+
Java
----
diff --git a/doc/sphinx-guides/source/api/auth.rst b/doc/sphinx-guides/source/api/auth.rst
index bbc81b595e3..eae3bd3c969 100644
--- a/doc/sphinx-guides/source/api/auth.rst
+++ b/doc/sphinx-guides/source/api/auth.rst
@@ -77,6 +77,11 @@ To test if bearer tokens are working, you can try something like the following (
.. code-block:: bash
- export TOKEN=`curl -s -X POST --location "http://keycloak.mydomain.com:8090/realms/oidc-realm/protocol/openid-connect/token" -H "Content-Type: application/x-www-form-urlencoded" -d "username=kcuser&password=kcpassword&grant_type=password&client_id=oidc-client&client_secret=ss6gE8mODCDfqesQaSG3gwUwZqZt547E" | jq '.access_token' -r | tr -d "\n"`
+ export TOKEN=`curl -s -X POST --location "http://keycloak.mydomain.com:8090/realms/test/protocol/openid-connect/token" -H "Content-Type: application/x-www-form-urlencoded" -d "username=user&password=user&grant_type=password&client_id=test&client_secret=94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8" | jq '.access_token' -r | tr -d "\n"`
curl -H "Authorization: Bearer $TOKEN" http://localhost:8080/api/users/:me
+
+Signed URLs
+-----------
+
+See :ref:`signed-urls`.
diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst
new file mode 100644
index 00000000000..d272086fa2e
--- /dev/null
+++ b/doc/sphinx-guides/source/api/changelog.rst
@@ -0,0 +1,24 @@
+API Changelog (Breaking Changes)
+================================
+
+This API changelog is experimental and we would love feedback on its usefulness. Its primary purpose is to inform API developers of any breaking changes. (We try not ship any backward incompatible changes, but it happens.) To see a list of new APIs and backward-compatible changes to existing API, please see each version's release notes at https://github.com/IQSS/dataverse/releases
+
+.. contents:: |toctitle|
+ :local:
+ :depth: 1
+
+v6.2
+----
+
+- **/api/datasets/{id}/versions/{versionId}**: The includeFiles parameter has been renamed to excludeFiles. The default behavior remains the same, which is to include files. However, when excludeFiles is set to true, the files will be excluded. A bug that caused the API to only return a deaccessioned dataset if the user had edit privileges has been fixed.
+- **/api/datasets/{id}/versions**: The includeFiles parameter has been renamed to excludeFiles. The default behavior remains the same, which is to include files. However, when excludeFiles is set to true, the files will be excluded.
+
+v6.1
+----
+
+- The metadata field "Alternative Title" now supports multiple values so you must pass an array rather than a string when populating that field via API. See https://github.com/IQSS/dataverse/pull/9440
+
+v6.0
+----
+
+- **/api/access/datafile**: When a null or invalid API token is provided to download a public (non-restricted) file with this API call, it will result on a ``401`` error response. Previously, the download was allowed (``200`` response). Please note that we noticed this change sometime between 5.9 and 6.0. If you can help us pinpoint the exact version (or commit!), please get in touch. See :doc:`dataaccess`.
\ No newline at end of file
diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst
index 9efcf7eb153..bd0aa55ba99 100755
--- a/doc/sphinx-guides/source/api/client-libraries.rst
+++ b/doc/sphinx-guides/source/api/client-libraries.rst
@@ -15,12 +15,16 @@ https://github.com/aeonSolutions/OpenScience-Dataverse-API-C-library is the offi
This C/C++ library was created and is currently maintained by `Miguel T. `_ To learn how to install and use it, see the project's `wiki page `_.
+Go
+--
+https://github.com/libis/rdm-dataverse-go-api is Go API library that can be used in your project by simply adding ``github.com/libis/rdm-dataverse-go-api`` as a dependency in your ``go.mod`` file. See the GitHub page for more details and usage examples.
+
Java
----
https://github.com/IQSS/dataverse-client-java is the official Java library for Dataverse APIs.
-`Richard Adams `_ from `ResearchSpace `_ created and maintains this library.
+`Richard Adams `_ from `ResearchSpace `_ created and maintains this library.
Javascript
----------
@@ -48,20 +52,25 @@ There are multiple Python modules for interacting with Dataverse APIs.
`EasyDataverse `_ is a Python library designed to simplify the management of Dataverse datasets in an object-oriented way, giving users the ability to upload, download, and update datasets with ease. By utilizing metadata block configurations, EasyDataverse automatically generates Python objects that contain all the necessary details required to create the native Dataverse JSON format used to create or edit datasets. Adding files and directories is also possible with EasyDataverse and requires no additional API calls. This library is particularly well-suited for client applications such as workflows and scripts as it minimizes technical complexities and facilitates swift development.
-`pyDataverse `_ primarily allows developers to manage Dataverse collections, datasets and datafiles. Its intention is to help with data migrations and DevOps activities such as testing and configuration management. The module is developed by `Stefan Kasberger `_ from `AUSSDA - The Austrian Social Science Data Archive `_.
+`python-dvuploader `_ implements Jim Myers' excellent `dv-uploader `_ as a Python module. It offers parallel direct uploads to Dataverse backend storage, streams files directly instead of buffering them in memory, and supports multi-part uploads, chunking data accordingly.
+
+`pyDataverse `_ primarily allows developers to manage Dataverse collections, datasets and datafiles. Its intention is to help with data migrations and DevOps activities such as testing and configuration management. The module is developed by `Stefan Kasberger `_ from `AUSSDA - The Austrian Social Science Data Archive `_.
+
+`UBC's Dataverse Utilities `_ are a set of Python console utilities which allow one to upload datasets from a tab-separated-value spreadsheet, bulk release multiple datasets, bulk delete unpublished datasets, quickly duplicate records. replace licenses, and more. For additional information see their `PyPi page `_.
`dataverse-client-python `_ had its initial release in 2015. `Robert Liebowitz `_ created this library while at the `Center for Open Science (COS) `_ and the COS uses it to integrate the `Open Science Framework (OSF) `_ with Dataverse installations via an add-on which itself is open source and listed on the :doc:`/api/apps` page.
`Pooch `_ is a Python library that allows library and application developers to download data. Among other features, it takes care of various protocols, caching in OS-specific locations, checksum verification and adds optional features like progress bars or log messages. Among other popular repositories, Pooch supports Dataverse in the sense that you can reference Dataverse-hosted datasets by just a DOI and Pooch will determine the data repository type, query the Dataverse API for contained files and checksums, giving you an easy interface to download them.
+`idsc.dataverse `_ reads metadata and files of datasets from a dataverse dataverse.example1.com and writes them into ~/.idsc/dataverse/api/dataverse.example1.com organized in directories PID_type/prefix/suffix, where PID_type is one of: hdl, doi or ark. It can then ''export'' the local copy of the dataverse from ~/.idsc/dataverse/api/dataverse.example1.com to ~/.idsc/.cache/dataverse.example2.com so that one can upload them to dataverse.example2.com.
+
R
-
https://github.com/IQSS/dataverse-client-r is the official R package for Dataverse APIs. The latest release can be installed from `CRAN `_.
The R client can search and download datasets. It is useful when automatically (instead of manually) downloading data files as part of a script. For bulk edit and upload operations, we currently recommend pyDataverse.
-The package is currently maintained by `Shiro Kuriwaki `_. It was originally created by `Thomas Leeper `_ and then formerly maintained by `Will Beasley `_.
-
+The package is currently maintained by `Shiro Kuriwaki `_. It was originally created by `Thomas Leeper `_ and then formerly maintained by `Will Beasley `_.
Ruby
----
diff --git a/doc/sphinx-guides/source/api/curation-labels.rst b/doc/sphinx-guides/source/api/curation-labels.rst
index 36950a37eb3..0675eeec398 100644
--- a/doc/sphinx-guides/source/api/curation-labels.rst
+++ b/doc/sphinx-guides/source/api/curation-labels.rst
@@ -93,3 +93,22 @@ To get the list of allowed curation labels allowed for a given Dataset
curl -H X-Dataverse-key:$API_TOKEN "$SERVER_URL/api/datasets/:persistentId/allowedCurationLabels?persistentId=$DATASET_PID"
You should expect a 200 ("OK") response with a comma-separated list of allowed labels contained in a JSON 'data' object.
+
+
+Get a Report on the Curation Status of All Datasets
+---------------------------------------------------
+
+To get a CSV file listing the curation label assigned to each Dataset with a draft version, along with the creation and last modification dates, and list of those with permissions to publish the version.
+
+This API call is restricted to superusers.
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+
+ Example: Get the report
+
+ curl -H X-Dataverse-key:$API_TOKEN "$SERVER_URL/api/datasets/listCurationStates"
+
+You should expect a 200 ("OK") response with a CSV formatted response.
diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst
index e76ea167587..f7aaa8f4ee4 100755
--- a/doc/sphinx-guides/source/api/dataaccess.rst
+++ b/doc/sphinx-guides/source/api/dataaccess.rst
@@ -83,7 +83,7 @@ Basic access URI:
``/api/access/datafile/$id``
-.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``.
+.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``. However, this file access method is only effective when the FilePIDsEnabled option is enabled, which can be authorized by the admin. For further information, refer to :ref:`:FilePIDsEnabled`.
Example: Getting the file whose DOI is *10.5072/FK2/J8SJZB* ::
@@ -403,3 +403,32 @@ This method returns a list of Authenticated Users who have requested access to t
A curl example using an ``id``::
curl -H "X-Dataverse-key:$API_TOKEN" -X GET http://$SERVER/api/access/datafile/{id}/listRequests
+
+User Has Requested Access to a File:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``/api/access/datafile/{id}/userFileAccessRequested``
+
+This method returns true or false depending on whether or not the calling user has requested access to a particular file.
+
+A curl example using an ``id``::
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X GET "http://$SERVER/api/access/datafile/{id}/userFileAccessRequested"
+
+
+Get User Permissions on a File:
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``/api/access/datafile/{id}/userPermissions``
+
+This method returns the permissions that the calling user has on a particular file.
+
+In particular, the user permissions that this method checks, returned as booleans, are the following:
+
+* Can download the file
+* Can manage the file permissions
+* Can edit the file owner dataset
+
+A curl example using an ``id``::
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X GET "http://$SERVER/api/access/datafile/{id}/userPermissions"
diff --git a/doc/sphinx-guides/source/api/external-tools.rst b/doc/sphinx-guides/source/api/external-tools.rst
index eec9944338f..ae0e44b36aa 100644
--- a/doc/sphinx-guides/source/api/external-tools.rst
+++ b/doc/sphinx-guides/source/api/external-tools.rst
@@ -11,7 +11,7 @@ Introduction
External tools are additional applications the user can access or open from your Dataverse installation to preview, explore, and manipulate data files and datasets. The term "external" is used to indicate that the tool is not part of the main Dataverse Software.
-Once you have created the external tool itself (which is most of the work!), you need to teach a Dataverse installation how to construct URLs that your tool needs to operate. For example, if you've deployed your tool to fabulousfiletool.com your tool might want the ID of a file and the siteUrl of the Dataverse installation like this: https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org
+Once you have created the external tool itself (which is most of the work!), you need to teach a Dataverse installation how to construct URLs that your tool needs to operate. For example, if you've deployed your tool to fabulousfiletool.com your tool might want the ID of a file and the siteUrl of the Dataverse installation like this: https://fabulousfiletool.com?fileId=42&siteUrl=https://demo.dataverse.org
In short, you will be creating a manifest in JSON format that describes not only how to construct URLs for your tool, but also what types of files your tool operates on, where it should appear in the Dataverse installation web interfaces, etc.
@@ -39,8 +39,8 @@ How External Tools Are Presented to Users
An external tool can appear in your Dataverse installation in a variety of ways:
-- as an explore, preview, or configure option for a file
-- as an explore option for a dataset
+- as an explore, preview, query or configure option for a file
+- as an explore or configure option for a dataset
- as an embedded preview on the file landing page
See also the :ref:`testing-external-tools` section of the Admin Guide for some perspective on how Dataverse installations will expect to test your tool before announcing it to their users.
@@ -88,11 +88,11 @@ Terminology
displayName The **name** of the tool in the Dataverse installation web interface. For example, "Data Explorer".
- description The **description** of the tool, which appears in a popup (for configure tools only) so the user who clicked the tool can learn about the tool before being redirected the tool in a new tab in their browser. HTML is supported.
+ description The **description** of the tool, which appears in a popup (for configure tools only) so the user who clicked the tool can learn about the tool before being redirected to the tool in a new tab in their browser. HTML is supported.
scope Whether the external tool appears and operates at the **file** level or the **dataset** level. Note that a file level tool much also specify the type of file it operates on (see "contentType" below).
- types Whether the external tool is an **explore** tool, a **preview** tool, a **configure** tool or any combination of these (multiple types are supported for a single tool). Configure tools require an API token because they make changes to data files (files within datasets). Configure tools are currently not supported at the dataset level. The older "type" keyword that allows you to pass a single type as a string is deprecated but still supported.
+ types Whether the external tool is an **explore** tool, a **preview** tool, a **query** tool, a **configure** tool or any combination of these (multiple types are supported for a single tool). Configure tools require an API token because they make changes to data files (files within datasets). The older "type" keyword that allows you to pass a single type as a string is deprecated but still supported.
toolUrl The **base URL** of the tool before query parameters are added.
@@ -102,7 +102,7 @@ Terminology
httpMethod Either ``GET`` or ``POST``.
- queryParameters **Key/value combinations** that can be appended to the toolUrl. For example, once substitution takes place (described below) the user may be redirected to ``https://fabulousfiletool.com?fileId=42&siteUrl=http://demo.dataverse.org``.
+ queryParameters **Key/value combinations** that can be appended to the toolUrl. For example, once substitution takes place (described below) the user may be redirected to ``https://fabulousfiletool.com?fileId=42&siteUrl=https://demo.dataverse.org``.
query parameter keys An **arbitrary string** to associate with a value that is populated with a reserved word (described below). As the author of the tool, you have control over what "key" you would like to be passed to your tool. For example, if you want to have your tool receive and operate on the query parameter "dataverseFileId=42" instead of just "fileId=42", that's fine.
@@ -160,17 +160,25 @@ Authorization Options
When called for datasets or data files that are not public (i.e. in a draft dataset or for a restricted file), external tools are allowed access via the user's credentials. This is accomplished by one of two mechanisms:
-* Signed URLs (more secure, recommended)
+.. _signed-urls:
- - Configured via the ``allowedApiCalls`` section of the manifest. The tool will be provided with signed URLs allowing the specified access to the given dataset or datafile for the specified amount of time. The tool will not be able to access any other datasets or files the user may have access to and will not be able to make calls other than those specified.
- - For tools invoked via a GET call, Dataverse will include a callback query parameter with a Base64 encoded value. The decoded value is a signed URL that can be called to retrieve a JSON response containing all of the queryParameters and allowedApiCalls specified in the manfiest.
- - For tools invoked via POST, Dataverse will send a JSON body including the requested queryParameters and allowedApiCalls. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool.
+Signed URLs
+^^^^^^^^^^^
-* API Token (deprecated, less secure, not recommended)
+The signed URL mechanism is more secure than exposing API tokens and therefore recommended.
- - Configured via the ``queryParameters`` by including an ``{apiToken}`` value. When this is present Dataverse will send the user's apiToken to the tool. With the user's API token, the tool can perform any action via the Dataverse API that the user could. External tools configured via this method should be assessed for their trustworthiness.
- - For tools invoked via GET, this will be done via a query parameter in the request URL which could be cached in the browser's history. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool.
- - For tools invoked via POST, Dataverse will send a JSON body including the apiToken.
+- Configured via the ``allowedApiCalls`` section of the manifest. The tool will be provided with signed URLs allowing the specified access to the given dataset or datafile for the specified amount of time. The tool will not be able to access any other datasets or files the user may have access to and will not be able to make calls other than those specified.
+- For tools invoked via a GET call, Dataverse will include a callback query parameter with a Base64 encoded value. The decoded value is a signed URL that can be called to retrieve a JSON response containing all of the queryParameters and allowedApiCalls specified in the manfiest.
+- For tools invoked via POST, Dataverse will send a JSON body including the requested queryParameters and allowedApiCalls. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool.
+
+API Token
+^^^^^^^^^
+
+The API token mechanism is deprecated. Because it is less secure than signed URLs, it is not recommended for new external tools.
+
+- Configured via the ``queryParameters`` by including an ``{apiToken}`` value. When this is present Dataverse will send the user's apiToken to the tool. With the user's API token, the tool can perform any action via the Dataverse API that the user could. External tools configured via this method should be assessed for their trustworthiness.
+- For tools invoked via GET, this will be done via a query parameter in the request URL which could be cached in the browser's history. Dataverse expects the response to the POST to indicate a redirect which Dataverse will use to open the tool.
+- For tools invoked via POST, Dataverse will send a JSON body including the apiToken.
Internationalization of Your External Tool
++++++++++++++++++++++++++++++++++++++++++
@@ -187,6 +195,7 @@ Using Example Manifests to Get Started
++++++++++++++++++++++++++++++++++++++
Again, you can use :download:`fabulousFileTool.json <../_static/installation/files/root/external-tools/fabulousFileTool.json>` or :download:`dynamicDatasetTool.json <../_static/installation/files/root/external-tools/dynamicDatasetTool.json>` as a starting point for your own manifest file.
+Additional working examples, including ones using :ref:`signed-urls`, are available at https://github.com/gdcc/dataverse-previewers .
Testing Your External Tool
--------------------------
diff --git a/doc/sphinx-guides/source/api/getting-started.rst b/doc/sphinx-guides/source/api/getting-started.rst
index c465b726421..a50f12d1381 100644
--- a/doc/sphinx-guides/source/api/getting-started.rst
+++ b/doc/sphinx-guides/source/api/getting-started.rst
@@ -9,9 +9,9 @@ If you are a researcher or curator who wants to automate parts of your workflow,
Servers You Can Test With
-------------------------
-Rather than using a production Dataverse installation, API users are welcome to use http://demo.dataverse.org for testing. You can email support@dataverse.org if you have any trouble with this server.
+Rather than using a production Dataverse installation, API users are welcome to use https://demo.dataverse.org for testing. You can email support@dataverse.org if you have any trouble with this server.
-If you would rather have full control over your own test server, deployments to AWS, Docker, Vagrant, and more are covered in the :doc:`/developers/index` and the :doc:`/installation/index`.
+If you would rather have full control over your own test server, deployments to AWS, Docker, and more are covered in the :doc:`/developers/index` and the :doc:`/installation/index`.
Getting an API Token
--------------------
@@ -52,6 +52,20 @@ If you ever want to check an environment variable, you can "echo" it like this:
echo $SERVER_URL
+With curl version 7.56.0 and higher, it is recommended to use --form-string with outer quote rather than -F flag without outer quote.
+
+For example, curl command parameter below might cause error such as ``warning: garbage at end of field specification: ,"categories":["Data"]}``.
+
+.. code-block:: bash
+
+ -F jsonData={\"description\":\"My description.\",\"categories\":[\"Data\"]}
+
+Instead, use --form-string with outer quote. See https://github.com/curl/curl/issues/2022
+
+.. code-block:: bash
+
+ --form-string 'jsonData={"description":"My description.","categories":["Data"]}'
+
If you don't like curl, don't have curl, or want to use a different programming language, you are encouraged to check out the Python, Javascript, R, and Java options in the :doc:`client-libraries` section.
.. _curl: https://curl.haxx.se
diff --git a/doc/sphinx-guides/source/api/index.rst b/doc/sphinx-guides/source/api/index.rst
index c9e79098546..dd195aa9d62 100755
--- a/doc/sphinx-guides/source/api/index.rst
+++ b/doc/sphinx-guides/source/api/index.rst
@@ -24,3 +24,4 @@ API Guide
linkeddatanotification
apps
faq
+ changelog
\ No newline at end of file
diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst
index 933932cd7b9..8eb11798dd7 100755
--- a/doc/sphinx-guides/source/api/intro.rst
+++ b/doc/sphinx-guides/source/api/intro.rst
@@ -187,6 +187,10 @@ Lists of Dataverse APIs
- Files
- etc.
+- :doc:`/developers/dataset-semantic-metadata-api`: For creating, reading, editing, and deleting dataset metadata using JSON-LD.
+- :doc:`/developers/dataset-migration-api`: For migrating datasets from other repositories while retaining the original persistent identifiers and publication date.
+- :doc:`/developers/s3-direct-upload-api`: For the transfer of larger files/larger numbers of files directly to an S3 bucket managed by Dataverse.
+- :doc:`/developers/globus-api`: For the Globus transfer of larger files/larger numbers of files directly via Globus endpoints managed by Dataverse or referencing files in remote endpoints.
- :doc:`metrics`: For query statistics about usage of a Dataverse installation.
- :doc:`sword`: For depositing data using a standards-based approach rather than the :doc:`native-api`.
@@ -237,7 +241,7 @@ Dataverse Software API questions are on topic in all the usual places:
- The dataverse-community Google Group: https://groups.google.com/forum/#!forum/dataverse-community
- The Dataverse Project community calls: https://dataverse.org/community-calls
-- The Dataverse Project chat room: http://chat.dataverse.org
+- The Dataverse Project chat room: https://chat.dataverse.org
- The Dataverse Project ticketing system: support@dataverse.org
After your question has been answered, you are welcome to help improve the :doc:`faq` section of this guide.
diff --git a/doc/sphinx-guides/source/api/metrics.rst b/doc/sphinx-guides/source/api/metrics.rst
index 28ac33ea228..613671e49d1 100755
--- a/doc/sphinx-guides/source/api/metrics.rst
+++ b/doc/sphinx-guides/source/api/metrics.rst
@@ -163,3 +163,10 @@ The following table lists the available metrics endpoints (not including the Mak
/api/info/metrics/uniquefiledownloads/toMonth/{yyyy-MM},"count by id, pid","json, csv",collection subtree,published,y,cumulative up to month specified,unique download counts per file id to the specified month. PIDs are also included in output if they exist
/api/info/metrics/tree,"id, ownerId, alias, depth, name, children",json,collection subtree,published,y,"tree of dataverses starting at the root or a specified parentAlias with their id, owner id, alias, name, a computed depth, and array of children dataverses","underlying code can also include draft dataverses, this is not currently accessible via api, depth starts at 0"
/api/info/metrics/tree/toMonth/{yyyy-MM},"id, ownerId, alias, depth, name, children",json,collection subtree,published,y,"tree of dataverses in existence as of specified date starting at the root or a specified parentAlias with their id, owner id, alias, name, a computed depth, and array of children dataverses","underlying code can also include draft dataverses, this is not currently accessible via api, depth starts at 0"
+
+Related API Endpoints
+---------------------
+
+The following endpoints are not under the metrics namespace but also return counts:
+
+- :ref:`file-download-count`
diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst
index 369e92ba129..6591c983824 100644
--- a/doc/sphinx-guides/source/api/native-api.rst
+++ b/doc/sphinx-guides/source/api/native-api.rst
@@ -9,7 +9,7 @@ The Dataverse Software exposes most of its GUI functionality via a REST-based AP
.. _CORS: https://www.w3.org/TR/cors/
-.. warning:: The Dataverse Software's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded.
+.. warning:: The Dataverse Software's API is versioned at the URI - all API calls may include the version number like so: ``https://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded.
.. contents:: |toctitle|
:local:
@@ -56,13 +56,13 @@ Next you need to figure out the alias or database id of the "parent" Dataverse c
export SERVER_URL=https://demo.dataverse.org
export PARENT=root
- curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$PARENT --upload-file dataverse-complete.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$PARENT" --upload-file dataverse-complete.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root --upload-file dataverse-complete.json
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/dataverses/root" --upload-file dataverse-complete.json
You should expect an HTTP 200 response and JSON beginning with "status":"OK" followed by a representation of the newly-created Dataverse collection.
@@ -80,13 +80,13 @@ To view a published Dataverse collection:
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl $SERVER_URL/api/dataverses/$ID
+ curl "$SERVER_URL/api/dataverses/$ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/dataverses/root
+ curl "https://demo.dataverse.org/api/dataverses/root"
To view an unpublished Dataverse collection:
@@ -96,13 +96,13 @@ To view an unpublished Dataverse collection:
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root"
Delete a Dataverse Collection
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -117,13 +117,13 @@ Deletes the Dataverse collection whose database ID or alias is given:
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN -X DELETE $SERVER_URL/api/dataverses/$ID
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/dataverses/$ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X DELETE https://demo.dataverse.org/api/dataverses/root
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root"
.. _show-contents-of-a-dataverse-api:
@@ -140,13 +140,13 @@ Show Contents of a Dataverse Collection
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/contents
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/contents"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/contents
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/contents"
Report the data (file) size of a Dataverse Collection
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -159,13 +159,13 @@ Shows the combined size in bytes of all the files uploaded into the Dataverse co
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/storagesize
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/storagesize"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/storagesize
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/storagesize"
The size of published and unpublished files will be summed both in the Dataverse collection specified and beneath all its sub-collections, recursively.
By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by the Dataverse installation - the resized thumbnail versions for image files, the metadata exports for published datasets, etc.
@@ -181,13 +181,13 @@ All the roles defined directly in the Dataverse collection identified by ``id``:
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/roles
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/roles"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/roles
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/roles"
List Facets Configured for a Dataverse Collection
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -200,13 +200,13 @@ List Facets Configured for a Dataverse Collection
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/facets
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/facets"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/facets
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/facets"
Set Facets for a Dataverse Collection
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -219,16 +219,18 @@ Assign search facets for a given Dataverse collection identified by ``id``:
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/dataverses/$ID/facets --upload-file dataverse-facets.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$ID/facets" --upload-file dataverse-facets.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/facets --upload-file dataverse-facets.json
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/dataverses/root/facets" --upload-file dataverse-facets.json
Where :download:`dataverse-facets.json <../_static/api/dataverse-facets.json>` contains a JSON encoded list of metadata keys (e.g. ``["authorName","authorAffiliation"]``).
+.. _metadata-block-facet-api:
+
List Metadata Block Facets Configured for a Dataverse Collection
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -240,13 +242,13 @@ List Metadata Block Facets Configured for a Dataverse Collection
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/metadatablockfacets
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/metadatablockfacets"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/metadatablockfacets
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/metadatablockfacets"
Set Metadata Block Facets for a Dataverse Collection
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -263,13 +265,13 @@ To clear the metadata blocks set by a parent collection, submit an empty array (
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN" -X POST -H "Content-type:application/json" $SERVER_URL/api/dataverses/$ID/metadatablockfacets --upload-file metadata-block-facets.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-type:application/json" "$SERVER_URL/api/dataverses/$ID/metadatablockfacets" --upload-file metadata-block-facets.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-type:application/json" https://demo.dataverse.org/api/dataverses/root/metadatablockfacets --upload-file metadata-block-facets.json
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-type:application/json" "https://demo.dataverse.org/api/dataverses/root/metadatablockfacets" --upload-file metadata-block-facets.json
Where :download:`metadata-block-facets.json <../_static/api/metadata-block-facets.json>` contains a JSON encoded list of metadata block names (e.g. ``["socialscience","geospatial"]``). This endpoint supports an empty list (e.g. ``[]``)
@@ -288,13 +290,15 @@ When updating the root to false, it will clear any metadata block facets from th
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN -X POST -H "Content-type:application/json" $SERVER_URL/api/dataverses/$ID/metadatablockfacets/isRoot -d 'true'
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-type:application/json" "$SERVER_URL/api/dataverses/$ID/metadatablockfacets/isRoot" -d 'true'
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-type:application/json" https://demo.dataverse.org/api/dataverses/root/metadatablockfacets/isRoot -d 'true'
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-type:application/json" "https://demo.dataverse.org/api/dataverses/root/metadatablockfacets/isRoot" -d 'true'
+
+.. _create-role-in-collection:
Create a New Role in a Dataverse Collection
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -307,24 +311,15 @@ Creates a new role under Dataverse collection ``id``. Needs a json file with the
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$ID/roles --upload-file roles.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$ID/roles" --upload-file roles.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-type:application/json" https://demo.dataverse.org/api/dataverses/root/roles --upload-file roles.json
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-type:application/json" "https://demo.dataverse.org/api/dataverses/root/roles" --upload-file roles.json
-Where ``roles.json`` looks like this::
-
- {
- "alias": "sys1",
- "name": “Restricted System Roleâ€,
- "description": “A person who may only add datasets.â€,
- "permissions": [
- "AddDataset"
- ]
- }
+For ``roles.json`` see :ref:`json-representation-of-a-role`
.. note:: Only a Dataverse installation account with superuser permissions is allowed to create roles in a Dataverse Collection.
@@ -341,13 +336,13 @@ List all the role assignments at the given Dataverse collection:
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/assignments
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/assignments"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/assignments
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/assignments"
Assign Default Role to User Creating a Dataset in a Dataverse Collection
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -361,13 +356,13 @@ Assign a default role to a user creating a dataset in a Dataverse collection ``i
export ID=root
export ROLE_ALIAS=curator
- curl -H X-Dataverse-key:$API_TOKEN -X PUT $SERVER_URL/api/dataverses/$ID/defaultContributorRole/$ROLE_ALIAS
+ curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/dataverses/$ID/defaultContributorRole/$ROLE_ALIAS"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X PUT https://demo.dataverse.org/api/dataverses/root/defaultContributorRole/curator
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/dataverses/root/defaultContributorRole/curator"
Note: You may use "none" as the ``ROLE_ALIAS``. This will prevent a user who creates a dataset from having any role on that dataset. It is not recommended for Dataverse collections with human contributors.
@@ -384,13 +379,13 @@ Assigns a new role, based on the POSTed JSON:
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN -X POST -H "Content-Type: application/json" $SERVER_URL/api/dataverses/$ID/assignments --upload-file role.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-Type: application/json" "$SERVER_URL/api/dataverses/$ID/assignments" --upload-file role.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-Type: application/json" https://demo.dataverse.org/api/dataverses/root/assignments --upload-file role.json
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-Type: application/json" "https://demo.dataverse.org/api/dataverses/root/assignments" --upload-file role.json
POSTed JSON example (the content of ``role.json`` file)::
@@ -413,13 +408,13 @@ Delete the assignment whose id is ``$id``:
export ID=root
export ASSIGNMENT_ID=6
- curl -H X-Dataverse-key:$API_TOKEN -X DELETE $SERVER_URL/api/dataverses/$ID/assignments/$ASSIGNMENT_ID
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/dataverses/$ID/assignments/$ASSIGNMENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X DELETE https://demo.dataverse.org/api/dataverses/root/assignments/6
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/dataverses/root/assignments/6"
List Metadata Blocks Defined on a Dataverse Collection
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -436,13 +431,13 @@ Please note that an API token is only required if the Dataverse collection has n
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/metadatablocks
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/metadatablocks"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/metadatablocks
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/metadatablocks"
Define Metadata Blocks for a Dataverse Collection
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -461,13 +456,13 @@ The metadata blocks that are available with a default Dataverse installation are
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$ID/metadatablocks -H \"Content-type:application/json\" --upload-file define-metadatablocks.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$ID/metadatablocks" -H \"Content-type:application/json\" --upload-file define-metadatablocks.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-type:application/json" --upload-file define-metadatablocks.json https://demo.dataverse.org/api/dataverses/root/metadatablocks
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-type:application/json" --upload-file define-metadatablocks.json "https://demo.dataverse.org/api/dataverses/root/metadatablocks"
Determine if a Dataverse Collection Inherits Its Metadata Blocks from Its Parent
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -480,13 +475,13 @@ Get whether the Dataverse collection is a metadata block root, or does it uses i
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/metadatablocks/isRoot
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/metadatablocks/isRoot"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/metadatablocks/isRoot
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/metadatablocks/isRoot"
Configure a Dataverse Collection to Inherit Its Metadata Blocks from Its Parent
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -500,16 +495,66 @@ values are ``true`` and ``false`` (both are valid JSON expressions):
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN -X PUT $SERVER_URL/api/dataverses/$ID/metadatablocks/isRoot
+ curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/dataverses/$ID/metadatablocks/isRoot"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/dataverses/root/metadatablocks/isRoot"
+
+.. note:: Previous endpoints ``$SERVER/api/dataverses/$id/metadatablocks/:isRoot`` and ``POST https://$SERVER/api/dataverses/$id/metadatablocks/:isRoot?key=$apiKey`` are deprecated, but supported.
+
+.. _get-dataset-json-schema:
+
+Retrieve a Dataset JSON Schema for a Collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Retrieves a JSON schema customized for a given collection in order to validate a dataset JSON file prior to creating the dataset. This
+first version of the schema only includes required elements and fields. In the future we plan to improve the schema by adding controlled
+vocabulary and more robust dataset field format testing:
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=root
+
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/datasetSchema"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X PUT https://demo.dataverse.org/api/dataverses/root/metadatablocks/isRoot
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/datasetSchema"
+
+Note: you must have "Add Dataset" permission in the given collection to invoke this endpoint.
+
+While it is recommended to download a copy of the JSON Schema from the collection (as above) to account for any fields that have been marked as required, you can also download a minimal :download:`dataset-schema.json <../_static/api/dataset-schema.json>` to get a sense of the schema when no customizations have been made.
+
+.. _validate-dataset-json:
+
+Validate Dataset JSON File for a Collection
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Validates a dataset JSON file customized for a given collection prior to creating the dataset. The validation only tests for json formatting
+and the presence of required elements:
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=root
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$ID/validateDatasetJson" -H 'Content-type:application/json' --upload-file dataset.json
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
-.. note:: Previous endpoints ``$SERVER/api/dataverses/$id/metadatablocks/:isRoot`` and ``POST http://$SERVER/api/dataverses/$id/metadatablocks/:isRoot?key=$apiKey`` are deprecated, but supported.
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/dataverses/root/validateDatasetJson" -H 'Content-type:application/json' --upload-file dataset.json
+Note: you must have "Add Dataset" permission in the given collection to invoke this endpoint.
.. _create-dataset-command:
@@ -530,10 +575,16 @@ Submit Incomplete Dataset
^^^^^^^^^^^^^^^^^^^^^^^^^
**Note:** This feature requires :ref:`dataverse.api.allow-incomplete-metadata` to be enabled and your Solr
-Schema to be up-to-date with the ``datasetValid`` field.
+Schema to be up-to-date with the ``datasetValid`` field. If not done yet with the version upgrade, you will
+also need to reindex all dataset after enabling the :ref:`dataverse.api.allow-incomplete-metadata` feature.
Providing a ``.../datasets?doNotValidate=true`` query parameter turns off the validation of metadata.
-In this case, only the "Author Name" is required. For example, a minimal JSON file would look like this:
+In this situation, only the "Author Name" is required, except for the case when the setting :ref:`:MetadataLanguages`
+is configured and the value of "Dataset Metadata Language" setting of a collection is left with the default
+"Chosen at Dataset Creation" value. In that case, a language that is a part of the :ref:`:MetadataLanguages` list must be
+declared in the incomplete dataset.
+
+For example, a minimal JSON file, without the language specification, would look like this:
.. code-block:: json
:name: dataset-incomplete.json
@@ -573,7 +624,7 @@ The following is an example HTTP call with deactivated validation:
export PARENT=root
export SERVER_URL=https://demo.dataverse.org
- curl -H X-Dataverse-key:$API_TOKEN -X POST "$SERVER_URL/api/dataverses/$PARENT/datasets?doNotValidate=true" --upload-file dataset-incomplete.json -H 'Content-type:application/json'
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$PARENT/datasets?doNotValidate=true" --upload-file dataset-incomplete.json -H 'Content-type:application/json'
**Note:** You may learn about an instance's support for deposition of incomplete datasets via :ref:`info-incomplete-metadata`.
@@ -594,7 +645,7 @@ Next you need to figure out the alias or database id of the "parent" Dataverse c
export PARENT=root
export SERVER_URL=https://demo.dataverse.org
- curl -H X-Dataverse-key:$API_TOKEN -X POST "$SERVER_URL/api/dataverses/$PARENT/datasets" --upload-file dataset-finch1.json -H 'Content-type:application/json'
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$PARENT/datasets" --upload-file dataset-finch1.json -H 'Content-type:application/json'
The fully expanded example above (without the environment variables) looks like this:
@@ -622,13 +673,13 @@ To import a dataset with an existing persistent identifier (PID), the dataset's
export DATAVERSE_ID=root
export PERSISTENT_IDENTIFIER=doi:ZZ7/MOSEISLEYDB94
- curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets/:import?pid=$PERSISTENT_IDENTIFIER&release=yes --upload-file dataset.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets/:import?pid=$PERSISTENT_IDENTIFIER&release=yes" --upload-file dataset.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/datasets/:import?pid=doi:ZZ7/MOSEISLEYDB94&release=yes --upload-file dataset.json
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/dataverses/root/datasets/:import?pid=doi:ZZ7/MOSEISLEYDB94&release=yes" --upload-file dataset.json
The ``pid`` parameter holds a persistent identifier (such as a DOI or Handle). The import will fail if no PID is provided, or if the provided PID fails validation.
@@ -663,13 +714,13 @@ To import a dataset with an existing persistent identifier (PID), you have to pr
export DATAVERSE_ID=root
export PERSISTENT_IDENTIFIER=doi:ZZ7/MOSEISLEYDB94
- curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets/:importddi?pid=$PERSISTENT_IDENTIFIER&release=yes --upload-file ddi_dataset.xml
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$DATAVERSE_ID/datasets/:importddi?pid=$PERSISTENT_IDENTIFIER&release=yes" --upload-file ddi_dataset.xml
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/datasets/:importddi?pid=doi:ZZ7/MOSEISLEYDB94&release=yes --upload-file ddi_dataset.xml
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/dataverses/root/datasets/:importddi?pid=doi:ZZ7/MOSEISLEYDB94&release=yes" --upload-file ddi_dataset.xml
The optional ``pid`` parameter holds a persistent identifier (such as a DOI or Handle). The import will fail if the provided PID fails validation.
@@ -699,13 +750,13 @@ In order to publish a Dataverse collection, you must know either its "alias" (wh
export SERVER_URL=https://demo.dataverse.org
export ID=root
- curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/dataverses/$ID/actions/:publish
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/dataverses/$ID/actions/:publish"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/dataverses/root/actions/:publish
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/dataverses/root/actions/:publish"
You should expect a 200 ("OK") response and JSON output.
@@ -728,13 +779,66 @@ In order to retrieve the Guestbook Responses for a Dataverse collection, you mus
export GUESTBOOK_ID=1
export FILENAME=myResponses.csv
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=$GUESTBOOK_ID -o $FILENAME
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/guestbookResponses?guestbookId=$GUESTBOOK_ID" -o $FILENAME
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/dataverses/root/guestbookResponses?guestbookId=1 -o myResponses.csv
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/dataverses/root/guestbookResponses?guestbookId=1" -o myResponses.csv
+
+.. _collection-attributes-api:
+
+Change Collection Attributes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block::
+
+ curl -X PUT -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/attribute/$ATTRIBUTE?value=$VALUE"
+
+The following attributes are supported:
+
+* ``alias`` Collection alias
+* ``name`` Name
+* ``description`` Description
+* ``affiliation`` Affiliation
+* ``filePIDsEnabled`` ("true" or "false") Restricted to use by superusers and only when the :ref:`:AllowEnablingFilePIDsPerCollection <:AllowEnablingFilePIDsPerCollection>` setting is true. Enables or disables registration of file-level PIDs in datasets within the collection (overriding the instance-wide setting).
+
+.. _collection-storage-quotas:
+
+Collection Storage Quotas
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block::
+
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/storage/quota"
+
+Will output the storage quota allocated (in bytes), or a message indicating that the quota is not defined for the specific collection. The user identified by the API token must have the ``Manage`` permission on the collection.
+
+
+To set or change the storage allocation quota for a collection:
+
+.. code-block::
+
+ curl -X PUT -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/storage/quota/$SIZE_IN_BYTES"
+
+This is API is superuser-only.
+
+
+To delete a storage quota configured for a collection:
+
+.. code-block::
+
+ curl -X DELETE -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/dataverses/$ID/storage/quota"
+
+This is API is superuser-only.
+
+Use the ``/settings`` API to enable or disable the enforcement of storage quotas that are defined across the instance via the following setting. For example,
+
+.. code-block::
+
+ curl -X PUT -d 'true' http://localhost:8080/api/admin/settings/:UseStorageQuotas
+
Datasets
--------
@@ -761,13 +865,13 @@ Example: Getting the dataset whose DOI is *10.5072/FK2/J8SJZB*:
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/datasets/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:$API_TOKEN" https://demo.dataverse.org/api/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB
+ curl -H "X-Dataverse-key:$API_TOKEN" "https://demo.dataverse.org/api/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB"
Getting its draft version:
@@ -776,13 +880,13 @@ Getting its draft version:
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB
- curl -H "X-Dataverse-key:$API_TOKEN" http://$SERVER/api/datasets/:persistentId/versions/:draft?persistentId=$PERSISTENT_IDENTIFIER
+ curl -H "X-Dataverse-key:$API_TOKEN" "https://$SERVER/api/datasets/:persistentId/versions/:draft?persistentId=$PERSISTENT_IDENTIFIER"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:$API_TOKEN" https://demo.dataverse.org/api/datasets/:persistentId/versions/:draft?persistentId=doi:10.5072/FK2/J8SJZB
+ curl -H "X-Dataverse-key:$API_TOKEN" "https://demo.dataverse.org/api/datasets/:persistentId/versions/:draft?persistentId=doi:10.5072/FK2/J8SJZB"
|CORS| Show the dataset whose database id is passed:
@@ -791,13 +895,13 @@ The fully expanded example above (without environment variables) looks like this
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl $SERVER_URL/api/datasets/$ID
+ curl "$SERVER_URL/api/datasets/$ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/datasets/24
+ curl "https://demo.dataverse.org/api/datasets/24"
The dataset id can be extracted from the response retrieved from the API which uses the persistent identifier (``/api/datasets/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER``).
@@ -811,13 +915,13 @@ List Versions of a Dataset
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl $SERVER_URL/api/datasets/$ID/versions
+ curl "$SERVER_URL/api/datasets/$ID/versions"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/datasets/24/versions
+ curl "https://demo.dataverse.org/api/datasets/24/versions"
It returns a list of versions with their metadata, and file list:
@@ -870,6 +974,10 @@ It returns a list of versions with their metadata, and file list:
]
}
+The optional ``includeFiles`` parameter specifies whether the files should be listed in the output. It defaults to ``true``, preserving backward compatibility. (Note that for a dataset with a large number of versions and/or files having the files included can dramatically increase the volume of the output). A separate ``/files`` API can be used for listing the files, or a subset thereof in a given version.
+
+The optional ``offset`` and ``limit`` parameters can be used to specify the range of the versions list to be shown. This can be used to paginate through the list in a dataset with a large number of versions.
+
Get Version of a Dataset
~~~~~~~~~~~~~~~~~~~~~~~~
@@ -882,13 +990,26 @@ Get Version of a Dataset
export ID=24
export VERSION=1.0
- curl $SERVER_URL/api/datasets/$ID/versions/$VERSION
+ curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION?includeFiles=false"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/datasets/24/versions/1.0
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0?includeFiles=false"
+
+The optional ``includeFiles`` parameter specifies whether the files should be listed in the output (defaults to ``true``). Note that a separate ``/files`` API can be used for listing the files, or a subset thereof in a given version.
+
+
+By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below.
+
+If you want to include deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0?includeDeaccessioned=true"
.. _export-dataset-metadata-api:
@@ -905,13 +1026,13 @@ See also :ref:`batch-exports-through-the-api` and the note below:
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB
export METADATA_FORMAT=ddi
- curl $SERVER_URL/api/datasets/export?exporter=$METADATA_FORMAT&persistentId=PERSISTENT_IDENTIFIER
+ curl "$SERVER_URL/api/datasets/export?exporter=$METADATA_FORMAT&persistentId=PERSISTENT_IDENTIFIER"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/datasets/export?exporter=ddi&persistentId=doi:10.5072/FK2/J8SJZB
+ curl "https://demo.dataverse.org/api/datasets/export?exporter=ddi&persistentId=doi:10.5072/FK2/J8SJZB"
.. note:: Supported exporters (export formats) are ``ddi``, ``oai_ddi``, ``dcterms``, ``oai_dc``, ``schema.org`` , ``OAI_ORE`` , ``Datacite``, ``oai_datacite`` and ``dataverse_json``. Descriptive names can be found under :ref:`metadata-export-formats` in the User Guide.
@@ -937,13 +1058,183 @@ List Files in a Dataset
export ID=24
export VERSION=1.0
- curl $SERVER_URL/api/datasets/$ID/versions/$VERSION/files
+ curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION/files"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/datasets/24/versions/1.0/files
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files"
+
+This endpoint supports optional pagination, through the ``limit`` and ``offset`` query parameters:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?limit=10&offset=20"
+
+Category name filtering is also optionally supported. To return files to which the requested category has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?categoryName=Data"
+
+Tabular tag name filtering is also optionally supported. To return files to which the requested tabular tag has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?tabularTagName=Survey"
+
+Content type filtering is also optionally supported. To return files matching the requested content type.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?contentType=image/png"
+
+Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return the files that contain the text searched in one of such fields.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?searchText=word"
+
+File access filtering is also optionally supported. In particular, by the following possible values:
+
+* ``Public``
+* ``Restricted``
+* ``EmbargoedThenRestricted``
+* ``EmbargoedThenPublic``
+
+If no filter is specified, the files will match all of the above categories.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?accessStatus=Public"
+
+Ordering criteria for sorting the results is also optionally supported. In particular, by the following possible values:
+
+* ``NameAZ`` (Default)
+* ``NameZA``
+* ``Newest``
+* ``Oldest``
+* ``Size``
+* ``Type``
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?orderCriteria=Newest"
+
+Please note that both filtering and ordering criteria values are case sensitive and must be correctly typed for the endpoint to recognize them.
+
+By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below.
+
+If you want to include deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files?includeDeaccessioned=true"
+
+.. note:: Keep in mind that you can combine all of the above query parameters depending on the results you are looking for.
+
+Get File Counts in a Dataset
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Get file counts, for the given dataset and version.
+
+The returned file counts are based on different criteria:
+
+- Total (The total file count)
+- Per content type
+- Per category name
+- Per tabular tag name
+- Per access status (Possible values: Public, Restricted, EmbargoedThenRestricted, EmbargoedThenPublic)
+
+.. code-block:: bash
+
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=24
+ export VERSION=1.0
+
+ curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION/files/counts"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts"
+
+Category name filtering is optionally supported. To return counts only for files to which the requested category has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?categoryName=Data"
+
+Tabular tag name filtering is also optionally supported. To return counts only for files to which the requested tabular tag has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?tabularTagName=Survey"
+
+Content type filtering is also optionally supported. To return counts only for files matching the requested content type.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?contentType=image/png"
+
+Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return counts only for files that contain the text searched in one of such fields.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?searchText=word"
+
+File access filtering is also optionally supported. In particular, by the following possible values:
+
+* ``Public``
+* ``Restricted``
+* ``EmbargoedThenRestricted``
+* ``EmbargoedThenPublic``
+
+If no filter is specified, the files will match all of the above categories.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?accessStatus=Public"
+
+By default, deaccessioned dataset versions are not supported by this endpoint and will be ignored in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a not found error if the version is deaccessioned and you do not enable the option described below.
+
+If you want to include deaccessioned dataset versions, you must specify this through the ``includeDeaccessioned`` query parameter.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/files/counts?includeDeaccessioned=true"
+
+Please note that filtering values are case sensitive and must be correctly typed for the endpoint to recognize them.
+
+Keep in mind that you can combine all of the above query parameters depending on the results you are looking for.
View Dataset Files and Folders as a Directory Index
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -952,9 +1243,9 @@ View Dataset Files and Folders as a Directory Index
.. code-block:: bash
- curl $SERVER_URL/api/datasets/${ID}/dirindex/
+ curl "$SERVER_URL/api/datasets/${ID}/dirindex/"
# or
- curl ${SERVER_URL}/api/datasets/:persistentId/dirindex?persistentId=doi:${PERSISTENT_ID}
+ curl "${SERVER_URL}/api/datasets/:persistentId/dirindex?persistentId=doi:${PERSISTENT_ID}"
Optional parameters:
@@ -1052,13 +1343,13 @@ List All Metadata Blocks for a Dataset
export ID=24
export VERSION=1.0
- curl $SERVER_URL/api/datasets/$ID/versions/$VERSION/metadata
+ curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION/metadata"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/datasets/24/versions/1.0/metadata
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/metadata"
List Single Metadata Block for a Dataset
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1072,13 +1363,13 @@ List Single Metadata Block for a Dataset
export VERSION=1.0
export METADATA_BLOCK=citation
- curl $SERVER_URL/api/datasets/$ID/versions/$VERSION/metadata/$METADATA_BLOCK
+ curl "$SERVER_URL/api/datasets/$ID/versions/$VERSION/metadata/$METADATA_BLOCK"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/datasets/24/versions/1.0/metadata/citation
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/metadata/citation"
Update Metadata For a Dataset
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1095,13 +1386,13 @@ For example, after making your edits, your JSON file might look like :download:`
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z
- curl -H "X-Dataverse-key: $API_TOKEN" -X PUT $SERVER_URL/api/datasets/:persistentId/versions/:draft?persistentId=$PERSISTENT_IDENTIFIER --upload-file dataset-update-metadata.json
+ curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/versions/:draft?persistentId=$PERSISTENT_IDENTIFIER" --upload-file dataset-update-metadata.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/datasets/:persistentId/versions/:draft?persistentId=doi:10.5072/FK2/BCCP9Z --upload-file dataset-update-metadata.json
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/versions/:draft?persistentId=doi:10.5072/FK2/BCCP9Z" --upload-file dataset-update-metadata.json
Note that in the example JSON file above, there are only two JSON objects with the ``license`` and ``metadataBlocks`` keys respectively. When you download a representation of your latest dataset version in JSON format, these objects will be nested inside another object called ``data`` in the API response. Note that there may be more objects in there, in addition to the ``license`` and ``metadataBlocks`` that you may need to preserve and re-import as well. Basically, you need everything in there except for the ``files``. This can be achived by downloading the metadata and selecting the sections you need with a JSON tool such as ``jq``, like this:
@@ -1111,13 +1402,13 @@ Note that in the example JSON file above, there are only two JSON objects with t
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z
- curl -H "X-Dataverse-key: $API_TOKEN" $SERVER_URL/api/datasets/:persistentId/versions/:latest?persistentId=$PERSISTENT_IDENTIFIER | jq '.data | del(.files)' > dataset-update-metadata.json
+ curl -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/versions/:latest?persistentId=$PERSISTENT_IDENTIFIER" | jq '.data | del(.files)' > dataset-update-metadata.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" https://demo.dataverse.org/api/datasets/:persistentId/versions/:latest?persistentId=doi:10.5072/FK2/BCCP9Z | jq '.data | {metadataBlocks: .metadataBlocks}' > dataset-update-metadata.json
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/:persistentId/versions/:latest?persistentId=doi:10.5072/FK2/BCCP9Z" | jq '.data | {metadataBlocks: .metadataBlocks}' > dataset-update-metadata.json
Now you can edit the JSON produced by the command above with a text editor of your choice. For example, with ``vi`` in the example below.
@@ -1141,13 +1432,13 @@ Alternatively to replacing an entire dataset version with its JSON representatio
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z
- curl -H "X-Dataverse-key: $API_TOKEN" -X PUT $SERVER_URL/api/datasets/:persistentId/editMetadata/?persistentId=$PERSISTENT_IDENTIFIER --upload-file dataset-add-metadata.json
+ curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/editMetadata/?persistentId=$PERSISTENT_IDENTIFIER" --upload-file dataset-add-metadata.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z --upload-file dataset-add-metadata.json
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z" --upload-file dataset-add-metadata.json
You may also replace existing metadata in dataset fields with the following (adding the parameter replace=true):
@@ -1157,13 +1448,13 @@ You may also replace existing metadata in dataset fields with the following (add
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z
- curl -H "X-Dataverse-key: $API_TOKEN" -X PUT $SERVER_URL/api/datasets/:persistentId/editMetadata?persistentId=$PERSISTENT_IDENTIFIER&replace=true --upload-file dataset-update-metadata.json
+ curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/editMetadata?persistentId=$PERSISTENT_IDENTIFIER&replace=true" --upload-file dataset-update-metadata.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z&replace=true --upload-file dataset-update-metadata.json
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/editMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z&replace=true" --upload-file dataset-update-metadata.json
For these edits your JSON file need only include those dataset fields which you would like to edit. A sample JSON file may be downloaded here: :download:`dataset-edit-metadata-sample.json <../_static/api/dataset-edit-metadata-sample.json>`
@@ -1178,13 +1469,13 @@ You may delete some of the metadata of a dataset version by supplying a file wit
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/BCCP9Z
- curl -H "X-Dataverse-key: $API_TOKEN" -X PUT $SERVER_URL/api/datasets/:persistentId/deleteMetadata/?persistentId=$PERSISTENT_IDENTIFIER --upload-file dataset-delete-author-metadata.json
+ curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/deleteMetadata/?persistentId=$PERSISTENT_IDENTIFIER" --upload-file dataset-delete-author-metadata.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/datasets/:persistentId/deleteMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z --upload-file dataset-delete-author-metadata.json
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/deleteMetadata/?persistentId=doi:10.5072/FK2/BCCP9Z" --upload-file dataset-delete-author-metadata.json
For these deletes your JSON file must include an exact match of those dataset fields which you would like to delete. A sample JSON file may be downloaded here: :download:`dataset-delete-author-metadata.json <../_static/api/dataset-delete-author-metadata.json>`
@@ -1233,13 +1524,46 @@ Deletes the draft version of dataset ``$ID``. Only the draft version can be dele
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE $SERVER_URL/api/datasets/$ID/versions/:draft
+ curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/$ID/versions/:draft"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/24/versions/:draft"
+
+Deaccession Dataset
+~~~~~~~~~~~~~~~~~~~
+
+Given a version of a dataset, updates its status to deaccessioned.
+
+The JSON body required to deaccession a dataset (``deaccession.json``) looks like this::
+
+ {
+ "deaccessionReason": "Description of the deaccession reason.",
+ "deaccessionForwardURL": "https://demo.dataverse.org"
+ }
+
+
+Note that the field ``deaccessionForwardURL`` is optional.
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=24
+ export VERSIONID=1.0
+ export FILE_PATH=deaccession.json
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/datasets/$ID/versions/$VERSIONID/deaccession" -H "Content-type:application/json" --upload-file $FILE_PATH
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/24/versions/:draft
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/24/versions/1.0/deaccession" -H "Content-type:application/json" --upload-file deaccession.json
+
+.. note:: You cannot deaccession a dataset more than once. If you call this endpoint twice for the same dataset version, you will get a not found error on the second call, since the dataset you are looking for will no longer be published since it is already deaccessioned.
Set Citation Date Field Type for a Dataset
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1254,13 +1578,13 @@ Note that the dataset citation date field type must be a date field.
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB
export DATASET_FIELD_TYPE_NAME=dateOfDeposit
- curl -H "X-Dataverse-key: $API_TOKEN" -X PUT $SERVER_URL/api/datasets/:persistentId/citationdate?persistentId=$PERSISTENT_IDENTIFIER --data "$DATASET_FIELD_TYPE_NAME"
+ curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/:persistentId/citationdate?persistentId=$PERSISTENT_IDENTIFIER" --data "$DATASET_FIELD_TYPE_NAME"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/datasets/:persistentId/citationdate?persistentId=doi:10.5072/FK2/J8SJZB --data "dateOfDeposit"
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/:persistentId/citationdate?persistentId=doi:10.5072/FK2/J8SJZB" --data "dateOfDeposit"
Revert Citation Date Field Type to Default for Dataset
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1273,13 +1597,13 @@ Restores the default citation date field type, ``:publicationDate``, for a given
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB
- curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE $SERVER_URL/api/datasets/:persistentId/citationdate?persistentId=$PERSISTENT_IDENTIFIER
+ curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/:persistentId/citationdate?persistentId=$PERSISTENT_IDENTIFIER"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/:persistentId/citationdate?persistentId=doi:10.5072/FK2/J8SJZB
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/:persistentId/citationdate?persistentId=doi:10.5072/FK2/J8SJZB"
.. _list-roles-on-a-dataset-api:
@@ -1294,13 +1618,13 @@ Lists all role assignments on a given dataset:
export SERVER_URL=https://demo.dataverse.org
export ID=2347
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/datasets/$ID/assignments
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/$ID/assignments"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/datasets/2347/assignments
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/2347/assignments"
.. _assign-role-on-a-dataset-api:
@@ -1315,13 +1639,13 @@ Assigns a new role, based on the POSTed JSON:
export SERVER_URL=https://demo.dataverse.org
export ID=2347
- curl -H X-Dataverse-key:$API_TOKEN -X POST -H "Content-Type: application/json" $SERVER_URL/api/datasets/$ID/assignments --upload-file role.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-Type: application/json" "$SERVER_URL/api/datasets/$ID/assignments" --upload-file role.json
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-Type: application/json" https://demo.dataverse.org/api/datasets/2347/assignments --upload-file role.json
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -H "Content-Type: application/json" "https://demo.dataverse.org/api/datasets/2347/assignments" --upload-file role.json
POSTed JSON example (the content of ``role.json`` file)::
@@ -1344,13 +1668,13 @@ Delete the assignment whose id is ``$id``:
export ID=2347
export ASSIGNMENT_ID=6
- curl -H X-Dataverse-key:$API_TOKEN -X DELETE $SERVER_URL/api/datasets/$ID/assignments/$ASSIGNMENT_ID
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/$ID/assignments/$ASSIGNMENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X DELETE https://demo.dataverse.org/api/datasets/2347/assignments/6
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/2347/assignments/6"
Create a Private URL for a Dataset
@@ -1364,20 +1688,20 @@ Create a Private URL (must be able to manage dataset permissions):
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key: $API_TOKEN" -X POST $SERVER_URL/api/datasets/$ID/privateUrl
+ curl -H "X-Dataverse-key: $API_TOKEN" -X POST "$SERVER_URL/api/datasets/$ID/privateUrl"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/24/privateUrl
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/24/privateUrl"
If Anonymized Access has been enabled on a Dataverse installation (see the :ref:`:AnonymizedFieldTypeNames` setting), an optional 'anonymizedAccess' query parameter is allowed.
Setting anonymizedAccess=true in your call will create a PrivateURL that only allows an anonymized view of the Dataset (see :ref:`privateurl`).
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/24/privateUrl?anonymizedAccess=true
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/24/privateUrl?anonymizedAccess=true"
Get the Private URL for a Dataset
@@ -1391,13 +1715,13 @@ Get a Private URL from a dataset (if available):
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key: $API_TOKEN" $SERVER_URL/api/datasets/$ID/privateUrl
+ curl -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/$ID/privateUrl"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" https://demo.dataverse.org/api/datasets/24/privateUrl
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/24/privateUrl"
Delete the Private URL from a Dataset
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1410,13 +1734,13 @@ Delete a Private URL from a dataset (if it exists):
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE $SERVER_URL/api/datasets/$ID/privateUrl
+ curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/$ID/privateUrl"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/24/privateUrl
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/24/privateUrl"
.. _add-file-api:
@@ -1443,13 +1767,13 @@ In the curl example below, all of the above are specified but they are optional.
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB
- curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "tabIngest":"false"}' "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_ID"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST -F "file=@$FILENAME" -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "tabIngest":"false"}' "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -F file=@data.tsv -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "tabIngest":"false"}' "https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB"
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -F file=@data.tsv -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "tabIngest":"false"}' "https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB"
You should expect a 201 ("CREATED") response and JSON indicating the database id that has been assigned to your newly uploaded file.
@@ -1567,7 +1891,7 @@ The fully expanded example above (without environment variables) looks like this
.. code-block:: bash
- curl -H X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB -F 'jsonData={"description":"A remote image.","storageIdentifier":"trsa://themes/custom/qdr/images/CoreTrustSeal-logo-transparent.png","checksumType":"MD5","md5Hash":"509ef88afa907eaf2c17c1c8d8fde77e","label":"testlogo.png","fileName":"testlogo.png","mimeType":"image/png"}'
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB" -F 'jsonData={"description":"A remote image.","storageIdentifier":"trsa://themes/custom/qdr/images/CoreTrustSeal-logo-transparent.png","checksumType":"MD5","md5Hash":"509ef88afa907eaf2c17c1c8d8fde77e","label":"testlogo.png","fileName":"testlogo.png","mimeType":"image/png"}'
.. _cleanup-storage-api:
@@ -1599,7 +1923,7 @@ The fully expanded example above (without environment variables) looks like this
.. code-block:: bash
- curl -H X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X GET https://demo.dataverse.org/api/datasets/:persistentId/cleanStorage?persistentId=doi:10.5072/FK2/J8SJZB&dryrun=true
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/datasets/:persistentId/cleanStorage?persistentId=doi:10.5072/FK2/J8SJZB&dryrun=true"
Adding Files To a Dataset via Other Tools
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1619,13 +1943,13 @@ Shows the combined size in bytes of all the files uploaded into the dataset ``id
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/datasets/$ID/storagesize
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/$ID/storagesize"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/datasets/24/storagesize
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/24/storagesize"
The size of published and unpublished files will be summed in the dataset specified.
By default, only the archival files are counted - i.e., the files uploaded by users (plus the tab-delimited versions generated for tabular data files on ingest). If the optional argument ``includeCached=true`` is specified, the API will also add the sizes of all the extra files generated and cached by the Dataverse installation - the resized thumbnail versions for image files, the metadata exports for published datasets, etc. Because this deals with unpublished files the token supplied must have permission to view unpublished drafts.
@@ -1643,17 +1967,84 @@ Shows the combined size in bytes of all the files available for download from ve
export ID=24
export VERSIONID=1.0
- curl -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/datasets/$ID/versions/$VERSIONID/downloadsize
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/$ID/versions/$VERSIONID/downloadsize"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize"
The size of all files available for download will be returned.
If :draft is passed as versionId the token supplied must have permission to view unpublished drafts. A token is not required for published datasets. Also restricted files will be included in this total regardless of whether the user has access to download the restricted file(s).
+There is an optional query parameter ``mode`` which applies a filter criteria to the operation. This parameter supports the following values:
+
+* ``All`` (Default): Includes both archival and original sizes for tabular files
+* ``Archival``: Includes only the archival size for tabular files
+* ``Original``: Includes only the original size for tabular files
+
+Usage example:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?mode=Archival"
+
+Category name filtering is also optionally supported. To return the size of all files available for download matching the requested category name.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?categoryName=Data"
+
+Tabular tag name filtering is also optionally supported. To return the size of all files available for download for which the requested tabular tag has been added.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?tabularTagName=Survey"
+
+Content type filtering is also optionally supported. To return the size of all files available for download matching the requested content type.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?contentType=image/png"
+
+Filtering by search text is also optionally supported. The search will be applied to the labels and descriptions of the dataset files, to return the size of all files available for download that contain the text searched in one of such fields.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?searchText=word"
+
+File access filtering is also optionally supported. In particular, by the following possible values:
+
+* ``Public``
+* ``Restricted``
+* ``EmbargoedThenRestricted``
+* ``EmbargoedThenPublic``
+
+If no filter is specified, the files will match all of the above categories.
+
+Please note that filtering query parameters are case sensitive and must be correctly typed for the endpoint to recognize them.
+
+By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below.
+
+If you want to include deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/datasets/24/versions/1.0/downloadsize?includeDeaccessioned=true"
+
+.. note:: Keep in mind that you can combine all of the above query parameters depending on the results you are looking for.
+
Submit a Dataset for Review
~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1718,13 +2109,13 @@ Creates a link between a dataset and a Dataverse collection (see :ref:`dataset-l
export DATASET_ID=24
export DATAVERSE_ID=test
- curl -H "X-Dataverse-key: $API_TOKEN" -X PUT $SERVER_URL/api/datasets/$DATASET_ID/link/$DATAVERSE_ID
+ curl -H "X-Dataverse-key: $API_TOKEN" -X PUT "$SERVER_URL/api/datasets/$DATASET_ID/link/$DATAVERSE_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/datasets/24/link/test
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/datasets/24/link/test"
Dataset Locks
~~~~~~~~~~~~~
@@ -1739,13 +2130,13 @@ To check if a dataset is locked:
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl $SERVER_URL/api/datasets/$ID/locks
+ curl "$SERVER_URL/api/datasets/$ID/locks"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/datasets/24/locks
+ curl "https://demo.dataverse.org/api/datasets/24/locks"
Optionally, you can check if there's a lock of a specific type on the dataset:
@@ -1795,13 +2186,13 @@ The following API end point will lock a Dataset with a lock of specified type. N
export ID=24
export LOCK_TYPE=Ingest
- curl -H "X-Dataverse-key: $API_TOKEN" -X POST $SERVER_URL/api/datasets/$ID/lock/$LOCK_TYPE
+ curl -H "X-Dataverse-key: $API_TOKEN" -X POST "$SERVER_URL/api/datasets/$ID/lock/$LOCK_TYPE"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/datasets/24/lock/Ingest
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/datasets/24/lock/Ingest"
Use the following API to unlock the dataset, by deleting all the locks currently on the dataset. Note that this requires “superuser†credentials:
@@ -1811,13 +2202,13 @@ Use the following API to unlock the dataset, by deleting all the locks currently
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE $SERVER_URL/api/datasets/$ID/locks
+ curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/$ID/locks"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/24/locks
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/24/locks"
Or, to delete a lock of the type specified only. Note that this requires “superuser†credentials:
@@ -1828,13 +2219,13 @@ Or, to delete a lock of the type specified only. Note that this requires “supe
export ID=24
export LOCK_TYPE=finalizePublication
- curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE $SERVER_URL/api/datasets/$ID/locks?type=$LOCK_TYPE
+ curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/$ID/locks?type=$LOCK_TYPE"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/24/locks?type=finalizePublication
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/24/locks?type=finalizePublication"
If the dataset is not locked (or if there is no lock of the specified type), the API will exit with a warning message.
@@ -1983,13 +2374,13 @@ Delete the dataset whose id is passed:
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE $SERVER_URL/api/datasets/$ID
+ curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/$ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/24
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/24"
Delete Published Dataset
~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2002,13 +2393,13 @@ Normally published datasets should not be deleted, but there exists a "destroy"
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
- curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE $SERVER_URL/api/datasets/:persistentId/destroy/?persistentId=$PERSISTENT_ID
+ curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/:persistentId/destroy/?persistentId=$PERSISTENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/:persistentId/destroy/?persistentId=doi:10.5072/FK2/AAA000
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/:persistentId/destroy/?persistentId=doi:10.5072/FK2/AAA000"
Delete with dataset identifier:
@@ -2018,13 +2409,13 @@ Delete with dataset identifier:
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE $SERVER_URL/api/datasets/$ID/destroy
+ curl -H "X-Dataverse-key: $API_TOKEN" -X DELETE "$SERVER_URL/api/datasets/$ID/destroy"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/datasets/24/destroy
+ curl -H "X-Dataverse-key: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/datasets/24/destroy"
Calling the destroy endpoint is permanent and irreversible. It will remove the dataset and its datafiles, then re-index the parent Dataverse collection in Solr. This endpoint requires the API token of a superuser.
@@ -2075,10 +2466,12 @@ The API call requires a Json body that includes the list of the fileIds that the
curl -H "X-Dataverse-key: $API_TOKEN" -H "Content-Type:application/json" "$SERVER_URL/api/datasets/:persistentId/files/actions/:unset-embargo?persistentId=$PERSISTENT_IDENTIFIER" -d "$JSON"
+.. _Archival Status API:
+
Get the Archival Status of a Dataset By Version
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Archiving is an optional feature that may be configured for a Dataverse installation. When that is enabled, this API call be used to retrieve the status. Note that this requires "superuser" credentials.
+Archival :ref:`BagIt Export` is an optional feature that may be configured for a Dataverse installation. When that is enabled, this API call be used to retrieve the status. Note that this requires "superuser" credentials.
``GET /api/datasets/$dataset-id/$version/archivalStatus`` returns the archival status of the specified dataset version.
@@ -2158,11 +2551,11 @@ Signposting involves the addition of a `Link ;rel="cite-as", ;rel="describedby";type="application/vnd.citationstyles.csl+json",;rel="describedby";type="application/json+ld", ;rel="type",;rel="type", https://demo.dataverse.org/api/datasets/:persistentId/versions/1.0/customlicense?persistentId=doi:10.5072/FK2/YD5QDG;rel="license", ; rel="linkset";type="application/linkset+json"``
+``Link: ;rel="cite-as", ;rel="describedby";type="application/vnd.citationstyles.csl+json",;rel="describedby";type="application/ld+json", ;rel="type",;rel="type", ;rel="license", ; rel="linkset";type="application/linkset+json"``
The URL for linkset information is discoverable under the ``rel="linkset";type="application/linkset+json`` entry in the "Link" header, such as in the example above.
-The reponse includes a JSON object conforming to the `Signposting `__ specification.
+The reponse includes a JSON object conforming to the `Signposting `__ specification. As part of this conformance, unlike most Dataverse API responses, the output is not wrapped in a ``{"status":"OK","data":{`` object.
Signposting is not supported for draft dataset versions.
.. code-block:: bash
@@ -2173,29 +2566,168 @@ Signposting is not supported for draft dataset versions.
curl -H "Accept:application/json" "$SERVER_URL/api/datasets/:persistentId/versions/$VERSION/linkset?persistentId=$PERSISTENT_IDENTIFIER"
-Files
------
-
-Get JSON Representation of a File
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``.
-
-Example: Getting the file whose DOI is *10.5072/FK2/J8SJZB*:
+Get Dataset By Private URL Token
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code-block:: bash
export SERVER_URL=https://demo.dataverse.org
- export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB
- export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export PRIVATE_URL_TOKEN=a56444bc-7697-4711-8964-e0577f055fd2
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/files/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER
+ curl "$SERVER_URL/api/datasets/privateUrlDatasetVersion/$PRIVATE_URL_TOKEN"
-The fully expanded example above (without environment variables) looks like this:
+.. _get-citation:
+
+Get Citation
+~~~~~~~~~~~~
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" https://demo.dataverse.org/api/files/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG
+ export VERSION=1.0
+
+ curl -H "Accept:application/json" "$SERVER_URL/api/datasets/:persistentId/versions/$VERSION/{version}/citation?persistentId=$PERSISTENT_IDENTIFIER"
+
+By default, deaccessioned dataset versions are not included in the search when applying the :latest or :latest-published identifiers. Additionally, when filtering by a specific version tag, you will get a "not found" error if the version is deaccessioned and you do not enable the ``includeDeaccessioned`` option described below.
+
+If you want to include deaccessioned dataset versions, you must set ``includeDeaccessioned`` query parameter to ``true``.
+
+Usage example:
+
+.. code-block:: bash
+
+ curl -H "Accept:application/json" "$SERVER_URL/api/datasets/:persistentId/versions/$VERSION/{version}/citation?persistentId=$PERSISTENT_IDENTIFIER&includeDeaccessioned=true"
+
+Get Citation by Private URL Token
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: bash
+
+ export SERVER_URL=https://demo.dataverse.org
+ export PRIVATE_URL_TOKEN=a56444bc-7697-4711-8964-e0577f055fd2
+
+ curl "$SERVER_URL/api/datasets/privateUrlDatasetVersion/$PRIVATE_URL_TOKEN/citation"
+
+.. _get-dataset-summary-field-names:
+
+Get Summary Field Names
+~~~~~~~~~~~~~~~~~~~~~~~
+
+See :ref:`:CustomDatasetSummaryFields` in the Installation Guide for how the list of dataset fields that summarize a dataset can be customized. Here's how to list them:
+
+.. code-block:: bash
+
+ export SERVER_URL=https://demo.dataverse.org
+
+ curl "$SERVER_URL/api/datasets/summaryFieldNames"
+
+.. _guestbook-at-request-api:
+
+Configure When a Dataset Guestbook Appears (If Enabled)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, users are asked to fill out a configured Guestbook when they down download files from a dataset. If enabled for a given Dataverse instance (see XYZ), users may instead be asked to fill out a Guestbook only when they request access to restricted files.
+This is configured by a global default, collection-level settings, or directly at the dataset level via these API calls (superuser access is required to make changes).
+
+To see the current choice for this dataset:
+
+.. code-block:: bash
+
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG
+
+ curl "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
+
+
+ The response will be true (guestbook displays when making a request), false (guestbook displays at download), or will indicate that the dataset inherits one of these settings.
+
+To set the behavior for this dataset:
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG
+
+ curl -X PUT -H "X-Dataverse-key:$API_TOKEN" -H Content-type:application/json -d true "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
+
+
+ This example uses true to set the behavior to guestbook at request. Note that this call will return a 403/Forbidden response if guestbook at request functionality is not enabled for this Dataverse instance.
+
+The API can also be used to reset the dataset to use the default/inherited value:
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/YD5QDG
+
+ curl -X DELETE -H "X-Dataverse-key:$API_TOKEN" -H Content-type:application/json "$SERVER_URL/api/datasets/:persistentId/guestbookEntryAtRequest?persistentId=$PERSISTENT_IDENTIFIER"
+
+Get User Permissions on a Dataset
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This API call returns the permissions that the calling user has on a particular dataset.
+
+In particular, the user permissions that this API call checks, returned as booleans, are the following:
+
+* Can view the unpublished dataset
+* Can edit the dataset
+* Can publish the dataset
+* Can manage the dataset permissions
+* Can delete the dataset draft
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=24
+
+ curl -H "X-Dataverse-key: $API_TOKEN" -X GET "$SERVER_URL/api/datasets/$ID/userPermissions"
+
+Know If a User Can Download at Least One File from a Dataset Version
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This API endpoint indicates if the calling user can download at least one file from a dataset version. Note that permissions based on :ref:`shib-groups` are not considered.
+
+.. code-block:: bash
+
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=24
+ export VERSION=1.0
+
+ curl -H "X-Dataverse-key: $API_TOKEN" -X GET "$SERVER_URL/api/datasets/$ID/versions/$VERSION/canDownloadAtLeastOneFile"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/datasets/24/versions/1.0/canDownloadAtLeastOneFile"
+
+Files
+-----
+
+Get JSON Representation of a File
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. note:: Files can be accessed using persistent identifiers. This is done by passing the constant ``:persistentId`` where the numeric id of the file is expected, and then passing the actual persistent id as a query parameter with the name ``persistentId``.
+
+Example: Getting the file whose DOI is *10.5072/FK2/J8SJZB*:
+
+.. code-block:: bash
+
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/files/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/files/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB"
You may get its draft version of an unpublished file if you pass an api token with view draft permissions:
@@ -2205,13 +2737,13 @@ You may get its draft version of an unpublished file if you pass an api token wi
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB
export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER/api/files/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER/api/files/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" https://demo.dataverse.org/api/files/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/files/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB"
|CORS| Show the file whose id is passed:
@@ -2221,13 +2753,13 @@ The fully expanded example above (without environment variables) looks like this
export SERVER_URL=https://demo.dataverse.org
export ID=408730
- curl $SERVER_URL/api/file/$ID
+ curl "$SERVER_URL/api/file/$ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/files/408730
+ curl "https://demo.dataverse.org/api/files/408730"
You may get its draft version of an published file if you pass an api token with view draft permissions and use the draft path parameter:
@@ -2237,13 +2769,13 @@ You may get its draft version of an published file if you pass an api token with
export PERSISTENT_IDENTIFIER=doi:10.5072/FK2/J8SJZB
export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER/api/files/:persistentId/draft/?persistentId=$PERSISTENT_IDENTIFIER
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER/api/files/:persistentId/draft/?persistentId=$PERSISTENT_IDENTIFIER"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" https://demo.dataverse.org/api/files/:persistentId/draft/?persistentId=doi:10.5072/FK2/J8SJZB
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/files/:persistentId/draft/?persistentId=doi:10.5072/FK2/J8SJZB"
The file id can be extracted from the response retrieved from the API which uses the persistent identifier (``/api/datasets/:persistentId/?persistentId=$PERSISTENT_IDENTIFIER``).
@@ -2294,13 +2826,13 @@ A curl example using an ``id``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" -X PUT -d true $SERVER_URL/api/files/$ID/restrict
+ curl -H "X-Dataverse-key:$API_TOKEN" -X PUT -d true "$SERVER_URL/api/files/$ID/restrict"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -d true https://demo.dataverse.org/api/files/24/restrict
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -d true "https://demo.dataverse.org/api/files/24/restrict"
A curl example using a ``pid``
@@ -2310,20 +2842,237 @@ A curl example using a ``pid``
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
- curl -H "X-Dataverse-key:$API_TOKEN" -X PUT -d true $SERVER_URL/api/files/:persistentId/restrict?persistentId=$PERSISTENT_ID
+ curl -H "X-Dataverse-key:$API_TOKEN" -X PUT -d true "$SERVER_URL/api/files/:persistentId/restrict?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -d true "https://demo.dataverse.org/api/files/:persistentId/restrict?persistentId=doi:10.5072/FK2/AAA000"
+
+Uningest a File
+~~~~~~~~~~~~~~~
+
+Reverse the tabular data ingest process performed on a file where ``ID`` is the database id or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file to process. Note that this requires "superuser" credentials.
+
+A curl example using an ``ID``:
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=24
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/$ID/uningest"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/24/uningest"
+
+A curl example using a ``PERSISTENT_ID``:
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/:persistentId/uningest?persistentId=$PERSISTENT_ID"
The fully expanded example above (without environment variables) looks like this:
-.. code-block:: bash
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/uningest?persistentId=doi:10.5072/FK2/AAA000"
+
+Reingest a File
+~~~~~~~~~~~~~~~
+
+Attempt to ingest an existing datafile as tabular data. This API can be used on a file that was not ingested as tabular back when it was uploaded. For example, a Stata v.14 file that was uploaded before ingest support for Stata 14 was added (in Dataverse Software v.4.9). It can also be used on a file that failed to ingest due to a bug in the ingest plugin that has since been fixed (hence the name "reingest").
+
+Note that this requires "superuser" credentials.
+
+A curl example using an ``ID``
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=24
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/$ID/reingest"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/24/reingest"
+
+A curl example using a ``PERSISTENT_ID``
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/:persistentId/reingest?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/reingest?persistentId=doi:10.5072/FK2/AAA000"
+
+Note: at present, the API cannot be used on a file that's already successfully ingested as tabular.
+
+.. _redetect-file-type:
+
+Redetect File Type
+~~~~~~~~~~~~~~~~~~
+
+The Dataverse Software uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have the Dataverse Software attempt to redetect the file type.
+
+When using the curl command below, you can pass ``dryRun=true`` if you don't want any changes to be saved to the database. Change this to ``dryRun=false`` (or omit it) to save the change.
+
+A curl example using an ``id``
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=24
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/$ID/redetect?dryRun=true"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/24/redetect?dryRun=true"
+
+A curl example using a ``pid``
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/:persistentId/redetect?persistentId=$PERSISTENT_ID&dryRun=true"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/redetect?persistentId=doi:10.5072/FK2/AAA000&dryRun=true"
+
+Currently the following methods are used to detect file types:
+
+- The file type detected by the browser (or sent via API).
+- JHOVE: https://jhove.openpreservation.org
+- The file extension (e.g. ".ipybn") is used, defined in a file called ``MimeTypeDetectionByFileExtension.properties``.
+- The file name (e.g. "Dockerfile") is used, defined in a file called ``MimeTypeDetectionByFileName.properties``.
+
+.. _extractNcml:
+
+Extract NcML
+~~~~~~~~~~~~
+
+As explained in the :ref:`netcdf-and-hdf5` section of the User Guide, when those file types are uploaded, an attempt is made to extract an NcML file from them and store it as an auxiliary file.
+
+This happens automatically but superusers can also manually trigger this NcML extraction process with the API endpoint below.
+
+Note that "true" will be returned if an NcML file was created. "false" will be returned if there was an error or if the NcML file already exists (check server.log for details).
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=24
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/$ID/extractNcml"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/24/extractNcml"
+
+A curl example using a PID:
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/:persistentId/extractNcml?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/extractNcml?persistentId=doi:10.5072/FK2/AAA000"
+
+Replacing Files
+~~~~~~~~~~~~~~~
+
+Replace an existing file where ``ID`` is the database id of the file to replace or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires the ``file`` to be passed as well as a ``jsonString`` expressing the new metadata. Note that metadata such as description, directoryLabel (File Path) and tags are not carried over from the file being replaced.
+
+Note that when a Dataverse installation is configured to use S3 storage with direct upload enabled, there is API support to send a replacement file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide.
+
+A curl example using an ``ID``
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=24
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST -F 'file=@file.extension' -F 'jsonData={json}' "$SERVER_URL/api/files/$ID/replace"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -F 'file=@data.tsv' \
+ -F 'jsonData={"description":"My description.","categories":["Data"],"forceReplace":false}' \
+ "https://demo.dataverse.org/api/files/24/replace"
+
+A curl example using a ``PERSISTENT_ID``
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST -F 'file=@file.extension' -F 'jsonData={json}' \
+ "$SERVER_URL/api/files/:persistentId/replace?persistentId=$PERSISTENT_ID"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -F 'file=@data.tsv' \
+ -F 'jsonData={"description":"My description.","categories":["Data"],"forceReplace":false}' \
+ "https://demo.dataverse.org/api/files/:persistentId/replace?persistentId=doi:10.5072/FK2/AAA000"
+
+Deleting Files
+~~~~~~~~~~~~~~
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT -d true "https://demo.dataverse.org/api/files/:persistentId/restrict?persistentId=doi:10.5072/FK2/AAA000"
+Delete an existing file where ``ID`` is the database id of the file to delete or ``PERSISTENT_ID`` is the persistent id (DOI or Handle, if it exists) of the file.
-Uningest a File
-~~~~~~~~~~~~~~~
+Note that the behavior of deleting files depends on if the dataset has ever been published or not.
-Reverse the tabular data ingest process performed on a file where ``ID`` is the database id or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file to process. Note that this requires "superuser" credentials.
+- If the dataset has never been published, the file will be deleted forever.
+- If the dataset has published, the file is deleted from the draft (and future published versions).
+- If the dataset has published, the deleted file can still be downloaded because it was part of a published version.
-A curl example using an ``ID``:
+A curl example using an ``ID``
.. code-block:: bash
@@ -2331,15 +3080,15 @@ A curl example using an ``ID``:
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/files/$ID/uningest
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/files/$ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/files/24/uningest
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/files/24"
-A curl example using a ``PERSISTENT_ID``:
+A curl example using a ``PERSISTENT_ID``
.. code-block:: bash
@@ -2347,20 +3096,18 @@ A curl example using a ``PERSISTENT_ID``:
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/:persistentId/uningest?persistentId=$PERSISTENT_ID"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/files/:persistentId?persistentId=$PERSISTENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/uningest?persistentId=doi:10.5072/FK2/AAA000"
-
-Reingest a File
-~~~~~~~~~~~~~~~
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/files/:persistentId?persistentId=doi:10.5072/FK2/AAA000"
-Attempt to ingest an existing datafile as tabular data. This API can be used on a file that was not ingested as tabular back when it was uploaded. For example, a Stata v.14 file that was uploaded before ingest support for Stata 14 was added (in Dataverse Software v.4.9). It can also be used on a file that failed to ingest due to a bug in the ingest plugin that has since been fixed (hence the name "reingest").
+Getting File Metadata
+~~~~~~~~~~~~~~~~~~~~~
-Note that this requires "superuser" credentials.
+Provides a json representation of the file metadata for an existing file where ``ID`` is the database id of the file to get metadata from or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file.
A curl example using an ``ID``
@@ -2370,13 +3117,13 @@ A curl example using an ``ID``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/files/$ID/reingest
+ curl "$SERVER_URL/api/files/$ID/metadata"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/files/24/reingest
+ curl "https://demo.dataverse.org/api/files/24/metadata"
A curl example using a ``PERSISTENT_ID``
@@ -2386,26 +3133,17 @@ A curl example using a ``PERSISTENT_ID``
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/files/:persistentId/reingest?persistentId=$PERSISTENT_ID
+ curl "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/reingest?persistentId=doi:10.5072/FK2/AAA000"
-
-Note: at present, the API cannot be used on a file that's already successfully ingested as tabular.
-
-.. _redetect-file-type:
-
-Redetect File Type
-~~~~~~~~~~~~~~~~~~
-
-The Dataverse Software uses a variety of methods for determining file types (MIME types or content types) and these methods (listed below) are updated periodically. If you have files that have an unknown file type, you can have the Dataverse Software attempt to redetect the file type.
+ curl "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
-When using the curl command below, you can pass ``dryRun=true`` if you don't want any changes to be saved to the database. Change this to ``dryRun=false`` (or omit it) to save the change.
+The current draft can also be viewed if you have permissions and pass your API token
-A curl example using an ``id``
+A curl example using an ``ID``
.. code-block:: bash
@@ -2413,15 +3151,15 @@ A curl example using an ``id``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/$ID/redetect?dryRun=true"
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/files/$ID/metadata/draft"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/24/redetect?dryRun=true"
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/files/24/metadata/draft"
-A curl example using a ``pid``
+A curl example using a ``PERSISTENT_ID``
.. code-block:: bash
@@ -2429,31 +3167,22 @@ A curl example using a ``pid``
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/:persistentId/redetect?persistentId=$PERSISTENT_ID&dryRun=true"
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/files/:persistentId/metadata/draft?persistentId=$PERSISTENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/redetect?persistentId=doi:10.5072/FK2/AAA000&dryRun=true"
-
-Currently the following methods are used to detect file types:
-
-- The file type detected by the browser (or sent via API).
-- JHOVE: http://jhove.openpreservation.org
-- The file extension (e.g. ".ipybn") is used, defined in a file called ``MimeTypeDetectionByFileExtension.properties``.
-- The file name (e.g. "Dockerfile") is used, defined in a file called ``MimeTypeDetectionByFileName.properties``.
-
-.. _extractNcml:
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/files/:persistentId/metadata/draft?persistentId=doi:10.5072/FK2/AAA000"
-Extract NcML
-~~~~~~~~~~~~
+Note: The ``id`` returned in the json response is the id of the file metadata version.
-As explained in the :ref:`netcdf-and-hdf5` section of the User Guide, when those file types are uploaded, an attempt is made to extract an NcML file from them and store it as an auxiliary file.
+Getting File Data Tables
+~~~~~~~~~~~~~~~~~~~~~~~~
-This happens automatically but superusers can also manually trigger this NcML extraction process with the API endpoint below.
+This endpoint is oriented toward tabular files and provides a JSON representation of the file data tables for an existing tabular file. ``ID`` is the database id of the file to get the data tables from or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file.
-Note that "true" will be returned if an NcML file was created. "false" will be returned if there was an error or if the NcML file already exists (check server.log for details).
+A curl example using an ``ID``
.. code-block:: bash
@@ -2461,15 +3190,15 @@ Note that "true" will be returned if an NcML file was created. "false" will be r
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/$ID/extractNcml"
+ curl $SERVER_URL/api/files/$ID/dataTables
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/24/extractNcml
+ curl https://demo.dataverse.org/api/files/24/dataTables
-A curl example using a PID:
+A curl example using a ``PERSISTENT_ID``
.. code-block:: bash
@@ -2477,20 +3206,22 @@ A curl example using a PID:
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/:persistentId/extractNcml?persistentId=$PERSISTENT_ID"
+ curl "$SERVER_URL/api/files/:persistentId/dataTables?persistentId=$PERSISTENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/extractNcml?persistentId=doi:10.5072/FK2/AAA000"
+ curl "https://demo.dataverse.org/api/files/:persistentId/dataTables?persistentId=doi:10.5072/FK2/AAA000"
-Replacing Files
-~~~~~~~~~~~~~~~
+Note that if the requested file is not tabular, the endpoint will return an error.
-Replace an existing file where ``ID`` is the database id of the file to replace or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires the ``file`` to be passed as well as a ``jsonString`` expressing the new metadata. Note that metadata such as description, directoryLabel (File Path) and tags are not carried over from the file being replaced.
+.. _file-download-count:
-Note that when a Dataverse installation is configured to use S3 storage with direct upload enabled, there is API support to send a replacement file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide.
+Getting File Download Count
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Provides the download count for a particular file, where ``ID`` is the database id of the file to get the download count from or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file.
A curl example using an ``ID``
@@ -2500,15 +3231,13 @@ A curl example using an ``ID``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST -F 'file=@file.extension' -F 'jsonData={json}' $SERVER_URL/api/files/$ID/replace
+ curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/downloadCount"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -F 'file=@data.tsv' \
- -F 'jsonData={"description":"My description.","categories":["Data"],"forceReplace":false}' \
- https://demo.dataverse.org/api/files/24/replace
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/24/downloadCount"
A curl example using a ``PERSISTENT_ID``
@@ -2518,27 +3247,20 @@ A curl example using a ``PERSISTENT_ID``
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST -F 'file=@file.extension' -F 'jsonData={json}' \
- "$SERVER_URL/api/files/:persistentId/replace?persistentId=$PERSISTENT_ID"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/downloadCount?persistentId=$PERSISTENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST -F 'file=@data.tsv' \
- -F 'jsonData={"description":"My description.","categories":["Data"],"forceReplace":false}' \
- "https://demo.dataverse.org/api/files/:persistentId/replace?persistentId=doi:10.5072/FK2/AAA000"
-
-Deleting Files
-~~~~~~~~~~~~~~
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/:persistentId/downloadCount?persistentId=doi:10.5072/FK2/AAA000"
-Delete an existing file where ``ID`` is the database id of the file to delete or ``PERSISTENT_ID`` is the persistent id (DOI or Handle, if it exists) of the file.
+If you are interested in download counts for multiple files, see :doc:`/api/metrics`.
-Note that the behavior of deleting files depends on if the dataset has ever been published or not.
+File Has Been Deleted
+~~~~~~~~~~~~~~~~~~~~~
-- If the dataset has never been published, the file will be deleted forever.
-- If the dataset has published, the file is deleted from the draft (and future published versions).
-- If the dataset has published, the deleted file can still be downloaded because it was part of a published version.
+Know if a particular file that existed in a previous version of the dataset no longer exists in the latest version.
A curl example using an ``ID``
@@ -2548,13 +3270,13 @@ A curl example using an ``ID``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE $SERVER_URL/api/files/$ID
+ curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/$ID/hasBeenDeleted"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/files/24
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/24/hasBeenDeleted"
A curl example using a ``PERSISTENT_ID``
@@ -2564,18 +3286,18 @@ A curl example using a ``PERSISTENT_ID``
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
- curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/files/:persistentId?persistentId=$PERSISTENT_ID"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/files/:persistentId/hasBeenDeleted?persistentId=$PERSISTENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/files/:persistentId?persistentId=doi:10.5072/FK2/AAA000"
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/files/:persistentId/hasBeenDeleted?persistentId=doi:10.5072/FK2/AAA000"
-Getting File Metadata
-~~~~~~~~~~~~~~~~~~~~~
+Updating File Metadata
+~~~~~~~~~~~~~~~~~~~~~~
-Provides a json representation of the file metadata for an existing file where ``ID`` is the database id of the file to get metadata from or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file.
+Updates the file metadata for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the new metadata. No metadata from the previous version of this file will be persisted, so if you want to update a specific field first get the json with the above command and alter the fields you want.
A curl example using an ``ID``
@@ -2585,13 +3307,17 @@ A curl example using an ``ID``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl $SERVER_URL/api/files/$ID/metadata
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \
+ "$SERVER_URL/api/files/$ID/metadata"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/files/24/metadata
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \
+ "https://demo.dataverse.org/api/files/24/metadata"
A curl example using a ``PERSISTENT_ID``
@@ -2601,15 +3327,39 @@ A curl example using a ``PERSISTENT_ID``
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
- curl "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \
+ "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+ -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"dataFileTags":["Survey"],"restrict":false}' \
+ "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
-The current draft can also be viewed if you have permissions and pass your API token
+Note: To update the 'tabularTags' property of file metadata, use the 'dataFileTags' key when making API requests. This property is used to update the 'tabularTags' of the file metadata.
+
+Also note that dataFileTags are not versioned and changes to these will update the published version of the file.
+
+.. _EditingVariableMetadata:
+
+Updating File Metadata Categories
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Updates the categories for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the category names.
+
+Although updating categories can also be done with the previous endpoint, this has been created to be more practical when it is only necessary to update categories and not other metadata fields.
+
+The JSON representation of file categories (``categories.json``) looks like this::
+
+ {
+ "categories": [
+ "Data",
+ "Custom"
+ ]
+ }
A curl example using an ``ID``
@@ -2618,14 +3368,19 @@ A curl example using an ``ID``
export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
export SERVER_URL=https://demo.dataverse.org
export ID=24
+ export FILE_PATH=categories.json
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/files/$ID/metadata/draft
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+ "$SERVER_URL/api/files/$ID/metadata/categories" \
+ -H "Content-type:application/json" --upload-file $FILE_PATH
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" https://demo.dataverse.org/api/files/24/metadata/draft
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+ "http://demo.dataverse.org/api/files/24/metadata/categories" \
+ -H "Content-type:application/json" --upload-file categories.json
A curl example using a ``PERSISTENT_ID``
@@ -2634,22 +3389,35 @@ A curl example using a ``PERSISTENT_ID``
export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+ export FILE_PATH=categories.json
- curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/files/:persistentId/metadata/draft?persistentId=$PERSISTENT_ID"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
+ "$SERVER_URL/api/files/:persistentId/metadata/categories?persistentId=$PERSISTENT_ID" \
+ -H "Content-type:application/json" --upload-file $FILE_PATH
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/files/:persistentId/metadata/draft?persistentId=doi:10.5072/FK2/AAA000"
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
+ "https://demo.dataverse.org/api/files/:persistentId/metadata/categories?persistentId=doi:10.5072/FK2/AAA000" \
+ -H "Content-type:application/json" --upload-file categories.json
-Note: The ``id`` returned in the json response is the id of the file metadata version.
+Note that if the specified categories do not exist, they will be created.
+Updating File Tabular Tags
+~~~~~~~~~~~~~~~~~~~~~~~~~~
-Updating File Metadata
-~~~~~~~~~~~~~~~~~~~~~~
+Updates the tabular tags for an existing tabular file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the tabular tag names.
-Updates the file metadata for an existing file where ``ID`` is the database id of the file to update or ``PERSISTENT_ID`` is the persistent id (DOI or Handle) of the file. Requires a ``jsonString`` expressing the new metadata. No metadata from the previous version of this file will be persisted, so if you want to update a specific field first get the json with the above command and alter the fields you want.
+The JSON representation of tabular tags (``tags.json``) looks like this::
+
+ {
+ "tabularTags": [
+ "Survey",
+ "Genomics"
+ ]
+ }
A curl example using an ``ID``
@@ -2658,18 +3426,19 @@ A curl example using an ``ID``
export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
export SERVER_URL=https://demo.dataverse.org
export ID=24
+ export FILE_PATH=tags.json
curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
- -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}' \
- $SERVER_URL/api/files/$ID/metadata
+ "$SERVER_URL/api/files/$ID/metadata/tabularTags" \
+ -H "Content-type:application/json" --upload-file $FILE_PATH
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
- -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}' \
- http://demo.dataverse.org/api/files/24/metadata
+ "http://demo.dataverse.org/api/files/24/metadata/tabularTags" \
+ -H "Content-type:application/json" --upload-file tags.json
A curl example using a ``PERSISTENT_ID``
@@ -2678,22 +3447,29 @@ A curl example using a ``PERSISTENT_ID``
export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
export SERVER_URL=https://demo.dataverse.org
export PERSISTENT_ID=doi:10.5072/FK2/AAA000
+ export FILE_PATH=tags.json
curl -H "X-Dataverse-key:$API_TOKEN" -X POST \
- -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}' \
- "$SERVER_URL/api/files/:persistentId/metadata?persistentId=$PERSISTENT_ID"
+ "$SERVER_URL/api/files/:persistentId/metadata/tabularTags?persistentId=$PERSISTENT_ID" \
+ -H "Content-type:application/json" --upload-file $FILE_PATH
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST \
- -F 'jsonData={"description":"My description bbb.","provFreeform":"Test prov freeform","categories":["Data"],"restrict":false}' \
- "https://demo.dataverse.org/api/files/:persistentId/metadata?persistentId=doi:10.5072/FK2/AAA000"
+ "https://demo.dataverse.org/api/files/:persistentId/metadata/tabularTags?persistentId=doi:10.5072/FK2/AAA000" \
+ -H "Content-type:application/json" --upload-file tags.json
-Also note that dataFileTags are not versioned and changes to these will update the published version of the file.
+Note that the specified tabular tags must be valid. The supported tags are:
-.. _EditingVariableMetadata:
+* ``Survey``
+* ``Time Series``
+* ``Panel``
+* ``Event``
+* ``Genomics``
+* ``Network``
+* ``Geospatial``
Editing Variable Level Metadata
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2709,13 +3485,13 @@ A curl example using an ``ID``
export ID=24
export FILE=dct.xml
- curl -H "X-Dataverse-key:$API_TOKEN" -X PUT $SERVER_URL/api/edit/$ID --upload-file $FILE
+ curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/edit/$ID" --upload-file $FILE
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT https://demo.dataverse.org/api/edit/24 --upload-file dct.xml
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X PUT "https://demo.dataverse.org/api/edit/24" --upload-file dct.xml
You can download :download:`dct.xml <../../../../src/test/resources/xml/dct.xml>` from the example above to see what the XML looks like.
@@ -2733,13 +3509,13 @@ A curl example using an ``ID``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/files/$ID/prov-json
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/files/$ID/prov-json"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" https://demo.dataverse.org/api/files/24/prov-json
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/files/24/prov-json"
A curl example using a ``PERSISTENT_ID``
@@ -2768,13 +3544,13 @@ A curl example using an ``ID``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/files/$ID/prov-freeform
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/files/$ID/prov-freeform"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" https://demo.dataverse.org/api/files/24/prov-freeform
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/files/24/prov-freeform"
A curl example using a ``PERSISTENT_ID``
@@ -2805,7 +3581,7 @@ A curl example using an ``ID``
export ENTITY_NAME="..."
export FILE_PATH=provenance.json
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/files/$ID/prov-json?entityName=$ENTITY_NAME -H "Content-type:application/json" --upload-file $FILE_PATH
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/$ID/prov-json?entityName=$ENTITY_NAME" -H "Content-type:application/json" --upload-file $FILE_PATH
The fully expanded example above (without environment variables) looks like this:
@@ -2845,13 +3621,13 @@ A curl example using an ``ID``
export ID=24
export FILE_PATH=provenance.json
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/files/$ID/prov-freeform -H "Content-type:application/json" --upload-file $FILE_PATH
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/files/$ID/prov-freeform" -H "Content-type:application/json" --upload-file $FILE_PATH
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST https://demo.dataverse.org/api/files/24/prov-freeform -H "Content-type:application/json" --upload-file provenance.json
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/24/prov-freeform" -H "Content-type:application/json" --upload-file provenance.json
A curl example using a ``PERSISTENT_ID``
@@ -2883,13 +3659,13 @@ A curl example using an ``ID``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE $SERVER_URL/api/files/$ID/prov-json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/files/$ID/prov-json"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/files/24/prov-json
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/files/24/prov-json"
A curl example using a ``PERSISTENT_ID``
@@ -2918,7 +3694,7 @@ Starting with the release 4.10 the size of the saved original file (for an inges
export SERVER_URL=https://localhost
- curl $SERVER_URL/api/admin/datafiles/integrity/fixmissingoriginalsizes
+ curl "$SERVER_URL/api/admin/datafiles/integrity/fixmissingoriginalsizes"
with limit parameter:
@@ -2933,13 +3709,13 @@ The fully expanded example above (without environment variables) looks like this
.. code-block:: bash
- curl https://localhost/api/admin/datafiles/integrity/fixmissingoriginalsizes"
+ curl "https://localhost/api/admin/datafiles/integrity/fixmissingoriginalsizes"
with limit parameter:
.. code-block:: bash
- curl https://localhost/api/admin/datafiles/integrity/fixmissingoriginalsizes?limit=10"
+ curl "https://localhost/api/admin/datafiles/integrity/fixmissingoriginalsizes?limit=10"
Note the optional "limit" parameter. Without it, the API will attempt to populate the sizes for all the saved originals that don't have them in the database yet. Otherwise it will do so for the first N such datafiles.
@@ -2961,7 +3737,23 @@ The response is a JSON object described in the :doc:`/api/external-tools` sectio
export FILEMETADATA_ID=1
export TOOL_ID=1
- curl -H "X-Dataverse-key: $API_TOKEN" -H "Accept:application/json" "$SERVER_URL/api/files/$FILE_ID/metadata/$FILEMETADATA_ID/toolparams/$TOOL_ID
+ curl -H "X-Dataverse-key: $API_TOKEN" -H "Accept:application/json" "$SERVER_URL/api/files/$FILE_ID/metadata/$FILEMETADATA_ID/toolparams/$TOOL_ID"
+
+.. _get-fixity-algorithm:
+
+Get Fixity Algorithm
+~~~~~~~~~~~~~~~~~~~~~~
+
+This API call can be used to discover the configured fixity/checksum algorithm being used by a Dataverse installation (as configured by - :ref:`:FileFixityChecksumAlgorithm`).
+Currently, the possible values are MD5, SHA-1, SHA-256, and SHA-512.
+This algorithm will be used when the Dataverse software manages a file upload and should be used by external clients uploading files to a Dataverse instance. (Existing files may or may not have checksums with this algorithm.)
+
+.. code-block:: bash
+
+ export SERVER_URL=https://demo.dataverse.org
+
+ curl "$SERVER_URL/api/files/fixityAlgorithm"
+
Users Token Management
----------------------
@@ -2973,21 +3765,21 @@ Find a Token's Expiration Date
In order to obtain the expiration date of a token use::
- curl -H X-Dataverse-key:$API_TOKEN -X GET $SERVER_URL/api/users/token
+ curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/users/token"
Recreate a Token
~~~~~~~~~~~~~~~~
In order to obtain a new token use::
- curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/users/token/recreate
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/users/token/recreate"
Delete a Token
~~~~~~~~~~~~~~
In order to delete a token use::
- curl -H X-Dataverse-key:$API_TOKEN -X DELETE $SERVER_URL/api/users/token
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/users/token"
@@ -3014,26 +3806,14 @@ Optionally, you may use a third query parameter "sendEmailNotification=false" to
Roles
-----
-Create a New Role in a Dataverse Collection
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Creates a new role under Dataverse collection ``id``. Needs a json file with the role description:
-
-.. code-block:: bash
-
- export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
- export SERVER_URL=https://demo.dataverse.org
- export ID=root
-
- curl -H X-Dataverse-key:$API_TOKEN -X POST -H "Content-type:application/json" $SERVER_URL/api/dataverses/$ID/roles --upload-file roles.json
-
-The fully expanded example above (without environment variables) looks like this:
+A role is a set of permissions.
-.. code-block:: bash
+.. _json-representation-of-a-role:
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -H "Content-type:application/json" https://demo.dataverse.org/api/dataverses/root/roles --upload-file roles.json
+JSON Representation of a Role
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Where ``roles.json`` looks like this::
+The JSON representation of a role (``roles.json``) looks like this::
{
"alias": "sys1",
@@ -3044,8 +3824,12 @@ Where ``roles.json`` looks like this::
]
}
-.. note:: Only a Dataverse installation account with superuser permissions is allowed to create roles in a Dataverse Collection.
+.. note:: alias is constrained to a length of 16 characters
+
+Create Role
+~~~~~~~~~~~
+Roles can be created globally (:ref:`create-global-role`) or for individual Dataverse collections (:ref:`create-role-in-collection`).
Show Role
~~~~~~~~~
@@ -3065,13 +3849,13 @@ A curl example using an ``ID``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE $SERVER_URL/api/roles/$ID
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/roles/$ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/roles/24
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/roles/24"
A curl example using a Role alias ``ALIAS``
@@ -3087,7 +3871,7 @@ The fully expanded example above (without environment variables) looks like this
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/roles/:alias?alias=roleAlias
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/roles/:alias?alias=roleAlias"
Explicit Groups
@@ -3180,13 +3964,13 @@ Show Dataverse Software Version and Build Number
export SERVER_URL=https://demo.dataverse.org
- curl $SERVER_URL/api/info/version
+ curl "$SERVER_URL/api/info/version"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/info/version
+ curl "https://demo.dataverse.org/api/info/version"
Show Dataverse Installation Server Name
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3199,13 +3983,13 @@ Get the server name. This is useful when a Dataverse installation is composed of
export SERVER_URL=https://demo.dataverse.org
- curl $SERVER_URL/api/info/server
+ curl "$SERVER_URL/api/info/server"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/info/server
+ curl "https://demo.dataverse.org/api/info/server"
Show Custom Popup Text for Publishing Datasets
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3218,13 +4002,13 @@ For now, only the value for the :ref:`:DatasetPublishPopupCustomText` setting fr
export SERVER_URL=https://demo.dataverse.org
- curl $SERVER_URL/api/info/settings/:DatasetPublishPopupCustomText
+ curl "$SERVER_URL/api/info/settings/:DatasetPublishPopupCustomText"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/info/settings/:DatasetPublishPopupCustomText
+ curl "https://demo.dataverse.org/api/info/settings/:DatasetPublishPopupCustomText"
Get API Terms of Use URL
~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3237,13 +4021,13 @@ Get API Terms of Use. The response contains the text value inserted as API Terms
export SERVER_URL=https://demo.dataverse.org
- curl $SERVER_URL/api/info/apiTermsOfUse
+ curl "$SERVER_URL/api/info/apiTermsOfUse"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/info/apiTermsOfUse
+ curl "https://demo.dataverse.org/api/info/apiTermsOfUse"
.. _info-incomplete-metadata:
@@ -3253,18 +4037,59 @@ Show Support Of Incomplete Metadata Deposition
Learn if an instance has been configured to allow deposition of incomplete datasets via the API.
See also :ref:`create-dataset-command` and :ref:`dataverse.api.allow-incomplete-metadata`
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
+
+.. code-block:: bash
+
+ export SERVER_URL=https://demo.dataverse.org
+
+ curl "$SERVER_URL/api/info/settings/incompleteMetadataViaApi"
+
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/info/settings/incompleteMetadataViaApi"
+
+Get Zip File Download Limit
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Get the configured zip file download limit. The response contains the long value of the limit in bytes.
+
+This limit comes from the database setting :ref:`:ZipDownloadLimit` if set, or the default value if the database setting is not set, which is 104857600 (100MB).
+
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
+
.. code-block:: bash
export SERVER_URL=https://demo.dataverse.org
- curl $SERVER_URL/api/info/settings/incompleteMetadataViaApi
+ curl "$SERVER_URL/api/info/zipDownloadLimit"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/info/settings/incompleteMetadataViaApi
+ curl "https://demo.dataverse.org/api/info/zipDownloadLimit"
+
+Get Maximum Embargo Duration In Months
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Get the maximum embargo duration in months, if available, configured through the database setting :ref:`:MaxEmbargoDurationInMonths` from the Configuration section of the Installation Guide.
+
+.. note:: See :ref:`curl-examples-and-environment-variables` if you are unfamiliar with the use of export below.
+
+.. code-block:: bash
+
+ export SERVER_URL=https://demo.dataverse.org
+
+ curl "$SERVER_URL/api/info/settings/:MaxEmbargoDurationInMonths"
+The fully expanded example above (without environment variables) looks like this:
+
+.. code-block:: bash
+
+ curl "https://demo.dataverse.org/api/info/settings/:MaxEmbargoDurationInMonths"
.. _metadata-blocks-api:
@@ -3282,13 +4107,13 @@ Show Info About All Metadata Blocks
export SERVER_URL=https://demo.dataverse.org
- curl $SERVER_URL/api/metadatablocks
+ curl "$SERVER_URL/api/metadatablocks"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/metadatablocks
+ curl "https://demo.dataverse.org/api/metadatablocks"
Show Info About Single Metadata Block
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3300,13 +4125,13 @@ Show Info About Single Metadata Block
export SERVER_URL=https://demo.dataverse.org
export IDENTIFIER=citation
- curl $SERVER_URL/api/metadatablocks/$IDENTIFIER
+ curl "$SERVER_URL/api/metadatablocks/$IDENTIFIER"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl https://demo.dataverse.org/api/metadatablocks/citation
+ curl "https://demo.dataverse.org/api/metadatablocks/citation"
.. _Notifications:
@@ -3322,7 +4147,7 @@ Each user can get a dump of their notifications by passing in their API token:
.. code-block:: bash
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/notifications/all
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/notifications/all"
Delete Notification by User
~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3333,7 +4158,7 @@ Each user can delete notifications by passing in their API token and specifying
export NOTIFICATION_ID=555
- curl -H X-Dataverse-key:$API_TOKEN -X DELETE "$SERVER_URL/api/notifications/$NOTIFICATION_ID"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/notifications/$NOTIFICATION_ID"
Get All Muted In-app Notifications by User
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3342,7 +4167,7 @@ Each user can get a list of their muted in-app notification types by passing in
.. code-block:: bash
- curl -H X-Dataverse-key:$API_TOKEN -X GET "$SERVER_URL/api/notifications/mutedNotifications"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/notifications/mutedNotifications"
Mute In-app Notification by User
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3353,7 +4178,7 @@ Each user can mute in-app notifications by passing in their API token and specif
export NOTIFICATION_TYPE=ASSIGNROLE
- curl -H X-Dataverse-key:$API_TOKEN -X PUT "$SERVER_URL/api/notifications/mutedNotifications/$NOTIFICATION_TYPE"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/notifications/mutedNotifications/$NOTIFICATION_TYPE"
Unmute In-app Notification by User
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3364,7 +4189,7 @@ Each user can unmute in-app notifications by passing in their API token and spec
export NOTIFICATION_TYPE=ASSIGNROLE
- curl -H X-Dataverse-key:$API_TOKEN -X DELETE "$SERVER_URL/api/notifications/mutedNotifications/$NOTIFICATION_TYPE"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/notifications/mutedNotifications/$NOTIFICATION_TYPE"
Get All Muted Email Notifications by User
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3373,7 +4198,7 @@ Each user can get a list of their muted email notification types by passing in t
.. code-block:: bash
- curl -H X-Dataverse-key:$API_TOKEN -X GET "$SERVER_URL/api/notifications/mutedEmails"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/notifications/mutedEmails"
Mute Email Notification by User
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3384,7 +4209,7 @@ Each user can mute email notifications by passing in their API token and specify
export NOTIFICATION_TYPE=ASSIGNROLE
- curl -H X-Dataverse-key:$API_TOKEN -X PUT "$SERVER_URL/api/notifications/mutedEmails/$NOTIFICATION_TYPE"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/notifications/mutedEmails/$NOTIFICATION_TYPE"
Unmute Email Notification by User
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3395,7 +4220,7 @@ Each user can unmute email notifications by passing in their API token and speci
export NOTIFICATION_TYPE=ASSIGNROLE
- curl -H X-Dataverse-key:$API_TOKEN -X DELETE "$SERVER_URL/api/notifications/mutedEmails/$NOTIFICATION_TYPE"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/notifications/mutedEmails/$NOTIFICATION_TYPE"
.. _User Information:
@@ -3407,7 +4232,7 @@ Get User Information in JSON Format
Each user can get a dump of their basic information in JSON format by passing in their API token::
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/users/:me
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/users/:me"
@@ -3454,7 +4279,7 @@ An example JSON file would look like this::
export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
export SERVER_URL=https://demo.dataverse.org
- curl -H X-Dataverse-key:$API_TOKEN -X POST "$SERVER_URL/api/harvest/server/oaisets/add" --upload-file harvestset-finch.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/harvest/server/oaisets/add" --upload-file harvestset-finch.json
The fully expanded example above (without the environment variables) looks like this:
@@ -3489,7 +4314,7 @@ An example JSON file would look like this::
export SERVER_URL=https://demo.dataverse.org
export SPECNAME=ffAuthor
- curl -H X-Dataverse-key:$API_TOKEN -X PUT "$SERVER_URL/api/harvest/server/oaisets/$SPECNAME" --upload-file modify-harvestset-finch.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X PUT "$SERVER_URL/api/harvest/server/oaisets/$SPECNAME" --upload-file modify-harvestset-finch.json
The fully expanded example above (without the environment variables) looks like this:
@@ -3510,7 +4335,7 @@ To delete a harvesting set, use the set's database name. For example, to delete
export SERVER_URL=https://demo.dataverse.org
export SPECNAME=ffAuthor
- curl -H X-Dataverse-key:$API_TOKEN -X DELETE "$SERVER_URL/api/harvest/server/oaisets/$SPECNAME"
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/harvest/server/oaisets/$SPECNAME"
The fully expanded example above (without the environment variables) looks like this:
@@ -3620,7 +4445,7 @@ Something important to keep in mind about this API is that, unlike the harvestin
export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
export SERVER_URL=http://localhost:8080
- curl -H X-Dataverse-key:$API_TOKEN -X POST -H "Content-Type: application/json" "$SERVER_URL/api/harvest/clients/zenodo" --upload-file client.json
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST -H "Content-Type: application/json" "$SERVER_URL/api/harvest/clients/zenodo" --upload-file client.json
The fully expanded example above (without the environment variables) looks like this:
@@ -3690,13 +4515,13 @@ Get information on a PID, especially its "state" such as "draft" or "findable".
export SERVER_URL=https://demo.dataverse.org
export PID=doi:10.70122/FK2/9BXT5O
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/pids?persistentId=$PID
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/pids?persistentId=$PID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/pids?persistentId=doi:10.70122/FK2/9BXT5O
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/pids?persistentId=doi:10.70122/FK2/9BXT5O"
List Unreserved PIDs
~~~~~~~~~~~~~~~~~~~~
@@ -3710,14 +4535,14 @@ Get a list of PIDs that have not been reserved on the PID provider side. This ca
export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
export SERVER_URL=https://demo.dataverse.org
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/pids/unreserved
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/pids/unreserved"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx https://demo.dataverse.org/api/pids/unreserved
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/pids/unreserved"
Reserve a PID
~~~~~~~~~~~~~
@@ -3732,13 +4557,13 @@ Reserved a PID for a dataset. A superuser API token is required.
export SERVER_URL=https://demo.dataverse.org
export PID=doi:10.70122/FK2/9BXT5O
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/pids/:persistentId/reserve?persistentId=$PID
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/pids/:persistentId/reserve?persistentId=$PID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST https://demo.dataverse.org/api/pids/:persistentId/reserve?persistentId=doi:10.70122/FK2/9BXT5O
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/pids/:persistentId/reserve?persistentId=doi:10.70122/FK2/9BXT5O"
Delete a PID
~~~~~~~~~~~~
@@ -3753,13 +4578,13 @@ Delete PID (this is only possible for PIDs that are in the "draft" state) and wi
export SERVER_URL=https://demo.dataverse.org
export PID=doi:10.70122/FK2/9BXT5O
- curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE $SERVER_URL/api/pids/:persistentId/delete?persistentId=$PID
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/pids/:persistentId/delete?persistentId=$PID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X DELETE https://demo.dataverse.org/api/pids/:persistentId/delete?persistentId=doi:10.70122/FK2/9BXT5O
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/pids/:persistentId/delete?persistentId=doi:10.70122/FK2/9BXT5O"
.. _admin:
@@ -3809,7 +4634,7 @@ Note that HTML can be included in banner messages.
Add a Banner Message::
- curl -H "Content-type:application/json" -X POST http://$SERVER/api/admin/bannerMessage --upload-file messages.json
+ curl -H "Content-type:application/json" -X POST "http://$SERVER/api/admin/bannerMessage" --upload-file messages.json
Where ``messages.json`` looks like this::
@@ -3829,15 +4654,15 @@ Where ``messages.json`` looks like this::
Get a list of active Banner Messages::
- curl -X GET http://$SERVER/api/admin/bannerMessage
+ curl -X GET "http://$SERVER/api/admin/bannerMessage"
Delete a Banner Message by its id::
- curl -X DELETE http://$SERVER/api/admin/bannerMessage/$id
+ curl -X DELETE "http://$SERVER/api/admin/bannerMessage/$id"
Deactivate a Banner Message by its id (allows you to hide a message while retaining information about which users have dismissed the banner)::
- curl -X PUT http://$SERVER/api/admin/bannerMessage/$id/deactivate
+ curl -X PUT "http://$SERVER/api/admin/bannerMessage/$id/deactivate"
List Authentication Provider Factories
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3890,7 +4715,7 @@ Check whether an authentication proider is enabled::
The body of the request should be either ``true`` or ``false``. Content type has to be ``application/json``, like so::
- curl -H "Content-type: application/json" -X POST -d"false" http://localhost:8080/api/admin/authenticationProviders/echo-dignified/:enabled
+ curl -H "Content-type: application/json" -X POST -d"false" "http://localhost:8080/api/admin/authenticationProviders/echo-dignified/:enabled"
Delete an Authentication Provider
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3906,13 +4731,25 @@ List all global roles in the system. ::
GET http://$SERVER/api/admin/roles
+.. _create-global-role:
+
Create Global Role
~~~~~~~~~~~~~~~~~~
Creates a global role in the Dataverse installation. The data POSTed are assumed to be a role JSON. ::
POST http://$SERVER/api/admin/roles
-
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ID=root
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/admin/roles" --upload-file roles.json
+
+``roles.json`` see :ref:`json-representation-of-a-role`
+
Delete Global Role
~~~~~~~~~~~~~~~~~~
@@ -3924,13 +4761,13 @@ A curl example using an ``ID``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE $SERVER_URL/api/admin/roles/$ID
+ curl -H "X-Dataverse-key:$API_TOKEN" -X DELETE "$SERVER_URL/api/admin/roles/$ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/admin/roles/24
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/admin/roles/24"
A curl example using a Role alias ``ALIAS``
@@ -3946,7 +4783,7 @@ The fully expanded example above (without environment variables) looks like this
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE https://demo.dataverse.org/api/admin/roles/:alias?alias=roleAlias
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X DELETE "https://demo.dataverse.org/api/admin/roles/:alias?alias=roleAlias"
List Users
~~~~~~~~~~
@@ -3964,7 +4801,7 @@ List users with the options to search and "page" through results. Only accessibl
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -H "X-Dataverse-key:$API_TOKEN" $SERVER_URL/api/admin/list-users
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/admin/list-users"
# sort by createdtime (the creation time of the account)
curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/admin/list-users?sortKey=createdtime"
@@ -3973,7 +4810,7 @@ The fully expanded example above (without environment variables) looks like this
.. code-block:: bash
- curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" https://demo.dataverse.org/api/admin/list-users
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/admin/list-users"
# sort by createdtime (the creation time of the account)
curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" "https://demo.dataverse.org/api/admin/list-users?sortKey=createdtime"
@@ -4124,7 +4961,7 @@ If a user has created multiple accounts and has been performed actions under bot
POST https://$SERVER/api/users/$toMergeIdentifier/mergeIntoUser/$continuingIdentifier
-Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST http://demo.dataverse.org/api/users/jsmith2/mergeIntoUser/jsmith``
+Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST "http://demo.dataverse.org/api/users/jsmith2/mergeIntoUser/jsmith"``
This action moves account data from jsmith2 into the account jsmith and deletes the account of jsmith2.
@@ -4139,7 +4976,7 @@ Changes identifier for user in ``AuthenticatedUser``, ``BuiltinUser``, ``Authent
POST http://$SERVER/api/users/$oldIdentifier/changeIdentifier/$newIdentifier
-Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST https://demo.dataverse.org/api/users/johnsmith/changeIdentifier/jsmith``
+Example: ``curl -H "X-Dataverse-key: $API_TOKEN" -X POST "https://demo.dataverse.org/api/users/johnsmith/changeIdentifier/jsmith"``
This action changes the identifier of user johnsmith to jsmith.
@@ -4179,13 +5016,13 @@ Deactivates a user. A superuser API token is not required but the command will o
export SERVER_URL=http://localhost:8080
export USERNAME=jdoe
- curl -X POST $SERVER_URL/api/admin/authenticatedUsers/$USERNAME/deactivate
+ curl -X POST "$SERVER_URL/api/admin/authenticatedUsers/$USERNAME/deactivate"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -X POST http://localhost:8080/api/admin/authenticatedUsers/jdoe/deactivate
+ curl -X POST "http://localhost:8080/api/admin/authenticatedUsers/jdoe/deactivate"
The database ID of the user can be passed instead of the username.
@@ -4194,7 +5031,7 @@ The database ID of the user can be passed instead of the username.
export SERVER_URL=http://localhost:8080
export USERID=42
- curl -X POST $SERVER_URL/api/admin/authenticatedUsers/id/$USERID/deactivate
+ curl -X POST "$SERVER_URL/api/admin/authenticatedUsers/id/$USERID/deactivate"
Note: A primary purpose of most Dataverse installations is to serve an archive. In the archival space, there are best practices around the tracking of data access and the tracking of modifications to data and metadata. In support of these key workflows, a simple mechanism to delete users that have performed edit or access actions in the system is not provided. Providing a Deactivate User endpoint for users who have taken certain actions in the system alongside a Delete User endpoint to remove users that haven't taken certain actions in the system is by design.
@@ -4233,13 +5070,13 @@ Show the traces that the user has left in the system, such as datasets created,
export SERVER_URL=https://demo.dataverse.org
export USERNAME=jdoe
- curl -H "X-Dataverse-key:$API_TOKEN" -X GET $SERVER_URL/api/users/$USERNAME/traces
+ curl -H "X-Dataverse-key:$API_TOKEN" -X GET "$SERVER_URL/api/users/$USERNAME/traces"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X GET https://demo.dataverse.org/api/users/jdoe/traces
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X GET "https://demo.dataverse.org/api/users/jdoe/traces"
Remove All Roles from a User
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -4254,13 +5091,13 @@ Removes all roles from the user. This is equivalent of clicking the "Remove All
export SERVER_URL=https://demo.dataverse.org
export USERNAME=jdoe
- curl -H "X-Dataverse-key:$API_TOKEN" -X POST $SERVER_URL/api/users/$USERNAME/removeRoles
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/users/$USERNAME/removeRoles"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST http://localhost:8080/api/users/jdoe/removeRoles
+ curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "http://localhost:8080/api/users/jdoe/removeRoles"
List Role Assignments of a Role Assignee
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -4324,11 +5161,11 @@ Datafile Integrity
Recalculate the check sum value value of a datafile, by supplying the file's database id and an algorithm (Valid values for $ALGORITHM include MD5, SHA-1, SHA-256, and SHA-512)::
- curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/admin/computeDataFileHashValue/{fileId}/algorithm/$ALGORITHM
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/admin/computeDataFileHashValue/{fileId}/algorithm/$ALGORITHM"
Validate an existing check sum value against one newly calculated from the saved file::
- curl -H X-Dataverse-key:$API_TOKEN -X POST $SERVER_URL/api/admin/validateDataFileHashValue/{fileId}
+ curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/admin/validateDataFileHashValue/{fileId}"
.. _dataset-files-validation-api:
@@ -4341,7 +5178,7 @@ The following validates all the physical files in the dataset specified, by reca
It will report the specific files that have failed the validation. For example::
- curl http://localhost:8080/api/admin/validate/dataset/files/:persistentId/?persistentId=doi:10.5072/FK2/XXXXX
+ curl "http://localhost:8080/api/admin/validate/dataset/files/:persistentId/?persistentId=doi:10.5072/FK2/XXXXX"
{"dataFiles": [
{"datafileId":2658,"storageIdentifier":"file://123-aaa","status":"valid"},
{"datafileId":2659,"storageIdentifier":"file://123-bbb","status":"invalid","errorMessage":"Checksum mismatch for datafile id 2669"},
@@ -4351,6 +5188,26 @@ It will report the specific files that have failed the validation. For example::
These are only available to super users.
+.. _UpdateChecksums:
+
+Update Checksums To Use New Algorithm
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The fixity algorithm used on existing files can be changed by a superuser using this API call. An optional query parameter (num) can be used to limit the number of updates attempted (i.e. to do processing in batches).
+The API call will only update the algorithm and checksum for a file if the existing checksum can be validated against the file.
+Statistics concerning the updates are returned in the response to the API call with details in the log.
+The primary use for this API call is to update existing files after the algorithm used when uploading new files is changes - see - :ref:`:FileFixityChecksumAlgorithm`.
+Allowed values are MD5, SHA-1, SHA-256, and SHA-512
+
+.. code-block:: bash
+
+ export ALG=SHA-256
+ export BATCHSIZE=1
+
+ curl "http://localhost:8080/api/admin/updateHashValues/$ALG"
+ curl "http://localhost:8080/api/admin/updateHashValues/$ALG?num=$BATCHSIZE"
+
+
.. _dataset-validation-api:
Dataset Validation
@@ -4358,7 +5215,7 @@ Dataset Validation
Validate the dataset and its components (DatasetVersion, FileMetadatas, etc.) for constraint violations::
- curl $SERVER_URL/api/admin/validate/dataset/{datasetId}
+ curl "$SERVER_URL/api/admin/validate/dataset/{datasetId}"
if validation fails, will report the specific database entity and the offending value. For example::
@@ -4368,7 +5225,7 @@ If the optional argument ``variables=true`` is specified, the API will also vali
Validate all the datasets in the Dataverse installation, report any constraint violations found::
- curl $SERVER_URL/api/admin/validate/datasets
+ curl "$SERVER_URL/api/admin/validate/datasets"
If the optional argument ``variables=true`` is specified, the API will also validate the metadata associated with any tabular data files. (For example: an invalid or empty variable name). Note that validating all the tabular metadata may significantly increase the run time of the full validation pass.
@@ -4477,14 +5334,14 @@ View the list of standard license terms that can be selected for a dataset:
.. code-block:: bash
export SERVER_URL=https://demo.dataverse.org
- curl $SERVER_URL/api/licenses
+ curl "$SERVER_URL/api/licenses"
View the details of the standard license with the database ID specified in ``$ID``:
.. code-block:: bash
export ID=1
- curl $SERVER_URL/api/licenses/$ID
+ curl "$SERVER_URL/api/licenses/$ID"
Superusers can add a new license by posting a JSON file adapted from this example :download:`add-license.json <../_static/api/add-license.json>`. The ``name`` and ``uri`` of the new license must be unique. Sort order field is mandatory. If you are interested in adding a Creative Commons license, you are encouarged to use the JSON files under :ref:`adding-creative-commons-licenses`:
@@ -4492,33 +5349,33 @@ Superusers can add a new license by posting a JSON file adapted from this exampl
.. code-block:: bash
export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
- curl -X POST -H 'Content-Type: application/json' -H X-Dataverse-key:$API_TOKEN --data-binary @add-license.json $SERVER_URL/api/licenses
+ curl -X POST -H 'Content-Type: application/json' -H "X-Dataverse-key:$API_TOKEN" --data-binary @add-license.json "$SERVER_URL/api/licenses"
Superusers can change whether an existing license is active (usable for new dataset versions) or inactive (only allowed on already-published versions) specified by the license ``$ID``:
.. code-block:: bash
export STATE=true
- curl -X PUT -H 'Content-Type: application/json' -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/licenses/$ID/:active/$STATE
+ curl -X PUT -H 'Content-Type: application/json' -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/licenses/$ID/:active/$STATE"
Superusers may change the default license by specifying the license ``$ID``:
.. code-block:: bash
- curl -X PUT -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/licenses/default/$ID
+ curl -X PUT -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/licenses/default/$ID"
Superusers can delete a license, provided it is not in use, by the license ``$ID``:
.. code-block:: bash
- curl -X DELETE -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/licenses/$ID
+ curl -X DELETE -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/licenses/$ID"
Superusers can change the sorting order of a license specified by the license ``$ID``:
.. code-block:: bash
export SORT_ORDER=100
- curl -X PUT -H 'Content-Type: application/json' -H X-Dataverse-key:$API_TOKEN $SERVER_URL/api/licenses/$ID/:sortOrder/$SORT_ORDER
+ curl -X PUT -H 'Content-Type: application/json' -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/licenses/$ID/:sortOrder/$SORT_ORDER"
List Dataset Templates
~~~~~~~~~~~~~~~~~~~~~~
@@ -4542,13 +5399,13 @@ A curl example using an ``ID``
export SERVER_URL=https://demo.dataverse.org
export ID=24
- curl -X DELETE $SERVER_URL/api/admin/template/$ID
+ curl -X DELETE "$SERVER_URL/api/admin/template/$ID"
The fully expanded example above (without environment variables) looks like this:
.. code-block:: bash
- curl -X DELETE https://demo.dataverse.org/api/admin/template/24
+ curl -X DELETE "https://demo.dataverse.org/api/admin/template/24"
.. _api-native-signed-url:
@@ -4575,7 +5432,7 @@ A curl example using allowing access to a dataset's metadata
export API_KEY=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
export JSON='{"url":"https://demo.dataverse.org/api/v1/datasets/:persistentId/?persistentId=doi:10.5072/FK2/J8SJZB","timeOut":5,"user":"alberteinstein"}'
- curl -H "X-Dataverse-key:$API_KEY" -H 'Content-Type:application/json' -d "$JSON" $SERVER_URL/api/admin/requestSignedUrl
+ curl -H "X-Dataverse-key:$API_KEY" -H 'Content-Type:application/json' -d "$JSON" "$SERVER_URL/api/admin/requestSignedUrl"
Please see :ref:`dataverse.api.signature-secret` for the configuration option to add a shared secret, enabling extra
security.
@@ -4602,6 +5459,72 @@ A curl example using an ``ID``
export SERVER_URL=http://localhost
export JSON='{"targetId":24, "subject":"Data Question", "body":"Please help me understand your data. Thank you!", "fromEmail":"dataverseSupport@mailinator.com"}'
- curl -X POST -H 'Content-Type:application/json' -d "$JSON" $SERVER_URL/api/admin/feedback
+ curl -X POST -H 'Content-Type:application/json' -d "$JSON" "$SERVER_URL/api/admin/feedback"
Note that this call could be useful in coordinating with dataset authors (assuming they are also contacts) as an alternative/addition to the functionality provided by :ref:`return-a-dataset`.
+
+.. _thumbnail_reset:
+
+Reset Thumbnail Failure Flags
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If Dataverse attempts to create a thumbnail image for an image or PDF file and the attempt fails, Dataverse will set a flag for the file to avoid repeated attempts to generate the thumbnail.
+For cases where the problem may have been temporary (or fixed in a later Dataverse release), the API calls below can be used to reset this flag for all files or for a given file.
+
+.. code-block:: bash
+
+ export SERVER_URL=https://demo.dataverse.org
+ export FILE_ID=1234
+
+ curl -X DELETE $SERVER_URL/api/admin/clearThumbnailFailureFlag
+
+ curl -X DELETE $SERVER_URL/api/admin/clearThumbnailFailureFlag/$FILE_ID
+
+.. _download-file-from-tmp:
+
+Download File from /tmp
+~~~~~~~~~~~~~~~~~~~~~~~
+
+As a superuser::
+
+ GET /api/admin/downloadTmpFile?fullyQualifiedPathToFile=/tmp/foo.txt
+
+Note that this API is probably only useful for testing.
+
+MyData
+------
+
+The MyData API is used to get a list of just the datasets, dataverses or datafiles an authenticated user can edit.
+
+A curl example listing objects
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export ROLE_IDS=6
+ export DVOBJECT_TYPES=Dataset
+ export PUBLISHED_STATES=Unpublished
+ export PER_PAGE=10
+
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/mydata/retrieve?role_ids=$ROLE_IDS&dvobject_types=$DVOBJECT_TYPES&published_states=$PUBLISHED_STATES&per_page=$PER_PAGE"
+
+Parameters:
+
+``role_id`` Roles are customizable. Standard roles include:
+
+- ``1`` = Admin
+- ``2`` = File Downloader
+- ``3`` = Dataverse + Dataset Creator
+- ``4`` = Dataverse Creator
+- ``5`` = Dataset Creator
+- ``6`` = Contributor
+- ``7`` = Curator
+- ``8`` = Member
+
+``dvobject_types`` Type of object, several possible values among: ``DataFile`` , ``Dataset`` & ``Dataverse`` .
+
+``published_states`` State of the object, several possible values among:``Published`` , ``Unpublished`` , ``Draft`` , ``Deaccessioned`` & ``In+Review`` .
+
+``per_page`` Number of results returned per page.
+
diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst
index 11b43e98774..51391784bde 100755
--- a/doc/sphinx-guides/source/api/sword.rst
+++ b/doc/sphinx-guides/source/api/sword.rst
@@ -9,19 +9,19 @@ SWORD_ stands for "Simple Web-service Offering Repository Deposit" and is a "pro
About
-----
-Introduced in Dataverse Network (DVN) `3.6 `_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse Software 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`.
+Introduced in Dataverse Network (DVN) `3.6 `_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse Software 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`.
-The Dataverse Software implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc.
+The Dataverse Software implements most of SWORDv2_, which is specified at https://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc.
As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse Software 4.0 datasets can also be created via JSON using the "native" API. SWORD is limited to the dozen or so fields listed below in the crosswalk, but the native API allows you to populate all metadata fields available in a Dataverse installation.
-.. _SWORD: http://en.wikipedia.org/wiki/SWORD_%28protocol%29
+.. _SWORD: https://en.wikipedia.org/wiki/SWORD_%28protocol%29
.. _SWORDv2: http://swordapp.org/sword-v2/sword-v2-specifications/
.. _RFC 5023: https://tools.ietf.org/html/rfc5023
-.. _SWORDv2 specification: http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html
+.. _SWORDv2 specification: https://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html
.. _sword-auth:
@@ -86,7 +86,7 @@ New features as of v1.1
- "Contact E-mail" is automatically populated from dataset owner's email.
-- "Subject" uses our controlled vocabulary list of subjects. This list is in the Citation Metadata of our User Guide > `Metadata References `_. Otherwise, if a term does not match our controlled vocabulary list, it will put any subject terms in "Keyword". If Subject is empty it is automatically populated with "N/A".
+- "Subject" uses our controlled vocabulary list of subjects. This list is in the Citation Metadata of our User Guide > `Metadata References `_. Otherwise, if a term does not match our controlled vocabulary list, it will put any subject terms in "Keyword". If Subject is empty it is automatically populated with "N/A".
- Zero-length files are now allowed (but not necessarily encouraged).
@@ -127,7 +127,7 @@ Dublin Core Terms (DC Terms) Qualified Mapping - Dataverse Project DB Element Cr
+-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
|dcterms:creator | authorName (LastName, FirstName) | Y | Author(s) for the Dataset. |
+-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
-|dcterms:subject | subject (Controlled Vocabulary) OR keyword | Y | Controlled Vocabulary list is in our User Guide > `Metadata References `_. |
+|dcterms:subject | subject (Controlled Vocabulary) OR keyword | Y | Controlled Vocabulary list is in our User Guide > `Metadata References `_. |
+-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
|dcterms:description | dsDescriptionValue | Y | Describing the purpose, scope or nature of the Dataset. Can also use dcterms:abstract. |
+-----------------------------+----------------------------------------------+--------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------+
diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py
index 736d86cacf5..64efc359e9a 100755
--- a/doc/sphinx-guides/source/conf.py
+++ b/doc/sphinx-guides/source/conf.py
@@ -66,9 +66,9 @@
# built documents.
#
# The short X.Y version.
-version = '5.13'
+version = '6.1'
# The full version, including alpha/beta/rc tags.
-release = '5.13'
+release = '6.1'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@@ -432,7 +432,7 @@
# Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {'https://docs.python.org/': None}
# Suppress "WARNING: unknown mimetype for ..." https://github.com/IQSS/dataverse/issues/3391
suppress_warnings = ['epub.unknown_project_files']
rst_prolog = """
diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst
index ed06314ef0e..1a47a8fc413 100644
--- a/doc/sphinx-guides/source/container/base-image.rst
+++ b/doc/sphinx-guides/source/container/base-image.rst
@@ -41,7 +41,7 @@ Image Contents
The base image provides:
-- `Eclipse Temurin JRE using Java 11 `_
+- `Eclipse Temurin JRE using Java 17 `_
- `Payara Community Application Server `_
- CLI tools necessary to run Dataverse (i. e. ``curl`` or ``jq`` - see also :doc:`../installation/prerequisites` in Installation Guide)
- Linux tools for analysis, monitoring and so on
diff --git a/doc/sphinx-guides/source/container/configbaker-image.rst b/doc/sphinx-guides/source/container/configbaker-image.rst
index 7218e2d8d14..d098bd46436 100644
--- a/doc/sphinx-guides/source/container/configbaker-image.rst
+++ b/doc/sphinx-guides/source/container/configbaker-image.rst
@@ -86,7 +86,7 @@ Maven modules packaging target with activated "container" profile from the proje
If you specifically want to build a config baker image *only*, try
-``mvn -Pct package -Ddocker.filter=dev_bootstrap``
+``mvn -Pct docker:build -Ddocker.filter=dev_bootstrap``
The build of config baker involves copying Solr configset files. The Solr version used is inherited from Maven,
acting as the single source of truth. Also, the tag of the image should correspond the application image, as
diff --git a/doc/sphinx-guides/source/container/dev-usage.rst b/doc/sphinx-guides/source/container/dev-usage.rst
index 5b8253b53f7..b2547306b03 100644
--- a/doc/sphinx-guides/source/container/dev-usage.rst
+++ b/doc/sphinx-guides/source/container/dev-usage.rst
@@ -9,15 +9,7 @@ Please note! This Docker setup is not for production!
Quickstart
----------
-After cloning the repo, try this:
-
-``mvn -Pct clean package docker:run``
-
-After some time you should be able to log in:
-
-- url: http://localhost:8080
-- username: dataverseAdmin
-- password: admin1
+See :ref:`container-dev-quickstart`.
Intro
-----
@@ -27,25 +19,182 @@ Assuming you have `Docker `_, `Docker D
you have Java and Maven installed, as you are at least about to develop code changes.
To test drive these local changes to the Dataverse codebase in a containerized application server (and avoid the
-setup described in :doc:`../developers/dev-environment`), you must a) build the application container and b)
-run it in addition to the necessary dependencies.
-
-Building and Running
---------------------
+setup described in :doc:`../developers/dev-environment`), you must a) build the application and b) run it in addition
+to the necessary dependencies. (Which might involve building a new local version of the :doc:`configbaker-image`.)
-To build the application image, run the following command, as described in :doc:`app-image`:
+.. _dev-build:
-``mvn -Pct clean package``
+Building
+--------
-Now, start all the containers with a single command:
+To build the :doc:`application ` and :doc:`config baker image `, run the following command:
-``mvn -Pct docker:run``
+``mvn -Pct clean package``
-(You could also concatenate both commands into one, as shown above in the quickstart.)
+Once this is done, you will see images ``gdcc/dataverse:unstable`` and ``gdcc/configbaker:unstable`` available in your
+Docker cache.
+
+**Note:** This will skip any unit tests. If you have built the code before for testing, etc. you might omit the
+``clean`` to avoid recompiling.
+
+**Note:** Also we have a ``docker-compose-dev.yml`` file, it's currently not possible to build the images without
+invoking Maven. This might change in the future.
+
+
+.. _dev-run:
+
+Running
+-------
+
+After building the app and config baker image containing your local changes to the Dataverse application, you want to
+run it together with all dependencies. There are four ways to do this (commands executed at root of project directory):
+
+.. list-table:: Cheatsheet: Running Containers
+ :widths: 15 40 45
+ :header-rows: 1
+ :stub-columns: 1
+ :align: left
+
+ * - \
+ - Using Maven
+ - Using Compose
+ * - In foreground
+ - ``mvn -Pct docker:run``
+ - ``docker compose -f docker-compose-dev.yml up``
+ * - In background
+ - ``mvn -Pct docker:start``
+ - ``docker compose -f docker-compose-dev.yml up -d``
+
+Both ways have their pros and cons:
+
+.. list-table:: Decision Helper: Fore- or Background?
+ :widths: 15 40 45
+ :header-rows: 1
+ :stub-columns: 1
+ :align: left
+
+ * - \
+ - Pros
+ - Cons
+ * - Foreground
+ - | Logs scroll by when interacting with API / UI
+ | To stop all containers simply hit ``Ctrl+C``
+ - | Lots and lots of logs scrolling by
+ | Must stop all containers to restart
+ * - Background
+ - | No logs scrolling by
+ | Easy to replace single containers
+ - | No logs scrolling by
+ | Stopping containers needs an extra command
+
+In case you want to concatenate building and running, here's a cheatsheet for you:
+
+.. list-table:: Cheatsheet: Building and Running Containers
+ :widths: 15 40 45
+ :header-rows: 1
+ :stub-columns: 1
+ :align: left
+
+ * - \
+ - Using Maven
+ - Using Compose
+ * - In foreground
+ - ``mvn -Pct package docker:run``
+ - ``mvn -Pct package && docker compose -f docker-compose-dev.yml up``
+ * - In background
+ - ``mvn -Pct package docker:start``
+ - ``mvn -Pct package && docker compose -f docker-compose-dev.yml up -d``
+
+Once all containers have been started, you can check if the application was deployed correctly by checking the version
+at http://localhost:8080/api/info/version or watch the logs.
+
+**Note:** To stop all containers you started in background, invoke ``mvn -Pct docker:stop`` or
+``docker compose -f docker-compose-dev.yml down``.
Check that you can log in to http://localhost:8080 using user ``dataverseAdmin`` and password ``admin1``.
-You can also access the Payara Admin Console if needed, which is available at http://localhost:4848. To log in, use user ``admin`` and password ``admin``. As a reminder, the application container is for development use only, so we are exposing the admin console for testing purposes. In a production environment, it may be more convenient to leave this console unopened.
+You can also access the Payara Admin Console if needed, which is available at http://localhost:4848. To log in, use
+user ``admin`` and password ``admin``. As a reminder, the application container is for development use only, so we
+are exposing the admin console for testing purposes. In a production environment, it may be more convenient to leave
+this console unopened.
Note that data is persisted in ``./docker-dev-volumes`` in the root of the Git repo. For a clean start, you should
remove this directory before running the ``mvn`` commands above.
+
+
+.. _dev-logs:
+
+Viewing Logs
+------------
+
+In case you started containers in background mode (see :ref:`dev-run`), you can use the following commands to view and/or
+watch logs from the containers.
+
+The safe bet for any running container's logs is to lookup the container name via ``docker ps`` and use it in
+``docker logs ``. You can tail logs by adding ``-n`` and follow them by adding ``-f`` (just like ``tail`` cmd).
+See ``docker logs --help`` for more.
+
+Alternatives:
+
+- In case you used Maven for running, you may use ``mvn -Pct docker:logs -Ddocker.filter=``.
+- If you used Docker Compose for running, you may use ``docker compose -f docker-compose-dev.yml logs ``.
+ Options are the same.
+
+
+Redeploying
+-----------
+
+Rebuild and Running Images
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The safest way to redeploy code is to stop the running containers (with Ctrl-c if you started them in the foreground) and then build and run them again with ``mvn -Pct clean package docker:run``.
+
+IntelliJ IDEA Ultimate and Payara Platform Tools
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you have IntelliJ IDEA Ultimate (note that `free educational licenses `_ are available), you can install `Payara Platform Tools `_ which can dramatically improve your feedback loop when iterating on code.
+
+The following steps are suggested:
+
+- Go to the Payara admin console (either at https://localhost:4848 or http://localhost:4849) and undeploy the dataverse application under "Applications".
+- Install Payara Platform Tools.
+- Under "Server":
+
+ - Click "Run" then "Edit Configurations".
+ - Click the plus sign and scroll down to Payara Server and click "Remote".
+ - For "Name" put "Payara in Docker" or something reasonable.
+ - Under "Application server" select a local directory that has the same version of Payara used in the container. This should match the version of Payara mentioned in the Installation Guide under :ref:`payara`.
+ - Change "Admin Server Port" to 4849.
+ - For username, put "admin".
+ - For password, put "admin".
+
+- Under "Deployment":
+
+ - Click the plus button and clien "Artifact" then "dataverse:war".
+
+- Under "Startup/Connection":
+
+ - Click "Debug" and change the port to 9009.
+
+- Click "Run" and then "Debug Payara in Docker". This initial deployment will take some time.
+- Go to http://localhost:8080/api/info/version and make sure the API is responding.
+- Edit ``Info.java`` and make a small change to the ``/api/info/version`` code.
+- Click "Run" then "Debugging Actions" then "Reload Changed Classes". The deployment should only take a few seconds.
+- Go to http://localhost:8080/api/info/version and verify the change you made.
+
+Using a Debugger
+----------------
+
+The :doc:`base-image` enables usage of the `Java Debugging Wire Protocol `_
+for remote debugging if you set ``ENABLE_JDWP=1`` as environment variable for the application container.
+The default configuration when executing containers with the commands listed at :ref:`dev-run` already enables this.
+
+There are a lot of tutorials how to connect your IDE's debugger to a remote endpoint. Please use ``localhost:9009``
+as the endpoint. Here are links to the most common IDEs docs on remote debugging:
+`Eclipse `_,
+`IntelliJ `_
+
+Building Your Own Base Image
+----------------------------
+
+If you find yourself tasked with upgrading Payara, you will need to create your own base image before running the :ref:`container-dev-quickstart`. For instructions, see :doc:`base-image`.
diff --git a/doc/sphinx-guides/source/developers/api-design.rst b/doc/sphinx-guides/source/developers/api-design.rst
new file mode 100755
index 00000000000..e7a7a6408bb
--- /dev/null
+++ b/doc/sphinx-guides/source/developers/api-design.rst
@@ -0,0 +1,63 @@
+==========
+API Design
+==========
+
+API design is a large topic. We expect this page to grow over time.
+
+.. contents:: |toctitle|
+ :local:
+
+Paths
+-----
+
+A reminder `from Wikipedia `_ of what a path is:
+
+.. code-block:: bash
+
+ userinfo host port
+ ┌──┴───┠┌──────┴──────┠┌┴â”
+ https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top
+ └─┬─┘ └─────────────┬────────────┘└───────┬───────┘ └────────────┬────────────┘ └┬┘
+ scheme authority path query fragment
+
+Exposing Settings
+~~~~~~~~~~~~~~~~~
+
+Since Dataverse 4, database settings have been exposed via API at http://localhost:8080/api/admin/settings
+
+(JVM options are probably available via the Payara REST API, but this is out of scope.)
+
+Settings need to be exposed outside to API clients outside of ``/api/admin`` (which is typically restricted to localhost). Here are some guidelines to follow when exposing settings.
+
+- When you are exposing a database setting as-is:
+
+ - Use ``/api/info/settings`` as the root path.
+
+ - Append the name of the setting including the colon (e.g. ``:DatasetPublishPopupCustomText``)
+
+ - Final path example: ``/api/info/settings/:DatasetPublishPopupCustomText``
+
+- If the absence of the database setting is filled in by a default value (e.g. ``:ZipDownloadLimit`` or ``:ApiTermsOfUse``):
+
+ - Use ``/api/info`` as the root path.
+
+ - Append the setting but remove the colon and downcase the first character (e.g. ``zipDownloadLimit``)
+
+ - Final path example: ``/api/info/zipDownloadLimit``
+
+- If the database setting you're exposing make more sense outside of ``/api/info`` because there's more context (e.g. ``:CustomDatasetSummaryFields``):
+
+ - Feel free to use a path outside of ``/api/info`` as the root path.
+
+ - Given additional context, append a shortened name (e.g. ``/api/datasets/summaryFieldNames``).
+
+ - Final path example: ``/api/datasets/summaryFieldNames``
+
+- If you need to expose a JVM option (MicroProfile setting) such as ``dataverse.api.allow-incomplete-metadata``:
+
+ - Use ``/api/info`` as the root path.
+
+ - Append a meaningful name for the setting (e.g. ``incompleteMetadataViaApi``).
+
+ - Final path example: ``/api/info/incompleteMetadataViaApi``
+
diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst
index 734f678ceb9..8d891e63317 100644
--- a/doc/sphinx-guides/source/developers/big-data-support.rst
+++ b/doc/sphinx-guides/source/developers/big-data-support.rst
@@ -36,6 +36,18 @@ At present, one potential drawback for direct-upload is that files are only part
``./asadmin create-jvm-options "-Ddataverse.files..ingestsizelimit="``
+.. _s3-direct-upload-features-disabled:
+
+Features that are Disabled if S3 Direct Upload is Enabled
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The following features are disabled when S3 direct upload is enabled.
+
+- Unzipping of zip files. (See :ref:`compressed-files`.)
+- Extraction of metadata from FITS files. (See :ref:`fits`.)
+- Creation of NcML auxiliary files (See :ref:`netcdf-and-hdf5`.)
+- Extraction of a geospatial bounding box from NetCDF and HDF5 files (see :ref:`netcdf-and-hdf5`) unless :ref:`dataverse.netcdf.geo-extract-s3-direct-upload` is set to true.
+
.. _cors-s3-bucket:
Allow CORS for S3 Buckets
@@ -137,30 +149,45 @@ Globus File Transfer
Note: Globus file transfer is still experimental but feedback is welcome! See :ref:`support`.
-Users can transfer files via `Globus `_ into and out of datasets when their Dataverse installation is configured to use a Globus accessible S3 store and a community-developed `dataverse-globus `_ "transfer" app has been properly installed and configured.
-
-Due to differences in the access control models of a Dataverse installation and Globus, enabling the Globus capability on a store will disable the ability to restrict and embargo files in that store.
+Users can transfer files via `Globus `_ into and out of datasets, or reference files on a remote Globus endpoint, when their Dataverse installation is configured to use a Globus accessible store(s)
+and a community-developed `dataverse-globus `_ app has been properly installed and configured.
-As Globus aficionados know, Globus endpoints can be in a variety of places, from data centers to personal computers. This means that from within the Dataverse software, a Globus transfer can feel like an upload or a download (with Globus Personal Connect running on your laptop, for example) or it can feel like a true transfer from one server to another (from a cluster in a data center into a Dataverse dataset or vice versa).
+Globus endpoints can be in a variety of places, from data centers to personal computers.
+This means that from within the Dataverse software, a Globus transfer can feel like an upload or a download (with Globus Personal Connect running on your laptop, for example) or it can feel like a true transfer from one server to another (from a cluster in a data center into a Dataverse dataset or vice versa).
-Globus transfer uses a very efficient transfer mechanism and has additional features that make it suitable for large files and large numbers of files:
+Globus transfer uses an efficient transfer mechanism and has additional features that make it suitable for large files and large numbers of files:
* robust file transfer capable of restarting after network or endpoint failures
* third-party transfer, which enables a user accessing a Dataverse installation in their desktop browser to initiate transfer of their files from a remote endpoint (i.e. on a local high-performance computing cluster), directly to an S3 store managed by the Dataverse installation
-Globus transfer requires use of the Globus S3 connector which requires a paid Globus subscription at the host institution. Users will need a Globus account which could be obtained via their institution or directly from Globus (at no cost).
+Note: Due to differences in the access control models of a Dataverse installation and Globus and the current Globus store model, Dataverse cannot enforce per-file-access restrictions.
+It is therefore recommended that a store be configured as public, which disables the ability to restrict and embargo files in that store, when Globus access is allowed.
+
+Dataverse supports three options for using Globus, two involving transfer to Dataverse-managed endpoints and one allowing Dataverse to reference files on remote endpoints.
+Dataverse-managed endpoints must be Globus 'guest collections' hosted on either a file-system-based endpoint or an S3-based endpoint (the latter requires use of the Globus
+S3 connector which requires a paid Globus subscription at the host institution). In either case, Dataverse is configured with the Globus credentials of a user account that can manage the endpoint.
+Users will need a Globus account, which can be obtained via their institution or directly from Globus (at no cost).
-The setup required to enable Globus is described in the `Community Dataverse-Globus Setup and Configuration document `_ and the references therein.
+With the file-system endpoint, Dataverse does not currently have access to the file contents. Thus, functionality related to ingest, previews, fixity hash validation, etc. are not available. (Using the S3-based endpoint, Dataverse has access via S3 and all functionality normally associated with direct uploads to S3 is available.)
+
+For the reference use case, Dataverse must be configured with a list of allowed endpoint/base paths from which files may be referenced. In this case, since Dataverse is not accessing the remote endpoint itself, it does not need Globus credentials.
+Users will need a Globus account in this case, and the remote endpoint must be configured to allow them access (i.e. be publicly readable, or potentially involving some out-of-band mechanism to request access (that could be described in the dataset's Terms of Use and Access).
+
+All of Dataverse's Globus capabilities are now store-based (see the store documentation) and therefore different collections/datasets can be configured to use different Globus-capable stores (or normal file, S3 stores, etc.)
+
+More details of the setup required to enable Globus is described in the `Community Dataverse-Globus Setup and Configuration document `_ and the references therein.
As described in that document, Globus transfers can be initiated by choosing the Globus option in the dataset upload panel. (Globus, which does asynchronous transfers, is not available during dataset creation.) Analogously, "Globus Transfer" is one of the download options in the "Access Dataset" menu and optionally the file landing page download menu (if/when supported in the dataverse-globus app).
An overview of the control and data transfer interactions between components was presented at the 2022 Dataverse Community Meeting and can be viewed in the `Integrations and Tools Session Video `_ around the 1 hr 28 min mark.
-See also :ref:`Globus settings <:GlobusBasicToken>`.
+See also :ref:`Globus settings <:GlobusSettings>`.
Data Capture Module (DCM)
-------------------------
+Please note: The DCM feature is deprecated.
+
Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync over ssh.
DCM was developed and tested using Glassfish but these docs have been updated with references to Payara.
@@ -197,7 +224,7 @@ The JSON that a DCM sends to your Dataverse installation on successful checksum
:language: json
- ``status`` - The valid strings to send are ``validation passed`` and ``validation failed``.
-- ``uploadFolder`` - This is the directory on disk where your Dataverse installation should attempt to find the files that a DCM has moved into place. There should always be a ``files.sha`` file and a least one data file. ``files.sha`` is a manifest of all the data files and their checksums. The ``uploadFolder`` directory is inside the directory where data is stored for the dataset and may have the same name as the "identifier" of the persistent id (DOI or Handle). For example, you would send ``"uploadFolder": "DNXV2H"`` in the JSON file when the absolute path to this directory is ``/usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/DNXV2H/DNXV2H``.
+- ``uploadFolder`` - This is the directory on disk where your Dataverse installation should attempt to find the files that a DCM has moved into place. There should always be a ``files.sha`` file and a least one data file. ``files.sha`` is a manifest of all the data files and their checksums. The ``uploadFolder`` directory is inside the directory where data is stored for the dataset and may have the same name as the "identifier" of the persistent id (DOI or Handle). For example, you would send ``"uploadFolder": "DNXV2H"`` in the JSON file when the absolute path to this directory is ``/usr/local/payara6/glassfish/domains/domain1/files/10.5072/FK2/DNXV2H/DNXV2H``.
- ``totalSize`` - Your Dataverse installation will use this value to represent the total size in bytes of all the files in the "package" that's created. If 360 data files and one ``files.sha`` manifest file are in the ``uploadFolder``, this value is the sum of the 360 data files.
@@ -219,9 +246,9 @@ Add Dataverse Installation settings to use mock (same as using DCM, noted above)
At this point you should be able to download a placeholder rsync script. Your Dataverse installation is then waiting for news from the DCM about if checksum validation has succeeded or not. First, you have to put files in place, which is usually the job of the DCM. You should substitute "X1METO" for the "identifier" of the dataset you create. You must also use the proper path for where you store files in your dev environment.
-- ``mkdir /usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/X1METO``
-- ``mkdir /usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO``
-- ``cd /usr/local/payara5/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO``
+- ``mkdir /usr/local/payara6/glassfish/domains/domain1/files/10.5072/FK2/X1METO``
+- ``mkdir /usr/local/payara6/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO``
+- ``cd /usr/local/payara6/glassfish/domains/domain1/files/10.5072/FK2/X1METO/X1METO``
- ``echo "hello" > file1.txt``
- ``shasum file1.txt > files.sha``
@@ -236,104 +263,11 @@ The following low level command should only be used when troubleshooting the "im
``curl -H "X-Dataverse-key: $API_TOKEN" -X POST "$DV_BASE_URL/api/batch/jobs/import/datasets/files/$DATASET_DB_ID?uploadFolder=$UPLOAD_FOLDER&totalSize=$TOTAL_SIZE"``
-Steps to set up a DCM via Docker for Development
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you need a fully operating DCM client for development purposes, these steps will guide you to setting one up. This includes steps to set up the DCM on S3 variant.
-
-Docker Image Set-up
-^^^^^^^^^^^^^^^^^^^
-
-See https://github.com/IQSS/dataverse/blob/develop/conf/docker-dcm/readme.md
-
-- Install docker if you do not have it
-
-Optional steps for setting up the S3 Docker DCM Variant
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-- Before: the default bucket for DCM to hold files in S3 is named test-dcm. It is coded into `post_upload_s3.bash` (line 30). Change to a different bucket if needed.
-- Also Note: With the new support for multiple file store in the Dataverse Software, DCM requires a store with id="s3" and DCM will only work with this store.
-
- - Add AWS bucket info to dcmsrv
- - Add AWS credentials to ``~/.aws/credentials``
-
- - ``[default]``
- - ``aws_access_key_id =``
- - ``aws_secret_access_key =``
-
-- Dataverse installation configuration (on dvsrv):
-
- - Set S3 as the storage driver
-
- - ``cd /opt/payara5/bin/``
- - ``./asadmin delete-jvm-options "\-Ddataverse.files.storage-driver-id=file"``
- - ``./asadmin create-jvm-options "\-Ddataverse.files.storage-driver-id=s3"``
- - ``./asadmin create-jvm-options "\-Ddataverse.files.s3.type=s3"``
- - ``./asadmin create-jvm-options "\-Ddataverse.files.s3.label=s3"``
-
-
- - Add AWS bucket info to your Dataverse installation
- - Add AWS credentials to ``~/.aws/credentials``
-
- - ``[default]``
- - ``aws_access_key_id =``
- - ``aws_secret_access_key =``
-
- - Also: set region in ``~/.aws/config`` to create a region file. Add these contents:
-
- - ``[default]``
- - ``region = us-east-1``
-
- - Add the S3 bucket names to your Dataverse installation
-
- - S3 bucket for your Dataverse installation
-
- - ``/usr/local/payara5/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.s3.bucket-name=iqsstestdcmbucket"``
-
- - S3 bucket for DCM (as your Dataverse installation needs to do the copy over)
-
- - ``/usr/local/payara5/glassfish/bin/asadmin create-jvm-options "-Ddataverse.files.dcm-s3-bucket-name=test-dcm"``
-
- - Set download method to be HTTP, as DCM downloads through S3 are over this protocol ``curl -X PUT "http://localhost:8080/api/admin/settings/:DownloadMethods" -d "native/http"``
-
-Using the DCM Docker Containers
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-For using these commands, you will need to connect to the shell prompt inside various containers (e.g. ``docker exec -it dvsrv /bin/bash``)
-
-- Create a dataset and download rsync upload script
-
- - connect to client container: ``docker exec -it dcm_client bash``
- - create dataset: ``cd /mnt ; ./create.bash`` ; this will echo the database ID to stdout
- - download transfer script: ``./get_transfer.bash $database_id_from_create_script``
- - execute the transfer script: ``bash ./upload-${database_id_from-create_script}.bash`` , and follow instructions from script.
-
-- Run script
-
- - e.g. ``bash ./upload-3.bash`` (``3`` being the database id from earlier commands in this example).
-
-- Manually run post upload script on dcmsrv
-
- - for posix implementation: ``docker exec -it dcmsrv /opt/dcm/scn/post_upload.bash``
- - for S3 implementation: ``docker exec -it dcmsrv /opt/dcm/scn/post_upload_s3.bash``
-
-Additional DCM docker development tips
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-- You can completely blow away all the docker images with these commands (including non DCM ones!)
- - ``docker-compose -f docmer-compose.yml down -v``
-
-- There are a few logs to tail
-
- - dvsrv : ``tail -n 2000 -f /opt/payara5/glassfish/domains/domain1/logs/server.log``
- - dcmsrv : ``tail -n 2000 -f /var/log/lighttpd/breakage.log``
- - dcmsrv : ``tail -n 2000 -f /var/log/lighttpd/access.log``
-
-- You may have to restart the app server domain occasionally to deal with memory filling up. If deployment is getting reallllllly slow, its a good time.
-
Repository Storage Abstraction Layer (RSAL)
-------------------------------------------
+Please note: The RSAL feature is deprecated.
+
Steps to set up a DCM via Docker for Development
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst
new file mode 100755
index 00000000000..d7b7f281634
--- /dev/null
+++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst
@@ -0,0 +1,266 @@
+=======================
+Classic Dev Environment
+=======================
+
+These are the old instructions we used for Dataverse 4 and 5. They should still work but these days we favor running Dataverse in Docker as described in :doc:`dev-environment`.
+
+These instructions are purposefully opinionated and terse to help you get your development environment up and running as quickly as possible! Please note that familiarity with running commands from the terminal is assumed.
+
+.. contents:: |toctitle|
+ :local:
+
+Quick Start (Docker)
+--------------------
+
+The quickest way to get Dataverse running is in Docker as explained in :doc:`../container/dev-usage` section of the Container Guide.
+
+
+Classic Dev Environment
+-----------------------
+
+Since before Docker existed, we have encouraged installing Dataverse and all its dependencies directly on your development machine, as described below. This can be thought of as the "classic" development environment for Dataverse.
+
+However, in 2023 we decided that we'd like to encourage all developers to start using Docker instead and opened https://github.com/IQSS/dataverse/issues/9616 to indicate that we plan to rewrite this page to recommend the use of Docker.
+
+There's nothing wrong with the classic instructions below and we don't plan to simply delete them. They are a valid alternative to running Dataverse in Docker. We will likely move them to another page.
+
+Set Up Dependencies
+-------------------
+
+Supported Operating Systems
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Mac OS X or Linux is required because the setup scripts assume the presence of standard Unix utilities.
+
+Windows is gaining support through Docker as described in the :doc:`windows` section.
+
+Install Java
+~~~~~~~~~~~~
+
+The Dataverse Software requires Java 11.
+
+We suggest downloading OpenJDK from https://adoptopenjdk.net
+
+On Linux, you are welcome to use the OpenJDK available from package managers.
+
+Install Netbeans or Maven
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+NetBeans IDE is recommended, and can be downloaded from https://netbeans.org . Developers may use any editor or IDE. We recommend NetBeans because it is free, works cross platform, has good support for Jakarta EE projects, and includes a required build tool, Maven.
+
+Below we describe how to build the Dataverse Software war file with Netbeans but if you prefer to use only Maven, you can find installation instructions in the :doc:`tools` section.
+
+Install Homebrew (Mac Only)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+On Mac, install Homebrew to simplify the steps below: https://brew.sh
+
+Clone the Dataverse Software Git Repo
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Fork https://github.com/IQSS/dataverse and then clone your fork like this:
+
+``git clone git@github.com:[YOUR GITHUB USERNAME]/dataverse.git``
+
+Build the Dataverse Software War File
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you installed Netbeans, follow these steps:
+
+- Launch Netbeans and click "File" and then "Open Project". Navigate to where you put the Dataverse Software code and double-click "Dataverse" to open the project.
+- If you see "resolve project problems," go ahead and let Netbeans try to resolve them. This will probably including downloading dependencies, which can take a while.
+- Allow Netbeans to install nb-javac (required for Java 8 and below).
+- Select "Dataverse" under Projects and click "Run" in the menu and then "Build Project (Dataverse)". Check back for "BUILD SUCCESS" at the end.
+
+If you installed Maven instead of Netbeans, run ``mvn package``. Check for "BUILD SUCCESS" at the end.
+
+NOTE: Do you use a locale different than ``en_US.UTF-8`` on your development machine? Are you in a different timezone
+than Harvard (Eastern Time)? You might experience issues while running tests that were written with these settings
+in mind. The Maven ``pom.xml`` tries to handle this for you by setting the locale to ``en_US.UTF-8`` and timezone
+``UTC``, but more, not yet discovered building or testing problems might lurk in the shadows.
+
+Install jq
+~~~~~~~~~~
+
+On Mac, run this command:
+
+``brew install jq``
+
+On Linux, install ``jq`` from your package manager or download a binary from https://stedolan.github.io/jq/
+
+Install Payara
+~~~~~~~~~~~~~~
+
+Payara 6.2023.8 or higher is required.
+
+To install Payara, run the following commands:
+
+``cd /usr/local``
+
+``sudo curl -O -L https://nexus.payara.fish/repository/payara-community/fish/payara/distributions/payara/6.2023.8/payara-6.2023.8.zip``
+
+``sudo unzip payara-6.2023.8.zip``
+
+``sudo chown -R $USER /usr/local/payara6``
+
+If nexus.payara.fish is ever down for maintenance, Payara distributions are also available from https://repo1.maven.org/maven2/fish/payara/distributions/payara/
+
+Install Service Dependencies Directly on localhost
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Install PostgreSQL
+^^^^^^^^^^^^^^^^^^
+
+The Dataverse Software has been tested with PostgreSQL versions up to 13. PostgreSQL version 10+ is required.
+
+On Mac, go to https://www.postgresql.org/download/macosx/ and choose "Interactive installer by EDB" option. Note that version 13.5 is used in the command line examples below, but the process should be similar for other versions. When prompted to set a password for the "database superuser (postgres)" just enter "password".
+
+After installation is complete, make a backup of the ``pg_hba.conf`` file like this:
+
+``sudo cp /Library/PostgreSQL/13/data/pg_hba.conf /Library/PostgreSQL/13/data/pg_hba.conf.orig``
+
+Then edit ``pg_hba.conf`` with an editor such as vi:
+
+``sudo vi /Library/PostgreSQL/13/data/pg_hba.conf``
+
+In the "METHOD" column, change all instances of "scram-sha-256" (or whatever is in that column) to "trust". This will make it so PostgreSQL doesn't require a password.
+
+In the Finder, click "Applications" then "PostgreSQL 13" and launch the "Reload Configuration" app. Click "OK" after you see "server signaled".
+
+Next, to confirm the edit worked, launch the "pgAdmin" application from the same folder. Under "Browser", expand "Servers" and double click "PostgreSQL 13". When you are prompted for a password, leave it blank and click "OK". If you have successfully edited "pg_hba.conf", you can get in without a password.
+
+On Linux, you should just install PostgreSQL using your favorite package manager, such as ``yum``. (Consult the PostgreSQL section of :doc:`/installation/prerequisites` in the main Installation guide for more info and command line examples). Find ``pg_hba.conf`` and set the authentication method to "trust" and restart PostgreSQL.
+
+Install Solr
+^^^^^^^^^^^^
+
+`Solr `_ 9.3.0 is required.
+
+To install Solr, execute the following commands:
+
+``sudo mkdir /usr/local/solr``
+
+``sudo chown $USER /usr/local/solr``
+
+``cd /usr/local/solr``
+
+``curl -O https://archive.apache.org/dist/solr/solr/9.3.0/solr-9.3.0.tgz``
+
+``tar xvfz solr-9.3.0.tgz``
+
+``cd solr-9.3.0/server/solr``
+
+``cp -r configsets/_default collection1``
+
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/9.3.0/schema.xml``
+
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/9.3.0/schema_dv_mdb_fields.xml``
+
+``mv schema*.xml collection1/conf``
+
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/9.3.0/solrconfig.xml``
+
+``mv solrconfig.xml collection1/conf/solrconfig.xml``
+
+``cd /usr/local/solr/solr-9.3.0``
+
+(Please note that the extra jetty argument below is a security measure to limit connections to Solr to only your computer. For extra security, run a firewall.)
+
+``bin/solr start -j "-Djetty.host=127.0.0.1"``
+
+``bin/solr create_core -c collection1 -d server/solr/collection1/conf``
+
+Install Service Dependencies Using Docker Compose
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To avoid having to install service dependencies like PostgreSQL or Solr directly on your localhost, there is the alternative of using the ``docker-compose-dev.yml`` file available in the repository root. For this option you need to have Docker and Docker Compose installed on your machine.
+
+The ``docker-compose-dev.yml`` can be configured to only run the service dependencies necessary to support a Dataverse installation running directly on localhost. In addition to PostgreSQL and Solr, it also runs a SMTP server.
+
+Before running the Docker Compose file, you need to update the value of the ``DATAVERSE_DB_USER`` environment variable to ``postgres``. The variable can be found inside the ``.env`` file in the repository root. This step is required as the Dataverse installation script expects that database user.
+
+To run the Docker Compose file, go to the Dataverse repository root, then run:
+
+``docker-compose -f docker-compose-dev.yml up -d --scale dev_dataverse=0``
+
+Note that this command omits the Dataverse container defined in the Docker Compose file, since Dataverse is going to be installed directly on localhost in the next section.
+
+The command runs the containers in detached mode, but if you want to run them attached and thus view container logs in real time, remove the ``-d`` option from the command.
+
+Data volumes of each dependency will be persisted inside the ``docker-dev-volumes`` folder, inside the repository root.
+
+If you want to stop the containers, then run (for detached mode only, otherwise use ``Ctrl + C``):
+
+``docker-compose -f docker-compose-dev.yml stop``
+
+If you want to remove the containers, then run:
+
+``docker-compose -f docker-compose-dev.yml down``
+
+If you want to run a single container (the mail server, for example) then run:
+
+``docker-compose -f docker-compose-dev.yml up dev_smtp``
+
+For a fresh installation, and before running the Software Installer Script, it is recommended to delete the docker-dev-env folder to avoid installation problems due to existing data in the containers.
+
+Run the Dataverse Software Installer Script
+-------------------------------------------
+
+Navigate to the directory where you cloned the Dataverse Software git repo change directories to the ``scripts/installer`` directory like this:
+
+``cd scripts/installer``
+
+Create a Python virtual environment, activate it, then install dependencies:
+
+``python3 -m venv venv``
+
+``source venv/bin/activate``
+
+``pip install psycopg2-binary``
+
+The installer will try to connect to the SMTP server you tell it to use. If you haven't used the Docker Compose option for setting up the dependencies, or you don't have a mail server handy, you can run ``nc -l 25`` in another terminal and choose "localhost" (the default) to get past this check.
+
+Finally, run the installer (see also :download:`README_python.txt <../../../../scripts/installer/README_python.txt>` if necessary):
+
+``python3 install.py``
+
+Verify the Dataverse Software is Running
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+After the script has finished, you should be able to log into your Dataverse installation with the following credentials:
+
+- http://localhost:8080
+- username: dataverseAdmin
+- password: admin
+
+Configure Your Development Environment for Publishing
+-----------------------------------------------------
+
+Run the following command:
+
+``curl http://localhost:8080/api/admin/settings/:DoiProvider -X PUT -d FAKE``
+
+This will disable DOI registration by using a fake (in-code) DOI provider. Please note that this feature is only available in Dataverse Software 4.10+ and that at present, the UI will give no indication that the DOIs thus minted are fake.
+
+Developers may also wish to consider using :ref:`PermaLinks `
+
+Configure Your Development Environment for GUI Edits
+----------------------------------------------------
+
+Out of the box, a JSF setting is configured for production use and prevents edits to the GUI (xhtml files) from being visible unless you do a full deployment.
+
+It is recommended that you run the following command so that simply saving the xhtml file in Netbeans is enough for the change to show up.
+
+``asadmin create-system-properties "dataverse.jsf.refresh-period=1"``
+
+For more on JSF settings like this, see :ref:`jsf-config`.
+
+Next Steps
+----------
+
+If you can log in to the Dataverse installation, great! If not, please see the :doc:`troubleshooting` section. For further assistance, please see "Getting Help" in the :doc:`intro` section.
+
+You're almost ready to start hacking on code. Now that the installer script has you up and running, you need to continue on to the :doc:`tips` section to get set up to deploy code from your IDE or the command line.
+
+----
+
+Previous: :doc:`intro` | Next: :doc:`tips`
diff --git a/doc/sphinx-guides/source/developers/containers.rst b/doc/sphinx-guides/source/developers/containers.rst
index b42f7f5a2e2..175b178b455 100755
--- a/doc/sphinx-guides/source/developers/containers.rst
+++ b/doc/sphinx-guides/source/developers/containers.rst
@@ -25,11 +25,6 @@ The primary community-lead projects (which the core team is drawing inspiration
- https://github.com/IQSS/dataverse-docker
- https://github.com/IQSS/dataverse-kubernetes (especially the https://github.com/EOSC-synergy/dataverse-kubernetes fork)
-Deprecated Projects
--------------------
-
-The :doc:`testing` section mentions using docker-aio for integration tests. We do not plan to keep this project alive.
-
Using Containers for Reproducible Research
------------------------------------------
diff --git a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst
index 52a6a283e9c..ded62288eb2 100644
--- a/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst
+++ b/doc/sphinx-guides/source/developers/dataset-semantic-metadata-api.rst
@@ -36,6 +36,8 @@ To get the json-ld formatted metadata for a Dataset, specify the Dataset ID (DAT
You should expect a 200 ("OK") response and JSON-LD mirroring the OAI-ORE representation in the returned 'data' object.
+.. _add-semantic-metadata:
+
Add Dataset Metadata
--------------------
diff --git a/doc/sphinx-guides/source/developers/debugging.rst b/doc/sphinx-guides/source/developers/debugging.rst
index 2088afe5521..50e8901b1ff 100644
--- a/doc/sphinx-guides/source/developers/debugging.rst
+++ b/doc/sphinx-guides/source/developers/debugging.rst
@@ -20,8 +20,8 @@ during development without recompiling. Changing the options will require at lea
how you get these options in. (Variable substitution only happens during deployment and when using system properties
or environment variables, you'll need to pass these into the domain, which usually will require an app server restart.)
-Please note that since Payara 5.2021.1 supporting MicroProfile Config 2.0, you can
-`use profiles `_
+Please note you can use
+`MicroProfile Config `_
to maintain your settings more easily for different environments.
.. list-table::
diff --git a/doc/sphinx-guides/source/developers/dependencies.rst b/doc/sphinx-guides/source/developers/dependencies.rst
index 65edfa3ffac..0208c49f90a 100644
--- a/doc/sphinx-guides/source/developers/dependencies.rst
+++ b/doc/sphinx-guides/source/developers/dependencies.rst
@@ -344,8 +344,7 @@ Repositories
------------
Maven receives all dependencies from *repositories*. These can be public like `Maven Central `_
-and others, but you can also use a private repository on premises or in the cloud. Last but not least, you can use
-local repositories, which can live next to your application code (see ``local_lib`` dir within the Dataverse Software codebase).
+and others, but you can also use a private repository on premises or in the cloud.
Repositories are defined within the Dataverse Software POM like this:
@@ -364,11 +363,6 @@ Repositories are defined within the Dataverse Software POM like this:
http://repository.primefaces.org
default
-
- dvn.private
- Local repository for hosting jars not available from network repositories.
- file://${project.basedir}/local_lib
-
You can also add repositories to your local Maven settings, see `docs `_.
diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst
index 58b25d8b941..1301994cc82 100755
--- a/doc/sphinx-guides/source/developers/dev-environment.rst
+++ b/doc/sphinx-guides/source/developers/dev-environment.rst
@@ -2,249 +2,81 @@
Development Environment
=======================
-These instructions are purposefully opinionated and terse to help you get your development environment up and running as quickly as possible! Please note that familiarity with running commands from the terminal is assumed.
+These instructions are oriented around Docker but the "classic" instructions we used for Dataverse 4 and 5 are still available at :doc:`classic-dev-env`.
.. contents:: |toctitle|
:local:
-Quick Start
------------
+.. _container-dev-quickstart:
-The quickest way to get the Dataverse Software running is to use Vagrant as described in the :doc:`tools` section, or use Docker containers as described the :doc:`../container/dev-usage` section of the Container Guide.
+Quickstart
+----------
+
+First, install Java 17, Maven, and Docker.
-For day to day development work, we recommended the following setup.
+After cloning the `dataverse repo `_, run this:
-Set Up Dependencies
--------------------
+``mvn -Pct clean package docker:run``
-Supported Operating Systems
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+After some time you should be able to log in:
-Mac OS X or Linux is required because the setup scripts assume the presence of standard Unix utilities.
+- url: http://localhost:8080
+- username: dataverseAdmin
+- password: admin1
-Windows is not well supported, unfortunately, but Vagrant and Minishift environments are described in the :doc:`windows` section.
+Detailed Steps
+--------------
Install Java
~~~~~~~~~~~~
-The Dataverse Software requires Java 11.
+The Dataverse Software requires Java 17.
-We suggest downloading OpenJDK from https://adoptopenjdk.net
+On Mac and Windows, we suggest downloading OpenJDK from https://adoptium.net (formerly `AdoptOpenJDK `_) or `SDKMAN `_.
On Linux, you are welcome to use the OpenJDK available from package managers.
-Install Netbeans or Maven
-~~~~~~~~~~~~~~~~~~~~~~~~~
+Install Maven
+~~~~~~~~~~~~~
-NetBeans IDE is recommended, and can be downloaded from http://netbeans.org . Developers may use any editor or IDE. We recommend NetBeans because it is free, works cross platform, has good support for Jakarta EE projects, and includes a required build tool, Maven.
+Follow instructions at https://maven.apache.org
-Below we describe how to build the Dataverse Software war file with Netbeans but if you prefer to use only Maven, you can find installation instructions in the :doc:`tools` section.
+Install and Start Docker
+~~~~~~~~~~~~~~~~~~~~~~~~
-Install Homebrew (Mac Only)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Follow instructions at https://www.docker.com
-On Mac, install Homebrew to simplify the steps below: https://brew.sh
+Be sure to start Docker.
-Clone the Dataverse Software Git Repo
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Git Clone Repo
+~~~~~~~~~~~~~~
Fork https://github.com/IQSS/dataverse and then clone your fork like this:
``git clone git@github.com:[YOUR GITHUB USERNAME]/dataverse.git``
-Build the Dataverse Software War File
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you installed Netbeans, follow these steps:
-
-- Launch Netbeans and click "File" and then "Open Project". Navigate to where you put the Dataverse Software code and double-click "Dataverse" to open the project.
-- If you see "resolve project problems," go ahead and let Netbeans try to resolve them. This will probably including downloading dependencies, which can take a while.
-- Allow Netbeans to install nb-javac (required for Java 8 and below).
-- Select "Dataverse" under Projects and click "Run" in the menu and then "Build Project (Dataverse)". Check back for "BUILD SUCCESS" at the end.
-
-If you installed Maven instead of Netbeans, run ``mvn package``. Check for "BUILD SUCCESS" at the end.
-
-NOTE: Do you use a locale different than ``en_US.UTF-8`` on your development machine? Are you in a different timezone
-than Harvard (Eastern Time)? You might experience issues while running tests that were written with these settings
-in mind. The Maven ``pom.xml`` tries to handle this for you by setting the locale to ``en_US.UTF-8`` and timezone
-``UTC``, but more, not yet discovered building or testing problems might lurk in the shadows.
-
-Install jq
-~~~~~~~~~~
-
-On Mac, run this command:
-
-``brew install jq``
-
-On Linux, install ``jq`` from your package manager or download a binary from http://stedolan.github.io/jq/
-
-Install Payara
-~~~~~~~~~~~~~~
-
-Payara 5.2022.3 or higher is required.
-
-To install Payara, run the following commands:
-
-``cd /usr/local``
-
-``sudo curl -O -L https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2022.3/payara-5.2022.3.zip``
-
-``sudo unzip payara-5.2022.3.zip``
-
-``sudo chown -R $USER /usr/local/payara5``
-
-Install Service Dependencies Directly on localhost
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Install PostgreSQL
-^^^^^^^^^^^^^^^^^^
-
-The Dataverse Software has been tested with PostgreSQL versions up to 13. PostgreSQL version 10+ is required.
-
-On Mac, go to https://www.postgresql.org/download/macosx/ and choose "Interactive installer by EDB" option. Note that version 13.5 is used in the command line examples below, but the process should be similar for other versions. When prompted to set a password for the "database superuser (postgres)" just enter "password".
-
-After installation is complete, make a backup of the ``pg_hba.conf`` file like this:
-
-``sudo cp /Library/PostgreSQL/13/data/pg_hba.conf /Library/PostgreSQL/13/data/pg_hba.conf.orig``
-
-Then edit ``pg_hba.conf`` with an editor such as vi:
-
-``sudo vi /Library/PostgreSQL/13/data/pg_hba.conf``
-
-In the "METHOD" column, change all instances of "scram-sha-256" (or whatever is in that column) to "trust". This will make it so PostgreSQL doesn't require a password.
-
-In the Finder, click "Applications" then "PostgreSQL 13" and launch the "Reload Configuration" app. Click "OK" after you see "server signaled".
-
-Next, to confirm the edit worked, launch the "pgAdmin" application from the same folder. Under "Browser", expand "Servers" and double click "PostgreSQL 13". When you are prompted for a password, leave it blank and click "OK". If you have successfully edited "pg_hba.conf", you can get in without a password.
-
-On Linux, you should just install PostgreSQL using your favorite package manager, such as ``yum``. (Consult the PostgreSQL section of :doc:`/installation/prerequisites` in the main Installation guide for more info and command line examples). Find ``pg_hba.conf`` and set the authentication method to "trust" and restart PostgreSQL.
-
-Install Solr
-^^^^^^^^^^^^
-
-`Solr `_ 8.11.1 is required.
-
-To install Solr, execute the following commands:
-
-``sudo mkdir /usr/local/solr``
-
-``sudo chown $USER /usr/local/solr``
-
-``cd /usr/local/solr``
-
-``curl -O http://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz``
-
-``tar xvfz solr-8.11.1.tgz``
-
-``cd solr-8.11.1/server/solr``
-
-``cp -r configsets/_default collection1``
+Build and Run
+~~~~~~~~~~~~~
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/schema.xml``
+Change into the ``dataverse`` directory you just cloned and run the following command:
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/schema_dv_mdb_fields.xml``
+``mvn -Pct clean package docker:run``
-``mv schema*.xml collection1/conf``
+Verify
+~~~~~~
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/solrconfig.xml``
+After some time you should be able to log in:
-``mv solrconfig.xml collection1/conf/solrconfig.xml``
-
-``cd /usr/local/solr/solr-8.11.1``
-
-(Please note that the extra jetty argument below is a security measure to limit connections to Solr to only your computer. For extra security, run a firewall.)
-
-``bin/solr start -j "-Djetty.host=127.0.0.1"``
-
-``bin/solr create_core -c collection1 -d server/solr/collection1/conf``
-
-Install Service Dependencies Using Docker Compose
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-To avoid having to install service dependencies like PostgreSQL or Solr directly on your localhost, there is the alternative of using the ``docker-compose-dev.yml`` file available in the repository root. For this option you need to have Docker and Docker Compose installed on your machine.
-
-The ``docker-compose-dev.yml`` can be configured to only run the service dependencies necessary to support a Dataverse installation running directly on localhost. In addition to PostgreSQL and Solr, it also runs a SMTP server.
-
-Before running the Docker Compose file, you need to update the value of the ``DATAVERSE_DB_USER`` environment variable to ``postgres``. The variable can be found inside the ``.env`` file in the repository root. This step is required as the Dataverse installation script expects that database user.
-
-To run the Docker Compose file, go to the Dataverse repository root, then run:
-
-``docker-compose -f docker-compose-dev.yml up -d --scale dev_dataverse=0``
-
-Note that this command omits the Dataverse container defined in the Docker Compose file, since Dataverse is going to be installed directly on localhost in the next section.
-
-The command runs the containers in detached mode, but if you want to run them attached and thus view container logs in real time, remove the ``-d`` option from the command.
-
-Data volumes of each dependency will be persisted inside the ``docker-dev-volumes`` folder, inside the repository root.
-
-If you want to stop the containers, then run (for detached mode only, otherwise use ``Ctrl + C``):
-
-``docker-compose -f docker-compose-dev.yml stop``
-
-If you want to remove the containers, then run:
-
-``docker-compose -f docker-compose-dev.yml down``
-
-For a fresh installation, and before running the Software Installer Script, it is recommended to delete the docker-dev-env folder to avoid installation problems due to existing data in the containers.
-
-Run the Dataverse Software Installer Script
--------------------------------------------
-
-Navigate to the directory where you cloned the Dataverse Software git repo change directories to the ``scripts/installer`` directory like this:
-
-``cd scripts/installer``
-
-Create a Python virtual environment, activate it, then install dependencies:
-
-``python3 -m venv venv``
-
-``source venv/bin/activate``
-
-``pip install psycopg2-binary``
-
-The installer will try to connect to the SMTP server you tell it to use. If you haven't used the Docker Compose option for setting up the dependencies, or you don't have a mail server handy, you can run ``nc -l 25`` in another terminal and choose "localhost" (the default) to get past this check.
-
-Finally, run the installer (see also :download:`README_python.txt <../../../../scripts/installer/README_python.txt>` if necessary):
-
-``python3 install.py``
-
-Verify the Dataverse Software is Running
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-After the script has finished, you should be able to log into your Dataverse installation with the following credentials:
-
-- http://localhost:8080
+- url: http://localhost:8080
- username: dataverseAdmin
-- password: admin
-
-Configure Your Development Environment for Publishing
------------------------------------------------------
-
-Run the following command:
-
-``curl http://localhost:8080/api/admin/settings/:DoiProvider -X PUT -d FAKE``
-
-This will disable DOI registration by using a fake (in-code) DOI provider. Please note that this feature is only available in Dataverse Software 4.10+ and that at present, the UI will give no indication that the DOIs thus minted are fake.
-
-Developers may also wish to consider using :ref:`PermaLinks `
-
-Configure Your Development Environment for GUI Edits
-----------------------------------------------------
-
-Out of the box, a JSF setting is configured for production use and prevents edits to the GUI (xhtml files) from being visible unless you do a full deployment.
-
-It is recommended that you run the following command so that simply saving the xhtml file in Netbeans is enough for the change to show up.
-
-``asadmin create-system-properties "dataverse.jsf.refresh-period=1"``
-
-For more on JSF settings like this, see :ref:`jsf-config`.
-
-Next Steps
-----------
+- password: admin1
-If you can log in to the Dataverse installation, great! If not, please see the :doc:`troubleshooting` section. For further assistance, please see "Getting Help" in the :doc:`intro` section.
+More Information
+----------------
-You're almost ready to start hacking on code. Now that the installer script has you up and running, you need to continue on to the :doc:`tips` section to get set up to deploy code from your IDE or the command line.
+See also the :doc:`/container/dev-usage` section of the Container Guide.
-----
+Getting Help
+------------
-Previous: :doc:`intro` | Next: :doc:`tips`
+Please feel free to reach out at https://chat.dataverse.org or https://groups.google.com/g/dataverse-dev if you have any difficulty setting up a dev environment!
diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst
index f0729c59dcf..d07b5b63f72 100755
--- a/doc/sphinx-guides/source/developers/documentation.rst
+++ b/doc/sphinx-guides/source/developers/documentation.rst
@@ -8,7 +8,7 @@ Writing Documentation
Quick Fix
-----------
-If you find a typo or a small error in the documentation you can fix it using GitHub's online web editor. Generally speaking, we will be following https://help.github.com/en/articles/editing-files-in-another-users-repository
+If you find a typo or a small error in the documentation you can fix it using GitHub's online web editor. Generally speaking, we will be following https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files#editing-files-in-another-users-repository
- Navigate to https://github.com/IQSS/dataverse/tree/develop/doc/sphinx-guides/source where you will see folders for each of the guides: `admin`_, `api`_, `developers`_, `installation`_, `style`_, `user`_.
- Find the file you want to edit under one of the folders above.
@@ -36,7 +36,7 @@ If you would like to read more about the Dataverse Project's use of GitHub, plea
Building the Guides with Sphinx
-------------------------------
-The Dataverse guides are written using Sphinx (http://sphinx-doc.org). We recommend installing Sphinx on your localhost or using a Sphinx Docker container to build the guides locally so you can get an accurate preview of your changes.
+The Dataverse guides are written using Sphinx (https://sphinx-doc.org). We recommend installing Sphinx on your localhost or using a Sphinx Docker container to build the guides locally so you can get an accurate preview of your changes.
In case you decide to use a Sphinx Docker container to build the guides, you can skip the next two installation sections, but you will need to have Docker installed.
@@ -62,7 +62,7 @@ In some parts of the documentation, graphs are rendered as images using the Sphi
Building the guides requires the ``dot`` executable from GraphViz.
-This requires having `GraphViz `_ installed and either having ``dot`` on the path or
+This requires having `GraphViz `_ installed and either having ``dot`` on the path or
`adding options to the make call `_.
Editing and Building the Guides
@@ -71,7 +71,7 @@ Editing and Building the Guides
To edit the existing documentation:
- Create a branch (see :ref:`how-to-make-a-pull-request`).
-- In ``doc/sphinx-guides/source`` you will find the .rst files that correspond to http://guides.dataverse.org.
+- In ``doc/sphinx-guides/source`` you will find the .rst files that correspond to https://guides.dataverse.org.
- Using your preferred text editor, open and edit the necessary files, or create new ones.
Once you are done, you can preview the changes by building the guides locally. As explained, you can build the guides with Sphinx locally installed, or with a Docker container.
diff --git a/doc/sphinx-guides/source/developers/globus-api.rst b/doc/sphinx-guides/source/developers/globus-api.rst
new file mode 100644
index 00000000000..de9df06a798
--- /dev/null
+++ b/doc/sphinx-guides/source/developers/globus-api.rst
@@ -0,0 +1,235 @@
+Globus Transfer API
+===================
+
+The Globus API addresses three use cases:
+* Transfer to a Dataverse-managed Globus endpoint (File-based or using the Globus S3 Connector)
+* Reference of files that will remain in a remote Globus endpoint
+* Transfer from a Dataverse-managed Globus endpoint
+
+The ability for Dataverse to interact with Globus endpoints is configured via a Globus store - see :ref:`globus-storage`.
+
+Globus transfers (or referencing a remote endpoint) for upload and download transfers involve a series of steps. These can be accomplished using the Dataverse and Globus APIs. (These are used internally by the `dataverse-globus app `_ when transfers are done via the Dataverse UI.)
+
+Requesting Upload or Download Parameters
+----------------------------------------
+
+The first step in preparing for a Globus transfer/reference operation is to request the parameters relevant for a given dataset:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/globusUploadParameters?locale=$LOCALE"
+
+The response will be of the form:
+
+.. code-block:: bash
+
+ {
+ "status": "OK",
+ "data": {
+ "queryParameters": {
+ "datasetId": 29,
+ "siteUrl": "http://ec2-34-204-169-194.compute-1.amazonaws.com",
+ "datasetVersion": ":draft",
+ "dvLocale": "en",
+ "datasetPid": "doi:10.5072/FK2/ILLPXE",
+ "managed": "true",
+ "endpoint": "d8c42580-6528-4605-9ad8-116a61982644"
+ },
+ "signedUrls": [
+ {
+ "name": "requestGlobusTransferPaths",
+ "httpMethod": "POST",
+ "signedUrl": "http://ec2-34-204-169-194.compute-1.amazonaws.com/api/v1/datasets/29/requestGlobusUploadPaths?until=2023-11-22T01:52:03.648&user=dataverseAdmin&method=POST&token=63ac4bb748d12078dded1074916508e19e6f6b61f64294d38e0b528010b07d48783cf2e975d7a1cb6d4a3c535f209b981c7c6858bc63afdfc0f8ecc8a139b44a",
+ "timeOut": 300
+ },
+ {
+ "name": "addGlobusFiles",
+ "httpMethod": "POST",
+ "signedUrl": "http://ec2-34-204-169-194.compute-1.amazonaws.com/api/v1/datasets/29/addGlobusFiles?until=2023-11-22T01:52:03.648&user=dataverseAdmin&method=POST&token=2aaa03f6b9f851a72e112acf584ffc0758ed0cc8d749c5a6f8c20494bb7bc13197ab123e1933f3dde2711f13b347c05e6cec1809a8f0b5484982570198564025",
+ "timeOut": 300
+ },
+ {
+ "name": "getDatasetMetadata",
+ "httpMethod": "GET",
+ "signedUrl": "http://ec2-34-204-169-194.compute-1.amazonaws.com/api/v1/datasets/29/versions/:draft?until=2023-11-22T01:52:03.649&user=dataverseAdmin&method=GET&token=1878d6a829cd5540e89c07bdaf647f1bea5314cc7a55433b0b506350dd330cad61ade3714a8ee199a7b464fb3b8cddaea0f32a89ac3bfc4a86cd2ea3004ecbb8",
+ "timeOut": 300
+ },
+ {
+ "name": "getFileListing",
+ "httpMethod": "GET",
+ "signedUrl": "http://ec2-34-204-169-194.compute-1.amazonaws.com/api/v1/datasets/29/versions/:draft/files?until=2023-11-22T01:52:03.650&user=dataverseAdmin&method=GET&token=78e8ca8321624f42602af659227998374ef3788d0feb43d696a0e19086e0f2b3b66b96981903a1565e836416c504b6248cd3c6f7c2644566979bd16e23a99622",
+ "timeOut": 300
+ }
+ ]
+ }
+ }
+
+The response includes the id for the Globus endpoint to use along with several signed URLs.
+
+The getDatasetMetadata and getFileListing URLs are just signed versions of the standard Dataset metadata and file listing API calls. The other two are Globus specific.
+
+If called for a dataset using a store that is configured with a remote Globus endpoint(s), the return response is similar but the response includes a
+the "managed" parameter will be false, the "endpoint" parameter is replaced with a JSON array of "referenceEndpointsWithPaths" and the
+requestGlobusTransferPaths and addGlobusFiles URLs are replaced with ones for requestGlobusReferencePaths and addFiles. All of these calls are
+described further below.
+
+The call to set up for a transfer out (download) is similar:
+
+.. code-block:: bash
+
+ curl -H "X-Dataverse-key:$API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/globusDownloadParameters?locale=$LOCALE"
+
+Note that this API call supports an additional downloadId query parameter. This is only used when the globus-dataverse app is called from the Dataverse user interface. There is no need to use it when calling the API directly.
+
+The returned response includes the same getDatasetMetadata and getFileListing URLs as in the upload case and includes "monitorGlobusDownload" and "requestGlobusDownload" URLs. The response will also indicate whether the store is "managed" and will provide the "endpoint" from which downloads can be made.
+
+
+Performing an Upload/Transfer In
+--------------------------------
+
+The information from the API call above can be used to provide a user with information about the dataset and to prepare to transfer or to reference files (based on the "managed" parameter).
+
+Once the user identifies which files are to be added, the requestGlobusTransferPaths or requestGlobusReferencePaths URLs can be called. These both reference the same API call but must be used with different entries in the JSON body sent:
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
+ export LOCALE=en-US
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:application/json" -X POST "$SERVER_URL/api/datasets/:persistentId/requestGlobusUpload"
+
+Note that when using the dataverse-globus app or the return from the previous call, the URL for this call will be signed and no API_TOKEN is needed.
+
+In the managed case, the JSON body sent must include the id of the Globus user that will perform the transfer and the number of files that will be transferred:
+
+.. code-block:: bash
+
+ {
+ "principal":"d15d4244-fc10-47f3-a790-85bdb6db9a75",
+ "numberOfFiles":2
+ }
+
+In the remote reference case, the JSON body sent must include the Globus endpoint/paths that will be referenced:
+
+.. code-block:: bash
+
+ {
+ "referencedFiles":[
+ "d8c42580-6528-4605-9ad8-116a61982644/hdc1/test1.txt"
+ ]
+ }
+
+The response will include a JSON object. In the managed case, the map is from new assigned file storageidentifiers and specific paths on the managed Globus endpoint:
+
+.. code-block:: bash
+
+ {
+ "status":"OK",
+ "data":{
+ "globusm://18b49d3688c-62137dcb06e4":"/hdc1/10.5072/FK2/ILLPXE/18b49d3688c-62137dcb06e4",
+ "globusm://18b49d3688c-5c17d575e820":"/hdc1/10.5072/FK2/ILLPXE/18b49d3688c-5c17d575e820"
+ }
+ }
+
+In the managed case, the specified Globus principal is granted write permission to the specified endpoint/path,
+which will allow initiation of a transfer from the external endpoint to the managed endpoint using the Globus API.
+The permission will be revoked if the transfer is not started and the next call to Dataverse to finish the transfer are not made within a short time (configurable, default of 5 minutes).
+
+In the remote/reference case, the map is from the initially supplied endpoint/paths to the new assigned file storageidentifiers:
+
+.. code-block:: bash
+
+ {
+ "status":"OK",
+ "data":{
+ "d8c42580-6528-4605-9ad8-116a61982644/hdc1/test1.txt":"globus://18bf8c933f4-ed2661e7d19b//d8c42580-6528-4605-9ad8-116a61982644/hdc1/test1.txt"
+ }
+ }
+
+
+
+Adding Files to the Dataset
+---------------------------
+
+In the managed case, once a Globus transfer has been initiated a final API call is made to Dataverse to provide it with the task identifier of the transfer and information about the files being transferred:
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
+ export JSON_DATA='{"taskIdentifier":"3f530302-6c48-11ee-8428-378be0d9c521", \
+ "files": [{"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"globusm://18b3972213f-f6b5c2221423", "fileName":"file1.txt", "mimeType":"text/plain", "checksum": {"@type": "MD5", "@value": "1234"}}, \
+ {"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "storageIdentifier":"globusm://18b39722140-50eb7d3c5ece", "fileName":"file2.txt", "mimeType":"text/plain", "checksum": {"@type": "MD5", "@value": "2345"}}]}'
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:multipart/form-data" -X POST "$SERVER_URL/api/datasets/:persistentId/addGlobusFiles -F "jsonData=$JSON_DATA"
+
+Note that the mimetype is multipart/form-data, matching the /addFiles API call. ALso note that the API_TOKEN is not needed when using a signed URL.
+
+With this information, Dataverse will begin to monitor the transfer and when it completes, will add all files for which the transfer succeeded.
+As the transfer can take significant time and the API call is asynchronous, the only way to determine if the transfer succeeded via API is to use the standard calls to check the dataset lock state and contents.
+
+Once the transfer completes, Dataverse will remove the write permission for the principal.
+
+Note that when using a managed endpoint that uses the Globus S3 Connector, the checksum should be correct as Dataverse can validate it. For file-based endpoints, the checksum should be included if available but Dataverse cannot verify it.
+
+In the remote/reference case, where there is no transfer to monitor, the standard /addFiles API call (see :ref:`direct-add-to-dataset-api`) is used instead. There are no changes for the Globus case.
+
+Downloading/Transfer Out Via Globus
+-----------------------------------
+
+To begin downloading files, the requestGlobusDownload URL is used:
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:application/json" -X POST "$SERVER_URL/api/datasets/:persistentId/requestGlobusDownload"
+
+The JSON body sent should include a list of file ids to download and, for a managed endpoint, the Globus principal that will make the transfer:
+
+.. code-block:: bash
+
+ {
+ "principal":"d15d4244-fc10-47f3-a790-85bdb6db9a75",
+ "fileIds":[60, 61]
+ }
+
+Note that this API call takes an optional downloadId parameter that is used with the dataverse-globus app. When downloadId is included, the list of fileIds is not needed.
+
+The response is a JSON object mapping the requested file Ids to Globus endpoint/paths. In the managed case, the principal will have been given read permissions for the specified paths:
+
+.. code-block:: bash
+
+ {
+ "status":"OK",
+ "data":{
+ "60": "d8c42580-6528-4605-9ad8-116a61982644/hdc1/10.5072/FK2/ILLPXE/18bf3af9c78-92b8e168090e",
+ "61": "d8c42580-6528-4605-9ad8-116a61982644/hdc1/10.5072/FK2/ILLPXE/18bf3af9c78-c8d81569305c"
+ }
+ }
+
+For the remote case, the use can perform the transfer without further contact with Dataverse. In the managed case, the user must initiate the transfer via the Globus API and then inform Dataverse.
+Dataverse will then monitor the transfer and revoke the read permission when the transfer is complete. (Not making this last call could result in failure of the transfer.)
+
+.. code-block:: bash
+
+ export API_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+ export SERVER_URL=https://demo.dataverse.org
+ export PERSISTENT_IDENTIFIER=doi:10.5072/FK27U7YBV
+
+ curl -H "X-Dataverse-key:$API_TOKEN" -H "Content-type:application/json" -X POST "$SERVER_URL/api/datasets/:persistentId/monitorGlobusDownload"
+
+The JSON body sent just contains the task identifier for the transfer:
+
+.. code-block:: bash
+
+ {
+ "taskIdentifier":"b5fd01aa-8963-11ee-83ae-d5484943e99a"
+ }
+
+
diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst
index d70b682fcda..25fea138736 100755
--- a/doc/sphinx-guides/source/developers/index.rst
+++ b/doc/sphinx-guides/source/developers/index.rst
@@ -19,7 +19,9 @@ Developer Guide
sql-upgrade-scripts
testing
documentation
+ api-design
security
+ performance
dependencies
debugging
coding-style
@@ -27,6 +29,7 @@ Developer Guide
deployment
containers
making-releases
+ making-library-releases
metadataexport
tools
unf/index
@@ -37,8 +40,10 @@ Developer Guide
big-data-support
aux-file-support
s3-direct-upload-api
+ globus-api
dataset-semantic-metadata-api
dataset-migration-api
workflows
fontcustom
+ classic-dev-env
diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst
index 7f4e8c1ba34..f446b73de09 100755
--- a/doc/sphinx-guides/source/developers/intro.rst
+++ b/doc/sphinx-guides/source/developers/intro.rst
@@ -2,7 +2,7 @@
Introduction
============
-Welcome! `The Dataverse Project `_ is an `open source `_ project that loves `contributors `_!
+Welcome! `The Dataverse Project `_ is an `open source `_ project that loves `contributors `_!
.. contents:: |toctitle|
:local:
@@ -19,7 +19,7 @@ To get started, you'll want to set up your :doc:`dev-environment` and make sure
Getting Help
------------
-If you have any questions at all, please reach out to other developers via the channels listed in https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md such as http://chat.dataverse.org, the `dataverse-dev `_ mailing list, `community calls `_, or support@dataverse.org.
+If you have any questions at all, please reach out to other developers via the channels listed in https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md such as https://chat.dataverse.org, the `dataverse-dev `_ mailing list, `community calls `_, or support@dataverse.org.
.. _core-technologies:
@@ -40,7 +40,7 @@ For the Dataverse Software development roadmap, please see https://www.iq.harvar
Kanban Board
------------
-You can get a sense of what's currently in flight (in dev, in QA, etc.) by looking at https://github.com/orgs/IQSS/projects/2
+You can get a sense of what's currently in flight (in dev, in QA, etc.) by looking at https://github.com/orgs/IQSS/projects/34
Issue Tracker
-------------
@@ -52,7 +52,9 @@ Related Guides
If you are a developer who wants to make use of the Dataverse Software APIs, please see the :doc:`/api/index`. If you have front-end UI questions, please see the :doc:`/style/index`.
-If you are a sysadmin who likes to code, you may be interested in hacking on installation scripts mentioned in the :doc:`/installation/index`. We validate the installation scripts with :doc:`/developers/tools` such as `Vagrant `_ and Docker (see the :doc:`containers` section).
+If you are a sysadmin who likes to code, you may be interested in hacking on installation scripts mentioned in the :doc:`/installation/index`.
+
+If you are a Docker enthusiasts, please check out the :doc:`/container/index`.
Related Projects
----------------
diff --git a/doc/sphinx-guides/source/developers/make-data-count.rst b/doc/sphinx-guides/source/developers/make-data-count.rst
index a3c0d10dc5e..8eaa5c0d7f8 100644
--- a/doc/sphinx-guides/source/developers/make-data-count.rst
+++ b/doc/sphinx-guides/source/developers/make-data-count.rst
@@ -30,15 +30,13 @@ Full Setup
The recommended way to work on the Make Data Count feature is to spin up an EC2 instance that has both the Dataverse Software and Counter Processor installed. Go to the :doc:`deployment` page for details on how to spin up an EC2 instance and make sure that your Ansible file is configured to install Counter Processor before running the "create" script.
-(Alternatively, you can try installing Counter Processor in Vagrant. :download:`setup-counter-processor.sh <../../../../scripts/vagrant/setup-counter-processor.sh>` might help you get it installed.)
-
After you have spun to your EC2 instance, set ``:MDCLogPath`` so that the Dataverse installation creates a log for Counter Processor to operate on. For more on this database setting, see the :doc:`/installation/config` section of the Installation Guide.
Next you need to have the Dataverse installation add some entries to the log that Counter Processor will operate on. To do this, click on some published datasets and download some files.
-Next you should run Counter Processor to convert the log into a SUSHI report, which is in JSON format. Before running Counter Processor, you need to put a configuration file into place. As a starting point use :download:`counter-processor-config.yaml <../../../../scripts/vagrant/counter-processor-config.yaml>` and edit the file, paying particular attention to the following settings:
+Next you should run Counter Processor to convert the log into a SUSHI report, which is in JSON format. Before running Counter Processor, you need to put a configuration file into place. As a starting point use :download:`counter-processor-config.yaml <../_static/developers/counter-processor-config.yaml>` and edit the file, paying particular attention to the following settings:
-- ``log_name_pattern`` You might want something like ``/usr/local/payara5/glassfish/domains/domain1/logs/counter_(yyyy-mm-dd).log``
+- ``log_name_pattern`` You might want something like ``/usr/local/payara6/glassfish/domains/domain1/logs/counter_(yyyy-mm-dd).log``
- ``year_month`` You should probably set this to the current month.
- ``output_file`` This needs to be a directory that the "dataverse" Unix user can read but that the "counter" user can write to. In dev, you can probably get away with "/tmp" as the directory.
- ``platform`` Out of the box from Counter Processor this is set to ``Dash`` but this should be changed to match the name of your Dataverse installation. Examples are "Harvard Dataverse Repository" for Harvard University or "LibraData" for the University of Virginia.
diff --git a/doc/sphinx-guides/source/developers/making-library-releases.rst b/doc/sphinx-guides/source/developers/making-library-releases.rst
new file mode 100755
index 00000000000..63b6eeb1c2a
--- /dev/null
+++ b/doc/sphinx-guides/source/developers/making-library-releases.rst
@@ -0,0 +1,93 @@
+=======================
+Making Library Releases
+=======================
+
+.. contents:: |toctitle|
+ :local:
+
+Introduction
+------------
+
+Note: See :doc:`making-releases` for Dataverse itself.
+
+We release Java libraries to Maven Central that are used by Dataverse (and perhaps `other `_ `software `_!):
+
+- https://central.sonatype.com/namespace/org.dataverse
+- https://central.sonatype.com/namespace/io.gdcc
+
+We release JavaScript/TypeScript libraries to npm:
+
+- https://www.npmjs.com/package/@iqss/dataverse-design-system
+
+Maven Central (Java)
+--------------------
+
+From the perspective of the Maven Central, we are both `producers `_ because we publish/release libraries there and `consumers `_ because we pull down those libraries (and many others) when we build Dataverse.
+
+Releasing Existing Libraries to Maven Central
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you need to release an existing library, all the setup should be done already. The steps below assume that GitHub Actions are in place to do the heavy lifting for you, such as signing artifacts with GPG.
+
+Releasing a Snapshot Version to Maven Central
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`Snapshot `_ releases are published automatically through GitHub Actions (e.g. through a `snapshot workflow `_ for the SWORD library) every time a pull request is merged (or the default branch, typically ``main``, is otherwise updated).
+
+That is to say, to make a snapshot release, you only need to get one or more commits into the default branch.
+
+Releasing a Release (Non-Snapshot) Version to Maven Central
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+From a pom.xml it may not be apparent that snapshots like ``6.0-SNAPSHOT`` might be changing under your feet. Browsing the snapshot repository (e.g. our `UNF 6.0-SNAPSHOT `_), may reveal versions changing over time. To finalize the code and stop it from changing, we publish/release what Maven calls a "`release version `_". This will remove ``-SNAPSHOT`` from the version (through an ``mvn`` command).
+
+Non-snapshot releases (`release `_ versions) are published automatically through GitHub Actions (e.g. through a `release workflow `_), kicked off locally by an ``mvn`` command that invokes the `Maven Release Plugin `_.
+
+First, run a clean:
+
+``mvn release:clean``
+
+Then run a prepare:
+
+``mvn release:prepare``
+
+The prepare step is interactive. You will be prompted for the following information:
+
+- the release version (e.g. `2.0.0 `_)
+- the git tag to create and push (e.g. `sword2-server-2.0.0 `_)
+- the next development (snapshot) version (e.g. `2.0.1-SNAPSHOT `_)
+
+These examples from the SWORD library. Below is what to expect from the interactive session. In many cases, you can just hit enter to accept the defaults.
+
+.. code-block:: bash
+
+ [INFO] 5/17 prepare:map-release-versions
+ What is the release version for "SWORD v2 Common Server Library (forked)"? (sword2-server) 2.0.0: :
+ [INFO] 6/17 prepare:input-variables
+ What is the SCM release tag or label for "SWORD v2 Common Server Library (forked)"? (sword2-server) sword2-server-2.0.0: :
+ [INFO] 7/17 prepare:map-development-versions
+ What is the new development version for "SWORD v2 Common Server Library (forked)"? (sword2-server) 2.0.1-SNAPSHOT: :
+ [INFO] 8/17 prepare:rewrite-poms-for-release
+
+It can take some time for the jar to be visible on Maven Central. You can start by looking on the repo1 server, like this: https://repo1.maven.org/maven2/io/gdcc/sword2-server/2.0.0/
+
+Don't bother putting the new version in a pom.xml until you see it on repo1.
+
+Note that the next snapshot release should be available as well, like this: https://s01.oss.sonatype.org/content/groups/staging/io/gdcc/sword2-server/2.0.1-SNAPSHOT/
+
+Releasing a New Library to Maven Central
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+At a high level:
+
+- Use an existing pom.xml as a starting point.
+- Use existing GitHub Actions workflows as a starting point.
+- Create secrets in the new library's GitHub repo used by the workflow.
+- If you need an entire new namespace, look at previous issues such as https://issues.sonatype.org/browse/OSSRH-94575 and https://issues.sonatype.org/browse/OSSRH-94577
+
+npm (JavaScript/TypeScript)
+---------------------------
+
+Currently, publishing `@iqss/dataverse-design-system `_ to npm done manually. We plan to automate this as part of https://github.com/IQSS/dataverse-frontend/issues/140
+
+https://www.npmjs.com/package/js-dataverse is the previous 1.0 version of js-dataverse. No 1.x releases are planned. We plan to publish 2.0 (used by the new frontend) as discussed in https://github.com/IQSS/dataverse-frontend/issues/13
\ No newline at end of file
diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst
index a2575bb5f50..6b94282d55e 100755
--- a/doc/sphinx-guides/source/developers/making-releases.rst
+++ b/doc/sphinx-guides/source/developers/making-releases.rst
@@ -8,20 +8,25 @@ Making Releases
Introduction
------------
+Note: See :doc:`making-library-releases` for how to publish our libraries to Maven Central.
+
See :doc:`version-control` for background on our branching strategy.
The steps below describe making both regular releases and hotfix releases.
+.. _write-release-notes:
+
Write Release Notes
-------------------
-Developers express the need for an addition to release notes by creating a file in ``/doc/release-notes`` containing the name of the issue they're working on. The name of the branch could be used for the filename with ".md" appended (release notes are written in Markdown) such as ``5053-apis-custom-homepage.md``.
+Developers express the need for an addition to release notes by creating a "release note snippet" in ``/doc/release-notes`` containing the name of the issue they're working on. The name of the branch could be used for the filename with ".md" appended (release notes are written in Markdown) such as ``5053-apis-custom-homepage.md``. See :ref:`writing-release-note-snippets` for how this is described for contributors.
-The task at or near release time is to collect these notes into a single doc.
+The task at or near release time is to collect these snippets into a single file.
- Create an issue in GitHub to track the work of creating release notes for the upcoming release.
-- Create a branch, add a .md file for the release (ex. 5.10.1 Release Notes) in ``/doc/release-notes`` and write the release notes, making sure to pull content from the issue-specific release notes mentioned above.
-- Delete the previously-created, issue-specific release notes as the content is added to the main release notes file.
+- Create a branch, add a .md file for the release (ex. 5.10.1 Release Notes) in ``/doc/release-notes`` and write the release notes, making sure to pull content from the release note snippets mentioned above.
+- Delete the release note snippets as the content is added to the main release notes file.
+- Include instructions to describe the steps required to upgrade the application from the previous version. These must be customized for release numbers and special circumstances such as changes to metadata blocks and infrastructure.
- Take the release notes .md through the regular Code Review and QA process.
Create a GitHub Issue and Branch for the Release
@@ -65,6 +70,19 @@ Once important tests have passed (compile, unit tests, etc.), merge the pull req
If this is a hotfix release, skip this whole "merge develop to master" step (the "develop" branch is not involved until later).
+(Optional) Test Docker Images
+-----------------------------
+
+After the "master" branch has been updated and the GitHub Action to build and push Docker images has run (see `PR #9776 `_), go to https://hub.docker.com/u/gdcc and make sure the "alpha" tag for the following images has been updated:
+
+- https://hub.docker.com/r/gdcc/base
+- https://hub.docker.com/r/gdcc/dataverse
+- https://hub.docker.com/r/gdcc/configbaker
+
+To test these images against our API test suite, go to the "alpha" workflow at https://github.com/gdcc/api-test-runner/actions/workflows/alpha.yml and run it.
+
+If there are failures, additional dependencies or settings may have been added to the "develop" workflow. Copy them over and try again.
+
Build the Guides for the Release
--------------------------------
@@ -110,9 +128,11 @@ Go to https://jenkins.dataverse.org/job/IQSS_Dataverse_Internal/ and make the fo
Click "Save" then "Build Now".
-The build number will appear in ``/api/info/version`` (along with the commit mentioned above) from a running installation (e.g. ``{"version":"5.10.1","build":"907-b844672``).
+This will build the war file, and then automatically deploy it on dataverse-internal. Verify that the application has deployed successfully.
+
+The build number will appear in ``/api/info/version`` (along with the commit mentioned above) from a running installation (e.g. ``{"version":"5.10.1","build":"907-b844672``).
-Note that the build number comes from script in an early build step...
+Note that the build number comes from the following script in an early Jenkins build step...
.. code-block:: bash
@@ -127,12 +147,16 @@ Build Installer (dvinstall.zip)
ssh into the dataverse-internal server and do the following:
- In a git checkout of the dataverse source switch to the master branch and pull the latest.
-- Copy the war file from the previous step to the ``target`` directory in the root of the repo (create it, if necessary).
+- Copy the war file from the previous step to the ``target`` directory in the root of the repo (create it, if necessary):
+- ``mkdir target``
+- ``cp /tmp/dataverse-5.10.1.war target``
- ``cd scripts/installer``
- ``make``
A zip file called ``dvinstall.zip`` should be produced.
+Alternatively, you can build the installer on your own dev. instance. But make sure you use the war file produced in the step above, not a war file build from master on your own system! That's because we want the released application war file to contain the build number described above. Download the war file directly from Jenkins, or from dataverse-internal.
+
Make Artifacts Available for Download
-------------------------------------
@@ -146,6 +170,11 @@ Upload the following artifacts to the draft release you created:
- metadata block tsv files
- config files
+Deploy on Demo
+--------------
+
+Now that you have the release ready to go, give it one final test by deploying it on https://demo.dataverse.org . Note that this is also an opportunity to re-test the upgrade checklist as described in the release note.
+
Publish the Release
-------------------
@@ -156,7 +185,14 @@ Update Guides Link
"latest" at https://guides.dataverse.org/en/latest/ is a symlink to the directory with the latest release. That directory (e.g. ``5.10.1``) was put into place by the Jenkins "guides" job described above.
-ssh into the guides server and update the symlink to point to the latest release.
+ssh into the guides server and update the symlink to point to the latest release, as in the example below.
+
+.. code-block:: bash
+
+ cd /var/www/html/en
+ ln -s 5.10.1 latest
+
+
Close Milestone on GitHub and Create a New One
----------------------------------------------
diff --git a/doc/sphinx-guides/source/developers/performance.rst b/doc/sphinx-guides/source/developers/performance.rst
new file mode 100644
index 00000000000..46c152f322e
--- /dev/null
+++ b/doc/sphinx-guides/source/developers/performance.rst
@@ -0,0 +1,196 @@
+Performance
+===========
+
+`Performance is a feature `_ was a mantra when Stack Overflow was being developed. We endeavor to do the same with Dataverse!
+
+In this section we collect ideas and share practices for improving performance.
+
+.. contents:: |toctitle|
+ :local:
+
+Problem Statement
+-----------------
+
+Performance has always been important to the Dataverse Project, but results have been uneven. We've seen enough success in the marketplace that performance must be adequate, but internally we sometimes refer to Dataverse as a pig. ðŸ·
+
+Current Practices
+-----------------
+
+We've adopted a number of practices to help us maintain our current level of performance and most should absolutely continue in some form, but challenges mentioned throughout should be addressed to further improve performance.
+
+Cache When You Can
+~~~~~~~~~~~~~~~~~~
+
+The Metrics API, for example, caches values for 7 days by default. We took a look at JSR 107 (JCache - Java Temporary Caching API) in `#2100 `_. We're aware of the benefits of caching.
+
+Use Async
+~~~~~~~~~
+
+We index datasets (and all objects) asynchronously. That is, we let changes persist in the database and afterward copy the data into Solr.
+
+Use a Queue
+~~~~~~~~~~~
+
+We use a JMS queue for when ingesting tabular files. We've talked about adding a queue (even `an external queue `_) for indexing, DOI registration, and other services.
+
+Offload Expensive Operations Outside the App Server
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When operations are computationally expensive, we have realized performance gains by offloading them to systems outside of the core code. For example, rather than having files pass through our application server when they are downloaded, we use direct download so that client machines download files directly from S3. (We use the same trick with upload.) When a client downloads multiple files, rather than zipping them within the application server as before, we now have a separate "zipper" process that does this work out of band.
+
+Drop to Raw SQL as Necessary
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We aren't shy about writing raw SQL queries when necessary. We've written `querycount `_ Â scripts to help identify problematic queries and mention slow query log at :doc:`/admin/monitoring`.
+
+Add Indexes to Database Tables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There was a concerted effort in `#1880 `_ to add indexes to a large number of columns, but it's something we're mindful of, generally. Perhaps we could use some better detection of when indexes would be valuable.
+
+Find Bottlenecks with a Profiler
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+VisualVM is popular and bundled with Netbeans. Many options are available including `JProfiler `_.
+
+Warn Developers in Code Comments
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For code that has been optimized for performance, warnings are sometimes inserted in the form of comments for future developers to prevent backsliding.
+
+Write Docs for Devs about Perf
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Like this doc. :)
+
+Sometimes perf is written about in other places, such as :ref:`avoid-efficiency-issues-with-render-logic-expressions`.
+
+Horizontal Scaling of App Server
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We've made it possible to run more than one application server, though it requires some special configuration. This way load can be spread out across multiple servers. For details, see :ref:`multiple-app-servers` in the Installation Guide.
+
+Code Review and QA
+~~~~~~~~~~~~~~~~~~
+
+Before code is merged, while it is in review or QA, if a performance problem is detected (usually on an ad hoc basis), the code is returned to the developer for improvement. Developers and reviewers typically do not have many tools at their disposal to test code changes against anything close to production data. QA maintains a machine with a copy of production data but tests against smaller data unless a performance problem is suspected.
+
+A new QA guide is coming in https://github.com/IQSS/dataverse/pull/10103
+
+Locust Testing at Release Time
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As one of the final steps in preparing for a release, QA runs performance tests using a tool called Locust as explained the Developer Guide (see :ref:`locust`). The tests are not comprehensive, testing only a handful of pages with anonymous users, but they increase confidence that the upcoming release is not drastically slower than previous releases.
+
+Issue Tracking and Prioritization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Performance issues are tracked in our issue tracker under the `Feature: Performance & Stability `_Â label (e.g. `#7788 `_). That way, we can track performance problems throughout the application. Unfortunately, the pain is often felt by users in production before we realize there is a problem. As needed, performance issues are prioritized to be included in a sprint, to \ `speed up the collection page `_, for example.
+
+Document Performance Tools
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the :doc:`/admin/monitoring` page section of the Admin Guide we describe how to set up Munin for monitoring performance of an operating system. We also explain how to set up Performance Insights to monitor AWS RDS (PostgreSQL as a service, in our case). In the :doc:`/developers/tools` section of the Developer Guide, we have documented how to use Eclipse Memory Analyzer Tool (MAT), SonarQube, jmap, and jstat.
+
+Google Analytics
+~~~~~~~~~~~~~~~~
+
+Emails go to a subset of the team monthly with subjects like "Your September Search performance for https://dataverse.harvard.edu" with a link to a report but it's mostly about the number clicks, not how fast the site is. It's unclear if it provides any value with regard to performance.
+
+Abandoned Tools and Practices
+-----------------------------
+
+New Relic
+~~~~~~~~~
+
+For many years Harvard Dataverse was hooked up to New Relic, a tool that promises all-in-one observability, according to their `website `_. In practice, we didn't do much with `the data `_.
+
+Areas of Particular Concern
+---------------------------
+
+Command Engine Execution Rate Metering
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We'd like to rate limit commands (CreateDataset, etc.) so that we can keep them at a reasonable level (`#9356 `_). This is similar to how many APIs are rate limited, such as the GitHub API.
+
+Solr
+~~~~
+
+While in the past Solr performance hasn't been much of a concern, in recent years we've noticed performance problems when Harvard Dataverse is under load. Improvements were made in `PR #10050 `_, for example.
+
+Datasets with Large Numbers of Files or Versions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We'd like to scale Dataverse to better handle large number of files or versions. Progress was made in `PR #9883 `_.
+
+Withstanding Bots
+~~~~~~~~~~~~~~~~~
+
+Google bot, etc.
+
+Suggested Practices
+-------------------
+
+Many of our current practices should remain in place unaltered. Others could use some refinement. Some new practices should be adopted as well. Here are some suggestions.
+
+Implement the Frontend Plan for Performance
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `Dataverse - SPA MVP Definition doc `_ Â has some ideas around how to achieve good performance for the new front end in the areas of rendering, monitoring,file upload/download, pagination, and caching. We should create as many issues as necessary in the frontend repo and work on them in time. The doc recommends the use of `React Profiler `_Â and other tools. Not mentioned is https://pagespeed.web.dev but we can investigate it as well. See also `#183 `_, a parent issue about performance. In `#184 `_ Â we plan to compare the performance of the old JSF UI vs. the new React UI. Cypress plugins for load testing could be investigated.
+
+Set up Query Counter in Jenkins
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+See countquery script above. See also https://jenkins.dataverse.org/job/IQSS-dataverse-develop/ws/target/query_count.out
+
+Show the plot over time. Make spikes easily apparent. 320,035 queries as of this writing.
+
+Count Database Queries per API Test
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Is it possible? Just a thought.
+
+Teach Developers How to Do Performance Testing Locally
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Do developers know how to use a profiler? Should they use `JMeter `_? `statsd-jvm-profiler `_? How do you run our :ref:`locust` tests? Should we continue using that tool? Give developers time and space to try out tools and document any tips along the way. For this stage, small data is fine.
+
+Automate Performance Testing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We are already using two excellent continuous integration (CI) tools, Jenkins and GitHub Actions, to test our code. We should add performance testing into the mix (`#4201 `_ is an old issue for this but we can open a fresh one). Currently we test every commit on every PR and we should consider if this model makes sense since performance testing will likely take longer to run than regular tests. Once developers are comfortable with their favorite tools, we can pick which ones to automate.
+
+Make Production Data or Equivalent Available to Developers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If developers are only testing small amounts of data on their laptops, it's hard to detect performance problems. Not every bug fix requires access to data similar to production, but it should be made available. This is not a trivial task! If we are to use actual production data, we need to be very careful to de-identify it. If we start with our `sample-data `_ Â repo instead, we'll need to figure out how to make sure we cover cases like many files, many versions, etc.
+
+Automate Performance Testing with Production Data or Equivalent
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Hopefully the environment developers use with production data or equivalent can be made available to our CI tools. Perhaps these tests don't need to be run on every commit to every pull request, but they should be run regularly.
+
+Use Monitoring as Performance Testing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Monitoring can be seen as a form of testing. How long is a round trip ping to production? What is the Time to First Byte? First Contentful Paint? Largest Contentful Paint? Time to Interactive? We now have a beta server that we could monitor continuously to know if our app is getting faster or slower over time. Should our monitoring of production servers be improved?
+
+Learn from Training and Conferences
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Most likely there is training available that is oriented toward performance. The subject of performance often comes up at conferences as well.
+
+Learn from the Community How They Monitor Performance
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some members of the Dataverse community are likely users of newish tools like the ELK stack (Elasticsearch, Logstash, and Kibana), the TICK stack (Telegraph InfluxDB Chronograph and Kapacitor), GoAccess, Prometheus, Graphite, and more we haven't even heard of. In the :doc:`/admin/monitoring` section of the Admin Guide, we already encourage the community to share findings, but we could dedicate time to this topic at our annual meeting or community calls.
+
+Teach the Community to Do Performance Testing
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We have a worldwide community of developers. We should do what we can in the form of documentation and other resources to help them develop performant code.
+
+Conclusion
+----------
+
+Given its long history, Dataverse has encountered many performance problems over the years. The core team is conversant in how to make the app more performant, but investment in learning additional tools and best practices would likely yield dividends. We should automate our performance testing, catching more problems before code is merged.
diff --git a/doc/sphinx-guides/source/developers/remote-users.rst b/doc/sphinx-guides/source/developers/remote-users.rst
index d8f90e9257f..484abe9ccf0 100755
--- a/doc/sphinx-guides/source/developers/remote-users.rst
+++ b/doc/sphinx-guides/source/developers/remote-users.rst
@@ -39,7 +39,7 @@ STOP! ``oidc-keycloak-auth-provider.json`` was changed from http://localhost:809
If you are working on the OpenID Connect (OIDC) user authentication flow, you do not need to connect to a remote provider (as explained in :doc:`/installation/oidc`) to test this feature. Instead, you can use the available configuration that allows you to run a test Keycloak OIDC identity management service locally through a Docker container.
-(Please note! The client secret (``ss6gE8mODCDfqesQaSG3gwUwZqZt547E``) is hard-coded in ``oidc-realm.json`` and ``oidc-keycloak-auth-provider.json``. Do not use this config in production! This is only for developers.)
+(Please note! The client secret (``94XHrfNRwXsjqTqApRrwWmhDLDHpIYV8``) is hard-coded in ``test-realm.json`` and ``oidc-keycloak-auth-provider.json``. Do not use this config in production! This is only for developers.)
You can find this configuration in ``conf/keycloak``. There are two options available in this directory to run a Keycloak container: bash script or docker-compose.
@@ -55,15 +55,27 @@ Now load the configuration defined in ``oidc-keycloak-auth-provider.json`` into
You should see the new provider, called "OIDC-Keycloak", under "Other options" on the Log In page.
-You should be able to log into Keycloak with the following credentials:
+You should be able to log into Keycloak with the one of the following credentials:
-- username: kcuser
-- password: kcpassword
+.. list-table::
+
+ * - Username
+ - Password
+ * - admin
+ - admin
+ * - curator
+ - curator
+ * - user
+ - user
+ * - affiliate
+ - affiliate
In case you want to stop and remove the Keycloak container, just run the other available bash script:
``./rm-keycloak.sh``
+Note: the Keycloak admin to login at the admin console is ``kcadmin:kcpassword``
+
----
Previous: :doc:`unf/index` | Next: :doc:`geospatial`
diff --git a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
index 4d323455d28..d7f270a4e38 100644
--- a/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
+++ b/doc/sphinx-guides/source/developers/s3-direct-upload-api.rst
@@ -69,8 +69,9 @@ In the single part case, only one call to the supplied URL is required:
.. code-block:: bash
- curl -H 'x-amz-tagging:dv-state=temp' -X PUT -T ""
+ curl -i -H 'x-amz-tagging:dv-state=temp' -X PUT -T ""
+Note that without the ``-i`` flag, you should not expect any output from the command above. With the ``-i`` flag, you should expect to see a "200 OK" response.
In the multipart case, the client must send each part and collect the 'eTag' responses from the server. The calls for this are the same as the one for the single part case except that each call should send a slice of the total file, with the last part containing the remaining bytes.
The responses from the S3 server for these calls will include the 'eTag' for the uploaded part.
@@ -115,8 +116,8 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data
curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
To add multiple Uploaded Files to the Dataset
---------------------------------------------
@@ -146,8 +147,8 @@ The allowed checksum algorithms are defined by the edu.harvard.iq.dataverse.Data
curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/datasets/:persistentId/addFiles?persistentId=$PERSISTENT_IDENTIFIER" -F "jsonData=$JSON_DATA"
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
Replacing an existing file in the Dataset
@@ -176,8 +177,8 @@ Note that the API call does not validate that the file matches the hash value su
curl -X POST -H "X-Dataverse-key: $API_TOKEN" "$SERVER_URL/api/files/$FILE_IDENTIFIER/replace" -F "jsonData=$JSON_DATA"
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the file already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
Replacing multiple existing files in the Dataset
------------------------------------------------
@@ -274,5 +275,5 @@ The JSON object returned as a response from this API call includes a "data" that
}
-Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method.
-With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifer must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
+Note that this API call can be used independently of the others, e.g. supporting use cases in which the files already exists in S3/has been uploaded via some out-of-band method. Enabling out-of-band uploads is described at :ref:`file-storage` in the Configuration Guide.
+With current S3 stores the object identifier must be in the correct bucket for the store, include the PID authority/identifier of the parent dataset, and be guaranteed unique, and the supplied storage identifier must be prefaced with the store identifier used in the Dataverse installation, as with the internally generated examples above.
diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst
index 55351b851bb..8e60378fd90 100755
--- a/doc/sphinx-guides/source/developers/testing.rst
+++ b/doc/sphinx-guides/source/developers/testing.rst
@@ -5,7 +5,7 @@ Testing
In order to keep our codebase healthy, the Dataverse Project encourages developers to write automated tests in the form of unit tests and integration tests. We also welcome ideas for how to improve our automated testing.
.. contents:: |toctitle|
- :local:
+ :local:
The Health of a Codebase
------------------------
@@ -46,13 +46,15 @@ The main takeaway should be that we care about unit testing enough to measure th
Writing Unit Tests with JUnit
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-We are aware that there are newer testing tools such as TestNG, but we use `JUnit `_ because it's tried and true.
-We support both (legacy) JUnit 4.x tests (forming the majority of our tests) and
-newer JUnit 5 based testing.
+We are aware that there are newer testing tools such as TestNG, but we use `JUnit `_ because it's tried and true.
+We support JUnit 5 based testing and require new tests written with it.
+(Since Dataverse 6.0, we migrated all of our tests formerly based on JUnit 4.)
-NOTE: When adding new tests, you should give JUnit 5 a go instead of adding more dependencies to JUnit 4.x.
-
-If writing tests is new to you, poke around existing unit tests which all end in ``Test.java`` and live under ``src/test``. Each test is annotated with ``@Test`` and should have at least one assertion which specifies the expected result. In Netbeans, you can run all the tests in it by clicking "Run" -> "Test File". From the test file, you should be able to navigate to the code that's being tested by right-clicking on the file and clicking "Navigate" -> "Go to Test/Tested class". Likewise, from the code, you should be able to use the same "Navigate" menu to go to the tests.
+If writing tests is new to you, poke around existing unit tests which all end in ``Test.java`` and live under ``src/test``.
+Each test is annotated with ``@Test`` and should have at least one assertion which specifies the expected result.
+In Netbeans, you can run all the tests in it by clicking "Run" -> "Test File".
+From the test file, you should be able to navigate to the code that's being tested by right-clicking on the file and clicking "Navigate" -> "Go to Test/Tested class".
+Likewise, from the code, you should be able to use the same "Navigate" menu to go to the tests.
NOTE: Please remember when writing tests checking possibly localized outputs to check against ``en_US.UTF-8`` and ``UTC``
l10n strings!
@@ -62,22 +64,24 @@ Refactoring Code to Make It Unit-Testable
Existing code is not necessarily written in a way that lends itself to easy testing. Generally speaking, it is difficult to write unit tests for both JSF "backing" beans (which end in ``Page.java``) and "service" beans (which end in ``Service.java``) because they require the database to be running in order to test them. If service beans can be exercised via API they can be tested with integration tests (described below) but a good technique for making the logic testable it to move code to "util beans" (which end in ``Util.java``) that operate on Plain Old Java Objects (POJOs). ``PrivateUrlUtil.java`` is a good example of moving logic from ``PrivateUrlServiceBean.java`` to a "util" bean to make the code testable.
-Parameterized Tests and JUnit Theories
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Parameterized Tests
+^^^^^^^^^^^^^^^^^^^
+
Often times you will want to test a method multiple times with similar values.
In order to avoid test bloat (writing a test for every data combination),
JUnit offers Data-driven unit tests. This allows a test to be run for each set
of defined data values.
-JUnit 4 uses ``Parameterized.class`` and ``Theories.class``. For reference, take a look at issue https://github.com/IQSS/dataverse/issues/5619.
-
-JUnit 5 doesn't offer theories (see `jqwik `_ for this), but
-greatly extended parameterized testing. Some guidance how to write those:
+JUnit 5 offers great parameterized testing. Some guidance how to write those:
- https://junit.org/junit5/docs/current/user-guide/#writing-tests-parameterized-tests
- https://www.baeldung.com/parameterized-tests-junit-5
- https://blog.codefx.org/libraries/junit-5-parameterized-tests/
-- See also some examples in our codebase.
+- See also many examples in our codebase.
+
+Note that JUnit 5 also offers support for custom test parameter resolvers. This enables keeping tests cleaner,
+as preparation might happen within some extension and the test code is more focused on the actual testing.
+See https://junit.org/junit5/docs/current/user-guide/#extensions-parameter-resolution for more information.
JUnit 5 Test Helper Extensions
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -85,22 +89,35 @@ JUnit 5 Test Helper Extensions
Our codebase provides little helpers to ease dealing with state during tests.
Some tests might need to change something which should be restored after the test ran.
-For unit tests, the most interesting part is to set a JVM setting just for the current test.
-Please use the ``@JvmSetting(key = JvmSettings.XXX, value = "")`` annotation on a test method or
-a test class to set and clear the property automatically.
+For unit tests, the most interesting part is to set a JVM setting just for the current test or a whole test class.
+(Which might be an inner class, too!). Please make use of the ``@JvmSetting(key = JvmSettings.XXX, value = "")``
+annotation and also make sure to annotate the test class with ``@LocalJvmSettings``.
+
+Inspired by JUnit's ``@MethodSource`` annotation, you may use ``@JvmSetting(key = JvmSettings.XXX, method = "zzz")``
+to reference a static method located in the same test class by name (i. e. ``private static String zzz() {}``) to allow
+retrieving dynamic data instead of String constants only. (Note the requirement for a *static* method!)
+
+If you want to delete a setting, simply provide a ``null`` value. This can be used to override a class-wide setting
+or some other default that is present for some reason.
-To set arbitrary system properties for the current test, a similar extension
-``@SystemProperty(key = "", value = "")`` has been added.
+To set arbitrary system properties for the current test, a similar extension ``@SystemProperty(key = "", value = "")``
+has been added. (Note: it does not support method references.)
Both extensions will ensure the global state of system properties is non-interfering for
test executions. Tests using these extensions will be executed in serial.
+This settings helper may be extended at a later time to manipulate settings in a remote instance during integration
+or end-to-end testing. Stay tuned!
+
Observing Changes to Code Coverage
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Once you've written some tests, you're probably wondering how much you've helped to increase the code coverage. In Netbeans, do a "clean and build." Then, under the "Projects" tab, right-click "dataverse" and click "Code Coverage" -> "Show Report". For each Java file you have open, you should be able to see the percentage of code that is covered by tests and every line in the file should be either green or red. Green indicates that the line is being exercised by a unit test and red indicates that it is not.
-In addition to seeing code coverage in Netbeans, you can also see code coverage reports by opening ``target/site/jacoco/index.html`` in your browser.
+In addition to seeing code coverage in Netbeans, you can also see code coverage reports by opening ``target/site/jacoco-X-test-coverage-report/index.html`` in your browser.
+Depending on the report type you want to look at, let ``X`` be one of ``unit``, ``integration`` or ``merged``.
+"Merged" will display combined coverage of both unit and integration test, but does currently not cover API tests.
+
Testing Commands
^^^^^^^^^^^^^^^^
@@ -116,11 +133,14 @@ In addition, there is a writeup on "The Testable Command" at https://github.com/
Running Non-Essential (Excluded) Unit Tests
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-You should be aware that some unit tests have been deemed "non-essential" and have been annotated with ``@Category(NonEssentialTests.class)`` and are excluded from the "dev" Maven profile, which is the default profile. All unit tests (that have not been annotated with ``@Ignore``), including these non-essential tests, are run from continuous integration systems such as Jenkins and GitHub Actions with the following ``mvn`` command that invokes a non-default profile:
+You should be aware that some unit tests have been deemed "non-essential" and have been annotated with ``@Tag(Tags.NOT_ESSENTIAL_UNITTESTS)`` and are excluded from the "dev" Maven profile, which is the default profile.
+All unit tests (that have not been annotated with ``@Disable``), including these non-essential tests, are run from continuous integration systems such as Jenkins and GitHub Actions with the following ``mvn`` command that invokes a non-default profile:
``mvn test -P all-unit-tests``
-Generally speaking, unit tests have been flagged as non-essential because they are slow or because they require an Internet connection. You should not feel obligated to run these tests continuously but you can use the ``mvn`` command above to run them. To iterate on the unit test in Netbeans and execute it with "Run -> Test File", you must temporarily comment out the annotation flagging the test as non-essential.
+Generally speaking, unit tests have been flagged as non-essential because they are slow or because they require an Internet connection.
+You should not feel obligated to run these tests continuously but you can use the ``mvn`` command above to run them.
+To iterate on the unit test in Netbeans and execute it with "Run -> Test File", you must temporarily comment out the annotation flagging the test as non-essential.
Integration Tests
-----------------
@@ -170,47 +190,38 @@ Finally, run the script:
$ ./ec2-create-instance.sh -g jenkins.yml -l log_dir
-Running the full API test suite using Docker
+Running the Full API Test Suite Using Docker
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-.. note::
- Sunsetting of this module is imminent.** There is no schedule yet, but expect it to go away.
- Please let the `Dataverse Containerization Working Group `_ know if you are a user and
- what should be preserved.
-
-To run the full suite of integration tests on your laptop, we recommend using the "all in one" Docker configuration described in ``conf/docker-aio/readme.md`` in the root of the repo.
-
-Alternatively, you can run tests against the app server running on your laptop by following the "getting set up" steps below.
-
-Getting Set Up to Run REST Assured Tests
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To run the full suite of integration tests on your laptop, we recommend running Dataverse and its dependencies in Docker, as explained in the :doc:`/container/dev-usage` section of the Container Guide. This environment provides additional services (such as S3) that are used in testing.
-Unit tests are run automatically on every build, but dev environments and servers require special setup to run REST Assured tests. In short, the Dataverse Software needs to be placed into an insecure mode that allows arbitrary users and datasets to be created and destroyed. This differs greatly from the out-of-the-box behavior of the Dataverse Software, which we strive to keep secure for sysadmins installing the software for their institutions in a production environment.
+Running the APIs Without Docker (Classic Dev Env)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The :doc:`dev-environment` section currently refers developers here for advice on getting set up to run REST Assured tests, but we'd like to add some sort of "dev" flag to the installer to put the Dataverse Software in "insecure" mode, with lots of scary warnings that this dev mode should not be used in production.
+While it is possible to run a good number of API tests without using Docker in our :doc:`classic-dev-env`, we are transitioning toward including additional services (such as S3) in our Dockerized development environment (:doc:`/container/dev-usage`), so you will probably find it more convenient to it instead.
-The instructions below assume a relatively static dev environment on a Mac. There is a newer "all in one" Docker-based approach documented in the :doc:`/developers/containers` section under "Docker" that you may like to play with as well.
+Unit tests are run automatically on every build, but dev environments and servers require special setup to run API (REST Assured) tests. In short, the Dataverse software needs to be placed into an insecure mode that allows arbitrary users and datasets to be created and destroyed (this is done automatically in the Dockerized environment, as well as the steps described below). This differs greatly from the out-of-the-box behavior of the Dataverse software, which we strive to keep secure for sysadmins installing the software for their institutions in a production environment.
The Burrito Key
^^^^^^^^^^^^^^^
-For reasons that have been lost to the mists of time, the Dataverse Software really wants you to to have a burrito. Specifically, if you're trying to run REST Assured tests and see the error "Dataverse config issue: No API key defined for built in user management", you must run the following curl command (or make an equivalent change to your database):
+For reasons that have been lost to the mists of time, the Dataverse software really wants you to to have a burrito. Specifically, if you're trying to run REST Assured tests and see the error "Dataverse config issue: No API key defined for built in user management", you must run the following curl command (or make an equivalent change to your database):
``curl -X PUT -d 'burrito' http://localhost:8080/api/admin/settings/BuiltinUsers.KEY``
-Without this "burrito" key in place, REST Assured will not be able to create users. We create users to create objects we want to test, such as Dataverse collections, datasets, and files.
+Without this "burrito" key in place, REST Assured will not be able to create users. We create users to create objects we want to test, such as collections, datasets, and files.
-Root Dataverse Collection Permissions
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Root Collection Permissions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
-In your browser, log in as dataverseAdmin (password: admin) and click the "Edit" button for your root Dataverse collection. Navigate to Permissions, then the Edit Access button. Under "Who can add to this Dataverse collection?" choose "Anyone with a Dataverse installation account can add sub Dataverse collections and datasets" if it isn't set to this already.
+In your browser, log in as dataverseAdmin (password: admin) and click the "Edit" button for your root collection. Navigate to Permissions, then the Edit Access button. Under "Who can add to this collection?" choose "Anyone with a Dataverse installation account can add sub collections and datasets" if it isn't set to this already.
Alternatively, this same step can be done with this script: ``scripts/search/tests/grant-authusers-add-on-root``
-Publish Root Dataverse Collection
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Publish Root Collection
+^^^^^^^^^^^^^^^^^^^^^^^
-The root Dataverse collection must be published for some of the REST Assured tests to run.
+The root collection must be published for some of the REST Assured tests to run.
dataverse.siteUrl
^^^^^^^^^^^^^^^^^
@@ -223,6 +234,20 @@ If ``dataverse.siteUrl`` is absent, you can add it with:
``./asadmin create-jvm-options "-Ddataverse.siteUrl=http\://localhost\:8080"``
+dataverse.oai.server.maxidentifiers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The OAI Harvesting tests require that the paging limit for ListIdentifiers must be set to 2, in order to be able to trigger this paging behavior without having to create and export too many datasets:
+
+``./asadmin create-jvm-options "-Ddataverse.oai.server.maxidentifiers=2"``
+
+dataverse.oai.server.maxrecords
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The OAI Harvesting tests require that the paging limit for ListRecords must be set to 2, in order to be able to trigger this paging behavior without having to create and export too many datasets:
+
+``./asadmin create-jvm-options "-Ddataverse.oai.server.maxrecords=2"``
+
Identifier Generation
^^^^^^^^^^^^^^^^^^^^^
@@ -243,17 +268,22 @@ Remember, it’s only a test (and it's not graded)! Some guidelines to bear in m
- Map out which logical functions you want to test
- Understand what’s being tested and ensure it’s repeatable
- Assert the conditions of success / return values for each operation
- * A useful resource would be `HTTP status codes `_
+ * A useful resource would be `HTTP status codes `_
- Let the code do the labor; automate everything that happens when you run your test file.
+- If you need to test an optional service (S3, etc.), add it to our docker compose file. See :doc:`/container/dev-usage`.
- Just as with any development, if you’re stuck: ask for help!
-To execute existing integration tests on your local Dataverse installation, a helpful command line tool to use is `Maven `_. You should have Maven installed as per the `Development Environment `_ guide, but if not it’s easily done via Homebrew: ``brew install maven``.
+To execute existing integration tests on your local Dataverse installation, a helpful command line tool to use is `Maven `_. You should have Maven installed as per the `Development Environment `_ guide, but if not it’s easily done via Homebrew: ``brew install maven``.
Once installed, you may run commands with ``mvn [options] [] []``.
-+ If you want to run just one particular API test, it’s as easy as you think:
++ If you want to run just one particular API test class:
- ``mvn test -Dtest=FileRecordJobIT``
+ ``mvn test -Dtest=UsersIT``
+
++ If you want to run just one particular API test method,
+
+ ``mvn test -Dtest=UsersIT#testMergeAccounts``
+ To run more than one test at a time, separate by commas:
@@ -282,35 +312,39 @@ To run a test with Testcontainers, you will need to write a JUnit 5 test.
Please make sure to:
1. End your test class with ``IT``
-2. Provide a ``@Tag("testcontainers")`` to be picked up during testing.
+2. Annotate the test class with two tags:
-.. code:: java
+ .. code:: java
- /** A very minimal example for a Testcontainers integration test class. */
- @Testcontainers
- @Tag("testcontainers")
- class MyExampleIT { /* ... */ }
+ /** A very minimal example for a Testcontainers integration test class. */
+ @Testcontainers(disabledWithoutDocker = true)
+ @Tag(edu.harvard.iq.dataverse.util.testing.Tags.INTEGRATION_TEST)
+ @Tag(edu.harvard.iq.dataverse.util.testing.Tags.USES_TESTCONTAINERS)
+ class MyExampleIT { /* ... */ }
-If using upstream Modules, e.g. for PostgreSQL or similar, you will need to add
+If using upstream modules, e.g. for PostgreSQL or similar, you will need to add
a dependency to ``pom.xml`` if not present. `See the PostgreSQL module example. `_
To run these tests, simply call out to Maven:
.. code::
- mvn -P tc verify
+ mvn verify
+
+Notes:
-.. note::
+1. Remember to have Docker ready to serve or tests will fail.
+2. You can skip running unit tests by adding ``-DskipUnitTests``
+3. You can choose to ignore test with Testcontainers by adding ``-Dit.groups='integration & !testcontainers'``
+ Learn more about `filter expressions in the JUnit 5 guide `_.
- 1. Remember to have Docker ready to serve or tests will fail.
- 2. This will not run any unit tests or API tests.
-Measuring Coverage of Integration Tests
----------------------------------------
+Measuring Coverage of API Tests
+-------------------------------
-Measuring the code coverage of integration tests with Jacoco requires several steps. In order to make these steps clear we'll use "/usr/local/payara5" as the Payara directory and "dataverse" as the Payara Unix user.
+Measuring the code coverage of API tests with Jacoco requires several steps. In order to make these steps clear we'll use "/usr/local/payara6" as the Payara directory and "dataverse" as the Payara Unix user.
-Please note that this was tested under Glassfish 4 but it is hoped that the same steps will work with Payara 5.
+Please note that this was tested under Glassfish 4 but it is hoped that the same steps will work with Payara.
Add jacocoagent.jar to Payara
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -329,9 +363,9 @@ Note that we are running the following commands as the user "dataverse". In shor
cd local/jacoco-0.8.1
wget https://github.com/jacoco/jacoco/releases/download/v0.8.1/jacoco-0.8.1.zip
unzip jacoco-0.8.1.zip
- /usr/local/payara5/bin/asadmin stop-domain
- cp /home/dataverse/local/jacoco-0.8.1/lib/jacocoagent.jar /usr/local/payara5/glassfish/lib
- /usr/local/payara5/bin/asadmin start-domain
+ /usr/local/payara6/bin/asadmin stop-domain
+ cp /home/dataverse/local/jacoco-0.8.1/lib/jacocoagent.jar /usr/local/payara6/glassfish/lib
+ /usr/local/payara6/bin/asadmin start-domain
Add jacococli.jar to the WAR File
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -354,21 +388,21 @@ Run this as the "dataverse" user.
.. code-block:: bash
- /usr/local/payara5/bin/asadmin deploy dataverse-jacoco.war
+ /usr/local/payara6/bin/asadmin deploy dataverse-jacoco.war
-Note that after deployment the file "/usr/local/payara5/glassfish/domains/domain1/config/jacoco.exec" exists and is empty.
+Note that after deployment the file "/usr/local/payara6/glassfish/domains/domain1/config/jacoco.exec" exists and is empty.
-Run Integration Tests
-~~~~~~~~~~~~~~~~~~~~~
+Run API Tests
+~~~~~~~~~~~~~
Note that even though you see "docker-aio" in the command below, we assume you are not necessarily running the test suite within Docker. (Some day we'll probably move this script to another directory.) For this reason, we pass the URL with the normal port (8080) that app servers run on to the ``run-test-suite.sh`` script.
-Note that "/usr/local/payara5/glassfish/domains/domain1/config/jacoco.exec" will become non-empty after you stop and start Payara. You must stop and start Payara before every run of the integration test suite.
+Note that "/usr/local/payara6/glassfish/domains/domain1/config/jacoco.exec" will become non-empty after you stop and start Payara. You must stop and start Payara before every run of the integration test suite.
.. code-block:: bash
- /usr/local/payara5/bin/asadmin stop-domain
- /usr/local/payara5/bin/asadmin start-domain
+ /usr/local/payara6/bin/asadmin stop-domain
+ /usr/local/payara6/bin/asadmin start-domain
git clone https://github.com/IQSS/dataverse.git
cd dataverse
conf/docker-aio/run-test-suite.sh http://localhost:8080
@@ -383,7 +417,7 @@ Run these commands as the "dataverse" user. The ``cd dataverse`` means that you
.. code-block:: bash
cd dataverse
- java -jar /home/dataverse/local/jacoco-0.8.1/lib/jacococli.jar report --classfiles target/classes --sourcefiles src/main/java --html target/coverage-it/ /usr/local/payara5/glassfish/domains/domain1/config/jacoco.exec
+ java -jar /home/dataverse/local/jacoco-0.8.1/lib/jacococli.jar report --classfiles target/classes --sourcefiles src/main/java --html target/coverage-it/ /usr/local/payara6/glassfish/domains/domain1/config/jacoco.exec
Read Code Coverage Report
~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -393,6 +427,8 @@ target/coverage-it/index.html is the place to start reading the code coverage re
Load/Performance Testing
------------------------
+.. _locust:
+
Locust
~~~~~~
@@ -492,7 +528,7 @@ Future Work on Integration Tests
- Automate testing of dataverse-client-python: https://github.com/IQSS/dataverse-client-python/issues/10
- Work with @leeper on testing the R client: https://github.com/IQSS/dataverse-client-r
- Review and attempt to implement "API Test Checklist" from @kcondon at https://docs.google.com/document/d/199Oq1YwQ4pYCguaeW48bIN28QAitSk63NbPYxJHCCAE/edit?usp=sharing
-- Generate code coverage reports for **integration** tests: https://github.com/pkainulainen/maven-examples/issues/3 and http://www.petrikainulainen.net/programming/maven/creating-code-coverage-reports-for-unit-and-integration-tests-with-the-jacoco-maven-plugin/
+- Generate code coverage reports for **integration** tests: https://github.com/pkainulainen/maven-examples/issues/3 and https://www.petrikainulainen.net/programming/maven/creating-code-coverage-reports-for-unit-and-integration-tests-with-the-jacoco-maven-plugin/
- Consistent logging of API Tests. Show test name at the beginning and end and status codes returned.
- expected passing and known/expected failing integration tests: https://github.com/IQSS/dataverse/issues/4438
@@ -504,7 +540,6 @@ Browser-Based Testing
Installation Testing
~~~~~~~~~~~~~~~~~~~~
-- Run `vagrant up` on a server to test the installer
- Work with @donsizemore to automate testing of https://github.com/GlobalDataverseCommunityConsortium/dataverse-ansible
Future Work on Load/Performance Testing
diff --git a/doc/sphinx-guides/source/developers/tips.rst b/doc/sphinx-guides/source/developers/tips.rst
index bf75a05f84e..764434d1896 100755
--- a/doc/sphinx-guides/source/developers/tips.rst
+++ b/doc/sphinx-guides/source/developers/tips.rst
@@ -19,20 +19,20 @@ Undeploy the war File from the Dataverse Software Installation Script
Because the initial deployment of the war file was done outside of Netbeans by the Dataverse Software installation script, it's a good idea to undeploy that war file to give Netbeans a clean slate to work with.
-Assuming you installed Payara in ``/usr/local/payara5``, run the following ``asadmin`` command to see the version of the Dataverse Software that the Dataverse Software installation script deployed:
+Assuming you installed Payara in ``/usr/local/payara6``, run the following ``asadmin`` command to see the version of the Dataverse Software that the Dataverse Software installation script deployed:
-``/usr/local/payara5/bin/asadmin list-applications``
+``/usr/local/payara6/bin/asadmin list-applications``
You will probably see something like ``dataverse-5.0 `` as the output. To undeploy, use whichever version you see like this:
-``/usr/local/payara5/bin/asadmin undeploy dataverse-5.0``
+``/usr/local/payara6/bin/asadmin undeploy dataverse-5.0``
Now that Payara doesn't have anything deployed, we can proceed with getting Netbeans set up to deploy the code.
Add Payara as a Server in Netbeans
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Launch Netbeans and click "Tools" and then "Servers". Click "Add Server" and select "Payara Server" and set the installation location to ``/usr/local/payara5``. The defaults are fine so you can click "Next" and "Finish".
+Launch Netbeans and click "Tools" and then "Servers". Click "Add Server" and select "Payara Server" and set the installation location to ``/usr/local/payara6``. The defaults are fine so you can click "Next" and "Finish".
Please note that if you are on a Mac, Netbeans may be unable to start Payara due to proxy settings in Netbeans. Go to the "General" tab in Netbeans preferences and click "Test connection" to see if you are affected. If you get a green checkmark, you're all set. If you get a red exclamation mark, change "Proxy Settings" to "No Proxy" and retest. A more complicated answer having to do with changing network settings is available at https://discussions.apple.com/thread/7680039?answerId=30715103022#30715103022 and the bug is also described at https://netbeans.org/bugzilla/show_bug.cgi?id=268076
@@ -117,7 +117,7 @@ Deploying With ``asadmin``
Sometimes you want to deploy code without using Netbeans or from the command line on a server you have ssh'ed into.
-For the ``asadmin`` commands below, we assume you have already changed directories to ``/usr/local/payara5/glassfish/bin`` or wherever you have installed Payara.
+For the ``asadmin`` commands below, we assume you have already changed directories to ``/usr/local/payara6/glassfish/bin`` or wherever you have installed Payara.
There are four steps to this process:
@@ -238,6 +238,8 @@ with the following code in ``SettingsWrapper.java``:
A more serious example would be direct calls to PermissionServiceBean methods used in render logic expressions. This is something that has happened and caused some problems in real life. A simple permission service lookup (for example, whether a user is authorized to create a dataset in the current dataverse) can easily take 15 database queries. Repeated multiple times, this can quickly become a measurable delay in rendering the page. PermissionsWrapper must be used exclusively for any such lookups from JSF pages.
+See also :doc:`performance`.
+
----
Previous: :doc:`dev-environment` | Next: :doc:`troubleshooting`
diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst
index cbd27d6e8d2..9d2740fab6a 100755
--- a/doc/sphinx-guides/source/developers/tools.rst
+++ b/doc/sphinx-guides/source/developers/tools.rst
@@ -25,38 +25,23 @@ Maven
With Maven installed you can run ``mvn package`` and ``mvn test`` from the command line. It can be downloaded from https://maven.apache.org
-.. _vagrant:
-
-Vagrant
-+++++++
-
-Vagrant allows you to spin up a virtual machine running the Dataverse Software on your development workstation. You'll need to install Vagrant from https://www.vagrantup.com and VirtualBox from https://www.virtualbox.org.
-
-We assume you have already cloned the repo from https://github.com/IQSS/dataverse as explained in the :doc:`/developers/dev-environment` section.
-
-From the root of the git repo (where the ``Vagrantfile`` is), run ``vagrant up`` and eventually you should be able to reach a Dataverse installation at http://localhost:8888 (the ``forwarded_port`` indicated in the ``Vagrantfile``).
-
-Please note that running ``vagrant up`` for the first time should run the ``downloads/download.sh`` script for you to download required software such as an app server, Solr, etc. However, these dependencies change over time so it's a place to look if ``vagrant up`` was working but later fails.
-
-On Windows if you see an error like ``/usr/bin/perl^M: bad interpreter`` you might need to run ``dos2unix`` on the installation scripts.
-
PlantUML
++++++++
-PlantUML is used to create diagrams in the guides and other places. Download it from http://plantuml.com and check out an example script at https://github.com/IQSS/dataverse/blob/v4.6.1/doc/Architecture/components.sh . Note that for this script to work, you'll need the ``dot`` program, which can be installed on Mac with ``brew install graphviz``.
+PlantUML is used to create diagrams in the guides and other places. Download it from https://plantuml.com and check out an example script at https://github.com/IQSS/dataverse/blob/v4.6.1/doc/Architecture/components.sh . Note that for this script to work, you'll need the ``dot`` program, which can be installed on Mac with ``brew install graphviz``.
Eclipse Memory Analyzer Tool (MAT)
++++++++++++++++++++++++++++++++++
The Memory Analyzer Tool (MAT) from Eclipse can help you analyze heap dumps, showing you "leak suspects" such as seen at https://github.com/payara/Payara/issues/350#issuecomment-115262625
-It can be downloaded from http://www.eclipse.org/mat
+It can be downloaded from https://www.eclipse.org/mat
If the heap dump provided to you was created with ``gcore`` (such as with ``gcore -o /tmp/app.core $app_pid``) rather than ``jmap``, you will need to convert the file before you can open it in MAT. Using ``app.core.13849`` as example of the original 33 GB file, here is how you could convert it into a 26 GB ``app.core.13849.hprof`` file. Please note that this operation took almost 90 minutes:
``/usr/java7/bin/jmap -dump:format=b,file=app.core.13849.hprof /usr/java7/bin/java app.core.13849``
-A file of this size may not "just work" in MAT. When you attempt to open it you may see something like "An internal error occurred during: "Parsing heap dump from '/tmp/heapdumps/app.core.13849.hprof'". Java heap space". If so, you will need to increase the memory allocated to MAT. On Mac OS X, this can be done by editing ``MemoryAnalyzer.app/Contents/MacOS/MemoryAnalyzer.ini`` and increasing the value "-Xmx1024m" until it's high enough to open the file. See also http://wiki.eclipse.org/index.php/MemoryAnalyzer/FAQ#Out_of_Memory_Error_while_Running_the_Memory_Analyzer
+A file of this size may not "just work" in MAT. When you attempt to open it you may see something like "An internal error occurred during: "Parsing heap dump from '/tmp/heapdumps/app.core.13849.hprof'". Java heap space". If so, you will need to increase the memory allocated to MAT. On Mac OS X, this can be done by editing ``MemoryAnalyzer.app/Contents/MacOS/MemoryAnalyzer.ini`` and increasing the value "-Xmx1024m" until it's high enough to open the file. See also https://wiki.eclipse.org/index.php/MemoryAnalyzer/FAQ#Out_of_Memory_Error_while_Running_the_Memory_Analyzer
PageKite
++++++++
@@ -73,7 +58,7 @@ The first time you run ``./pagekite.py`` a file at ``~/.pagekite.rc`` will be
created. You can edit this file to configure PageKite to serve up port 8080
(the default app server HTTP port) or the port of your choosing.
-According to https://pagekite.net/support/free-for-foss/ PageKite (very generously!) offers free accounts to developers writing software the meets http://opensource.org/docs/definition.php such as the Dataverse Project.
+According to https://pagekite.net/support/free-for-foss/ PageKite (very generously!) offers free accounts to developers writing software the meets https://opensource.org/docs/definition.php such as the Dataverse Project.
MSV
+++
@@ -111,7 +96,7 @@ Download SonarQube from https://www.sonarqube.org and start look in the `bin` di
-Dsonar.test.exclusions='src/test/**,src/main/webapp/resources/**' \
-Dsonar.issuesReport.html.enable=true \
-Dsonar.issuesReport.html.location='sonar-issues-report.html' \
- -Dsonar.jacoco.reportPath=target/jacoco.exec
+ -Dsonar.jacoco.reportPath=target/coverage-reports/jacoco-unit.exec
Once the analysis is complete, you should be able to access http://localhost:9000/dashboard?id=edu.harvard.iq%3Adataverse to see the report. To learn about resource leaks, for example, click on "Bugs", the "Tag", then "leak" or "Rule", then "Resources should be closed".
diff --git a/doc/sphinx-guides/source/developers/unf/index.rst b/doc/sphinx-guides/source/developers/unf/index.rst
index 2423877348f..856de209e82 100644
--- a/doc/sphinx-guides/source/developers/unf/index.rst
+++ b/doc/sphinx-guides/source/developers/unf/index.rst
@@ -27,7 +27,7 @@ with Dataverse Software 2.0 and throughout the 3.* lifecycle, UNF v.5
UNF v.6. Two parallel implementation, in R and Java, will be
available, for cross-validation.
-Learn more: Micah Altman and Gary King. 2007. “A Proposed Standard for the Scholarly Citation of Quantitative Data.†D-Lib Magazine, 13. Publisher’s Version Copy at http://j.mp/2ovSzoT
+Learn more: Micah Altman and Gary King. 2007. “A Proposed Standard for the Scholarly Citation of Quantitative Data.†D-Lib Magazine, 13. Publisher’s Version Copy at https://j.mp/2ovSzoT
**Contents:**
diff --git a/doc/sphinx-guides/source/developers/unf/unf-v3.rst b/doc/sphinx-guides/source/developers/unf/unf-v3.rst
index 3f0018d7fa5..98c07b398e0 100644
--- a/doc/sphinx-guides/source/developers/unf/unf-v3.rst
+++ b/doc/sphinx-guides/source/developers/unf/unf-v3.rst
@@ -34,11 +34,11 @@ For example, the number pi at five digits is represented as -3.1415e+, and the n
1. Terminate character strings representing nonmissing values with a POSIX end-of-line character.
-2. Encode each character string with `Unicode bit encoding `_. Versions 3 through 4 use UTF-32BE; Version 4.1 uses UTF-8.
+2. Encode each character string with `Unicode bit encoding `_. Versions 3 through 4 use UTF-32BE; Version 4.1 uses UTF-8.
3. Combine the vector of character strings into a single sequence, with each character string separated by a POSIX end-of-line character and a null byte.
-4. Compute a hash on the resulting sequence using the standard MD5 hashing algorithm for Version 3 and using `SHA256 `_ for Version 4. The resulting hash is `base64 `_ encoded to support readability.
+4. Compute a hash on the resulting sequence using the standard MD5 hashing algorithm for Version 3 and using `SHA256 `_ for Version 4. The resulting hash is `base64 `_ encoded to support readability.
5. Calculate the UNF for each lower-level data object, using a consistent UNF version and level of precision across the individual UNFs being combined.
@@ -49,4 +49,4 @@ For example, the number pi at five digits is represented as -3.1415e+, and the n
8. Combine UNFs from multiple variables to form a single UNF for an entire data frame, and then combine UNFs for a set of data frames to form a single UNF that represents an entire research study.
Learn more:
-Software for computing UNFs is available in an R Module, which includes a Windows standalone tool and code for Stata and SAS languages. Also see the following for more details: Micah Altman and Gary King. 2007. "A Proposed Standard for the Scholarly Citation of Quantitative Data," D-Lib Magazine, Vol. 13, No. 3/4 (March). (Abstract: `HTML `_ | Article: `PDF `_)
+Software for computing UNFs is available in an R Module, which includes a Windows standalone tool and code for Stata and SAS languages. Also see the following for more details: Micah Altman and Gary King. 2007. "A Proposed Standard for the Scholarly Citation of Quantitative Data," D-Lib Magazine, Vol. 13, No. 3/4 (March). (Abstract: `HTML `_ | Article: `PDF `_)
diff --git a/doc/sphinx-guides/source/developers/unf/unf-v6.rst b/doc/sphinx-guides/source/developers/unf/unf-v6.rst
index 9648bae47c8..b2495ff3dd9 100644
--- a/doc/sphinx-guides/source/developers/unf/unf-v6.rst
+++ b/doc/sphinx-guides/source/developers/unf/unf-v6.rst
@@ -156,7 +156,7 @@ For example, to specify a non-default precision the parameter it is specified us
| Allowed values are {``128`` , ``192`` , ``196`` , ``256``} with ``128`` being the default.
| ``R1`` - **truncate** numeric values to ``N`` digits, **instead of rounding**, as previously described.
-`Dr. Micah Altman's classic UNF v5 paper `_ mentions another optional parameter ``T###``, for specifying rounding of date and time values (implemented as stripping the values of entire components - fractional seconds, seconds, minutes, hours... etc., progressively) - but it doesn't specify its syntax. It is left as an exercise for a curious reader to contact the author and work out the details, if so desired. (Not implemented in UNF Version 6 by the Dataverse Project).
+`Dr. Micah Altman's classic UNF v5 paper `_ mentions another optional parameter ``T###``, for specifying rounding of date and time values (implemented as stripping the values of entire components - fractional seconds, seconds, minutes, hours... etc., progressively) - but it doesn't specify its syntax. It is left as an exercise for a curious reader to contact the author and work out the details, if so desired. (Not implemented in UNF Version 6 by the Dataverse Project).
Note: we do not recommend truncating character strings at fewer bytes than the default ``128`` (the ``X`` parameter). At the very least this number **must** be high enough so that the printable UNFs of individual variables or files are not truncated, when calculating combined UNFs of files or datasets, respectively.
diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst
index aacc245af5a..12f3d5b81fd 100644
--- a/doc/sphinx-guides/source/developers/version-control.rst
+++ b/doc/sphinx-guides/source/developers/version-control.rst
@@ -24,7 +24,7 @@ The goals of the Dataverse Software branching strategy are:
- allow for concurrent development
- only ship stable code
-We follow a simplified "git flow" model described at http://nvie.com/posts/a-successful-git-branching-model/ involving a "master" branch, a "develop" branch, and feature branches such as "1234-bug-fix".
+We follow a simplified "git flow" model described at https://nvie.com/posts/a-successful-git-branching-model/ involving a "master" branch, a "develop" branch, and feature branches such as "1234-bug-fix".
Branches
~~~~~~~~
@@ -65,21 +65,65 @@ The example of creating a pull request below has to do with fixing an important
Find or Create a GitHub Issue
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-For guidance on which issue to work on, please ask! Also, see https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md
+An issue represents a bug (unexpected behavior) or a new feature in Dataverse. We'll use the issue number in the branch we create for our pull request.
-Let's say you want to tackle https://github.com/IQSS/dataverse/issues/3728 which points out a typo in a page of the Dataverse Software's documentation.
+Finding GitHub Issues to Work On
+********************************
+
+Assuming this is your first contribution to Dataverse, you should start with something small. The following issue labels might be helpful in your search:
+
+- `good first issue `_ (these appear at https://github.com/IQSS/dataverse/contribute )
+- `hacktoberfest `_
+- `Help Wanted: Code `_
+- `Help Wanted: Documentation `_
+
+For guidance on which issue to work on, please ask! :ref:`getting-help-developers` explains how to get in touch.
+
+Creating GitHub Issues to Work On
+*********************************
+
+You are very welcome to create a GitHub issue to work on. However, for significant changes, please reach out (see :ref:`getting-help-developers`) to make sure the team and community agree with the proposed change.
+
+For small changes and especially typo fixes, please don't worry about reaching out first.
+
+Communicate Which Issue You Are Working On
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the issue you can simply leave a comment to say you're working on it.
If you tell us your GitHub username we are happy to add you to the "read only" team at https://github.com/orgs/IQSS/teams/dataverse-readonly/members so that we can assign the issue to you while you're working on it. You can also tell us if you'd like to be added to the `Dataverse Community Contributors spreadsheet `_.
-Create a New Branch off the develop Branch
+Create a New Branch Off the develop Branch
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Always create your feature branch from the latest code in develop, pulling the latest code if necessary. As mentioned above, your branch should have a name like "3728-doc-apipolicy-fix" that starts with the issue number you are addressing, and ends with a short, descriptive name. Dashes ("-") and underscores ("_") in your branch name are ok, but please try to avoid other special characters such as ampersands ("&") that have special meaning in Unix shells.
+Always create your feature branch from the latest code in develop, pulling the latest code if necessary. As mentioned above, your branch should have a name like "3728-doc-apipolicy-fix" that starts with the issue number you are addressing (e.g. `#3728 `_) and ends with a short, descriptive name. Dashes ("-") and underscores ("_") in your branch name are ok, but please try to avoid other special characters such as ampersands ("&") that have special meaning in Unix shells.
Commit Your Change to Your New Branch
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Making a commit (or several commits) to that branch. Ideally the first line of your commit message includes the number of the issue you are addressing, such as ``Fixed BlockedApiPolicy #3728``.
+For each commit to that branch, try to include the issue number along with a summary in the first line of the commit message, such as ``Fixed BlockedApiPolicy #3728``. You are welcome to write longer descriptions in the body as well!
+
+.. _writing-release-note-snippets:
+
+Writing a Release Note Snippet
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We highly value your insight as a contributor when in comes to describing your work in our release notes. Not every pull request will be mentioned in release notes but most are.
+
+As described at :ref:`write-release-notes`, at release time we compile together release note "snippets" into the final release notes.
+
+Here's how to add a release note snippet to your pull request:
+
+- Create a Markdown file under ``doc/release-notes``. You can reuse the name of your branch and append ".md" to it, e.g. ``3728-doc-apipolicy-fix.md``
+- Edit the snippet to include anything you think should be mentioned in the release notes, such as:
+
+ - Descriptions of new features
+ - Explanations of bugs fixed
+ - New configuration settings
+ - Upgrade instructions
+ - Etc.
+
+Release note snippets do not need to be long. For a new feature, a single line description might be enough. Please note that your release note will likely be edited (expanded or shortened) when the final release notes are being created.
Push Your Branch to GitHub
~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -89,7 +133,9 @@ Push your feature branch to your fork of the Dataverse Software. Your git comman
Make a Pull Request
~~~~~~~~~~~~~~~~~~~
-Make a pull request to get approval to merge your changes into the develop branch. Note that once a pull request is created, we'll remove the corresponding issue from our kanban board so that we're only tracking one card.
+Make a pull request to get approval to merge your changes into the develop branch.
+If the pull request notes indicate that release notes are necessary, the workflow can then verify the existence of a corresponding file and respond with a 'thank you!' message. On the other hand, if no release notes are detected, the contributor can be gently reminded of their absence. Please see :doc:`making-releases` for guidance on writing release notes.
+Note that once a pull request is created, we'll remove the corresponding issue from our kanban board so that we're only tracking one card.
Feedback on the pull request template we use is welcome! Here's an example of a pull request for issue #3827: https://github.com/IQSS/dataverse/pull/3827
diff --git a/doc/sphinx-guides/source/developers/windows.rst b/doc/sphinx-guides/source/developers/windows.rst
index e278b193e12..53578fe980c 100755
--- a/doc/sphinx-guides/source/developers/windows.rst
+++ b/doc/sphinx-guides/source/developers/windows.rst
@@ -2,86 +2,17 @@
Windows Development
===================
-Development on Windows is not well supported, unfortunately. You will have a much easier time if you develop on Mac or Linux as described under :doc:`dev-environment` section.
-
-Vagrant commands appear below and were tested on Windows 10 but the Vagrant environment is currently broken. Please see https://github.com/IQSS/dataverse/issues/6849
-
-There is a newer, experimental Docker option described under :doc:`/container/dev-usage` in the Container Guide.
+Historically, development on Windows is `not well supported `_ but as of 2023 a container-based approach is recommended.
.. contents:: |toctitle|
:local:
-Running the Dataverse Software in Vagrant
------------------------------------------
-
-Install Vagrant
-~~~~~~~~~~~~~~~
-
-Download and install Vagrant from https://www.vagrantup.com
-
-Vagrant advises you to reboot but let's install VirtualBox first.
-
-Install VirtualBox
-~~~~~~~~~~~~~~~~~~
-
-Download and install VirtualBox from https://www.virtualbox.org
-
-Note that we saw an error saying "Oracle VM VirtualBox 5.2.8 Setup Wizard ended prematurely" but then we re-ran the installer and it seemed to work.
-
-Reboot
-~~~~~~
-
-Again, Vagrant asks you to reboot, so go ahead.
-
-Install Git
-~~~~~~~~~~~
-
-Download and install Git from https://git-scm.com
-
-Configure Git to use Unix Line Endings
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Launch Git Bash and run the following commands:
-
-``git config --global core.autocrlf input``
-
-Pro tip: Use Shift-Insert to paste into Git Bash.
-
-See also https://help.github.com/articles/dealing-with-line-endings/
-
-If you skip this step you are likely to see the following error when you run ``vagrant up``.
-
-``/tmp/vagrant-shell: ./install: /usr/bin/perl^M: bad interpreter: No such file or directory``
-
-Clone Git Repo
-~~~~~~~~~~~~~~
-
-From Git Bash, run the following command:
-
-``git clone https://github.com/IQSS/dataverse.git``
-
-vagrant up
-~~~~~~~~~~
-
-From Git Bash, run the following commands:
-
-``cd dataverse``
-
-The ``dataverse`` directory you changed is the one you just cloned. Vagrant will operate on a file called ``Vagrantfile``.
-
-``vagrant up``
-
-After a long while you hopefully will have a Dataverse installation available at http://localhost:8888
-
-Improving Windows Support
--------------------------
-
-Windows Subsystem for Linux
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Running Dataverse in Docker on Windows
+--------------------------------------
-We have been unable to get Windows Subsystem for Linux (WSL) to work. We tried following the steps at https://docs.microsoft.com/en-us/windows/wsl/install-win10 but the "Get" button was greyed out when we went to download Ubuntu.
+See the `post `_ by Akio Sone for additional details, but please observe the following:
-Discussion and Feedback
-~~~~~~~~~~~~~~~~~~~~~~~
+- In git, the line-ending setting should be set to always LF (line feed, ``core.autocrlf=input``)
+- You must have jq installed: https://jqlang.github.io/jq/download/
-For more discussion of Windows support for Dataverse Software development see our community list thread `"Do you want to develop on Windows?" `_ We would be happy to incorporate feedback from Windows developers into this page. The :doc:`documentation` section describes how.
+One the above is all set you can move on to :doc:`/container/dev-usage` in the Container Guide.
diff --git a/doc/sphinx-guides/source/index.rst b/doc/sphinx-guides/source/index.rst
index f6eda53d718..e4eeea9b6d0 100755
--- a/doc/sphinx-guides/source/index.rst
+++ b/doc/sphinx-guides/source/index.rst
@@ -45,7 +45,7 @@ Other Resources
Additional information about the Dataverse Project itself
including presentations, information about upcoming releases, data
management and citation, and announcements can be found at
-`http://dataverse.org/ `__
+`https://dataverse.org/ `__
**User Group**
@@ -68,7 +68,7 @@ The support email address is `support@dataverse.org `__
-or use `GitHub pull requests `__,
+or use `GitHub pull requests `__,
if you have some code, scripts or documentation that you'd like to share.
If you have a **security issue** to report, please email `security@dataverse.org `__. See also :ref:`reporting-security-issues`.
diff --git a/doc/sphinx-guides/source/installation/advanced.rst b/doc/sphinx-guides/source/installation/advanced.rst
index a842d566595..3de5d0ea07c 100644
--- a/doc/sphinx-guides/source/installation/advanced.rst
+++ b/doc/sphinx-guides/source/installation/advanced.rst
@@ -7,14 +7,16 @@ Advanced installations are not officially supported but here we are at least doc
.. contents:: |toctitle|
:local:
+.. _multiple-app-servers:
+
Multiple App Servers
--------------------
You should be conscious of the following when running multiple app servers.
- Only one app server can be the dedicated timer server, as explained in the :doc:`/admin/timers` section of the Admin Guide.
-- When users upload a logo or footer for their Dataverse collection using the "theme" feature described in the :doc:`/user/dataverse-management` section of the User Guide, these logos are stored only on the app server the user happened to be on when uploading the logo. By default these logos and footers are written to the directory ``/usr/local/payara5/glassfish/domains/domain1/docroot/logos``.
-- When a sitemap is created by an app server it is written to the filesystem of just that app server. By default the sitemap is written to the directory ``/usr/local/payara5/glassfish/domains/domain1/docroot/sitemap``.
+- When users upload a logo or footer for their Dataverse collection using the "theme" feature described in the :doc:`/user/dataverse-management` section of the User Guide, these logos are stored only on the app server the user happened to be on when uploading the logo. By default these logos and footers are written to the directory ``/usr/local/payara6/glassfish/domains/domain1/docroot/logos``.
+- When a sitemap is created by an app server it is written to the filesystem of just that app server. By default the sitemap is written to the directory ``/usr/local/payara6/glassfish/domains/domain1/docroot/sitemap``.
- If Make Data Count is used, its raw logs must be copied from each app server to single instance of Counter Processor. See also :ref:`:MDCLogPath` section in the Configuration section of this guide and the :doc:`/admin/make-data-count` section of the Admin Guide.
- Dataset draft version logging occurs separately on each app server. See :ref:`edit-draft-versions-logging` section in Monitoring of the Admin Guide for details.
- Password aliases (``dataverse.db.password``, etc.) are stored per app server.
diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst
index c0eb576d7f5..a7d7905ca4a 100644
--- a/doc/sphinx-guides/source/installation/config.rst
+++ b/doc/sphinx-guides/source/installation/config.rst
@@ -1,4 +1,3 @@
-=============
Configuration
=============
@@ -34,6 +33,12 @@ It is very important to keep the block in place for the "admin" endpoint, and to
It's also possible to prevent file uploads via API by adjusting the :ref:`:UploadMethods` database setting.
+If you are using a load balancer or a reverse proxy, there are some additional considerations. If no additional configurations are made and the upstream is configured to redirect to localhost, the API will be accessible from the outside, as your installation will register as origin the localhost for any requests to the endpoints "admin" and "builtin-users". To prevent this, you have two options:
+
+- If your upstream is configured to redirect to localhost, you will need to set the :ref:`JVM option ` to one of the following values ``%client.name% %datetime% %request% %status% %response.length% %header.referer% %header.x-forwarded-for%`` and configure from the load balancer side the chosen header to populate with the client IP address.
+
+- Another solution is to set the upstream to the client IP address. In this case no further configuration is needed.
+
Forcing HTTPS
+++++++++++++
@@ -137,7 +142,7 @@ The need to redirect port HTTP (port 80) to HTTPS (port 443) for security has al
Your decision to proxy or not should primarily be driven by which features of the Dataverse Software you'd like to use. If you'd like to use Shibboleth, the decision is easy because proxying or "fronting" Payara with Apache is required. The details are covered in the :doc:`shibboleth` section.
-Even if you have no interest in Shibboleth, you may want to front your Dataverse installation with Apache or nginx to simply the process of installing SSL certificates. There are many tutorials on the Internet for adding certs to Apache, including a some `notes used by the Dataverse Project team `_, but the process of adding a certificate to Payara is arduous and not for the faint of heart. The Dataverse Project team cannot provide much help with adding certificates to Payara beyond linking to `tips `_ on the web.
+Even if you have no interest in Shibboleth, you may want to front your Dataverse installation with Apache or nginx to simply the process of installing SSL certificates. There are many tutorials on the Internet for adding certs to Apache, including a some `notes used by the Dataverse Project team `_, but the process of adding a certificate to Payara is arduous and not for the faint of heart. The Dataverse Project team cannot provide much help with adding certificates to Payara beyond linking to `tips `_ on the web.
Still not convinced you should put Payara behind another web server? Even if you manage to get your SSL certificate into Payara, how are you going to run Payara on low ports such as 80 and 443? Are you going to run Payara as root? Bad idea. This is a security risk. Under "Additional Recommendations" under "Securing Your Installation" above you are advised to configure Payara to run as a user other than root.
@@ -149,7 +154,7 @@ If you really don't want to front Payara with any proxy (not recommended), you c
``./asadmin set server-config.network-config.network-listeners.network-listener.http-listener-2.port=443``
-What about port 80? Even if you don't front your Dataverse installation with Apache, you may want to let Apache run on port 80 just to rewrite HTTP to HTTPS as described above. You can use a similar command as above to change the HTTP port that Payara uses from 8080 to 80 (substitute ``http-listener-1.port=80``). Payara can be used to enforce HTTPS on its own without Apache, but configuring this is an exercise for the reader. Answers here may be helpful: http://stackoverflow.com/questions/25122025/glassfish-v4-java-7-port-unification-error-not-able-to-redirect-http-to
+What about port 80? Even if you don't front your Dataverse installation with Apache, you may want to let Apache run on port 80 just to rewrite HTTP to HTTPS as described above. You can use a similar command as above to change the HTTP port that Payara uses from 8080 to 80 (substitute ``http-listener-1.port=80``). Payara can be used to enforce HTTPS on its own without Apache, but configuring this is an exercise for the reader. Answers here may be helpful: https://stackoverflow.com/questions/25122025/glassfish-v4-java-7-port-unification-error-not-able-to-redirect-http-to
If you are running an installation with Apache and Payara on the same server, and would like to restrict Payara from responding to any requests to port 8080 from external hosts (in other words, not through Apache), you can restrict the AJP listener to localhost only with:
@@ -248,7 +253,7 @@ this provider.
- :ref:`:Shoulder <:Shoulder>`
- :ref:`:IdentifierGenerationStyle <:IdentifierGenerationStyle>` (optional)
- :ref:`:DataFilePIDFormat <:DataFilePIDFormat>` (optional)
-- :ref:`:FilePIDsEnabled <:FilePIDsEnabled>` (optional, defaults to true)
+- :ref:`:FilePIDsEnabled <:FilePIDsEnabled>` (optional, defaults to false)
.. _pids-handle-configuration:
@@ -273,7 +278,7 @@ change the ``:Protocol`` setting, as it defaults to DOI usage.
- :ref:`:IndependentHandleService <:IndependentHandleService>` (optional)
- :ref:`:HandleAuthHandle <:HandleAuthHandle>` (optional)
-Note: If you are **minting your own handles** and plan to set up your own handle service, please refer to `Handle.Net documentation `_.
+Note: If you are **minting your own handles** and plan to set up your own handle service, please refer to `Handle.Net documentation `_.
.. _permalinks:
@@ -297,7 +302,9 @@ Here are the configuration options for PermaLinks:
- :ref:`:Shoulder <:Shoulder>`
- :ref:`:IdentifierGenerationStyle <:IdentifierGenerationStyle>` (optional)
- :ref:`:DataFilePIDFormat <:DataFilePIDFormat>` (optional)
-- :ref:`:FilePIDsEnabled <:FilePIDsEnabled>` (optional, defaults to true)
+- :ref:`:FilePIDsEnabled <:FilePIDsEnabled>` (optional, defaults to false)
+
+You must restart Payara after making changes to these settings.
.. _auth-modes:
@@ -492,14 +499,18 @@ Logging & Slow Performance
.. _file-storage:
-File Storage: Using a Local Filesystem and/or Swift and/or Object Stores and/or Trusted Remote Stores
------------------------------------------------------------------------------------------------------
+File Storage
+------------
-By default, a Dataverse installation stores all data files (files uploaded by end users) on the filesystem at ``/usr/local/payara5/glassfish/domains/domain1/files``. This path can vary based on answers you gave to the installer (see the :ref:`dataverse-installer` section of the Installation Guide) or afterward by reconfiguring the ``dataverse.files.\.directory`` JVM option described below.
+By default, a Dataverse installation stores all data files (files uploaded by end users) on the filesystem at ``/usr/local/payara6/glassfish/domains/domain1/files``. This path can vary based on answers you gave to the installer (see the :ref:`dataverse-installer` section of the Installation Guide) or afterward by reconfiguring the ``dataverse.files.\.directory`` JVM option described below.
-A Dataverse installation can alternately store files in a Swift or S3-compatible object store, and can now be configured to support multiple stores at once. With a multi-store configuration, the location for new files can be controlled on a per-Dataverse collection basis.
+A Dataverse installation can alternately store files in a Swift or S3-compatible object store, or on a Globus endpoint, and can now be configured to support multiple stores at once. With a multi-store configuration, the location for new files can be controlled on a per-Dataverse collection basis.
-A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web-accessible trusted remote store.
+A Dataverse installation may also be configured to reference some files (e.g. large and/or sensitive data) stored in a web or Globus accessible trusted remote store.
+
+A Dataverse installation can be configured to allow out of band upload by setting the ``dataverse.files.\.upload-out-of-band`` JVM option to ``true``.
+By default, Dataverse supports uploading files via the :ref:`add-file-api`. With S3 stores, a direct upload process can be enabled to allow sending the file directly to the S3 store (without any intermediate copies on the Dataverse server).
+With the upload-out-of-band option enabled, it is also possible for file upload to be managed manually or via third-party tools, with the :ref:`Adding the Uploaded file to the Dataset ` API call (described in the :doc:`/developers/s3-direct-upload-api` page) used to add metadata and inform Dataverse that a new file has been added to the relevant store.
The following sections describe how to set up various types of stores and how to configure for multiple stores.
@@ -528,6 +539,27 @@ If you wish to change which store is used by default, you'll need to delete the
It is also possible to set maximum file upload size limits per store. See the :ref:`:MaxFileUploadSizeInBytes` setting below.
+.. _labels-file-stores:
+
+Labels for File Stores
+++++++++++++++++++++++
+
+If you find yourself adding many file stores with various configurations such as per-file limits and direct upload, you might find it helpful to make the label descriptive.
+
+For example, instead of simply labeling an S3 store as "S3"...
+
+.. code-block:: none
+
+ ./asadmin create-jvm-options "\-Ddataverse.files.s3xl.label=S3"
+
+... you might want to include some extra information such as the example below.
+
+.. code-block:: none
+
+ ./asadmin create-jvm-options "\-Ddataverse.files.s3xl.label=S3XL, Filesize limit: 100GB, direct-upload"
+
+Please keep in mind that the UI will only show so many characters, so labels are best kept short.
+
.. _storage-files-dir:
File Storage
@@ -544,7 +576,7 @@ Multiple file stores should specify different directories (which would nominally
Swift Storage
+++++++++++++
-Rather than storing data files on the filesystem, you can opt for an experimental setup with a `Swift Object Storage `_ backend. Each dataset that users create gets a corresponding "container" on the Swift side, and each data file is saved as a file within that container.
+Rather than storing data files on the filesystem, you can opt for an experimental setup with a `Swift Object Storage `_ backend. Each dataset that users create gets a corresponding "container" on the Swift side, and each data file is saved as a file within that container.
**In order to configure a Swift installation,** you need to complete these steps to properly modify the JVM options:
@@ -560,7 +592,7 @@ First, run all the following create commands with your Swift endpoint informatio
./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files..username.endpoint1=your-username"
./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files..endpoint.endpoint1=your-swift-endpoint"
-``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that looks something like ``http://rdgw.swift.example.org/swift/v1``.
+``auth_type`` can either be ``keystone``, ``keystone_v3``, or it will assumed to be ``basic``. ``auth_url`` should be your keystone authentication URL which includes the tokens (e.g. for keystone, ``https://openstack.example.edu:35357/v2.0/tokens`` and for keystone_v3, ``https://openstack.example.edu:35357/v3/auth/tokens``). ``swift_endpoint`` is a URL that looks something like ``https://rdgw.swift.example.org/swift/v1``.
Then create a password alias by running (without changes):
@@ -656,7 +688,7 @@ You'll need an AWS account with an associated S3 bucket for your installation to
**Make note** of the **bucket's name** and the **region** its data is hosted in.
To **create a user** with full S3 access and nothing more for security reasons, we recommend using IAM
-(Identity and Access Management). See `IAM User Guide `_
+(Identity and Access Management). See `IAM User Guide `_
for more info on this process.
To use programmatic access, **Generate the user keys** needed for a Dataverse installation afterwards by clicking on the created user.
@@ -727,7 +759,7 @@ Additional profiles can be added to these files by appending the relevant inform
aws_access_key_id =
aws_secret_access_key =
-Place these two files in a folder named ``.aws`` under the home directory for the user running your Dataverse Installation on Payara. (From the `AWS Command Line Interface Documentation `_:
+Place these two files in a folder named ``.aws`` under the home directory for the user running your Dataverse Installation on Payara. (From the `AWS Command Line Interface Documentation `_:
"In order to separate credentials from less sensitive options, region and output format are stored in a separate file
named config in the same folder")
@@ -793,27 +825,28 @@ List of S3 Storage Options
.. table::
:align: left
- =========================================== ================== ========================================================================== =============
- JVM Option Value Description Default value
- =========================================== ================== ========================================================================== =============
- dataverse.files.storage-driver-id Enable as the default storage driver. ``file``
- dataverse.files..type ``s3`` **Required** to mark this storage as S3 based. (none)
- dataverse.files..label **Required** label to be shown in the UI for this storage (none)
- dataverse.files..bucket-name The bucket name. See above. (none)
- dataverse.files..download-redirect ``true``/``false`` Enable direct download or proxy through Dataverse. ``false``
- dataverse.files..upload-redirect ``true``/``false`` Enable direct upload of files added to a dataset to the S3 store. ``false``
- dataverse.files..ingestsizelimit Maximum size of directupload files that should be ingested (none)
- dataverse.files..url-expiration-minutes If direct uploads/downloads: time until links expire. Optional. 60
- dataverse.files..min-part-size Multipart direct uploads will occur for files larger than this. Optional. ``1024**3``
- dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none)
- dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse``
- dataverse.files..profile Allows the use of AWS profiles for storage spanning multiple AWS accounts. (none)
- dataverse.files..proxy-url URL of a proxy protecting the S3 store. Optional. (none)
- dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false``
- dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false``
- dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true``
- dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256``
- =========================================== ================== ========================================================================== =============
+ =========================================== ================== =================================================================================== =============
+ JVM Option Value Description Default value
+ =========================================== ================== =================================================================================== =============
+ dataverse.files.storage-driver-id Enable as the default storage driver. ``file``
+ dataverse.files..type ``s3`` **Required** to mark this storage as S3 based. (none)
+ dataverse.files..label **Required** label to be shown in the UI for this storage (none)
+ dataverse.files.