From ec68220fd8a4b653fdc041157fbbf08aa280d305 Mon Sep 17 00:00:00 2001 From: "c.dumitru@orcid.org" Date: Fri, 27 Oct 2023 10:52:11 +0300 Subject: [PATCH 1/3] Added the @JsonIgnoreProperties(ignoreUnknown = true) for all the zenodo json and updated the ROR url as the original one has a 301 --- .../org/orcid/scheduler/loader/io/OrgDataClient.java | 9 ++++++++- .../scheduler/loader/source/ror/RorOrgLoadSource.java | 2 +- .../loader/source/zenodo/api/ZenodoRecords.java | 2 +- .../loader/source/zenodo/api/ZenodoRecordsFile.java | 2 ++ .../loader/source/zenodo/api/ZenodoRecordsFileLinks.java | 2 ++ .../loader/source/zenodo/api/ZenodoRecordsHit.java | 2 +- .../loader/source/zenodo/api/ZenodoRecordsHitLinks.java | 2 ++ properties/development.properties | 8 ++++---- 8 files changed, 21 insertions(+), 8 deletions(-) diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/io/OrgDataClient.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/io/OrgDataClient.java index 0355cb11b20..85e440b0b4f 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/io/OrgDataClient.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/io/OrgDataClient.java @@ -33,8 +33,10 @@ public class OrgDataClient { public T get(String url, String userAgent, Class type) { JerseyClientResponse response = jerseyClientHelperForOrgLoaders.executeGetRequest(url, null, null, false, Map.of(), Map.of("User-Agent", userAgent), type, String.class); int status = response.getStatus(); + LOGGER.error("!!!!!!!!!!!!!!!!!!!! download file URL: " + url ); + if (status != 200) { - LOGGER.warn("Unable to fetch file {}: {}", new Object[] { url, status }); + LOGGER.error("Unable to fetch file {}: {}", new Object[] { url, status }); return null; } return response.getEntity(); @@ -47,8 +49,13 @@ public T get(String url, String userAgent, Class type) { * @return boolean indicator of success */ public boolean downloadFile(String url, String userAgent, String localFilePath) { + LOGGER.error("!!!!!!!!!!!!!!!!!!!! download file URL: " + url + " localFile path " + localFilePath); JerseyClientResponse response = jerseyClientHelperForOrgLoaders.executeGetRequest(url, null, null, false, Map.of(), Map.of("User-Agent", userAgent), InputStream.class, String.class); + System.out.println("!!!!! Inside download file"); int status = response.getStatus(); + String statusStrs = response.toString(); + System.out.println("!!!!! download file: " + statusStrs); + LOGGER.error("!!!!!!!!!!!!!!!!!!!! download file: " + statusStrs); if (status != 200) { LOGGER.warn("Unable to fetch file {}: {}", new Object[] { url, status }); return false; diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java index 37a9840c5f1..a42ddc906c5 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java @@ -79,7 +79,7 @@ public class RorOrgLoadSource implements OrgLoadSource { @Resource private OrgDisambiguatedExternalIdentifierDao orgDisambiguatedExternalIdentifierDao; - @Value("${org.orcid.core.orgs.ror.zenodoRecordsUrl:https://zenodo.org/api/records/?communities=ror-data}") + @Value("${org.orcid.core.orgs.ror.zenodoRecordsUrl:https://zenodo.org/api/records?communities=ror-data}") private String rorZenodoRecordsUrl; @Resource diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecords.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecords.java index f3227f51c6c..9056f740542 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecords.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecords.java @@ -6,7 +6,7 @@ @JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(value = { "aggregations", "links" }) +@JsonIgnoreProperties(ignoreUnknown = true) public class ZenodoRecords { @JsonProperty("hits") diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsFile.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsFile.java index 130fedfb488..3b38579f44e 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsFile.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsFile.java @@ -1,7 +1,9 @@ package org.orcid.scheduler.loader.source.zenodo.api; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +@JsonIgnoreProperties(ignoreUnknown = true) public class ZenodoRecordsFile { @JsonProperty("bucket") diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsFileLinks.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsFileLinks.java index 50e505ed29a..c101e896b74 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsFileLinks.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsFileLinks.java @@ -1,7 +1,9 @@ package org.orcid.scheduler.loader.source.zenodo.api; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +@JsonIgnoreProperties(ignoreUnknown = true) public class ZenodoRecordsFileLinks { @JsonProperty("self") diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsHit.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsHit.java index 31a0b352a7e..6c8caaafb7d 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsHit.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsHit.java @@ -7,7 +7,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; @JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(value = { "metadata", "owners", "stats", "revision" }) +@JsonIgnoreProperties(ignoreUnknown = true) public class ZenodoRecordsHit { @JsonProperty("conceptdoi") diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsHitLinks.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsHitLinks.java index 57ba1b1e46f..f33ec19d1c9 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsHitLinks.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/zenodo/api/ZenodoRecordsHitLinks.java @@ -1,7 +1,9 @@ package org.orcid.scheduler.loader.source.zenodo.api; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +@JsonIgnoreProperties(ignoreUnknown = true) public class ZenodoRecordsHitLinks { @JsonProperty("badge") diff --git a/properties/development.properties b/properties/development.properties index bfcef35f40a..df81f58df1e 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -195,10 +195,10 @@ org.orcid.core.orgs.fundref.latestReleaseUrl=https://some/path org.orcid.core.orgs.fundref.localFilePath=/tmp/some/path # ROR org imports -org.orcid.core.orgs.ror.enabled=false -org.orcid.core.orgs.ror.zenodoRecordsUrl=https://zenodo.org/api/records/?communities=ror-data -org.orcid.core.orgs.ror.localDataPath=/tmp/ror/ror.json -org.orcid.core.orgs.ror.localZipPath=/tmp/ror/ror.zip +org.orcid.core.orgs.ror.enabled=true +org.orcid.core.orgs.ror.zenodoRecordsUrl=https://zenodo.org/api/records?communities=ror-data +org.orcid.core.orgs.ror.localDataPath=/Users/camelia/Desktop/tmp/ror.json +org.orcid.core.orgs.ror.localZipPath=/Users/camelia/Desktop/tmp/ror.zip ################ # UI # From cb45d87103d5b554c91ff60c641eac1570216dba Mon Sep 17 00:00:00 2001 From: "c.dumitru@orcid.org" Date: Fri, 27 Oct 2023 10:54:01 +0300 Subject: [PATCH 2/3] removed debugging code --- .../org/orcid/scheduler/loader/io/OrgDataClient.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/io/OrgDataClient.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/io/OrgDataClient.java index 85e440b0b4f..8b1a791a14b 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/io/OrgDataClient.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/io/OrgDataClient.java @@ -32,9 +32,7 @@ public class OrgDataClient { */ public T get(String url, String userAgent, Class type) { JerseyClientResponse response = jerseyClientHelperForOrgLoaders.executeGetRequest(url, null, null, false, Map.of(), Map.of("User-Agent", userAgent), type, String.class); - int status = response.getStatus(); - LOGGER.error("!!!!!!!!!!!!!!!!!!!! download file URL: " + url ); - + int status = response.getStatus(); if (status != 200) { LOGGER.error("Unable to fetch file {}: {}", new Object[] { url, status }); return null; @@ -49,13 +47,8 @@ public T get(String url, String userAgent, Class type) { * @return boolean indicator of success */ public boolean downloadFile(String url, String userAgent, String localFilePath) { - LOGGER.error("!!!!!!!!!!!!!!!!!!!! download file URL: " + url + " localFile path " + localFilePath); JerseyClientResponse response = jerseyClientHelperForOrgLoaders.executeGetRequest(url, null, null, false, Map.of(), Map.of("User-Agent", userAgent), InputStream.class, String.class); - System.out.println("!!!!! Inside download file"); int status = response.getStatus(); - String statusStrs = response.toString(); - System.out.println("!!!!! download file: " + statusStrs); - LOGGER.error("!!!!!!!!!!!!!!!!!!!! download file: " + statusStrs); if (status != 200) { LOGGER.warn("Unable to fetch file {}: {}", new Object[] { url, status }); return false; From ccc9e8270af26c1046f6a618fa3ec39f9653684a Mon Sep 17 00:00:00 2001 From: Angel Montenegro Date: Mon, 30 Oct 2023 09:01:43 -0600 Subject: [PATCH 3/3] Update development.properties --- properties/development.properties | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/properties/development.properties b/properties/development.properties index df81f58df1e..de9a4064834 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -195,10 +195,10 @@ org.orcid.core.orgs.fundref.latestReleaseUrl=https://some/path org.orcid.core.orgs.fundref.localFilePath=/tmp/some/path # ROR org imports -org.orcid.core.orgs.ror.enabled=true +org.orcid.core.orgs.ror.enabled=false org.orcid.core.orgs.ror.zenodoRecordsUrl=https://zenodo.org/api/records?communities=ror-data -org.orcid.core.orgs.ror.localDataPath=/Users/camelia/Desktop/tmp/ror.json -org.orcid.core.orgs.ror.localZipPath=/Users/camelia/Desktop/tmp/ror.zip +org.orcid.core.orgs.ror.localDataPath=/tmp/ror/ror.json +org.orcid.core.orgs.ror.localZipPath=/tmp/ror/ror.zip ################ # UI #