diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/OrgDisambiguatedEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/OrgDisambiguatedEntity.java index 431fd0263a7..9379cd2db44 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/OrgDisambiguatedEntity.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/OrgDisambiguatedEntity.java @@ -211,4 +211,28 @@ public void setMemberChosenOrgDisambiguatedEntity(MemberChosenOrgDisambiguatedEn this.memberChosenOrgDisambiguatedEntity = memberChosenOrgDisambiguatedEntity; } -} + @Column(name = "locations_json") + private String locationsJson; + + @Column(name = "locations_json") + public String getLocationsJson() { + return locationsJson; + } + + public void setLocationsJson(String locationsJson) { + this.locationsJson = locationsJson; + } + + @Column(name = "names_json") + private String namesJson; + + @Column(name = "names_json") + public String getNamesJson() { + return namesJson; + } + + public void setNamesJson(String namesJson) { + this.namesJson = namesJson; + } + +} \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/db-master.xml b/orcid-persistence/src/main/resources/db-master.xml index 034c50c878d..50751220853 100644 --- a/orcid-persistence/src/main/resources/db-master.xml +++ b/orcid-persistence/src/main/resources/db-master.xml @@ -387,5 +387,6 @@ - + + diff --git a/orcid-persistence/src/main/resources/db/updates/org_disambiguated_extra_columns.xml b/orcid-persistence/src/main/resources/db/updates/org_disambiguated_extra_columns.xml new file mode 100644 index 00000000000..e0b2613f4c1 --- /dev/null +++ b/orcid-persistence/src/main/resources/db/updates/org_disambiguated_extra_columns.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java index 8e9b4f2bd46..28da2f9a727 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java @@ -46,473 +46,456 @@ @Component public class RorOrgLoadSource implements OrgLoadSource { - private static final Logger LOGGER = LoggerFactory.getLogger(RorOrgLoadSource.class); - - private static final String WIKIPEDIA_URL = "wikipedia_url"; - - @Value("${org.orcid.core.orgs.ror.enabled:true}") - private boolean enabled; - - @Value("${org.orcid.core.orgs.clients.userAgent}") - private String userAgent; - - @Resource(name = "rorOrgDataClient") - private OrgDataClient orgDataClient; - - @Value("${org.orcid.core.orgs.ror.localZipPath:/tmp/grid/ror.zip}") - private String zipFilePath; - - @Value("${org.orcid.core.orgs.ror.localDataPath:/tmp/grid/ror.json}") - private String localDataPath; - - @Value("${org.orcid.core.orgs.ror.indexAllEnabled:false}") - private boolean indexAllEnabled; - - @Resource - private OrgDisambiguatedDao orgDisambiguatedDao; - - @Resource - private OrgDisambiguatedManager orgDisambiguatedManager; - - @Resource - private OrgDisambiguatedExternalIdentifierDao orgDisambiguatedExternalIdentifierDao; - - @Value("${org.orcid.core.orgs.ror.zenodoRecordsUrl:https://zenodo.org/api/records?communities=ror-data}") - private String rorZenodoRecordsUrl; - - @Resource - private FileRotator fileRotator; - - private Set UPDATED_RORS; - - @Override - public String getSourceName() { - return "ROR"; - } - - @Override - public boolean loadOrgData() { - if (!enabled) { - throw new LoadSourceDisabledException(getSourceName()); - } - - return loadData(); - } - - @Override - public boolean downloadOrgData() { - try { - fileRotator.removeFileIfExists(zipFilePath); - fileRotator.removeFileIfExists(localDataPath); - - ZenodoRecords zenodoRecords = orgDataClient.get(rorZenodoRecordsUrl + "&sort=mostrecent&size=1", userAgent, - ZenodoRecords.class); - ZenodoRecordsHit zenodoHit = zenodoRecords.getHits().getHits().get(0); - - boolean success = false; - - // we are returning the collection ordered by mostrecent and size 1, we need to - // get the last element in the list that has the last version - String zenodoUrl = zenodoHit.getFiles() - .get(zenodoHit.getFiles().size() > 0 ? zenodoHit.getFiles().size() - 1 : 0).getLinks().getSelf(); - LOGGER.info("Retrieving ROR data from: " + zenodoUrl); - success = orgDataClient.downloadFile(zenodoUrl, userAgent, zipFilePath); - - try { - LOGGER.info("Unzipping ROR ...."); - unzipData(); - } catch (IOException e) { - LOGGER.error("Error unzipping Zenodo ROR data", e); - throw new RuntimeException(e); - } - return success; - } catch (Exception e) { - LOGGER.error("Error downloading Zenodo ROR data", e); - return false; - } - } - - private void unzipData() throws IOException { - byte[] buffer = new byte[1024]; - ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFilePath)); - ZipEntry zipEntry = zis.getNextEntry(); - while (zipEntry != null) { - String zipEntryName = zipEntry.getName(); - if (zipEntryName.endsWith("v2.json")) { - File jsonData = new File(localDataPath); - FileOutputStream fos = new FileOutputStream(jsonData); - int len; - while ((len = zis.read(buffer)) > 0) { - fos.write(buffer, 0, len); - } - fos.close(); - break; - } - zipEntry = zis.getNextEntry(); - } - zis.closeEntry(); - zis.close(); - } - - private boolean loadData() { - try { - LOGGER.info("Loading ROR data..."); - Instant start = Instant.now(); - File fileToLoad = new File(localDataPath); - if (!fileToLoad.exists()) { - LOGGER.error("File {} doesn't exist", localDataPath); - return false; - } - - // ror returns the JSON as Array of institutes - JsonNode rootNode = JsonUtils.read(fileToLoad); - UPDATED_RORS = new HashSet(); - - rootNode.forEach(institute -> { - String sourceId = institute.get("id").isNull() ? null : institute.get("id").asText(); - String status = institute.get("status").isNull() ? null : institute.get("status").asText(); - if ("active".equalsIgnoreCase(status) || "inactive".equalsIgnoreCase(status)) { - ArrayNode namesNode = institute.get("names").isNull() ? null : (ArrayNode) institute.get("names"); - String name = null; - if (namesNode != null) { - for (JsonNode nameJson : namesNode) { - ArrayNode nameTypes = nameJson.get("types").isNull() ? null - : (ArrayNode) nameJson.get("types"); - for (JsonNode nameType : nameTypes) { - if (StringUtils.equalsIgnoreCase(nameType.asText(), "ror_display")) { - name = nameJson.get("value").asText(); - break; - } - } - } - } - - StringJoiner sj = new StringJoiner(","); - String orgType = null; - if (!institute.get("types").isNull()) { - ((ArrayNode) institute.get("types")).forEach(x -> sj.add(x.textValue())); - orgType = sj.toString(); - } - - - //location node - - ArrayNode locationsNode = institute.get("locations").isNull() ? null : (ArrayNode) institute.get("locations"); - Iso3166Country country = null; - String region = null; - String city = null; - if (locationsNode != null) { - for (JsonNode locationJson : locationsNode) { - JsonNode geoDetailsNode = locationJson.get("geonames_details").isNull() ? null - : (JsonNode) locationJson.get("geonames_details"); - - if(geoDetailsNode !=null) { - String countryCode = geoDetailsNode.get("country_code").isNull() ? null - : geoDetailsNode.get("country_code").asText(); - country = StringUtils.isBlank(countryCode) ? null : Iso3166Country.fromValue(countryCode); - //for now storing just the first location - city = geoDetailsNode.get("name").isNull() ? null - : geoDetailsNode.get("name").asText(); - if(country != null) { - break; - } - } - - } - } - - - ArrayNode urls = institute.get("links").isNull() ? null : (ArrayNode) institute.get("links"); - // Use the first URL - String url = (urls != null && urls.size() > 0) ? urls.get(0).asText() : null; - - // Creates or updates an institute - OrgDisambiguatedEntity entity = processInstitute(sourceId, name, country, city, region, url, - orgType); - - // Creates external identifiers - processExternalIdentifiers(entity, institute); - } else if ("redirected".equals(status)) { - String primaryId = institute.get("redirect").isNull() ? null : institute.get("redirect").asText(); - deprecateOrg(sourceId, primaryId); - } else if ("withdrawn".equals(status) || "obsolete".equals(status)) { - obsoleteOrg(sourceId); - } else { - LOGGER.error("Illegal status '" + status + "' for institute " + sourceId); - } - }); - - // Check if any RORs with external identifiers updated and group them - groupRORsWithUpdatedExternalModifiers(); - - LOGGER.info("Time taken to process the data: {}", Duration.between(start, Instant.now()).toString()); - return true; - } catch (Exception e) { - LOGGER.error("Error loading ROR data", e); - return false; - } - } - - private OrgDisambiguatedEntity processInstitute(String sourceId, String name, Iso3166Country country, String city, - String region, String url, String orgType) { - OrgDisambiguatedEntity existingBySourceId = orgDisambiguatedDao.findBySourceIdAndSourceType(sourceId, - OrgDisambiguatedSourceType.ROR.name()); - if (existingBySourceId != null) { - if (entityChanged(existingBySourceId, name, country.value(), city, region, url, orgType) - || indexAllEnabled) { - existingBySourceId.setCity(city); - existingBySourceId.setCountry(country.name()); - existingBySourceId.setName(name); - existingBySourceId.setOrgType(orgType); - existingBySourceId.setRegion(region); - existingBySourceId.setUrl(url); - existingBySourceId.setIndexingStatus(IndexingStatus.PENDING); - try { - // mark group for indexing - new OrgGrouping(existingBySourceId, orgDisambiguatedManager) - .markGroupForIndexing(orgDisambiguatedDao); - - } catch (Exception ex) { - LOGGER.error( - "Error when grouping by ROR and marking group orgs for reindexing, eating the exception", - ex); - } - orgDisambiguatedManager.updateOrgDisambiguated(existingBySourceId); - } - return existingBySourceId; - } - - // Create a new disambiguated org - OrgDisambiguatedEntity newOrg = createDisambiguatedOrg(sourceId, name, orgType, country, city, region, url); - try { - // mark group for indexing - new OrgGrouping(newOrg, orgDisambiguatedManager).markGroupForIndexing(orgDisambiguatedDao); - } catch (Exception ex) { - LOGGER.error("Error when grouping by ROR and removing related orgs solr index, eating the exception", ex); - } - return newOrg; - } - - private void processExternalIdentifiers(OrgDisambiguatedEntity org, JsonNode institute) { - ArrayNode nodes = institute.get("external_ids") == null ? null : (ArrayNode) institute.get("external_ids"); - if (nodes!= null) { - for(JsonNode entry:nodes){ - String identifierTypeName = entry.get("type").asText().toUpperCase(); - String preferredId = entry.get("preferred").isNull() ? null - : entry.get("preferred").asText(); - if (StringUtils.equalsIgnoreCase(OrgDisambiguatedSourceType.GRID.name(), identifierTypeName)) { - JsonNode extId = (JsonNode) entry.get("all"); - setExternalId(org, identifierTypeName, preferredId, extId); - UPDATED_RORS.add(org.getId()); - } else { - ArrayNode elements = (ArrayNode) entry.get("all"); - for (JsonNode extId : elements) { - setExternalId(org, identifierTypeName, preferredId, extId); - UPDATED_RORS.add(org.getId()); - } - } - } - } - } - - private void setExternalId(OrgDisambiguatedEntity org, String identifierTypeName, String preferredId, - JsonNode extId) { - // If the external identifier doesn't exists yet - OrgDisambiguatedExternalIdentifierEntity existingExternalId = orgDisambiguatedExternalIdentifierDao - .findByDetails(org.getId(), extId.asText(), identifierTypeName); - Boolean preferred = extId.asText().equals(preferredId); - if (existingExternalId == null) { - if (preferred) { - createExternalIdentifier(org, extId.asText(), identifierTypeName, true); - } else { - createExternalIdentifier(org, extId.asText(), identifierTypeName, false); - } - } else { - if (existingExternalId.getPreferred() != preferred) { - existingExternalId.setPreferred(preferred); - orgDisambiguatedManager.updateOrgDisambiguatedExternalIdentifier(existingExternalId); - LOGGER.info("External identifier for {} with ext id {} and type {} was updated", - new Object[] { org.getId(), extId.asText(), identifierTypeName }); - } else { - LOGGER.info("External identifier for {} with ext id {} and type {} already exists", - new Object[] { org.getId(), extId.asText(), identifierTypeName }); - } - } - } - - /** - * Indicates if an entity changed his address, url or org type - * - * @return true if the entity has changed. - */ - private boolean entityChanged(OrgDisambiguatedEntity entity, String name, String countryCode, String city, - String region, String url, String orgType) { - // Check name - if (StringUtils.isNotBlank(name)) { - if (!name.equalsIgnoreCase(entity.getName())) - return true; - } else if (StringUtils.isNotBlank(entity.getName())) { - return true; - } - // Check country - if (StringUtils.isNotBlank(countryCode)) { - if (entity.getCountry() == null || !countryCode.equals(entity.getCountry())) { - return true; - } - } else if (entity.getCountry() != null) { - return true; - } - // Check city - if (StringUtils.isNotBlank(city)) { - if (entity.getCity() == null || !city.equals(entity.getCity())) { - return true; - } - } else if (StringUtils.isNotBlank(entity.getCity())) { - return true; - } - // Check region - if (StringUtils.isNotBlank(region)) { - if (entity.getRegion() == null || !region.equals(entity.getRegion())) { - return true; - } - } else if (StringUtils.isNotBlank(entity.getRegion())) { - return true; - } - // Check url - if (StringUtils.isNotBlank(url)) { - if (entity.getUrl() == null || !url.equals(entity.getUrl())) { - return true; - } - } else if (StringUtils.isNotBlank(entity.getUrl())) { - return true; - } - // Check org_type - if (StringUtils.isNotBlank(orgType)) { - if (entity.getOrgType() == null || !orgType.equals(entity.getOrgType())) { - return true; - } - } else if (StringUtils.isNotBlank(entity.getOrgType())) { - return true; - } - - return false; - } - - /** - * Creates a disambiguated ORG in the org_disambiguated table - */ - private OrgDisambiguatedEntity createDisambiguatedOrg(String sourceId, String name, String orgType, - Iso3166Country country, String city, String region, String url) { - LOGGER.info("Creating disambiguated org {}", name); - OrgDisambiguatedEntity orgDisambiguatedEntity = new OrgDisambiguatedEntity(); - orgDisambiguatedEntity.setName(name); - orgDisambiguatedEntity.setCountry(country != null ? country.name() : null); - orgDisambiguatedEntity.setCity(city); - orgDisambiguatedEntity.setRegion(region); - orgDisambiguatedEntity.setUrl(url); - orgDisambiguatedEntity.setOrgType(orgType); - orgDisambiguatedEntity.setSourceId(sourceId); - orgDisambiguatedEntity.setSourceType(OrgDisambiguatedSourceType.ROR.name()); - orgDisambiguatedManager.createOrgDisambiguated(orgDisambiguatedEntity); - return orgDisambiguatedEntity; - } - - /** - * Creates an external identifier in the org_disambiguated_external_identifier - * table - */ - private boolean createExternalIdentifier(OrgDisambiguatedEntity disambiguatedOrg, String identifier, - String externalIdType, Boolean preferred) { - LOGGER.info("Creating external identifier for {}", disambiguatedOrg.getId()); - OrgDisambiguatedExternalIdentifierEntity externalIdentifier = new OrgDisambiguatedExternalIdentifierEntity(); - externalIdentifier.setIdentifier(identifier); - externalIdentifier.setIdentifierType(externalIdType); - externalIdentifier.setOrgDisambiguated(disambiguatedOrg); - externalIdentifier.setPreferred(preferred); - orgDisambiguatedManager.createOrgDisambiguatedExternalIdentifier(externalIdentifier); - return true; - } - - /** - * Mark an existing org as DEPRECATED - */ - private void deprecateOrg(String sourceId, String primarySourceId) { - LOGGER.info("Deprecating org {} for {}", sourceId, primarySourceId); - OrgDisambiguatedEntity existingEntity = orgDisambiguatedDao.findBySourceIdAndSourceType(sourceId, - OrgDisambiguatedSourceType.ROR.name()); - if (existingEntity != null) { - if (existingEntity.getStatus() == null - || !existingEntity.getStatus().equals(OrganizationStatus.DEPRECATED.name()) - || !existingEntity.getSourceParentId().equals(primarySourceId)) { - existingEntity.setStatus(OrganizationStatus.DEPRECATED.name()); - existingEntity.setSourceParentId(primarySourceId); - existingEntity.setIndexingStatus(IndexingStatus.PENDING); - orgDisambiguatedManager.updateOrgDisambiguated(existingEntity); - } - } else { - OrgDisambiguatedEntity deprecatedEntity = new OrgDisambiguatedEntity(); - deprecatedEntity.setSourceType(OrgDisambiguatedSourceType.ROR.name()); - deprecatedEntity.setStatus(OrganizationStatus.DEPRECATED.name()); - deprecatedEntity.setSourceId(sourceId); - deprecatedEntity.setSourceParentId(primarySourceId); - // We don't need to index it - deprecatedEntity.setIndexingStatus(IndexingStatus.DONE); - orgDisambiguatedManager.createOrgDisambiguated(deprecatedEntity); - } - } - - /** - * Mark an existing org as OBSOLETE - */ - private void obsoleteOrg(String sourceId) { - LOGGER.info("Marking or as obsolete {}", sourceId); - OrgDisambiguatedEntity existingEntity = orgDisambiguatedDao.findBySourceIdAndSourceType(sourceId, - OrgDisambiguatedSourceType.ROR.name()); - if (existingEntity != null) { - if (existingEntity.getStatus() == null - || !existingEntity.getStatus().equals(OrganizationStatus.OBSOLETE.name())) { - existingEntity.setStatus(OrganizationStatus.OBSOLETE.name()); - existingEntity.setIndexingStatus(IndexingStatus.PENDING); - orgDisambiguatedManager.updateOrgDisambiguated(existingEntity); - new OrgGrouping(existingEntity, orgDisambiguatedManager) - .ungroupObsoleteRorForIndexing(orgDisambiguatedDao); - } - } else { - OrgDisambiguatedEntity obsoletedEntity = new OrgDisambiguatedEntity(); - obsoletedEntity.setSourceType(OrgDisambiguatedSourceType.ROR.name()); - obsoletedEntity.setStatus(OrganizationStatus.OBSOLETE.name()); - obsoletedEntity.setSourceId(sourceId); - // We don't need to index it - obsoletedEntity.setIndexingStatus(IndexingStatus.DONE); - orgDisambiguatedManager.createOrgDisambiguated(obsoletedEntity); - new OrgGrouping(obsoletedEntity, orgDisambiguatedManager) - .ungroupObsoleteRorForIndexing(orgDisambiguatedDao); - } - } - - @Override - public boolean isEnabled() { - return enabled; - } - - private void groupRORsWithUpdatedExternalModifiers() { - for (Long id : UPDATED_RORS) { - OrgDisambiguatedEntity entity = orgDisambiguatedDao.find(id); - if (entity != null) { - entity.setIndexingStatus(IndexingStatus.PENDING); - try { - // mark group for indexing - new OrgGrouping(entity, orgDisambiguatedManager).markGroupForIndexing(orgDisambiguatedDao); - - } catch (Exception ex) { - LOGGER.error( - "Error when grouping by ROR and marking group orgs for reindexing, eating the exception", - ex); - } - entity = orgDisambiguatedManager.updateOrgDisambiguated(entity); - - } - } - } + private static final Logger LOGGER = LoggerFactory.getLogger(RorOrgLoadSource.class); + + private static final String WIKIPEDIA_URL = "wikipedia_url"; + + @Value("${org.orcid.core.orgs.ror.enabled:true}") + private boolean enabled; + + @Value("${org.orcid.core.orgs.clients.userAgent}") + private String userAgent; + + @Resource(name = "rorOrgDataClient") + private OrgDataClient orgDataClient; + + @Value("${org.orcid.core.orgs.ror.localZipPath:/tmp/grid/ror.zip}") + private String zipFilePath; + + @Value("${org.orcid.core.orgs.ror.localDataPath:/tmp/grid/ror.json}") + private String localDataPath; + + @Value("${org.orcid.core.orgs.ror.indexAllEnabled:false}") + private boolean indexAllEnabled; + + @Resource + private OrgDisambiguatedDao orgDisambiguatedDao; + + @Resource + private OrgDisambiguatedManager orgDisambiguatedManager; + + @Resource + private OrgDisambiguatedExternalIdentifierDao orgDisambiguatedExternalIdentifierDao; + + @Value("${org.orcid.core.orgs.ror.zenodoRecordsUrl:https://zenodo.org/api/records?communities=ror-data}") + private String rorZenodoRecordsUrl; + + @Resource + private FileRotator fileRotator; + + private Set UPDATED_RORS; + + @Override + public String getSourceName() { + return "ROR"; + } + + @Override + public boolean loadOrgData() { + if (!enabled) { + throw new LoadSourceDisabledException(getSourceName()); + } + + return loadData(); + } + + @Override + public boolean downloadOrgData() { + try { + fileRotator.removeFileIfExists(zipFilePath); + fileRotator.removeFileIfExists(localDataPath); + + ZenodoRecords zenodoRecords = orgDataClient.get(rorZenodoRecordsUrl + "&sort=mostrecent&size=1", userAgent, ZenodoRecords.class); + ZenodoRecordsHit zenodoHit = zenodoRecords.getHits().getHits().get(0); + + boolean success = false; + + // we are returning the collection ordered by mostrecent and size 1, + // we need to + // get the last element in the list that has the last version + String zenodoUrl = zenodoHit.getFiles().get(zenodoHit.getFiles().size() > 0 ? zenodoHit.getFiles().size() - 1 : 0).getLinks().getSelf(); + LOGGER.info("Retrieving ROR data from: " + zenodoUrl); + success = orgDataClient.downloadFile(zenodoUrl, userAgent, zipFilePath); + + try { + LOGGER.info("Unzipping ROR ...."); + unzipData(); + } catch (IOException e) { + LOGGER.error("Error unzipping Zenodo ROR data", e); + throw new RuntimeException(e); + } + return success; + } catch (Exception e) { + LOGGER.error("Error downloading Zenodo ROR data", e); + return false; + } + } + + private void unzipData() throws IOException { + byte[] buffer = new byte[1024]; + ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFilePath)); + ZipEntry zipEntry = zis.getNextEntry(); + while (zipEntry != null) { + String zipEntryName = zipEntry.getName(); + if (zipEntryName.endsWith("v2.json")) { + File jsonData = new File(localDataPath); + FileOutputStream fos = new FileOutputStream(jsonData); + int len; + while ((len = zis.read(buffer)) > 0) { + fos.write(buffer, 0, len); + } + fos.close(); + break; + } + zipEntry = zis.getNextEntry(); + } + zis.closeEntry(); + zis.close(); + } + + private boolean loadData() { + try { + LOGGER.info("Loading ROR data..."); + Instant start = Instant.now(); + File fileToLoad = new File(localDataPath); + if (!fileToLoad.exists()) { + LOGGER.error("File {} doesn't exist", localDataPath); + return false; + } + + // ror returns the JSON as Array of institutes + JsonNode rootNode = JsonUtils.read(fileToLoad); + UPDATED_RORS = new HashSet(); + + rootNode.forEach(institute -> { + String sourceId = institute.get("id").isNull() ? null : institute.get("id").asText(); + String status = institute.get("status").isNull() ? null : institute.get("status").asText(); + if ("active".equalsIgnoreCase(status) || "inactive".equalsIgnoreCase(status)) { + ArrayNode namesNode = institute.get("names").isNull() ? null : (ArrayNode) institute.get("names"); + String name = null; + String namesJson = null; + + if (namesNode != null) { + for (JsonNode nameJson : namesNode) { + ArrayNode nameTypes = nameJson.get("types").isNull() ? null : (ArrayNode) nameJson.get("types"); + for (JsonNode nameType : nameTypes) { + if (StringUtils.equalsIgnoreCase(nameType.asText(), "ror_display")) { + name = nameJson.get("value").asText(); + break; + } + } + } + namesJson = namesNode.toString(); + } + + StringJoiner sj = new StringJoiner(","); + String orgType = null; + if (!institute.get("types").isNull()) { + ((ArrayNode) institute.get("types")).forEach(x -> sj.add(x.textValue())); + orgType = sj.toString(); + } + + // location node + + ArrayNode locationsNode = institute.get("locations").isNull() ? null : (ArrayNode) institute.get("locations"); + Iso3166Country country = null; + String region = null; + String city = null; + + String locationsJson = null; + if (locationsNode != null) { + for (JsonNode locationJson : locationsNode) { + JsonNode geoDetailsNode = locationJson.get("geonames_details").isNull() ? null : (JsonNode) locationJson.get("geonames_details"); + + if (geoDetailsNode != null) { + String countryCode = geoDetailsNode.get("country_code").isNull() ? null : geoDetailsNode.get("country_code").asText(); + country = StringUtils.isBlank(countryCode) ? null : Iso3166Country.fromValue(countryCode); + // for now storing just the first location + city = geoDetailsNode.get("name").isNull() ? null : geoDetailsNode.get("name").asText(); + if (country != null) { + break; + } + } + + } + locationsJson = locationsNode.toString(); + } + + ArrayNode urls = institute.get("links").isNull() ? null : (ArrayNode) institute.get("links"); + // Use the first URL + String url = (urls != null && urls.size() > 0) ? urls.get(0).asText() : null; + + // Creates or updates an institute + OrgDisambiguatedEntity entity = processInstitute(sourceId, name, country, city, region, url, orgType, locationsJson, namesJson); + + // Creates external identifiers + processExternalIdentifiers(entity, institute); + } else if ("redirected".equals(status)) { + String primaryId = institute.get("redirect").isNull() ? null : institute.get("redirect").asText(); + deprecateOrg(sourceId, primaryId); + } else if ("withdrawn".equals(status) || "obsolete".equals(status)) { + obsoleteOrg(sourceId); + } else { + LOGGER.error("Illegal status '" + status + "' for institute " + sourceId); + } + }); + + // Check if any RORs with external identifiers updated and group + // them + groupRORsWithUpdatedExternalModifiers(); + + LOGGER.info("Time taken to process the data: {}", Duration.between(start, Instant.now()).toString()); + return true; + } catch (Exception e) { + LOGGER.error("Error loading ROR data", e); + return false; + } + } + + private OrgDisambiguatedEntity processInstitute(String sourceId, String name, Iso3166Country country, String city, + + String region, String url, String orgType, String locationsJson, String namesJson) { + OrgDisambiguatedEntity existingBySourceId = orgDisambiguatedDao.findBySourceIdAndSourceType(sourceId, OrgDisambiguatedSourceType.ROR.name()); + if (existingBySourceId != null) { + if (entityChanged(existingBySourceId, name, country.value(), city, region, url, orgType) || indexAllEnabled) { + existingBySourceId.setCity(city); + existingBySourceId.setCountry(country.name()); + existingBySourceId.setName(name); + existingBySourceId.setOrgType(orgType); + existingBySourceId.setRegion(region); + existingBySourceId.setUrl(url); + existingBySourceId.setLocationsJson(locationsJson); + existingBySourceId.setNamesJson(namesJson); + + existingBySourceId.setIndexingStatus(IndexingStatus.PENDING); + try { + // mark group for indexing + new OrgGrouping(existingBySourceId, orgDisambiguatedManager).markGroupForIndexing(orgDisambiguatedDao); + + } catch (Exception ex) { + LOGGER.error("Error when grouping by ROR and marking group orgs for reindexing, eating the exception", ex); + } + orgDisambiguatedManager.updateOrgDisambiguated(existingBySourceId); + } + return existingBySourceId; + } + + // Create a new disambiguated org + OrgDisambiguatedEntity newOrg = createDisambiguatedOrg(sourceId, name, orgType, country, city, region, url); + try { + // mark group for indexing + new OrgGrouping(newOrg, orgDisambiguatedManager).markGroupForIndexing(orgDisambiguatedDao); + } catch (Exception ex) { + LOGGER.error("Error when grouping by ROR and removing related orgs solr index, eating the exception", ex); + } + return newOrg; + } + + private void processExternalIdentifiers(OrgDisambiguatedEntity org, JsonNode institute) { + ArrayNode nodes = institute.get("external_ids") == null ? null : (ArrayNode) institute.get("external_ids"); + if (nodes != null) { + for (JsonNode entry : nodes) { + String identifierTypeName = entry.get("type").asText().toUpperCase(); + String preferredId = entry.get("preferred").isNull() ? null : entry.get("preferred").asText(); + if (StringUtils.equalsIgnoreCase(OrgDisambiguatedSourceType.GRID.name(), identifierTypeName)) { + JsonNode extId = (JsonNode) entry.get("all"); + setExternalId(org, identifierTypeName, preferredId, extId); + UPDATED_RORS.add(org.getId()); + } else { + ArrayNode elements = (ArrayNode) entry.get("all"); + for (JsonNode extId : elements) { + setExternalId(org, identifierTypeName, preferredId, extId); + UPDATED_RORS.add(org.getId()); + } + } + } + } + } + + private void setExternalId(OrgDisambiguatedEntity org, String identifierTypeName, String preferredId, JsonNode extId) { + // If the external identifier doesn't exists yet + OrgDisambiguatedExternalIdentifierEntity existingExternalId = orgDisambiguatedExternalIdentifierDao.findByDetails(org.getId(), extId.asText(), + identifierTypeName); + Boolean preferred = extId.asText().equals(preferredId); + if (existingExternalId == null) { + if (preferred) { + createExternalIdentifier(org, extId.asText(), identifierTypeName, true); + } else { + createExternalIdentifier(org, extId.asText(), identifierTypeName, false); + } + } else { + if (existingExternalId.getPreferred() != preferred) { + existingExternalId.setPreferred(preferred); + orgDisambiguatedManager.updateOrgDisambiguatedExternalIdentifier(existingExternalId); + LOGGER.info("External identifier for {} with ext id {} and type {} was updated", new Object[] { org.getId(), extId.asText(), identifierTypeName }); + } else { + LOGGER.info("External identifier for {} with ext id {} and type {} already exists", new Object[] { org.getId(), extId.asText(), identifierTypeName }); + } + } + } + + /** + * Indicates if an entity changed his address, url or org type + * + * @return true if the entity has changed. + */ + private boolean entityChanged(OrgDisambiguatedEntity entity, String name, String countryCode, String city, String region, String url, String orgType) { + // Check name + if (StringUtils.isNotBlank(name)) { + if (!name.equalsIgnoreCase(entity.getName())) + return true; + } else if (StringUtils.isNotBlank(entity.getName())) { + return true; + } + // Check country + if (StringUtils.isNotBlank(countryCode)) { + if (entity.getCountry() == null || !countryCode.equals(entity.getCountry())) { + return true; + } + } else if (entity.getCountry() != null) { + return true; + } + // Check city + if (StringUtils.isNotBlank(city)) { + if (entity.getCity() == null || !city.equals(entity.getCity())) { + return true; + } + } else if (StringUtils.isNotBlank(entity.getCity())) { + return true; + } + // Check region + if (StringUtils.isNotBlank(region)) { + if (entity.getRegion() == null || !region.equals(entity.getRegion())) { + return true; + } + } else if (StringUtils.isNotBlank(entity.getRegion())) { + return true; + } + // Check url + if (StringUtils.isNotBlank(url)) { + if (entity.getUrl() == null || !url.equals(entity.getUrl())) { + return true; + } + } else if (StringUtils.isNotBlank(entity.getUrl())) { + return true; + } + // Check org_type + if (StringUtils.isNotBlank(orgType)) { + if (entity.getOrgType() == null || !orgType.equals(entity.getOrgType())) { + return true; + } + } else if (StringUtils.isNotBlank(entity.getOrgType())) { + return true; + } + + return false; + } + + /** + * Creates a disambiguated ORG in the org_disambiguated table + */ + private OrgDisambiguatedEntity createDisambiguatedOrg(String sourceId, String name, String orgType, Iso3166Country country, String city, String region, String url) { + LOGGER.info("Creating disambiguated org {}", name); + OrgDisambiguatedEntity orgDisambiguatedEntity = new OrgDisambiguatedEntity(); + orgDisambiguatedEntity.setName(name); + orgDisambiguatedEntity.setCountry(country != null ? country.name() : null); + orgDisambiguatedEntity.setCity(city); + orgDisambiguatedEntity.setRegion(region); + orgDisambiguatedEntity.setUrl(url); + orgDisambiguatedEntity.setOrgType(orgType); + orgDisambiguatedEntity.setSourceId(sourceId); + orgDisambiguatedEntity.setSourceType(OrgDisambiguatedSourceType.ROR.name()); + orgDisambiguatedManager.createOrgDisambiguated(orgDisambiguatedEntity); + return orgDisambiguatedEntity; + } + + /** + * Creates an external identifier in the + * org_disambiguated_external_identifier table + */ + private boolean createExternalIdentifier(OrgDisambiguatedEntity disambiguatedOrg, String identifier, String externalIdType, Boolean preferred) { + LOGGER.info("Creating external identifier for {}", disambiguatedOrg.getId()); + OrgDisambiguatedExternalIdentifierEntity externalIdentifier = new OrgDisambiguatedExternalIdentifierEntity(); + externalIdentifier.setIdentifier(identifier); + externalIdentifier.setIdentifierType(externalIdType); + externalIdentifier.setOrgDisambiguated(disambiguatedOrg); + externalIdentifier.setPreferred(preferred); + orgDisambiguatedManager.createOrgDisambiguatedExternalIdentifier(externalIdentifier); + return true; + } + + /** + * Mark an existing org as DEPRECATED + */ + private void deprecateOrg(String sourceId, String primarySourceId) { + LOGGER.info("Deprecating org {} for {}", sourceId, primarySourceId); + OrgDisambiguatedEntity existingEntity = orgDisambiguatedDao.findBySourceIdAndSourceType(sourceId, OrgDisambiguatedSourceType.ROR.name()); + if (existingEntity != null) { + if (existingEntity.getStatus() == null || !existingEntity.getStatus().equals(OrganizationStatus.DEPRECATED.name()) + || !existingEntity.getSourceParentId().equals(primarySourceId)) { + existingEntity.setStatus(OrganizationStatus.DEPRECATED.name()); + existingEntity.setSourceParentId(primarySourceId); + existingEntity.setIndexingStatus(IndexingStatus.PENDING); + orgDisambiguatedManager.updateOrgDisambiguated(existingEntity); + } + } else { + OrgDisambiguatedEntity deprecatedEntity = new OrgDisambiguatedEntity(); + deprecatedEntity.setSourceType(OrgDisambiguatedSourceType.ROR.name()); + deprecatedEntity.setStatus(OrganizationStatus.DEPRECATED.name()); + deprecatedEntity.setSourceId(sourceId); + deprecatedEntity.setSourceParentId(primarySourceId); + // We don't need to index it + deprecatedEntity.setIndexingStatus(IndexingStatus.DONE); + orgDisambiguatedManager.createOrgDisambiguated(deprecatedEntity); + } + } + + /** + * Mark an existing org as OBSOLETE + */ + private void obsoleteOrg(String sourceId) { + LOGGER.info("Marking or as obsolete {}", sourceId); + OrgDisambiguatedEntity existingEntity = orgDisambiguatedDao.findBySourceIdAndSourceType(sourceId, OrgDisambiguatedSourceType.ROR.name()); + if (existingEntity != null) { + if (existingEntity.getStatus() == null || !existingEntity.getStatus().equals(OrganizationStatus.OBSOLETE.name())) { + existingEntity.setStatus(OrganizationStatus.OBSOLETE.name()); + existingEntity.setIndexingStatus(IndexingStatus.PENDING); + orgDisambiguatedManager.updateOrgDisambiguated(existingEntity); + new OrgGrouping(existingEntity, orgDisambiguatedManager).ungroupObsoleteRorForIndexing(orgDisambiguatedDao); + } + } else { + OrgDisambiguatedEntity obsoletedEntity = new OrgDisambiguatedEntity(); + obsoletedEntity.setSourceType(OrgDisambiguatedSourceType.ROR.name()); + obsoletedEntity.setStatus(OrganizationStatus.OBSOLETE.name()); + obsoletedEntity.setSourceId(sourceId); + // We don't need to index it + obsoletedEntity.setIndexingStatus(IndexingStatus.DONE); + orgDisambiguatedManager.createOrgDisambiguated(obsoletedEntity); + new OrgGrouping(obsoletedEntity, orgDisambiguatedManager).ungroupObsoleteRorForIndexing(orgDisambiguatedDao); + } + } + + @Override + public boolean isEnabled() { + return enabled; + } + + private void groupRORsWithUpdatedExternalModifiers() { + for (Long id : UPDATED_RORS) { + OrgDisambiguatedEntity entity = orgDisambiguatedDao.find(id); + if (entity != null) { + entity.setIndexingStatus(IndexingStatus.PENDING); + try { + // mark group for indexing + new OrgGrouping(entity, orgDisambiguatedManager).markGroupForIndexing(orgDisambiguatedDao); + + } catch (Exception ex) { + LOGGER.error("Error when grouping by ROR and marking group orgs for reindexing, eating the exception", ex); + } + entity = orgDisambiguatedManager.updateOrgDisambiguated(entity); + + } + } + } } \ No newline at end of file