Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CurationManager performance fixes #11

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open

CurationManager performance fixes #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
a142354
Changes were made to stop the Curation manager submitting messages to
ozej8y Jan 7, 2013
d2ba46e
Prevent log from displaying null
ozej8y Jan 8, 2013
91cc60c
Merge branch 'master' into curationFix1
ozej8y Jan 10, 2013
3b600c8
Revert "Merge branch 'master' into curationFix1"
ozej8y Feb 25, 2013
81d648a
Merge 1.5.2.2 to 1.6.1
ozej8y Jun 5, 2013
6ea74d0
Merge to 1.7
ozej8y Mar 3, 2014
26c595e
Changed from 1.7-SNAPSHOT to 1.7
ozej8y Mar 3, 2014
43ed452
CurationManager.java code fix from retrofit
ozej8y Mar 3, 2014
63e4720
Backed out REDBOXHELP-22 fix.
ozej8y Apr 16, 2014
ebb7ea6
REDBOX-266 #done
andrewbrazzatti May 29, 2014
b50279f
Back out Base curation changes between 1.6.1 & 1.7
ozej8y Aug 6, 2014
ec215c2
Updated from redbox-mint/master
ozej8y Aug 12, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions config/src/main/config/home/templates/people/rif.vm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,6 @@
<party type="person">
<identifier type="uri">$util.encodeXml($pid)</identifier>

#set ($nla_id = $util.getMetadata($object, "nlaPid"))
#if ("$!nla_id" != "")
<identifier type="AU-ANL:PEAU">$util.encodeXml($nla_id)</identifier>
#end


### Name
<name type="primary">
<namePart type="given">$util.encodeXml($util.get($item, 'data', 'Given_Name'))</namePart>
Expand Down
120 changes: 63 additions & 57 deletions ...src/main/java/com/googlecode/fascinator/redbox/plugins/curation/mint/CurationManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@ public class CurationManager extends GenericTransactionManager {
/** Logging **/
private static Logger log = LoggerFactory.getLogger(CurationManager.class);

/** System configuration */
private JsonSimpleConfig systemConfig;

/** Storage */
private Storage storage;

Expand Down Expand Up @@ -113,9 +116,6 @@ public class CurationManager extends GenericTransactionManager {
/** NLA Integration - Property Name */
private String nlaIdProperty;

/** NLA Integration - whether to use the NlaId for relationships*/
private boolean useNlaIdForRelationships;

/** NLA Integration - Test for execution */
private Map<String, String> nlaIncludeTest;

Expand All @@ -137,10 +137,10 @@ public CurationManager() {
*/
@Override
public void init() throws TransactionException {
JsonSimpleConfig config = getJsonConfig();
systemConfig = getJsonConfig();

// Load the storage plugin
String storageId = config.getString("file-system", "storage", "type");
String storageId = systemConfig.getString("file-system", "storage", "type");
if (storageId == null) {
throw new TransactionException("No Storage ID provided");
}
Expand All @@ -150,14 +150,14 @@ public void init() throws TransactionException {
+ storageId + "'");
}
try {
storage.init(config.toString());
storage.init(systemConfig.toString());
} catch (PluginException ex) {
log.error("Unable to initialise storage layer!", ex);
throw new TransactionException(ex);
}

// Load the indexer plugin
String indexerId = config.getString("solr", "indexer", "type");
String indexerId = systemConfig.getString("solr", "indexer", "type");
if (indexerId == null) {
throw new TransactionException("No Indexer ID provided");
}
Expand All @@ -167,49 +167,47 @@ public void init() throws TransactionException {
+ indexerId + "'");
}
try {
indexer.init(config.toString());
indexer.init(systemConfig.toString());
} catch (PluginException ex) {
log.error("Unable to initialise indexer!", ex);
throw new TransactionException(ex);
}

// External facing URL
urlBase = config.getString(null, "urlBase");
urlBase = systemConfig.getString(null, "urlBase");
if (urlBase == null) {
throw new TransactionException("URL Base in config cannot be null");
}

// Where should emails be sent?
emailAddress = config.getString(null,
emailAddress = systemConfig.getString(null,
"curation", "curationEmailAddress");
if (emailAddress == null) {
throw new TransactionException("An admin email is required!");
}

// Where are PIDs stored?
pidProperty = config.getString(null, "curation", "pidProperty");
pidProperty = systemConfig.getString(null, "curation", "pidProperty");
if (pidProperty == null) {
throw new TransactionException("An admin email is required!");
}

// Do admin staff want to confirm each curation?
manualConfirmation = config.getBoolean(false,
manualConfirmation = systemConfig.getBoolean(false,
"curation", "curationRequiresConfirmation");

// Find the address of our broker
brokerUrl = config.getString(null, "messaging", "url");
brokerUrl = systemConfig.getString(null, "messaging", "url");
if (brokerUrl == null) {
throw new TransactionException("Cannot find the message broker.");
}

// National Library Integration
nlaIntegrationEnabled = config.getBoolean(false,
nlaIntegrationEnabled = systemConfig.getBoolean(false,
"curation", "nlaIntegration", "enabled");
nlaIdProperty = config.getString(NLA_ID_PROPERTY_DEFAULT,
nlaIdProperty = systemConfig.getString(NLA_ID_PROPERTY_DEFAULT,
"curation", "nlaIntegration", "pidProperty");
useNlaIdForRelationships = config.getBoolean(true,
"curation", "nlaIntegration", "useNlaIdForRelationships");
JsonObject nlaIncludeTestNode = config.getObject(
JsonObject nlaIncludeTestNode = systemConfig.getObject(
"curation", "nlaIntegration", "includeTest");
nlaIncludeTest = new HashMap<String, String>();
if (nlaIncludeTestNode != null) {
Expand Down Expand Up @@ -255,7 +253,7 @@ public void shutdown() throws PluginException {
* @returns JsonSimple The response object to send back to the
* queue consumer
*/
private JsonSimple curation(JsonSimple message, String task, String oid) {
private JsonSimple curation(JsonSimple message, String task, String oid) throws TransactionException {
JsonSimple response = new JsonSimple();

//*******************
Expand Down Expand Up @@ -367,13 +365,16 @@ private JsonSimple curation(JsonSimple message, String task, String oid) {
}
responseObj.put("originOid", oid);
// If NLA Integration is enabled, use the NLA ID instead
if (nlaIntegrationEnabled && metadata.containsKey(nlaIdProperty) && useNlaIdForRelationships) {
if (nlaIntegrationEnabled && metadata.containsKey(nlaIdProperty)) {
responseObj.put("curatedPid", metadata.getProperty(nlaIdProperty));
} else {
responseObj.put("curatedPid", thisPid);
}
}

//JCU: now that the responses have been sent, remove them, so they are not sent again. Otherwise, they just keep getting resent and performance suffers greatly.
responses.clear();
saveObjectData(data, oid);

// Set a flag to let publish events that may come in later
// that this is ready to publish (if not already set)
if (!metadata.containsKey(READY_PROPERTY)) {
Expand Down Expand Up @@ -545,8 +546,6 @@ private JsonSimple curation(JsonSimple message, String task, String oid) {
if (task.equals("curation-request")) {
JsonObject taskObj = createTask(response, oid, "curation");
taskObj.put("alreadyCurated", true);
return response;

// Queries
} else {
// Rather then push to 'curation-response' we are just
Expand All @@ -562,12 +561,12 @@ private JsonSimple curation(JsonSimple message, String task, String oid) {
responseObj.put("originOid", oid);
responseObj.put("curatedPid", thisPid);
}
return response;
}

// Same as above, but this is a second stage request, let's be a
// little sterner in case log filtering is occurring
if (task.equals("curation")) {
alreadyCurated = message.getBoolean(false, "alreadyCurated");
log.info("Request to curate ignored. This object '{}' has"
+ " already been curated.", oid);
JsonObject taskObj = createTask(response, oid,
Expand Down Expand Up @@ -644,11 +643,10 @@ private JsonSimple curation(JsonSimple message, String task, String oid) {
for (String id : list) {
JsonObject order = newTransform(response, id, oid);
JsonObject config = (JsonObject) order.get("config");
// Make sure it even has an override...
JsonObject override = itemConfig.getObject(
JsonObject overrides = itemConfig.getObject(
"transformerOverrides", id);
if (override != null) {
config.putAll(override);
if (overrides != null) {
config.putAll(overrides);
}
}

Expand Down Expand Up @@ -723,13 +721,13 @@ private boolean checkChildren(JsonSimple response, JsonSimple data,
boolean localRecord = broker.equals(brokerUrl);
String relatedId = json.getString(null, "identifier");

// We need to find OIDs to match IDs... for local records
// We need to find OIDs to match IDs (only for local records)
String relatedOid = json.getString(null, "oid");
if (relatedOid == null && localRecord) {
String identifier = json.getString(null, "identifier");
if (identifier == null) {
throw new TransactionException(
"Cannot resolve identifer: " + identifier);
"NULL identifer provided!");
}
relatedOid = idToOid(identifier);
if (relatedOid == null) {
Expand Down Expand Up @@ -760,14 +758,13 @@ private boolean checkChildren(JsonSimple response, JsonSimple data,
// Only send out curation requests if asked to
if (sendRequests) {
JsonObject task;
broker = json.getString(null, "broker");
// It is a local object
if (broker == null) {
if (localRecord) {
task = createTask(response, relatedOid,
"curation-query");
// Or remote
} else {
task = createTask(response, broker,relatedOid,
task = createTask(response, broker, relatedOid,
"curation-query");
}

Expand Down Expand Up @@ -807,7 +804,7 @@ private boolean checkChildren(JsonSimple response, JsonSimple data,
task.put("respond", msgResponse);
}
} else {
log.debug(" * Already curated '{}'", relatedOid);
log.debug(" * Already curated '{}'", relatedId);
}
}

Expand Down Expand Up @@ -918,30 +915,19 @@ private void storeRequestData(JsonSimple message, String oid)
JSONArray relations = metadata.writeArray("relationships");
for (JsonSimple newRelation : JsonSimple.toJavaList(newRelations)) {
boolean duplicate = false;
// Relationships have multiple keys. String comparison of
// the JSON will catch this sometimes, but a housekeeping
// job periodically cleans up dupes that make it through.

// When building the string for comparison is needs to be
// done before any alterations, so basically as it it was
// recieved.
String uniqueString = newRelation.toString();

String identifier = newRelation.getString(null, "identifier");
// Compare to each existing relationship
for (JsonSimple relation : JsonSimple.toJavaList(relations)) {
String storedUnique = relation.getString(null,
"uniqueString");
if (uniqueString.equals(storedUnique)) {
log.debug("Ignoring duplicate relationship '{}'", oid);
String storedId = relation.getString(null, "identifier");
if (identifier.equals(storedId)) {
log.debug("Ignoring duplicate relationship '{}'", identifier);
duplicate = true;
}
}

// Store new entries
if (!duplicate) {
log.debug("New relationship added to '{}'", oid);
newRelation.getJsonObject().put(
"uniqueString", uniqueString);
relations.add(newRelation.getJsonObject());
}
}
Expand Down Expand Up @@ -990,7 +976,7 @@ private JsonSimple publish(JsonSimple message, String oid)
throw new TransactionException(
"Error setting publish property: ", ex);
}

// Make a final pass through the curation tool(s),
// allows for external publication. eg. VITAL
JsonSimple itemConfig = getConfigFromStorage(oid);
Expand Down Expand Up @@ -1022,8 +1008,9 @@ private JsonSimple publish(JsonSimple message, String oid)
* Send out requests to all relations to publish
*
* @param oid The object identifier to publish
* @throws TransactionException
*/
private void publishRelations(JsonSimple response, String oid) {
private void publishRelations(JsonSimple response, String oid) throws TransactionException {
log.debug("Publishing Children of '{}'", oid);

JsonSimple data = getDataFromStorage(oid);
Expand All @@ -1035,6 +1022,8 @@ private void publishRelations(JsonSimple response, String oid) {
return;
}

boolean saveData = false;

JSONArray relations = data.writeArray("relationships");
for (Object relation : relations) {
JsonSimple json = new JsonSimple((JsonObject) relation);
Expand All @@ -1047,7 +1036,7 @@ private void publishRelations(JsonSimple response, String oid) {
if (relatedOid == null && localRecord) {
String identifier = json.getString(null, "identifier");
if (identifier == null) {
log.error("Cannot resolve identifer: '{}'", identifier);
log.error("NULL identifer provided!");
}
relatedOid = idToOid(identifier);
if (relatedOid == null) {
Expand All @@ -1059,26 +1048,43 @@ private void publishRelations(JsonSimple response, String oid) {
if (authority) {
// Is this relationship using a curated ID?
boolean isCurated = json.getBoolean(false, "isCurated");
if (isCurated) {
//JCU: adding check for publishMsgSent
boolean publishMsgSent = json.getBoolean(false, "publishMsgSent");
if (isCurated && !publishMsgSent) {
log.debug(" * Publishing '{}'", relatedId);
JsonObject task;
// It is a local object
if (localRecord) {
task = createTask(response, relatedOid, "publish");
createTask(response, relatedOid, "publish");

// Or remote
} else {
task = createTask(response, broker, relatedOid,
JsonObject task = createTask(response, broker, relatedOid,
"publish");
// We won't know OIDs for remote systems
task.remove("oid") ;
task.put("identifier", relatedId);
}
} else {

//JCU: Adding tag to indicate the publish message has been sent.
((JsonObject) relation).put("publishMsgSent", "true");
saveData = true;

} else if (publishMsgSent){
log.debug(" * Ignoring already published relationship '{}'",
relatedId);
}
else {
log.debug(" * Ignoring non-curated relationship '{}'",
relatedId);
}
}
}

if (saveData){
//updating the relations with publishMsgSent
saveObjectData(data, oid);
}

}

/**
Expand Down
Loading