Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

no need to call if esindexer not config #56

Merged
merged 3 commits into from
Jul 8, 2024
Merged

no need to call if esindexer not config #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1951758
no need to call if esindexer not config
utas-raymondng Jul 5, 2024
47eada7
show warning if not config
utas-raymondng Jul 5, 2024
2534c5e
Fix bug after PR
utas-raymondng Jul 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions geonetwork-core/src/main/java/au/org/aodn/geonetwork4/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,8 +144,13 @@ public RestTemplate createRestTemplate() {
}

@Bean
public GenericEntityListener createGenericEntityListener() {
return new GenericEntityListener();
public GenericEntityListener createGenericEntityListener(
@Value("${aodn.geonetwork4.esIndexer.apikey}") String apiKey,
@Value("${aodn.geonetwork4.esIndexer.host}") String host,
@Value("${aodn.geonetwork4.esIndexer.urlIndex}") String indexUrl,
RestTemplate restTemplate) {

return new GenericEntityListener(apiKey, host, indexUrl, restTemplate);
}
/**
* Must use prototype scope as there is a XSRF-TOKEN header for each api, that cannot share
Expand Down
188 changes: 98 additions & 90 deletions geonetwork-core/src/main/java/au/org/aodn/geonetwork4/handler/GenericEntityListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import org.fao.geonet.entitylistener.GeonetworkEntityListener;
import org.fao.geonet.entitylistener.PersistentEventType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.*;
import org.springframework.web.client.HttpServerErrorException;
import org.springframework.web.client.RestTemplate;
Expand All @@ -33,13 +32,10 @@ public class GenericEntityListener implements GeonetworkEntityListener<Metadata>

protected static final String UUID = "uuid";

@Value("${aodn.geonetwork4.esIndexer.urlIndex}")
protected String indexUrl;

@Value("${aodn.geonetwork4.esIndexer.apikey}")
protected String apiKey;

@Autowired
protected RestTemplate restTemplate;

@Override
Expand All @@ -54,90 +50,98 @@ public void cleanUp() {

protected int delayStart = 5;

@PostConstruct
public void init() {
// We pick up the items in map and then post trigger indexer call, this thread keep execute every 5 secs
service.scheduleWithFixedDelay(() -> {

// If the updateMap contain items that is going do delete, then there is no point to update
deleteMap.forEach((key, value) -> updateMap.remove(key));

// Noted, our geonetwork setup never use un-publish, therefore it will be always
// public readable.
for(String uuid : updateMap.keySet()) {
boolean needRemoveFromMap = true;
@Autowired
public GenericEntityListener(String apiKey, String host, String indexUrl, RestTemplate restTemplate) {

try {
logger.info("Call indexer on metadata {} after metadata updated.", uuid);
Map<String, Object> variables = new HashMap<>();
variables.put(UUID, uuid);
this.apiKey = apiKey;
this.indexUrl = host != null && !host.isEmpty() ? indexUrl : null;
this.restTemplate = restTemplate;
}

callApiUpdate(indexUrl, variables);
}
catch(HttpServerErrorException server) {
if(server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
// Error may due to indexer reboot, so we just need to keep retry
logger.warn("Indexer not available, will keep retry update operation");
needRemoveFromMap = false;
}
}
catch (Exception e1) {
// Must not throw exception, can only print log and handle it manually
logger.error("Fail to call indexer on metadata {} after transaction committed. {}",
uuid, e1.getMessage());
}
finally {
if(needRemoveFromMap) {
updateMap.remove(uuid);
@PostConstruct
public void init() {
if(indexUrl == null) {
logger.warn("Call to es-indexer is off due to config missing");
}
else {
// We pick up the items in map and then post trigger indexer call, this thread keep execute every 5 secs
service.scheduleWithFixedDelay(() -> {

// If the updateMap contain items that is going do delete, then there is no point to update
deleteMap.forEach((key, value) -> updateMap.remove(key));

// Noted, our geonetwork setup never use un-publish, therefore it will be always
// public readable.
for (String uuid : updateMap.keySet()) {
boolean needRemoveFromMap = true;

try {
logger.info("Call indexer on metadata {} after metadata updated.", uuid);
Map<String, Object> variables = new HashMap<>();
variables.put(UUID, uuid);

callApiUpdate(indexUrl, variables);
} catch (HttpServerErrorException server) {
if (server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
// Error may due to indexer reboot, so we just need to keep retry
logger.warn("Indexer not available, will keep retry update operation");
needRemoveFromMap = false;
}
} catch (Exception e1) {
// Must not throw exception, can only print log and handle it manually
logger.error("Fail to call indexer on metadata {} after transaction committed. {}",
uuid, e1.getMessage());
} finally {
if (needRemoveFromMap) {
updateMap.remove(uuid);
}
}
}
}

for(String uuid : deleteMap.keySet()) {
boolean needRemoveFromMap = true;

try {
logger.info("Call indexer to delete metadata {} after transaction committed.", uuid);
Map<String, Object> variables = new HashMap<>();
variables.put(UUID, uuid);

callApiDelete(indexUrl, variables);
}
catch(HttpServerErrorException server) {
if(server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
// Error may due to indexer reboot, so we just need to keep retry
logger.warn("Indexer not available, will keep retry delete operation");
needRemoveFromMap = false;
for (String uuid : deleteMap.keySet()) {
boolean needRemoveFromMap = true;

try {
logger.info("Call indexer to delete metadata {} after transaction committed.", uuid);
Map<String, Object> variables = new HashMap<>();
variables.put(UUID, uuid);

callApiDelete(indexUrl, variables);
} catch (HttpServerErrorException server) {
if (server.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
// Error may due to indexer reboot, so we just need to keep retry
logger.warn("Indexer not available, will keep retry delete operation");
needRemoveFromMap = false;
}
} catch (Exception e1) {
// Must not throw exception, can only print log and handle it manually
logger.error("Fail to call indexer to delete metadata {} after transaction committed. {}",
uuid, e1.getMessage());
} finally {
if (needRemoveFromMap) {
deleteMap.remove(uuid);
}
}
}
catch (Exception e1) {
// Must not throw exception, can only print log and handle it manually
logger.error("Fail to call indexer to delete metadata {} after transaction committed. {}",
uuid, e1.getMessage());
}
finally {
if(needRemoveFromMap) {
deleteMap.remove(uuid);
}
}
}

}, delayStart,10, TimeUnit.SECONDS);
}, delayStart, 10, TimeUnit.SECONDS);
}
}

@Override
public void handleEvent(PersistentEventType persistentEventType, Metadata metaData) {
if(persistentEventType == PersistentEventType.PostUpdate) {
logger.info("{} handler for {}", persistentEventType, metaData);
// We see same even fired multiple times, this map will combine the event into one
// using a map with same key.
updateMap.put(metaData.getUuid(), metaData);
}
else if(persistentEventType == PersistentEventType.PostRemove) {
logger.info("{} handler for {}", persistentEventType, metaData);
// We see same even fired multiple times, this map will combine the event into one
// using a map with same key.
deleteMap.put(metaData.getUuid(), metaData);
if(indexUrl != null) {
if (persistentEventType == PersistentEventType.PostUpdate) {
logger.info("PostUpdate handler for {}", metaData);
// We see same even fired multiple times, this map will combine the event into one
// using a map with same key.
updateMap.put(metaData.getUuid(), metaData);
} else if (persistentEventType == PersistentEventType.PostRemove) {
logger.info("PostRemove handler for {}", metaData);
// We see same even fired multiple times, this map will combine the event into one
// using a map with same key.
deleteMap.put(metaData.getUuid(), metaData);
}
}
}
/**
Expand All @@ -147,13 +151,15 @@ else if(persistentEventType == PersistentEventType.PostRemove) {
* @param variables - The variable for the template URL.
*/
protected void callApiUpdate(String indexUrl, Map<String, Object> variables) {
HttpHeaders headers = new HttpHeaders();
headers.set("X-API-Key", apiKey.trim());
headers.setContentType(MediaType.APPLICATION_JSON);

HttpEntity<Void> request = new HttpEntity<>(null, headers);
logger.info("Call indexer update {} metadata {}", indexUrl, variables.get(UUID));
restTemplate.postForEntity(indexUrl, request, Void.class, variables);
if(indexUrl != null) {
HttpHeaders headers = new HttpHeaders();
headers.set("X-API-Key", apiKey.trim());
headers.setContentType(MediaType.APPLICATION_JSON);

HttpEntity<Void> request = new HttpEntity<>(null, headers);
logger.info("Call indexer update {} metadata {}", indexUrl, variables.get(UUID));
restTemplate.postForEntity(indexUrl, request, Void.class, variables);
}
}
/**
* Call indexer rest api to delete index.
Expand All @@ -162,12 +168,14 @@ protected void callApiUpdate(String indexUrl, Map<String, Object> variables) {
* @param variables - The variable for the template URL.
*/
protected void callApiDelete(String indexUrl, Map<String, Object> variables) {
HttpHeaders headers = new HttpHeaders();
headers.set("X-API-Key", apiKey.trim());
headers.setContentType(MediaType.APPLICATION_JSON);

HttpEntity<Void> request = new HttpEntity<>(null, headers);
logger.info("Call indexer delete {} metadata {}", indexUrl, variables.get(UUID));
restTemplate.exchange(indexUrl, HttpMethod.DELETE, request, Void.class, variables);
if(indexUrl != null) {
HttpHeaders headers = new HttpHeaders();
headers.set("X-API-Key", apiKey.trim());
headers.setContentType(MediaType.APPLICATION_JSON);

HttpEntity<Void> request = new HttpEntity<>(null, headers);
logger.info("Call indexer delete {} metadata {}", indexUrl, variables.get(UUID));
restTemplate.exchange(indexUrl, HttpMethod.DELETE, request, Void.class, variables);
}
}
}
2 changes: 1 addition & 1 deletion geonetwork-core/src/main/resources/application.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
aodn.geonetwork4.esIndexer.host=${INDEXER_HOST:http://localhost}
aodn.geonetwork4.esIndexer.host=${INDEXER_HOST:}
aodn.geonetwork4.esIndexer.port=${INDEXER_PORT:80}
aodn.geonetwork4.esIndexer.apikey=${INDEXER_APIKEY}
aodn.geonetwork4.esIndexer.urlIndex=${aodn.geonetwork4.esIndexer.host}:${aodn.geonetwork4.esIndexer.port}/api/v1/indexer/index/{uuid}
Expand Down
5 changes: 5 additions & 0 deletions geonetwork-core/src/main/resources/log4j-imos.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,11 @@
<AppenderRef ref="File"/>
</Logger>

<Logger name="au.org.aodn.geonetwork4.handler" level="debug" additivity="false">
<AppenderRef ref="Console"/>
<AppenderRef ref="File"/>
</Logger>

<Logger name="au.org.aodn.geonetwork_api" level="debug" additivity="false">
<AppenderRef ref="Console"/>
<AppenderRef ref="File"/>
Expand Down
38 changes: 29 additions & 9 deletions geonetwork-core/src/test/java/au/org/aodn/geonetwork4/handler/GenericEntityListenerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@ public class GenericEntityListenerTest {
@Test
public void verifyUpdateDeleteBehavior() throws InterruptedException {
CountDownLatch latch = new CountDownLatch(1);
GenericEntityListener listener = new GenericEntityListener();
RestTemplate template = Mockito.mock(RestTemplate.class);

// Set of test only, a mock to count what have been called
listener.indexUrl = "http://localhost/api/v1/indexer/index/{uuid}";
listener.apiKey = "test-key";
listener.restTemplate = template;

GenericEntityListener listener = new GenericEntityListener(
"test-key",
"localhost",
"http://localhost/api/v1/indexer/index/{uuid}",
template);
listener.init();

// Test data
Expand Down Expand Up @@ -80,7 +80,6 @@ public void verifyUpdateDeleteBehavior() throws InterruptedException {
@Test
public void verifyRetryBehavior() throws InterruptedException {
CountDownLatch latch = new CountDownLatch(1);
GenericEntityListener listener = new GenericEntityListener();
RestTemplate template = Mockito.mock(RestTemplate.class);

// Throw exception on first call
Expand All @@ -105,9 +104,11 @@ public void verifyRetryBehavior() throws InterruptedException {
.thenReturn(ResponseEntity.ok(null));

// Set of test only, a mock to count what have been called
listener.indexUrl = "http://localhost/api/v1/indexer/index/{uuid}";
listener.apiKey = "test-key";
listener.restTemplate = template;
GenericEntityListener listener = new GenericEntityListener(
"test-key",
"localhost",
"http://localhost/api/v1/indexer/index/{uuid}",
template);

listener.init();

Expand All @@ -132,4 +133,23 @@ public void verifyRetryBehavior() throws InterruptedException {
assertEquals("Map not contains uuid", 0, listener.updateMap.size());
assertEquals("Delete not contains uuid", 0, listener.deleteMap.size());
}
/**
* If host null, then we disable api call to indexer
*/
@Test
public void verifyIndexerCanBeDisabled() {
RestTemplate template = Mockito.mock(RestTemplate.class);
GenericEntityListener listener = new GenericEntityListener(
"test-key",
null,
"http://localhost/api/v1/indexer/index/{uuid}",
template);

listener.init();
listener.handleEvent(PersistentEventType.PostUpdate, new Metadata());
listener.handleEvent(PersistentEventType.PostRemove, new Metadata());

assertTrue("Internal update map empty", listener.updateMap.isEmpty());
assertTrue("Internal delete map empty", listener.deleteMap.isEmpty());
}
}
Loading