From 34941219f4d4fecc9634cd79af4e2c264f6418e2 Mon Sep 17 00:00:00 2001 From: Juraj Roka Date: Fri, 15 Nov 2024 13:31:29 +0100 Subject: [PATCH 01/12] Updated the version of matomo dependency and tried to change request from Custom Variables to Dimension --- dspace-api/pom.xml | 5 +++-- .../statistics/clarin/ClarinMatomoBitstreamTracker.java | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index c7b93f6ac377..4f6e4257b083 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -339,9 +339,10 @@ org.piwik.java.tracking - matomo-java-tracker - 2.0 + matomo-java-tracker-java11 + 3.4.0 + org.apache.logging.log4j log4j-api diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java index 838a2650586a..b6d503883962 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java @@ -9,6 +9,7 @@ import java.sql.SQLException; import java.text.MessageFormat; +import java.util.LinkedHashMap; import java.util.List; import java.util.Objects; import javax.servlet.http.HttpServletRequest; @@ -26,7 +27,6 @@ import org.dspace.eperson.EPerson; import org.dspace.services.ConfigurationService; import org.dspace.services.factory.DSpaceServicesFactory; -import org.matomo.java.tracking.CustomVariable; import org.matomo.java.tracking.MatomoException; import org.matomo.java.tracking.MatomoRequest; import org.springframework.beans.factory.annotation.Autowired; @@ -69,7 +69,6 @@ public ClarinMatomoBitstreamTracker() { @Override protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, HttpServletRequest request) { super.preTrack(context, matomoRequest, item, request); - matomoRequest.setSiteId(siteId); log.debug("Logging to site " + matomoRequest.getSiteId()); String itemIdentifier = getItemIdentifier(item); @@ -81,7 +80,9 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, matomoRequest.setActionUrl(itemIdentifier); } try { - matomoRequest.setPageCustomVariable(new CustomVariable("source", "bitstream"), 1); + LinkedHashMap handle = new LinkedHashMap<>(); + handle.put(1L, "bitstream"); + matomoRequest.setDimensions(handle); } catch (MatomoException e) { log.error(e); } From 97d3c1ef1213effe6ff807a1456ab1a4db83de52 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Wed, 20 Nov 2024 16:29:07 +0100 Subject: [PATCH 02/12] Added a custom dimension with item's handle URL --- .../clarin/ClarinMatomoBitstreamTracker.java | 9 +++++--- .../clarin/ClarinMatomoTracker.java | 21 +++++++------------ 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java index b6d503883962..3cf49e944e5b 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java @@ -27,6 +27,7 @@ import org.dspace.eperson.EPerson; import org.dspace.services.ConfigurationService; import org.dspace.services.factory.DSpaceServicesFactory; +import org.matomo.java.tracking.CustomVariable; import org.matomo.java.tracking.MatomoException; import org.matomo.java.tracking.MatomoRequest; import org.springframework.beans.factory.annotation.Autowired; @@ -80,9 +81,11 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, matomoRequest.setActionUrl(itemIdentifier); } try { - LinkedHashMap handle = new LinkedHashMap<>(); - handle.put(1L, "bitstream"); - matomoRequest.setDimensions(handle); + matomoRequest.setPageCustomVariable(new CustomVariable("source", "bitstream"), 1); + // Add the Item handle into the request as a custom dimension + LinkedHashMap handleDimension = new LinkedHashMap<>(); + handleDimension.put(10L, item.getHandle()); + matomoRequest.setDimensions(handleDimension); } catch (MatomoException e) { log.error(e); } diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java index 360b4efe8e93..85f5e5ecfccd 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java @@ -9,12 +9,10 @@ import java.util.Calendar; import java.util.Objects; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; +import java.util.concurrent.CompletableFuture; import javax.servlet.http.HttpServletRequest; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpResponse; import org.apache.logging.log4j.Logger; import org.dspace.content.Item; import org.dspace.content.factory.ClarinServiceFactory; @@ -134,18 +132,13 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, * @param matomoRequest prepared MatomoRequest for sending */ public void sendTrackingRequest(MatomoRequest matomoRequest) { - try { - Future response = tracker.sendRequestAsync(matomoRequest); - // usually not needed: - HttpResponse httpResponse = response.get(); - int statusCode = httpResponse.getStatusLine().getStatusCode(); - if (statusCode > 399) { - // problem - log.error("Matomo tracker error the response has status code: " + statusCode); + CompletableFuture completableFuture = tracker.sendRequestAsync(matomoRequest); + + completableFuture.whenComplete((result, exception) -> { + if (exception != null) { + log.error("Matomo tracker error - the response exception message: {}", exception.getMessage()); } - } catch (ExecutionException | InterruptedException e) { - e.printStackTrace(); - } + }); } protected String getFullURL(HttpServletRequest request) { From 7fddbd00e21edaf6a533220ab3aab636d5d77d3a Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Wed, 20 Nov 2024 16:30:52 +0100 Subject: [PATCH 03/12] Send custom dimension also in oai tracker --- .../dspace/app/statistics/clarin/ClarinMatomoOAITracker.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java index c6240b4da8a6..5e0ca5703b52 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java @@ -7,6 +7,7 @@ */ package org.dspace.app.statistics.clarin; +import java.util.LinkedHashMap; import java.util.Objects; import javax.servlet.http.HttpServletRequest; @@ -56,6 +57,10 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, log.debug("Logging to site " + matomoRequest.getSiteId()); try { matomoRequest.setPageCustomVariable(new CustomVariable("source", "oai"), 1); + // Add the Item handle into the request as a custom dimension + LinkedHashMap handleDimension = new LinkedHashMap<>(); + handleDimension.put(11L, item.getHandle()); + matomoRequest.setDimensions(handleDimension); } catch (MatomoException e) { log.error(e); } From 00f40488d88b4f8870b491d0e96c6833fc21ce74 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Mon, 25 Nov 2024 20:55:59 +0100 Subject: [PATCH 04/12] Use only IPv4 address, the Matomo tracker has a problem with IPv6 --- .../clarin/ClarinMatomoTracker.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java index 85f5e5ecfccd..e3838c6101da 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java @@ -7,6 +7,8 @@ */ package org.dspace.app.statistics.clarin; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Calendar; import java.util.Objects; import java.util.concurrent.CompletableFuture; @@ -21,6 +23,7 @@ import org.dspace.services.factory.DSpaceServicesFactory; import org.matomo.java.tracking.MatomoException; import org.matomo.java.tracking.MatomoRequest; +import org.matomo.java.tracking.parameters.AcceptLanguage; /** * The statistics Tracker for Matomo. This class prepare and send the track GET request to the `/matomo.php` @@ -97,13 +100,13 @@ protected MatomoRequest createMatomoRequest(HttpServletRequest request, String p */ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, HttpServletRequest request) { if (StringUtils.isNotBlank(request.getHeader("referer"))) { - matomoRequest.setHeaderUserAgent(request.getHeader("referer")); + matomoRequest.setReferrerUrl(request.getHeader("referer")); } if (StringUtils.isNotBlank(request.getHeader("user-agent"))) { matomoRequest.setHeaderUserAgent(request.getHeader("user-agent")); } if (StringUtils.isNotBlank(request.getHeader("accept-language"))) { - matomoRequest.setHeaderUserAgent(request.getHeader("accept-language")); + matomoRequest.setHeaderAcceptLanguage(AcceptLanguage.fromHeader(request.getHeader("accept-language"))); } // Creating a calendar using getInstance method @@ -157,10 +160,11 @@ protected String getFullURL(HttpServletRequest request) { } /** - * Get IpAddress of the current user which throws this statistic event + * Get IpAddress of the current user which throws this statistic event. Return only the first valid IPv4 address + * because the Matomo tracker has a problem with IPv6 addresses. * * @param request current request - * @return + * @return only the first valid IPv4 address */ protected String getIpAddress(HttpServletRequest request) { String ip = ""; @@ -170,8 +174,23 @@ protected String getIpAddress(HttpServletRequest request) { } if (header != null) { String[] ips = header.split(", "); - ip = ips.length > 0 ? ips[0] : ""; + for (String candidateIp : ips) { + // Validate if it's an IPv4 address + if (isIPv4Address(candidateIp)) { + ip = candidateIp; + break; + } + } } return ip; } + + private boolean isIPv4Address(String ip) { + try { + InetAddress inetAddress = InetAddress.getByName(ip); + return inetAddress.getHostAddress().equals(ip) && inetAddress instanceof java.net.Inet4Address; + } catch (UnknownHostException e) { + return false; // Not a valid IP address + } + } } From 8deab9ecf5f7b889b94989bea3cdf93c27db0754 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Mon, 25 Nov 2024 21:13:32 +0100 Subject: [PATCH 05/12] Do not change custom dimension when the Item is null --- .../dspace/app/statistics/clarin/ClarinMatomoOAITracker.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java index 5e0ca5703b52..465aa82f9f7a 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java @@ -57,10 +57,6 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, log.debug("Logging to site " + matomoRequest.getSiteId()); try { matomoRequest.setPageCustomVariable(new CustomVariable("source", "oai"), 1); - // Add the Item handle into the request as a custom dimension - LinkedHashMap handleDimension = new LinkedHashMap<>(); - handleDimension.put(11L, item.getHandle()); - matomoRequest.setDimensions(handleDimension); } catch (MatomoException e) { log.error(e); } From 08a1cd89fb35b0ea4d9f24e984ab9bc1a79abf7d Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Tue, 26 Nov 2024 09:12:18 +0100 Subject: [PATCH 06/12] First custom dimension should have ID '1'. --- .../app/statistics/clarin/ClarinMatomoBitstreamTracker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java index 3cf49e944e5b..518320e3845a 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java @@ -84,7 +84,7 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, matomoRequest.setPageCustomVariable(new CustomVariable("source", "bitstream"), 1); // Add the Item handle into the request as a custom dimension LinkedHashMap handleDimension = new LinkedHashMap<>(); - handleDimension.put(10L, item.getHandle()); + handleDimension.put(1L, item.getHandle()); matomoRequest.setDimensions(handleDimension); } catch (MatomoException e) { log.error(e); From 46b3cadc6bb5f32027dab5894e7685a0bf1bc7a1 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Tue, 26 Nov 2024 09:32:58 +0100 Subject: [PATCH 07/12] Fixed checkstyle issue. --- .../org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java index 465aa82f9f7a..c6240b4da8a6 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoOAITracker.java @@ -7,7 +7,6 @@ */ package org.dspace.app.statistics.clarin; -import java.util.LinkedHashMap; import java.util.Objects; import javax.servlet.http.HttpServletRequest; From 36399304789c257760455eed930e89f0aec32d7b Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Tue, 26 Nov 2024 12:05:17 +0100 Subject: [PATCH 08/12] Use a valid URL for Matomo tracker in the IT --- dspace-api/src/test/data/dspaceFolder/config/local.cfg | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/test/data/dspaceFolder/config/local.cfg b/dspace-api/src/test/data/dspaceFolder/config/local.cfg index 51d050c296a2..b7b701b27847 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/local.cfg +++ b/dspace-api/src/test/data/dspaceFolder/config/local.cfg @@ -310,4 +310,7 @@ webui.supported.locales = en # Type bind configuration for the submission form with special type-bind field # When title is something like "Type-bind test" the type-bind field will popped up -submit.type-bind.field = dc.type,dc.identifier.citation=>dc.title \ No newline at end of file +submit.type-bind.field = dc.type,dc.identifier.citation=>dc.title + +# The configuration for the Matomo tracker must have a valid URL, as it will throw an exception if it does not. +matomo.tracker.host.url = http://localhost:8135/matomo.php \ No newline at end of file From 4950dfc66762585d417c7e173629fcd40ca2ad3a Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Tue, 26 Nov 2024 12:08:25 +0100 Subject: [PATCH 09/12] Removed empty row --- dspace-api/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index 4f6e4257b083..6ac36d129ade 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -342,7 +342,6 @@ matomo-java-tracker-java11 3.4.0 - org.apache.logging.log4j log4j-api From 0fa70f81266a8963868272b5e3b4764b65e88822 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Tue, 26 Nov 2024 12:09:29 +0100 Subject: [PATCH 10/12] Configure handle custom dimension ID in the clarin-dspace.cfg --- .../app/statistics/clarin/ClarinMatomoBitstreamTracker.java | 3 ++- dspace/config/clarin-dspace.cfg | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java index 518320e3845a..5b90ab7fc740 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java @@ -84,7 +84,8 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, matomoRequest.setPageCustomVariable(new CustomVariable("source", "bitstream"), 1); // Add the Item handle into the request as a custom dimension LinkedHashMap handleDimension = new LinkedHashMap<>(); - handleDimension.put(1L, item.getHandle()); + handleDimension.put(configurationService.getLongProperty("matomo.custom.dimension.handle.id", + 1L), item.getHandle()); matomoRequest.setDimensions(handleDimension); } catch (MatomoException e) { log.error(e); diff --git a/dspace/config/clarin-dspace.cfg b/dspace/config/clarin-dspace.cfg index 6c00f45b5a1e..a0c8dc4794d3 100644 --- a/dspace/config/clarin-dspace.cfg +++ b/dspace/config/clarin-dspace.cfg @@ -155,6 +155,7 @@ matomo.site.id = 1 matomo.tracker.bitstream.site_id = 1 matomo.tracker.oai.site_id = 1 matomo.tracker.host.url = http://url:port/matomo.php +matomo.custom.dimension.handle.id = 1 statistics.cache-server.uri = http://cache-server.none #### Statistic usage reports #### From 6448775bdf8676950e7c35565619ef9b37773ff7 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Fri, 29 Nov 2024 09:16:20 +0100 Subject: [PATCH 11/12] Refactored ipv4 method to be more readable --- .../org/dspace/app/statistics/clarin/ClarinMatomoTracker.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java index e3838c6101da..92a6706bd7aa 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java @@ -177,8 +177,7 @@ protected String getIpAddress(HttpServletRequest request) { for (String candidateIp : ips) { // Validate if it's an IPv4 address if (isIPv4Address(candidateIp)) { - ip = candidateIp; - break; + return candidateIp; } } } From 5a658641e651f4df306c25796b74773fc0571ff8 Mon Sep 17 00:00:00 2001 From: milanmajchrak Date: Fri, 29 Nov 2024 09:37:33 +0100 Subject: [PATCH 12/12] Refactored ipv4 method to be more readable - return null --- .../org/dspace/app/statistics/clarin/ClarinMatomoTracker.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java index 92a6706bd7aa..34615bc2ed24 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java @@ -167,7 +167,6 @@ protected String getFullURL(HttpServletRequest request) { * @return only the first valid IPv4 address */ protected String getIpAddress(HttpServletRequest request) { - String ip = ""; String header = request.getHeader("X-Forwarded-For"); if (header == null) { header = request.getRemoteAddr(); @@ -181,7 +180,7 @@ protected String getIpAddress(HttpServletRequest request) { } } } - return ip; + return null; } private boolean isIPv4Address(String ip) {