From a264c6116e668df7fb542915a701c99ba556e367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Wed, 30 Dec 2020 21:53:09 +0100 Subject: [PATCH 01/19] Refactoring removes unreachable code; more readable structure; no changes in functionality --- .../cuni/mff/ufal/PiwikStatisticsReader.java | 278 ++++++++++-------- 1 file changed, 148 insertions(+), 130 deletions(-) diff --git a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java index 63ae62945ad0..4b05bfecddfe 100644 --- a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java +++ b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java @@ -1,31 +1,14 @@ package cz.cuni.mff.ufal; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.StringReader; -import java.io.StringWriter; -import java.io.Writer; -import java.net.URL; +import java.io.*; import java.net.URLEncoder; import java.sql.SQLException; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.Map; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathFactory; - import org.apache.avalon.framework.parameters.Parameters; import org.apache.cocoon.ProcessingException; import org.apache.cocoon.ResourceNotFoundException; @@ -37,21 +20,12 @@ import org.apache.log4j.Logger; import org.dspace.app.xmlui.utils.ContextUtil; import org.dspace.authorize.AuthorizeException; -import org.dspace.authorize.AuthorizeManager; import org.dspace.content.DSpaceObject; import org.dspace.content.Item; import org.dspace.core.ConfigurationManager; import org.dspace.core.Context; import org.dspace.eperson.EPerson; import org.dspace.handle.HandleManager; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; import org.xml.sax.SAXException; public class PiwikStatisticsReader extends AbstractReader { @@ -105,7 +79,7 @@ public void setup(SourceResolver resolver, Map objectModel, String src, Paramete this.isSpider = par.getParameter("userAgent", "").equals("spider"); // Reference by an item's handle. - DSpaceObject dso = dso = HandleManager.resolveToObject(context, handle); + DSpaceObject dso = HandleManager.resolveToObject(context, handle); if (dso instanceof Item) { item = (Item)dso; @@ -131,116 +105,160 @@ public void setup(SourceResolver resolver, Map objectModel, String src, Paramete @Override - public void generate() throws IOException, SAXException, ProcessingException { - + public void generate() throws ProcessingException { try { - - String mergedResult = ""; - - SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); - - // should contain the period - String queryString = request.getQueryString(); - String period = request.getParameter("period"); - - if(PIWIK_API_MODE==null || PIWIK_API_MODE.equals("direct")) { - - Calendar cal = Calendar.getInstance(); - // default start and end data - Date startDate = df.parse("2014-01-01"); - Date endDate = cal.getTime(); - - if(request.getParameter("date")!=null) { - String sdate = request.getParameter("date"); - String edate = request.getParameter("date"); - if(sdate.length()==4) { - sdate += "-01-01"; - edate += "-12-31"; - } else - if(sdate.length()==7) { - cal.set(Calendar.YEAR, Integer.parseInt(sdate.substring(0,4))); - cal.set(Calendar.MONTH, Integer.parseInt(sdate.substring(5,7))-1); - cal.set(Calendar.DATE, cal.getActualMaximum(Calendar.DATE)); - sdate += "-01"; - edate = df.format(cal.getTime()); - } - startDate = df.parse(sdate); - endDate = df.parse(edate); - } - - String dspaceURL = ConfigurationManager.getProperty("dspace.url"); - - String urlParams = - "&date=" + df.format(startDate) + "," + df.format(endDate) - + "&period=" + period - + "&idSite=" + PIWIK_SITE_ID - + "&token_auth=" + PIWIK_AUTH_TOKEN - + "&segment=pageUrl=@" + dspaceURL + "/handle/" + item.getHandle() - + "&showColumns=label,url,nb_visits,nb_hits"; - String downloadUrlParams = - "&date=" + df.format(startDate) + "," + df.format(endDate) - + "&period=" + period - + "&idSite=" + PIWIK_DOWNLOAD_SITE_ID - + "&token_auth=" + PIWIK_AUTH_TOKEN - + "&segment=pageUrl=@" + dspaceURL + "/bitstream/handle/" + item.getHandle() - + "&showColumns=label,url,nb_visits,nb_hits"; - - - final boolean multi_requests = false; - queryString += "&token_auth=" + PIWIK_AUTH_TOKEN + "&module=API"; - String piwikApiGetQuery = "method=Actions.getPageUrls&expanded=1&flat=1"; - - if ( multi_requests ) { - - String report = PiwikHelper.readFromURL( - PIWIK_API_URL + rest + "?" + queryString + "&" + piwikApiGetQuery + urlParams - ); - String downloadReport = PiwikHelper.readFromURL( - PIWIK_API_URL + rest + "?" + queryString + "&" + piwikApiGetQuery + downloadUrlParams - ); - mergedResult = PiwikHelper.mergeJSON(report, downloadReport); - }else { - String piwikBulkApiGetQuery = "module=API&method=API.getBulkRequest&format=JSON" - + "&token_auth=" + PIWIK_AUTH_TOKEN; - - - String url0 = URLEncoder.encode(piwikApiGetQuery + urlParams, "UTF-8"); - String url1 = URLEncoder.encode(piwikApiGetQuery + downloadUrlParams, "UTF-8"); - String report = PiwikHelper.readFromURL( - PIWIK_API_URL + rest + "?" - + piwikBulkApiGetQuery - + "&urls[0]=" + url0 - + "&urls[1]=" + url1 - ); - //mergedResult = PiwikHelper.mergeJSONResults(report); - mergedResult = PiwikHelper.transformJSONResults(report); - //mergedResult = report; - - } - } else - if(PIWIK_API_MODE.equals("cached")) { - - String url = PIWIK_API_URL_CACHED + "handle?h=" + item.getHandle() + "&period=" + period; - - if(request.getParameter("date")!=null) { - String date = request.getParameter("date"); - url += "&date=" + date; - } - - mergedResult = PiwikHelper.readFromURL(url); - + mergedResult = getDataFromPiwikServer(); + } else if(PIWIK_API_MODE.equals("cached")) { + mergedResult = getDataFromLindatPiwikCacheServer(); } - out.write(mergedResult.getBytes()); out.flush(); - } catch (Exception e) { throw new ProcessingException("Unable to read piwik statisitcs", e); } - } - + } + + private String getDataFromPiwikServer() throws Exception { + String viewsURL = buildViewsURL(); + String downloadURL = buildDownloadsURL(); + String bulkApiGetRequestURL = buildBulkApiGetRequestURL(viewsURL, downloadURL); + + log.debug(String.format("Fetching data from piwik server; requesting \"%s\"", bulkApiGetRequestURL)); + + String report = PiwikHelper.readFromURL(bulkApiGetRequestURL); + return PiwikHelper.transformJSONResults(report); + } + + private String buildBulkApiGetRequestURL(String... urls){ + String piwikBulkApiGetQuery = "module=API&method=API.getBulkRequest&format=JSON" + + "&token_auth=" + PIWIK_AUTH_TOKEN; + StringBuilder sb = new StringBuilder(); + sb.append(PIWIK_API_URL) + .append(rest) + .append("?") + .append(piwikBulkApiGetQuery); + for(int i=0; ipageUrl=^https%253A%252F%252Fdivezone.net%252Fdiving + - lvl 2: pageUrl==https%253A%252F%252Fdivezone.net%252Fdiving%252Fbali; pageUrl==https%253A%252F%252Fdivezone.net%252Fdiving%252Fthailand + - lvl 1 has no url, ie. it's cummulative for the "underlying" urls + flat: + - some API functions have a parameter 'expanded', which means that the data is hierarchical. For such API function, if 'flat' is set to 1, the returned data will contain the flattened view of the table data set. The children of all first level rows will be aggregated under one row. This is useful for example to see all Custom Variables names and values at once, for example, Matomo forum user status, or to see the full URLs not broken down by directory or structure. + - this will remove the cummulative results; all rows will be of the same type (having url field) + */ + String piwikApiGetQuery = "method=Actions.getPageUrls&expanded=1&flat=1"; + String commonParams = + "&date=" + dateRange + + "&period=" + period + + "&token_auth=" + PIWIK_AUTH_TOKEN + + "&showColumns=label,url,nb_visits,nb_hits"; + return URLEncoder.encode(piwikApiGetQuery + commonParams + params, "UTF-8"); + } + + private String getDataFromLindatPiwikCacheServer() throws IOException { + String period = request.getParameter("period"); + String url = PIWIK_API_URL_CACHED + "handle?h=" + item.getHandle() + "&period=" + period; + + if(request.getParameter("date")!=null) { + String date = request.getParameter("date"); + url += "&date=" + date; + } + + return PiwikHelper.readFromURL(url); + } + + private static class DateRange { + private static final SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); + + private final Date startDate; + private final Date endDate; + + private DateRange(Date startDate, Date endDate){ + this.startDate = startDate; + this.endDate = endDate; + } + + private static DateRange fromDateString(String date) throws ParseException { + Date startDate; + Date endDate; + if(date!=null) { + String sdate = date; + String edate = date; + if(sdate.length()==4) { + sdate += "-01-01"; + edate += "-12-31"; + } else if(sdate.length()==7) { + Calendar cal = Calendar.getInstance(); + cal.set(Calendar.YEAR, Integer.parseInt(sdate.substring(0,4))); + cal.set(Calendar.MONTH, Integer.parseInt(sdate.substring(5,7))-1); + cal.set(Calendar.DATE, cal.getActualMaximum(Calendar.DATE)); + sdate += "-01"; + edate = df.format(cal.getTime()); + } + startDate = df.parse(sdate); + endDate = df.parse(edate); + } else { + // default start and end data + startDate = df.parse("2014-01-01"); + endDate = Calendar.getInstance().getTime(); + } + return new DateRange(startDate, endDate); + } + + public String getFormattedStartDate(){ + return df.format(startDate); + } + + public String getFormattedEndDate(){ + return df.format(endDate); + } + + @Override + public String toString(){ + return getFormattedStartDate() + "," + getFormattedEndDate(); + } + + } + } From b486de774f963951fe0f416ad4cb12d57a43cbb2 Mon Sep 17 00:00:00 2001 From: Ondra Kosarko Date: Wed, 30 Dec 2020 23:33:10 +0100 Subject: [PATCH 02/19] direct mode as default and debug logging --- .../java/cz/cuni/mff/ufal/PiwikStatisticsReader.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java index 4b05bfecddfe..8a6487803e4f 100644 --- a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java +++ b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java @@ -108,10 +108,13 @@ public void setup(SourceResolver resolver, Map objectModel, String src, Paramete public void generate() throws ProcessingException { try { String mergedResult = ""; - if(PIWIK_API_MODE==null || PIWIK_API_MODE.equals("direct")) { - mergedResult = getDataFromPiwikServer(); - } else if(PIWIK_API_MODE.equals("cached")) { + if(PIWIK_API_MODE.equals("cached")) { + log.debug("========CACHED MODE"); mergedResult = getDataFromLindatPiwikCacheServer(); + } else { + // direct mode as default + log.debug("========DIRECT MODE"); + mergedResult = getDataFromPiwikServer(); } out.write(mergedResult.getBytes()); out.flush(); From 6645038590f7d3b0a62d29590c8875a1e36f74c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Thu, 31 Dec 2020 00:14:56 +0100 Subject: [PATCH 03/19] use filter_pattern and filter_column --- .../cuni/mff/ufal/PiwikStatisticsReader.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java index 8a6487803e4f..a8e3ccd8a4fc 100644 --- a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java +++ b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java @@ -153,21 +153,15 @@ private String buildBulkApiGetRequestURL(String... urls){ private String buildViewsURL() throws UnsupportedEncodingException, ParseException { String dspaceURL = ConfigurationManager.getProperty("dspace.url"); - String params = - "&idSite=" + PIWIK_SITE_ID - + "&segment=pageUrl=@" + dspaceURL + "/handle/" + item.getHandle(); - return buildURL(params); + return buildURL(PIWIK_SITE_ID, dspaceURL + "/handle/" + item.getHandle()); } private String buildDownloadsURL() throws UnsupportedEncodingException, ParseException { String dspaceURL = ConfigurationManager.getProperty("dspace.url"); - String params = - "&idSite=" + PIWIK_DOWNLOAD_SITE_ID - + "&segment=pageUrl=@" + dspaceURL + "/bitstream/handle/" + item.getHandle(); - return buildURL(params); + return buildURL(PIWIK_DOWNLOAD_SITE_ID, dspaceURL + "/bitstream/handle/" + item.getHandle()); } - private String buildURL(String params) throws UnsupportedEncodingException, ParseException { + private String buildURL(String siteID, String filterPattern) throws UnsupportedEncodingException, ParseException { // should contain the period String period = request.getParameter("period"); String dateRange = DateRange.fromDateString(request.getParameter("date")).toString(); @@ -189,12 +183,17 @@ private String buildURL(String params) throws UnsupportedEncodingException, Pars - this will remove the cummulative results; all rows will be of the same type (having url field) */ String piwikApiGetQuery = "method=Actions.getPageUrls&expanded=1&flat=1"; - String commonParams = + String params = "&date=" + dateRange + "&period=" + period + "&token_auth=" + PIWIK_AUTH_TOKEN - + "&showColumns=label,url,nb_visits,nb_hits"; - return URLEncoder.encode(piwikApiGetQuery + commonParams + params, "UTF-8"); + + "&showColumns=label,url,nb_visits,nb_hits" + // don't want to handle paging + + "&filter_limit=-1" + + "&filter_column=url" + + "&filter_pattern=" + filterPattern + + "&idSite=" + siteID; + return URLEncoder.encode(piwikApiGetQuery + params, "UTF-8"); } private String getDataFromLindatPiwikCacheServer() throws IOException { From 775f57da896bd52d3f3ccfe7513cf50253690602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Mon, 15 Feb 2021 14:27:53 +0100 Subject: [PATCH 04/19] code cleanup + debug logs --- .../main/java/cz/cuni/mff/ufal/PiwikHelper.java | 10 ++++++++++ .../cz/cuni/mff/ufal/PiwikStatisticsReader.java | 14 ++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java index 2021289b8d8b..c465bc08fec1 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java @@ -27,6 +27,8 @@ import org.xml.sax.InputSource; public class PiwikHelper { + private static org.apache.log4j.Logger log = Logger.getLogger(PiwikHelper.class); + private static final String[] requiredStatsNames = {"nb_visits", "nb_uniq_visitors", "nb_pageviews", "nb_uniq_pageviews", "nb_downloads", "nb_uniq_downloads"}; @@ -396,12 +398,20 @@ public static String readFromURL(String url) throws IOException { old_value = System.getProperty("jsse.enableSNIExtension"); System.setProperty("jsse.enableSNIExtension", "false"); + long fetchingStart = 0L; + if(log.isDebugEnabled()){ + fetchingStart = System.currentTimeMillis(); + } BufferedReader in = new BufferedReader(new InputStreamReader(widget.openStream())); String inputLine; while ((inputLine = in.readLine()) != null) { output.append(inputLine).append("\n"); } in.close(); + if(log.isDebugEnabled()) { + long fetchingEnd = System.currentTimeMillis(); + log.debug(String.format("PiwikHelper fetching took %s", fetchingEnd - fetchingStart)); + } }finally { //true is the default http://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html old_value = (old_value == null) ? "true" : old_value; diff --git a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java index a8e3ccd8a4fc..1b6f1088826c 100644 --- a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java +++ b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java @@ -45,16 +45,16 @@ public class PiwikStatisticsReader extends AbstractReader { private boolean isSpider = false; private String rest = ""; - - - /** Piwik configurations */ + + private String dspaceURL = ConfigurationManager.getProperty("dspace.url"); + + /** Piwik configurations */ private static final String PIWIK_API_MODE = ConfigurationManager.getProperty("lr", "lr.statistics.api.mode"); private static final String PIWIK_API_URL = ConfigurationManager.getProperty("lr", "lr.statistics.api.url"); private static final String PIWIK_API_URL_CACHED = ConfigurationManager.getProperty("lr", "lr.statistics.api.cached.url"); private static final String PIWIK_AUTH_TOKEN = ConfigurationManager.getProperty("lr", "lr.statistics.api.auth.token"); private static final String PIWIK_SITE_ID = ConfigurationManager.getProperty("lr", "lr.statistics.api.site_id"); private static final String PIWIK_DOWNLOAD_SITE_ID = ConfigurationManager.getProperty("lr", "lr.tracker.bitstream.site_id"); - //private static final int PIWIK_SHOW_LAST_N_DAYS = ConfigurationManager.getIntProperty("lr", "lr.statistics.show_last_n", 7); /** * Set up the PiwikStatisticsReader @@ -93,10 +93,6 @@ public void setup(SourceResolver resolver, Map objectModel, String src, Paramete throw new AuthorizeException(); } - /*if(!(AuthorizeManager.isAdmin(context) || item.getSubmitter().getID()==eperson.getID())) { - throw new AuthorizeException(); - }*/ - } catch (AuthorizeException | SQLException | IllegalStateException e) { throw new ProcessingException("Unable to read piwik statistics", e); } @@ -152,12 +148,10 @@ private String buildBulkApiGetRequestURL(String... urls){ } private String buildViewsURL() throws UnsupportedEncodingException, ParseException { - String dspaceURL = ConfigurationManager.getProperty("dspace.url"); return buildURL(PIWIK_SITE_ID, dspaceURL + "/handle/" + item.getHandle()); } private String buildDownloadsURL() throws UnsupportedEncodingException, ParseException { - String dspaceURL = ConfigurationManager.getProperty("dspace.url"); return buildURL(PIWIK_DOWNLOAD_SITE_ID, dspaceURL + "/bitstream/handle/" + item.getHandle()); } From 867cb9d3de053ca3529755446241899f5ce70009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Fri, 19 Feb 2021 16:59:18 +0100 Subject: [PATCH 05/19] PiwikStatisticsReader (json api) without segments --- .../java/cz/cuni/mff/ufal/PiwikHelper.java | 68 +++++++++++-------- .../cz/cuni/mff/ufal/PiwikHelperTest.java | 32 +++++++++ .../cuni/mff/ufal/PiwikStatisticsReader.java | 55 +++++++++------ 3 files changed, 107 insertions(+), 48 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java index c465bc08fec1..9f9a264ccbf5 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java @@ -7,6 +7,7 @@ import java.io.StringWriter; import java.io.Writer; import java.net.URL; +import java.util.Set; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -111,37 +112,27 @@ public static String mergeJSON(String report, String downloadReport) throws Exce return reportJSON.toJSONString(); } - public static String mergeJSONResults(String report) throws Exception { + /** + * + * @param keys - The order of keys should match the json object indexes + * @param report - The downloaded json + * @return + * @throws Exception + */ + public static String transformJSONResults(Set keys, String report) throws Exception { JSONParser parser = new JSONParser(); JSONArray json = (JSONArray)parser.parse(report); - JSONObject views = (JSONObject)json.get(0); - JSONObject downloads = (JSONObject)json.get(1); - JSONObject result = new JSONObject(); - - for(Object key : views.keySet()) { - JSONObject view_data = new JSONObject(); - // any valid view data? - try { - view_data = (JSONObject) views.get(key); - } catch (ClassCastException e) { + JSONObject views = null; + JSONObject downloads = null; + int i = 0; + for(String key: keys){ + if(key.toLowerCase().contains("itemview")){ + views = mergeJSONReports(views, (JSONObject)json.get(i)); + }else if(key.toLowerCase().contains("downloads")){ + downloads = mergeJSONReports(downloads, (JSONObject)json.get(i)); } - // any valid download data? - try { - JSONObject download_data = (JSONObject)downloads.get(key); - view_data.put("nb_downloads", download_data.get("nb_pageviews")); - view_data.put("nb_uniq_downloads", download_data.get("nb_uniq_pageviews")); - } catch (ClassCastException e) { - } - result.put(key, view_data); + i++; } - return result.toJSONString(); - } - - public static String transformJSONResults(String report) throws Exception { - JSONParser parser = new JSONParser(); - JSONArray json = (JSONArray)parser.parse(report); - JSONObject views = (JSONObject)json.get(0); - JSONObject downloads = (JSONObject)json.get(1); JSONObject response = new JSONObject(); JSONObject result = new JSONObject(); response.put("response", result); @@ -151,6 +142,29 @@ public static String transformJSONResults(String report) throws Exception { return response.toJSONString().replace("\\/", "/"); } + + @SuppressWarnings("unchecked") + private static JSONObject mergeJSONReports(JSONObject o1, JSONObject o2) { + if (o1 == null) { + return o2; + } else { + //just concatenate the dmy arrays, transformJSON should do the rest + Set keys = o1.keySet(); + for (String dmyKey : keys) { + if(o2.containsKey(dmyKey)){ + JSONArray a = (JSONArray)o1.get(dmyKey); + a.addAll((JSONArray)o2.get(dmyKey)); + } + } + keys = o2.keySet(); + for (String dmyKey : keys) { + if(!o1.containsKey(dmyKey)){ + o1.put(dmyKey, o2.get(dmyKey)); + } + } + return o1; + } + } public static JSONObject transformJSON(JSONObject views) { diff --git a/dspace-api/src/test/java/cz/cuni/mff/ufal/PiwikHelperTest.java b/dspace-api/src/test/java/cz/cuni/mff/ufal/PiwikHelperTest.java index d36e22f057a6..b74ffcdc1bad 100644 --- a/dspace-api/src/test/java/cz/cuni/mff/ufal/PiwikHelperTest.java +++ b/dspace-api/src/test/java/cz/cuni/mff/ufal/PiwikHelperTest.java @@ -15,6 +15,8 @@ import javax.xml.xpath.XPathFactory; import java.io.File; import java.net.URL; +import java.util.SortedSet; +import java.util.TreeSet; /** * Created by okosarko on 3.8.15. @@ -205,4 +207,34 @@ public void issue_917() throws Exception { File vrXml = new File(url.getFile()); PiwikHelper.mergeXML(FileUtils.readFileToString(vrXml), FileUtils.readFileToString(drXml)); } + + @Test + public void testTransformJSONResults() throws Exception { + SortedSet set = new TreeSet(); + set.add("1simpleItemView"); + set.add("2fullItemView"); + set.add("3downloads"); + String simple = "{\"2021-01-01\":[],\"2021-01-02\":[],\"2021-01-03\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-04\":[{\"label\":\"/1-1827\",\"nb_visits\":2,\"nb_hits\":2,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-05\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-06\":[{\"label\":\"/1-1827\",\"nb_visits\":2,\"nb_hits\":2,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-07\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":2,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-08\":[{\"label\":\"/1-1827\",\"nb_visits\":2,\"nb_hits\":2,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-09\":[],\"2021-01-10\":[],\"2021-01-11\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-12\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-13\":[{\"label\":\"/1-1827\",\"nb_visits\":3,\"nb_hits\":3,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-14\":[],\"2021-01-15\":[{\"label\":\"/1-1827\",\"nb_visits\":2,\"nb_hits\":3,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-16\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-17\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-18\":[{\"label\":\"/1-1827\",\"nb_visits\":3,\"nb_hits\":3,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-19\":[],\"2021-01-20\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-21\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-22\":[{\"label\":\"/1-1827\",\"nb_visits\":2,\"nb_hits\":3,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-23\":[],\"2021-01-24\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-25\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-26\":[{\"label\":\"/1-1827\",\"nb_visits\":3,\"nb_hits\":3,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-27\":[],\"2021-01-28\":[{\"label\":\"/1-1827\",\"nb_visits\":2,\"nb_hits\":2,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-29\":[{\"label\":\"/1-1827\",\"nb_visits\":2,\"nb_hits\":2,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-30\":[{\"label\":\"/1-1827\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827\"}],\"2021-01-31\":[]}"; + String showFull = "{\"2021-01-01\":[],\"2021-01-02\":[],\"2021-01-03\":[],\"2021-01-04\":[]," + + "\"2021-01-05\":[],\"2021-01-06\":[{\"label\":\"/1-1827?show=full\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827?show=full\"}],\"2021-01-07\":[{\"label\":\"/1-1827?show=full\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827?show=full\"}],\"2021-01-08\":[],\"2021-01-09\":[],\"2021-01-10\":[],\"2021-01-11\":[],\"2021-01-12\":[],\"2021-01-13\":[],\"2021-01-14\":[],\"2021-01-15\":[],\"2021-01-16\":[],\"2021-01-17\":[],\"2021-01-18\":[],\"2021-01-19\":[],\"2021-01-20\":[],\"2021-01-21\":[],\"2021-01-22\":[{\"label\":\"/1-1827?show=full\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827?show=full\"}],\"2021-01-23\":[],\"2021-01-24\":[{\"label\":\"/1-1827?show=full\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827?show=full\"}],\"2021-01-25\":[],\"2021-01-26\":[],\"2021-01-27\":[],\"2021-01-28\":[{\"label\":\"/1-1827?show=full\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827?show=full\"}],\"2021-01-29\":[{\"label\":\"/1-1827?show=full\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827?show=full\"}],\"2021-01-30\":[{\"label\":\"/1-1827?show=full\",\"nb_visits\":1,\"nb_hits\":1,\"url\":\"https://lindat.mff.cuni.cz/repository/xmlui/handle/11234/1-1827?show=full\"}],\"2021-01-31\":[]}\n"; + String json = String.format("[%s, %s, {}]", simple, showFull); + String result = PiwikHelper.transformJSONResults(set, json); + System.out.println(result); + final JSONParser parser = new JSONParser(); + final JSONObject jsonResult = (JSONObject)parser.parse(result); + JSONObject response = (JSONObject) jsonResult.get("response"); + JSONObject downloads = (JSONObject) response.get("downloads"); + assertTrue(downloads.isEmpty()); + JSONObject views = (JSONObject) response.get("views"); + final long totalHits = + Long.parseLong( + ((JSONObject) + ((JSONObject) + ((JSONObject) + ((JSONObject) + views.get("total")).get("2021")).get("1")).get("29")).get("nb_hits").toString()); + assertEquals(2+1, totalHits); + + } + } \ No newline at end of file diff --git a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java index 1b6f1088826c..5b151633744f 100644 --- a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java +++ b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java @@ -5,9 +5,7 @@ import java.sql.SQLException; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.Map; +import java.util.*; import org.apache.avalon.framework.parameters.Parameters; import org.apache.cocoon.ProcessingException; @@ -120,17 +118,17 @@ public void generate() throws ProcessingException { } private String getDataFromPiwikServer() throws Exception { - String viewsURL = buildViewsURL(); - String downloadURL = buildDownloadsURL(); - String bulkApiGetRequestURL = buildBulkApiGetRequestURL(viewsURL, downloadURL); + SortedMap urls = buildViewsURL(); + urls.put("downloads", buildDownloadsURL()); + String bulkApiGetRequestURL = buildBulkApiGetRequestURL(urls); log.debug(String.format("Fetching data from piwik server; requesting \"%s\"", bulkApiGetRequestURL)); String report = PiwikHelper.readFromURL(bulkApiGetRequestURL); - return PiwikHelper.transformJSONResults(report); + return PiwikHelper.transformJSONResults(urls.keySet(), report); } - private String buildBulkApiGetRequestURL(String... urls){ + private String buildBulkApiGetRequestURL(SortedMap urls){ String piwikBulkApiGetQuery = "module=API&method=API.getBulkRequest&format=JSON" + "&token_auth=" + PIWIK_AUTH_TOKEN; StringBuilder sb = new StringBuilder(); @@ -138,24 +136,41 @@ private String buildBulkApiGetRequestURL(String... urls){ .append(rest) .append("?") .append(piwikBulkApiGetQuery); - for(int i=0; i buildViewsURL() throws UnsupportedEncodingException, ParseException { + // use Actions.getPageUrl; call it twice; once with ?show=full + String paramsFmt = "method=Actions.getPageUrl&pageUrl=%s"; + String summaryItemView = buildURL(PIWIK_SITE_ID, + String.format(paramsFmt, URLEncoder.encode(dspaceURL + "/handle/" + item.getHandle(), "UTF-8"))); + String fullItemView = buildURL(PIWIK_SITE_ID, + String.format(paramsFmt, URLEncoder.encode( + dspaceURL + "/handle" + "/" + item.getHandle() + "?show=full", "UTF-8"))); + SortedMap ret = new TreeMap<>(); + ret.put("summaryItemView", summaryItemView); + ret.put("fullItemView", fullItemView); + return ret; } private String buildDownloadsURL() throws UnsupportedEncodingException, ParseException { - return buildURL(PIWIK_DOWNLOAD_SITE_ID, dspaceURL + "/bitstream/handle/" + item.getHandle()); + String filterPattern = URLEncoder.encode(dspaceURL + "/bitstream/handle/" + item.getHandle(), "UTF-8"); + String params = + "method=Actions.getPageUrls" + + "&expanded=1&flat=1" + + "&filter_column=url" + + "&filter_pattern=" + filterPattern; + return buildURL(PIWIK_DOWNLOAD_SITE_ID, params); } - private String buildURL(String siteID, String filterPattern) throws UnsupportedEncodingException, ParseException { + private String buildURL(String siteID, String specificParams) throws UnsupportedEncodingException, ParseException { // should contain the period String period = request.getParameter("period"); String dateRange = DateRange.fromDateString(request.getParameter("date")).toString(); @@ -176,18 +191,16 @@ private String buildURL(String siteID, String filterPattern) throws UnsupportedE - some API functions have a parameter 'expanded', which means that the data is hierarchical. For such API function, if 'flat' is set to 1, the returned data will contain the flattened view of the table data set. The children of all first level rows will be aggregated under one row. This is useful for example to see all Custom Variables names and values at once, for example, Matomo forum user status, or to see the full URLs not broken down by directory or structure. - this will remove the cummulative results; all rows will be of the same type (having url field) */ - String piwikApiGetQuery = "method=Actions.getPageUrls&expanded=1&flat=1"; String params = - "&date=" + dateRange + specificParams + + "&date=" + dateRange + "&period=" + period + "&token_auth=" + PIWIK_AUTH_TOKEN + "&showColumns=label,url,nb_visits,nb_hits" - // don't want to handle paging + // don't want to handle "paging" (summary views) + "&filter_limit=-1" - + "&filter_column=url" - + "&filter_pattern=" + filterPattern + "&idSite=" + siteID; - return URLEncoder.encode(piwikApiGetQuery + params, "UTF-8"); + return URLEncoder.encode(params, "UTF-8"); } private String getDataFromLindatPiwikCacheServer() throws IOException { From 22221136cd855a597059bf8eac275600c7ee119b Mon Sep 17 00:00:00 2001 From: Ondrej Kosarko Date: Fri, 28 Jan 2022 15:33:48 +0100 Subject: [PATCH 06/19] Code reorg - PiwikPDFExporter uses the same source as PiwikStatisticsReader. In addition it can now pull the handle-country data from the cache server. --- .../java/cz/cuni/mff/ufal/PiwikHelper.java | 385 +++++++++++++----- .../cz/cuni/mff/ufal/PiwikPDFExporter.java | 249 ++++------- .../cz/cuni/mff/ufal/PiwikHelperTest.java | 185 --------- .../test/resources/piwik/download_report.json | 1 - .../test/resources/piwik/download_report.xml | 221 ---------- .../piwik/issue_917_download_report.xml | 45 -- .../resources/piwik/issue_917_view_report.xml | 45 -- .../src/test/resources/piwik/view_report.json | 1 - .../src/test/resources/piwik/view_report.xml | 289 ------------- .../cuni/mff/ufal/PiwikStatisticsReader.java | 175 +------- 10 files changed, 364 insertions(+), 1232 deletions(-) delete mode 100644 dspace-api/src/test/resources/piwik/download_report.json delete mode 100644 dspace-api/src/test/resources/piwik/download_report.xml delete mode 100644 dspace-api/src/test/resources/piwik/issue_917_download_report.xml delete mode 100644 dspace-api/src/test/resources/piwik/issue_917_view_report.xml delete mode 100644 dspace-api/src/test/resources/piwik/view_report.json delete mode 100644 dspace-api/src/test/resources/piwik/view_report.xml diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java index 9f9a264ccbf5..3154f90bc26e 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java @@ -1,117 +1,39 @@ package cz.cuni.mff.ufal; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.StringReader; -import java.io.StringWriter; -import java.io.Writer; +import java.io.*; import java.net.URL; -import java.util.Set; +import java.net.URLEncoder; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; +import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.*; +import org.apache.tools.ant.filters.StringInputStream; +import org.dspace.content.Item; +import org.dspace.core.ConfigurationManager; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.w3c.dom.Document; import org.w3c.dom.Element; -import org.w3c.dom.Node; import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; public class PiwikHelper { private static org.apache.log4j.Logger log = Logger.getLogger(PiwikHelper.class); - private static final String[] requiredStatsNames = {"nb_visits", "nb_uniq_visitors", "nb_pageviews", - "nb_uniq_pageviews", "nb_downloads", "nb_uniq_downloads"}; + private String dspaceURL = ConfigurationManager.getProperty("dspace.url"); + /** Piwik configurations */ + private static final String PIWIK_API_MODE = ConfigurationManager.getProperty("lr", "lr.statistics.api.mode"); + private static final String PIWIK_API_URL = ConfigurationManager.getProperty("lr", "lr.statistics.api.url"); + private static final String PIWIK_API_URL_CACHED = ConfigurationManager.getProperty("lr", "lr.statistics.api.cached.url"); + private static final String PIWIK_AUTH_TOKEN = ConfigurationManager.getProperty("lr", "lr.statistics.api.auth.token"); + private static final String PIWIK_SITE_ID = ConfigurationManager.getProperty("lr", "lr.statistics.api.site_id"); + private static final String PIWIK_DOWNLOAD_SITE_ID = ConfigurationManager.getProperty("lr", "lr.tracker.bitstream.site_id"); - - public static String mergeXML(String report, String downloadReport) throws Exception { - /** - * add page views from downloadReport as nb_downloads to report - */ - Document reportDoc = loadXMLFromString(report); - Document downloadReportDoc = loadXMLFromString(downloadReport); - - XPathFactory xpathFactory = XPathFactory.newInstance(); - XPath xpath = xpathFactory.newXPath(); - XPathExpression resExpr = xpath.compile("//result"); - XPathExpression downExpr = xpath.compile("./nb_downloads"); - XPathExpression uniqDownExpr = xpath.compile("./nb_uniq_downloads"); - - NodeList rRows = (NodeList)resExpr.evaluate(reportDoc, XPathConstants.NODESET); - - for(int i=0;i getCountryData() throws Exception{ + if(PIWIK_API_MODE.equals("cached")) { + log.debug("========CACHED MODE"); + return getCountryDataFromLindatPiwikCacheServer(); + } else { + // direct mode as default + log.debug("========DIRECT MODE"); + return getCountryDataFromPiwik(); + } + } + + private List getCountryDataFromLindatPiwikCacheServer() throws Exception { + String url = PIWIK_API_URL_CACHED + "handle?h=" + item.getHandle() + "&period=month&country=true"; + + if(date!=null) { + url += "&date=" + date; + } + JSONParser parser = new JSONParser(); + JSONObject countriesReport = (JSONObject)parser.parse(PiwikHelper.readFromURL(url)); + + List result = new ArrayList<>(10); + + for (Object key : countriesReport.keySet() ) { + if(key instanceof String){ + String date = (String) key; + JSONArray countryData = (JSONArray) countriesReport.get(date); + for(Object country: countryData){ + if(country instanceof JSONObject){ + JSONObject c = (JSONObject) country; + String label = (String) c.get("label"); + String count = ((Integer) c.get("nb_visits")).toString(); + result.add(new String[]{label, count}); + } + } + // expecting only one date key + break; + } + } + return result; + } + + private List getCountryDataFromPiwik() throws Exception { + + String countryReportURL = PIWIK_API_URL + "index.php" + + "?module=API" + + "&method=UserCountry.getCountry" + + "&idSite=" + PIWIK_SITE_ID + + "&period=month" + + "&date=" + date + + "&expanded=1" + + "&token_auth=" + PIWIK_AUTH_TOKEN + + "&filter_limit=10" + + "&format=xml" + + "&segment=pageUrl=@" + item.getHandle(); + + + String xml = PiwikHelper.readFromURL(countryReportURL); + + Document doc = parseXML(xml); + + if(doc==null) throw new Exception("Unable to parse XML"); + + List data = new ArrayList(); + + XPath xPath = XPathFactory.newInstance().newXPath(); + XPathExpression eachResultNode = xPath.compile("//result/row"); + + NodeList results = (NodeList)eachResultNode.evaluate(doc, XPathConstants.NODESET); + + for(int i=0;i urls){ + String piwikBulkApiGetQuery = "module=API&method=API.getBulkRequest&format=JSON" + + "&token_auth=" + PIWIK_AUTH_TOKEN; + StringBuilder sb = new StringBuilder(); + sb.append(PIWIK_API_URL) + .append(rest) + .append("?") + .append(piwikBulkApiGetQuery); + int i = 0; + for(String url : urls.values()){ + sb.append("&urls[") + .append(i++) + .append("]=") + .append(url); + } + return sb.toString(); + } + + private SortedMap buildViewsURL() throws UnsupportedEncodingException, ParseException { + // use Actions.getPageUrl; call it twice; once with ?show=full + String paramsFmt = "method=Actions.getPageUrl&pageUrl=%s"; + String summaryItemView = buildURL(PIWIK_SITE_ID, + String.format(paramsFmt, URLEncoder.encode(dspaceURL + "/handle/" + item.getHandle(), "UTF-8"))); + String fullItemView = buildURL(PIWIK_SITE_ID, + String.format(paramsFmt, URLEncoder.encode( + dspaceURL + "/handle" + "/" + item.getHandle() + "?show=full", "UTF-8"))); + SortedMap ret = new TreeMap<>(); + ret.put("summaryItemView", summaryItemView); + ret.put("fullItemView", fullItemView); + return ret; + } + + private String buildDownloadsURL() throws UnsupportedEncodingException, ParseException { + String filterPattern = URLEncoder.encode(dspaceURL + "/bitstream/handle/" + item.getHandle(), "UTF-8"); + String params = + "method=Actions.getPageUrls" + + "&expanded=1&flat=1" + + "&filter_column=url" + + "&filter_pattern=" + filterPattern; + return buildURL(PIWIK_DOWNLOAD_SITE_ID, params); + } + + private String buildURL(String siteID, String specificParams) throws UnsupportedEncodingException, + ParseException { + String dateRange = DateRange.fromDateString(date).toString(); + + /* + The Actions API lets you request reports for all your Visitor Actions: Page URLs, Page titles, Events, + Content Tracking, File Downloads and Clicks on external websites. + Actions.getPageUrls: + - stats(nb_visits, nb_hits, etc) per url, in given date broken down by period + expanded: + - some API functions have a parameter 'expanded'. If 'expanded' is set to 1, the returned data will contain the first level results, as well as all sub-tables. + - basically fetches subtable (if present as idsubdatatable) + - eg. urls broken down by directory structure: + - lvl 1: pageUrl=^https%253A%252F%252Fdivezone.net%252Fdiving + - lvl 2: pageUrl==https%253A%252F%252Fdivezone.net%252Fdiving%252Fbali; pageUrl==https%253A%252F%252Fdivezone.net%252Fdiving%252Fthailand + - lvl 1 has no url, ie. it's cummulative for the "underlying" urls + flat: + - some API functions have a parameter 'expanded', which means that the data is hierarchical. For such API function, if 'flat' is set to 1, the returned data will contain the flattened view of the table data set. The children of all first level rows will be aggregated under one row. This is useful for example to see all Custom Variables names and values at once, for example, Matomo forum user status, or to see the full URLs not broken down by directory or structure. + - this will remove the cummulative results; all rows will be of the same type (having url field) + */ + String params = + specificParams + + "&date=" + dateRange + + "&period=" + period + + "&token_auth=" + PIWIK_AUTH_TOKEN + + "&showColumns=label,url,nb_visits,nb_hits" + // don't want to handle "paging" (summary views) + + "&filter_limit=-1" + + "&idSite=" + siteID; + return URLEncoder.encode(params, "UTF-8"); + } + + private static class DateRange { + private static final SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); + + private final Date startDate; + private final Date endDate; + + private DateRange(Date startDate, Date endDate){ + this.startDate = startDate; + this.endDate = endDate; + } + + private static DateRange fromDateString(String date) throws ParseException { + Date startDate; + Date endDate; + if(date!=null) { + String sdate = date; + String edate = date; + if(sdate.length()==4) { + sdate += "-01-01"; + edate += "-12-31"; + } else if(sdate.length()==7) { + Calendar cal = Calendar.getInstance(); + cal.set(Calendar.YEAR, Integer.parseInt(sdate.substring(0,4))); + cal.set(Calendar.MONTH, Integer.parseInt(sdate.substring(5,7))-1); + cal.set(Calendar.DATE, cal.getActualMaximum(Calendar.DATE)); + sdate += "-01"; + edate = df.format(cal.getTime()); + } + startDate = df.parse(sdate); + endDate = df.parse(edate); + } else { + // default start and end data + startDate = df.parse("2014-01-01"); + endDate = Calendar.getInstance().getTime(); + } + return new DateRange(startDate, endDate); + } + + public String getFormattedStartDate(){ + return df.format(startDate); + } + + public String getFormattedEndDate(){ + return df.format(endDate); + } + + @Override + public String toString(){ + return getFormattedStartDate() + "," + getFormattedEndDate(); + } + } - } diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index e2c0d18daf74..a086020460de 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -6,15 +6,12 @@ import java.awt.BasicStroke; import java.awt.geom.Ellipse2D; import java.awt.geom.Rectangle2D; -import java.io.BufferedReader; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.net.URL; import java.sql.SQLException; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.HashMap; @@ -22,22 +19,12 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.ResourceBundle; import javax.mail.MessagingException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathExpression; -import javax.xml.xpath.XPathFactory; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; -import org.apache.tools.ant.filters.StringInputStream; import org.dspace.content.Item; -import org.dspace.content.ItemIterator; import org.dspace.core.ConfigurationManager; import org.dspace.core.Context; import org.dspace.core.Email; @@ -57,10 +44,8 @@ import org.jfree.data.time.TimeSeriesCollection; import org.jfree.ui.RectangleEdge; import org.jfree.ui.RectangleInsets; -import org.jfree.util.ShapeUtilities; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.NodeList; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; import com.itextpdf.awt.PdfGraphics2D; import com.itextpdf.text.BaseColor; @@ -87,10 +72,6 @@ public class PiwikPDFExporter { private static String PIWIK_REPORTS_OUTPUT_PATH; /** Piwik configurations */ - private static String PIWIK_API_URL; - private static String PIWIK_AUTH_TOKEN; - private static String PIWIK_SITE_ID; - private static String PIWIK_DOWNLOAD_SITE_ID; private static boolean PIWIK_KEEP_REPORTS; private static String LINDAT_LOGO; @@ -111,10 +92,6 @@ public static void main(String args[]) throws Exception { public static void initialize() { PIWIK_REPORTS_OUTPUT_PATH = ConfigurationManager.getProperty("lr", "lr.statistics.report.path"); PIWIK_KEEP_REPORTS = ConfigurationManager.getBooleanProperty("lr", "lr.statistics.keep.reports", true); - PIWIK_API_URL = ConfigurationManager.getProperty("lr", "lr.statistics.api.url"); - PIWIK_AUTH_TOKEN = ConfigurationManager.getProperty("lr", "lr.statistics.api.auth.token"); - PIWIK_SITE_ID = ConfigurationManager.getProperty("lr", "lr.statistics.api.site_id"); - PIWIK_DOWNLOAD_SITE_ID = ConfigurationManager.getProperty("lr", "lr.tracker.bitstream.site_id"); LINDAT_LOGO = ConfigurationManager.getProperty("lr", "lr.lindat.logo.mono"); } @@ -180,8 +157,7 @@ public static void sendEmail(Context context, EPerson to, Item item) throws IOEx Locale supportedLocale = I18nUtil.getEPersonLocale(to); String itemTitle = item.getMetadata("dc", "title", null, Item.ANY)[0].value; - String hdlURL = item.getMetadata("dc", "identifier", "uri", Item.ANY)[0].value; - + Email email = Email.getEmail(I18nUtil.getEmailFilename(supportedLocale, "piwik_report")); email.addArgument(itemTitle); email.addArgument(to.getName()); @@ -196,139 +172,91 @@ public static void generateItemReport(Item item) throws Exception { cal.add(Calendar.MONTH, -1); cal.set(Calendar.DATE, 1); Date firstDay = cal.getTime(); - cal.set(Calendar.DATE, cal.getActualMaximum(Calendar.DAY_OF_MONTH)); - Date lastDay = cal.getTime(); - - - String viewsReportURL = PIWIK_API_URL + "index.php" - + "?module=API" - + "&method=API.get" - + "&idSite=" + PIWIK_SITE_ID - + "&period=day" - + "&date=" + inputDateFormat.format(firstDay) + "," + inputDateFormat.format(lastDay) - + "&token_auth=" + PIWIK_AUTH_TOKEN - + "&format=xml" - + "&segment=pageUrl=@" + item.getHandle(); - - String downloadReportURL = PIWIK_API_URL + "index.php" - + "?module=API" - + "&method=API.get" - + "&idSite=" + PIWIK_DOWNLOAD_SITE_ID - + "&period=day" - + "&date=" + inputDateFormat.format(firstDay) + "," + inputDateFormat.format(lastDay) - + "&token_auth=" + PIWIK_AUTH_TOKEN - + "&format=xml" - + "&segment=pageUrl=@" + item.getHandle(); - - String countryReportURL = PIWIK_API_URL + "index.php" - + "?module=API" - + "&method=UserCountry.getCountry" - + "&idSite=" + PIWIK_SITE_ID - + "&period=month" - + "&date=" + inputDateFormat.format(firstDay) - + "&expanded=1" - + "&token_auth=" + PIWIK_AUTH_TOKEN - + "&filter_limit=10" - + "&format=xml" - + "&segment=pageUrl=@" + item.getHandle(); - - - String viewsXML = PiwikHelper.readFromURL(viewsReportURL); - String downloadXML = PiwikHelper.readFromURL(downloadReportURL); - - viewsXML = PiwikHelper.mergeXML(viewsXML, downloadXML); - String countriesXML = PiwikHelper.readFromURL(countryReportURL); - + + // use just the yyyy-MM part for the date param + PiwikHelper piwikHelper = new PiwikHelper("day", inputDateFormat.format(firstDay).substring(0,7), item, ""); + + String json = piwikHelper.getDataAsJsonString(); + JSONParser parser = new JSONParser(); + JSONObject report = (JSONObject) parser.parse(json); + + + Map summary = new HashMap(); - JFreeChart viewsChart = createViewsChart(viewsXML, summary); - List countryData = getCountryData(countriesXML); + JFreeChart viewsChart = createViewsChart(report, summary); + List countryData = piwikHelper.getCountryData(); geneartePDF(item, firstDay, viewsChart, summary, countryData); } - public static List getCountryData(String xml) throws Exception { - - Document doc = parseXML(xml); - - if(doc==null) throw new Exception("Unable to parse XML"); - - List data = new ArrayList(); - - XPath xPath = XPathFactory.newInstance().newXPath(); - XPathExpression eachResultNode = xPath.compile("//result/row"); - - NodeList results = (NodeList)eachResultNode.evaluate(doc, XPathConstants.NODESET); - - for(int i=0;i extractStats(JSONObject statsObject, TimeSeries series, + String keyForSeries, String[] keysForTotals){ + HashMap totals = new HashMap<>(); + for(String key: keysForTotals){ + totals.put(key, 0); } - - return data; - + + int max = Integer.MIN_VALUE; + + for(Object yo : statsObject.keySet()){ + if(yo instanceof String){ + String year = (String)yo; + JSONObject months = (JSONObject) statsObject.get(year); + for(Object mo : months.keySet()){ + if(mo instanceof String){ + String month = (String)mo; + JSONObject days = (JSONObject) months.get(month); + for(Object dayo: days.keySet()){ + if(dayo instanceof String){ + String day = (String) dayo; + JSONObject stats = (JSONObject) days.get(day); + int valueForSeries = (Integer)stats.get(keyForSeries); + if(max < valueForSeries){ + max = valueForSeries; + } + series.add(Day.parseDay(String.format("%s-%s-%s", year, month, day)), valueForSeries); + for(String key : keysForTotals){ + int valueForTotal = (Integer)stats.get(key); + int number = totals.get(key); + totals.put(key, number + valueForTotal); + } + } + } + } + } + } + } + totals.put("max", max); + return totals; } - public static JFreeChart createViewsChart(String xml, Map summary) throws Exception { + public static JFreeChart createViewsChart(JSONObject report, Map summary) throws Exception { - Document doc = parseXML(xml); - - if(doc==null) throw new Exception("Unable to parse XML"); - JFreeChart lineChart = null; - XPath xPath = XPathFactory.newInstance().newXPath(); - XPathExpression eachResultNode = xPath.compile("//result"); - - NodeList results = (NodeList)eachResultNode.evaluate(doc, XPathConstants.NODESET); - - int maxPageViews = Integer.MIN_VALUE; - TimeSeries viewsSeries = new TimeSeries("Views"); TimeSeries downloadsSeries = new TimeSeries("Downloads"); - int totalViews = 0; - int totalUniqueViews = 0; - int totalVisitors = 0; - int totalUniqueVisitors = 0; - int totalDownloads = 0; - int totalUniqueDownloads = 0; - - for(int i=0;i viewsTotals = extractStats(views, viewsSeries, "nb_hits", new String[]{"nb_hits", + "nb_uniq_pageviews", "nb_visits", "nb_uniq_visitors"}); + Map downloadsTotals = extractStats(downloads, downloadsSeries, "nb_hits", new String[]{ "nb_hits", "nb_uniq_pageviews"}); + + int maxPageViews = viewsTotals.get("max"); + + summary.put("pageviews", viewsTotals.get("nb_hits")); + summary.put("unique pageviews", viewsTotals.get("nb_uniq_pageviews")); + summary.put("visits", viewsTotals.get("nb_visits")); + // XXX IMO this number makes no sense (if there are 10 uniq visitors on 1st and 10 on 2nd, the actual number + // of uniq visitors is any number between 10 and 20, inclusive). + summary.put("unique visitors", viewsTotals.get("nb_uniq_visitors")); + summary.put("downloads", downloadsTotals.get("nb_hits")); + summary.put("unique downloads", downloadsTotals.get("nb_uniq_pageviews")); TimeSeriesCollection dataset = new TimeSeriesCollection(); dataset.addSeries(viewsSeries); @@ -469,7 +397,7 @@ public static void geneartePDF(Item item, Date date, JFreeChart viewsChart, Map< Paragraph byCountry = new Paragraph(); byCountry.setFont(FONT[5]); - byCountry.add("Visitors By Country"); + byCountry.add("Visits By Country"); PdfPCell statsC1 = new PdfPCell(); statsC1.setBorder(0); @@ -594,34 +522,5 @@ public static void geneartePDF(Item item, Date date, JFreeChart viewsChart, Map< pdf.close(); writer.close(); } - - public static Document parseXML(String xml) { - DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = null; - Document doc = null; - try { - builder = builderFactory.newDocumentBuilder(); - } catch (ParserConfigurationException e) { - e.printStackTrace(); - } - try { - doc = builder.parse(new StringInputStream(xml)); - } catch (Exception e) { - log.error(e); - } - return doc; - } - - private static String readFromURL(String url) throws IOException { - StringBuilder output = new StringBuilder(); - URL widget = new URL(url); - BufferedReader in = new BufferedReader(new InputStreamReader(widget.openStream())); - String inputLine; - while ((inputLine = in.readLine()) != null) { - output.append(inputLine).append("\n"); - } - in.close(); - return output.toString(); - } } diff --git a/dspace-api/src/test/java/cz/cuni/mff/ufal/PiwikHelperTest.java b/dspace-api/src/test/java/cz/cuni/mff/ufal/PiwikHelperTest.java index b74ffcdc1bad..bd27ff52ae39 100644 --- a/dspace-api/src/test/java/cz/cuni/mff/ufal/PiwikHelperTest.java +++ b/dspace-api/src/test/java/cz/cuni/mff/ufal/PiwikHelperTest.java @@ -23,191 +23,6 @@ */ public class PiwikHelperTest { - @Test - public void testJSONmerge() throws Exception{ - URL url = this.getClass().getResource("/piwik/view_report.json"); - File vrJson = new File(url.getFile()); - url = this.getClass().getResource("/piwik/download_report.json"); - File drJson = new File(url.getFile()); - String jsonResult = PiwikHelper.mergeJSON(FileUtils.readFileToString(vrJson), FileUtils.readFileToString(drJson)); - int mergedPVS = 0; - int mergedUPVS = 0; - int mergedDS = 0; - int mergedUDS = 0; - JSONParser parser = new JSONParser(); - JSONObject merged = (JSONObject)parser.parse(jsonResult); - for(Object key : merged.keySet()) { - if(merged.get(key) instanceof JSONObject) { - JSONObject result = (JSONObject) merged.get(key); - for (Object resultKey : result.keySet()) { - if ("nb_pageviews".equals(resultKey)) { - mergedPVS += ((Long) result.get(resultKey)).intValue(); - } else if ("nb_uniq_pageviews".equals(resultKey)) { - mergedUPVS += ((Long) result.get(resultKey)).intValue(); - } else if ("nb_downloads".equals(resultKey)) { - mergedDS += ((Long) result.get(resultKey)).intValue(); - } else if ("nb_uniq_downloads".equals(resultKey)) { - mergedUDS += ((Long) result.get(resultKey)).intValue(); - } - } - }else if(merged.get(key) instanceof JSONArray){ - JSONArray arr = (JSONArray) merged.get(key); - assertEquals("Expecting empty array", 0, arr.size()); - } - } - - int ds = 0; - int uds = 0; - int pvs = 0; - int upvs = 0; - JSONObject download = (JSONObject)parser.parse(FileUtils.readFileToString(drJson)); - for(Object key : download.keySet()) { - if(download.get(key) instanceof JSONObject) { - JSONObject result = (JSONObject) download.get(key); - for (Object resultKey : result.keySet()) { - if ("nb_pageviews".equals(resultKey)) { - ds += ((Long) result.get(resultKey)).intValue(); - } else if ("nb_uniq_pageviews".equals(resultKey)) { - uds += ((Long) result.get(resultKey)).intValue(); - } - } - }else if(download.get(key) instanceof JSONArray){ - JSONArray arr = (JSONArray) download.get(key); - assertEquals("Expecting empty array", 0, arr.size()); - } - } - - JSONObject views = (JSONObject)parser.parse(FileUtils.readFileToString(vrJson)); - for(Object key : views.keySet()) { - if(views.get(key) instanceof JSONObject) { - JSONObject result = (JSONObject) views.get(key); - for (Object resultKey : result.keySet()) { - if ("nb_pageviews".equals(resultKey)) { - pvs += ((Long) result.get(resultKey)).intValue(); - } else if ("nb_uniq_pageviews".equals(resultKey)) { - upvs += ((Long) result.get(resultKey)).intValue(); - } - } - }else if(views.get(key) instanceof JSONArray){ - JSONArray arr = (JSONArray) views.get(key); - assertEquals("Expecting empty array", 0, arr.size()); - } - } - assertEquals("Downloads not merged properly", ds, mergedDS); - assertEquals("Uniq downloads not merged properly", uds, mergedUDS); - assertEquals("Page views not merged properly", pvs, mergedPVS); - assertEquals("Uniq page views not merged properly", upvs, mergedUPVS); - - } - - @Test - public void testXMLMerge() throws Exception{ - URL url = this.getClass().getResource("/piwik/download_report.xml"); - File drXml = new File(url.getFile()); - url = this.getClass().getResource("/piwik/view_report.xml"); - File vrXml = new File(url.getFile()); - String xmlResult = PiwikHelper.mergeXML(FileUtils.readFileToString(vrXml), FileUtils.readFileToString(drXml)); - - XPathFactory xpathFactory = XPathFactory.newInstance(); - XPath xpath = xpathFactory.newXPath(); - XPathExpression resCountX = xpath.compile("count(//result)"); - XPathExpression pageViewsSumX = xpath.compile("sum(//nb_pageviews/text())"); - XPathExpression uniqPageViewsSumX = xpath.compile("sum(//nb_uniq_pageviews/text())"); - XPathExpression downSumX = xpath.compile("sum(//nb_downloads/text())"); - XPathExpression uniqDownSumX = xpath.compile("sum(//nb_uniq_downloads/text())"); - - Document merged = PiwikHelper.loadXMLFromString(xmlResult); - int mergedResCount = ((Double)resCountX.evaluate(merged, XPathConstants.NUMBER)).intValue(); - int mergedPageViewsSum = ((Double)pageViewsSumX.evaluate(merged, XPathConstants.NUMBER)).intValue(); - int mergedUniqPageViewsSum = ((Double)uniqPageViewsSumX.evaluate(merged, XPathConstants.NUMBER)).intValue(); - int mergedDownSum = ((Double)downSumX.evaluate(merged, XPathConstants.NUMBER)).intValue(); - int mergedUniqDownSum = ((Double)uniqDownSumX.evaluate(merged, XPathConstants.NUMBER)).intValue(); - - Document downloads = PiwikHelper.loadXMLFromString(FileUtils.readFileToString(drXml)); - int downSum = ((Double)pageViewsSumX.evaluate(downloads, XPathConstants.NUMBER)).intValue(); - int uniqDownSum = ((Double)uniqPageViewsSumX.evaluate(downloads, XPathConstants.NUMBER)).intValue(); - - Document views = PiwikHelper.loadXMLFromString(FileUtils.readFileToString(vrXml)); - int pageViewsSum = ((Double)pageViewsSumX.evaluate(views, XPathConstants.NUMBER)).intValue(); - int uniqPageViewsSum = ((Double)uniqPageViewsSumX.evaluate(views, XPathConstants.NUMBER)).intValue(); - - assertEquals("Downloads not merged properly", downSum, mergedDownSum); - assertEquals("Uniq downloads not merged properly", uniqDownSum, mergedUniqDownSum); - assertEquals("Page views not merged properly", pageViewsSum, mergedPageViewsSum); - assertEquals("Uniq page views not merged properly", uniqPageViewsSum, mergedUniqPageViewsSum); - } - - @Test - public void testMergeSanity() throws Exception{ - URL url = this.getClass().getResource("/piwik/download_report.xml"); - File drXml = new File(url.getFile()); - url = this.getClass().getResource("/piwik/view_report.xml"); - File vrXml = new File(url.getFile()); - url = this.getClass().getResource("/piwik/view_report.json"); - File vrJson = new File(url.getFile()); - url = this.getClass().getResource("/piwik/download_report.json"); - File drJson = new File(url.getFile()); - - String xmlResult = PiwikHelper.mergeXML(FileUtils.readFileToString(vrXml), FileUtils.readFileToString(drXml)); - String jsonResult = PiwikHelper.mergeJSON(FileUtils.readFileToString(vrJson), FileUtils.readFileToString(drJson)); - - XPathFactory xpathFactory = XPathFactory.newInstance(); - XPath xpath = xpathFactory.newXPath(); - XPathExpression resCountX = xpath.compile("count(//result)"); - XPathExpression pageViewsSumX = xpath.compile("sum(//nb_pageviews/text())"); - XPathExpression uniqPageViewsSumX = xpath.compile("sum(//nb_uniq_pageviews/text())"); - XPathExpression downSumX = xpath.compile("sum(//nb_downloads/text())"); - XPathExpression uniqDownSumX = xpath.compile("sum(//nb_uniq_downloads/text())"); - - Document doc = PiwikHelper.loadXMLFromString(xmlResult); - int xmlResCount = ((Double)resCountX.evaluate(doc, XPathConstants.NUMBER)).intValue(); - int xmlPageViewsSum = ((Double)pageViewsSumX.evaluate(doc, XPathConstants.NUMBER)).intValue(); - int xmlUniqPageViewsSum = ((Double)uniqPageViewsSumX.evaluate(doc, XPathConstants.NUMBER)).intValue(); - int xmlDownSum = ((Double)downSumX.evaluate(doc, XPathConstants.NUMBER)).intValue(); - int xmlUniqDownSum = ((Double)uniqDownSumX.evaluate(doc, XPathConstants.NUMBER)).intValue(); - - JSONParser parser = new JSONParser(); - JSONObject reportJSON = (JSONObject)parser.parse(jsonResult); - int jsonResCount = reportJSON.size(); - int jsonPageViewsSum = 0; - int jsonUniqPageViewsSum = 0; - int jsonDownSum = 0; - int jsonUniqDownSum = 0; - for(Object key : reportJSON.keySet()) { - if(reportJSON.get(key) instanceof JSONObject) { - JSONObject result = (JSONObject) reportJSON.get(key); - for (Object resultKey : result.keySet()) { - if ("nb_pageviews".equals(resultKey)) { - jsonPageViewsSum += ((Long) result.get(resultKey)).intValue(); - } else if ("nb_uniq_pageviews".equals(resultKey)) { - jsonUniqPageViewsSum += ((Long) result.get(resultKey)).intValue(); - } else if ("nb_downloads".equals(resultKey)) { - jsonDownSum += ((Long) result.get(resultKey)).intValue(); - } else if ("nb_uniq_downloads".equals(resultKey)) { - jsonUniqDownSum += ((Long) result.get(resultKey)).intValue(); - } - } - }else if(reportJSON.get(key) instanceof JSONArray){ - JSONArray arr = (JSONArray) reportJSON.get(key); - assertEquals("Expecting empty array", 0, arr.size()); - } - } - assertEquals("Result count differs", jsonResCount, xmlResCount); - assertEquals("Page views sum differs", jsonPageViewsSum, xmlPageViewsSum); - assertEquals("Uniq page views sum differs", jsonUniqPageViewsSum, xmlUniqPageViewsSum); - assertEquals("Download sum differs", jsonDownSum, xmlDownSum); - assertEquals("Uniq download sum differs", jsonUniqDownSum, xmlUniqDownSum); - } - - @Test - public void issue_917() throws Exception { - URL url = this.getClass().getResource("/piwik/issue_917_download_report.xml"); - File drXml = new File(url.getFile()); - url = this.getClass().getResource("/piwik/issue_917_view_report.xml"); - File vrXml = new File(url.getFile()); - PiwikHelper.mergeXML(FileUtils.readFileToString(vrXml), FileUtils.readFileToString(drXml)); - } - @Test public void testTransformJSONResults() throws Exception { SortedSet set = new TreeSet(); diff --git a/dspace-api/src/test/resources/piwik/download_report.json b/dspace-api/src/test/resources/piwik/download_report.json deleted file mode 100644 index 3e626ba6b20b..000000000000 --- a/dspace-api/src/test/resources/piwik/download_report.json +++ /dev/null @@ -1 +0,0 @@ -{"2015-07-27":{"nb_uniq_visitors":2,"nb_visits":2,"nb_users":0,"nb_actions":5,"max_actions":3,"bounce_count":0,"sum_visit_length":197,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_pageviews":5,"nb_uniq_pageviews":3,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"bounce_rate":"0%","nb_actions_per_visit":2.5,"avg_time_on_site":99,"avg_time_on_site_returning":0,"nb_actions_per_visit_returning":0,"bounce_rate_returning":"0%"},"2015-07-28":{"nb_uniq_visitors":10,"nb_visits":10,"nb_users":0,"nb_actions":17,"max_actions":5,"bounce_count":8,"sum_visit_length":63,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_pageviews":17,"nb_uniq_pageviews":13,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"bounce_rate":"80%","nb_actions_per_visit":1.7,"avg_time_on_site":6,"avg_time_on_site_returning":0,"nb_actions_per_visit_returning":0,"bounce_rate_returning":"0%"},"2015-07-29":{"nb_uniq_visitors":4,"nb_visits":4,"nb_users":0,"nb_actions":4,"max_actions":1,"bounce_count":4,"sum_visit_length":0,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_pageviews":4,"nb_uniq_pageviews":4,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"bounce_rate":"100%","nb_actions_per_visit":1,"avg_time_on_site":0,"avg_time_on_site_returning":0,"nb_actions_per_visit_returning":0,"bounce_rate_returning":"0%"},"2015-07-30":{"nb_uniq_visitors":5,"nb_visits":5,"nb_users":0,"nb_actions":14,"max_actions":4,"bounce_count":1,"sum_visit_length":284,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_pageviews":14,"nb_uniq_pageviews":10,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"bounce_rate":"20%","nb_actions_per_visit":2.8,"avg_time_on_site":57,"avg_time_on_site_returning":0,"nb_actions_per_visit_returning":0,"bounce_rate_returning":"0%"},"2015-07-31":{"nb_uniq_visitors":4,"nb_visits":4,"nb_users":0,"nb_actions":4,"max_actions":1,"bounce_count":4,"sum_visit_length":0,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_pageviews":4,"nb_uniq_pageviews":4,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"bounce_rate":"100%","nb_actions_per_visit":1,"avg_time_on_site":0,"avg_time_on_site_returning":0,"nb_actions_per_visit_returning":0,"bounce_rate_returning":"0%"},"2015-08-01":[],"2015-08-02":{"nb_uniq_visitors":1,"nb_visits":1,"nb_users":0,"nb_actions":1,"max_actions":1,"bounce_count":1,"sum_visit_length":0,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_pageviews":1,"nb_uniq_pageviews":1,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"bounce_rate":"100%","nb_actions_per_visit":1,"avg_time_on_site":0,"avg_time_on_site_returning":0,"nb_actions_per_visit_returning":0,"bounce_rate_returning":"0%"},"2015-08-03":{"nb_uniq_visitors":4,"nb_visits":4,"nb_users":0,"nb_actions":6,"max_actions":3,"bounce_count":3,"sum_visit_length":218,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_pageviews":6,"nb_uniq_pageviews":6,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"bounce_rate":"75%","nb_actions_per_visit":1.5,"avg_time_on_site":55,"avg_time_on_site_returning":0,"nb_actions_per_visit_returning":0,"bounce_rate_returning":"0%"}} \ No newline at end of file diff --git a/dspace-api/src/test/resources/piwik/download_report.xml b/dspace-api/src/test/resources/piwik/download_report.xml deleted file mode 100644 index c0588763d927..000000000000 --- a/dspace-api/src/test/resources/piwik/download_report.xml +++ /dev/null @@ -1,221 +0,0 @@ - - - - 2 - 2 - 0 - 5 - 3 - 0 - 197 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 5 - 3 - 0 - 0 - 0 - 0 - 0 - 0 - 0% - 2.5 - 99 - 0 - 0 - 0% - - - 10 - 10 - 0 - 17 - 5 - 8 - 63 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 17 - 13 - 0 - 0 - 0 - 0 - 0 - 0 - 80% - 1.7 - 6 - 0 - 0 - 0% - - - 4 - 4 - 0 - 4 - 1 - 4 - 0 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 4 - 4 - 0 - 0 - 0 - 0 - 0 - 0 - 100% - 1 - 0 - 0 - 0 - 0% - - - 5 - 5 - 0 - 14 - 4 - 1 - 284 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 14 - 10 - 0 - 0 - 0 - 0 - 0 - 0 - 20% - 2.8 - 57 - 0 - 0 - 0% - - - 4 - 4 - 0 - 4 - 1 - 4 - 0 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 4 - 4 - 0 - 0 - 0 - 0 - 0 - 0 - 100% - 1 - 0 - 0 - 0 - 0% - - - - 1 - 1 - 0 - 1 - 1 - 1 - 0 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 1 - 1 - 0 - 0 - 0 - 0 - 0 - 0 - 100% - 1 - 0 - 0 - 0 - 0% - - - 4 - 4 - 0 - 6 - 3 - 3 - 218 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 6 - 6 - 0 - 0 - 0 - 0 - 0 - 0 - 75% - 1.5 - 55 - 0 - 0 - 0% - - \ No newline at end of file diff --git a/dspace-api/src/test/resources/piwik/issue_917_download_report.xml b/dspace-api/src/test/resources/piwik/issue_917_download_report.xml deleted file mode 100644 index 3b15f1f7d7f0..000000000000 --- a/dspace-api/src/test/resources/piwik/issue_917_download_report.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - 0 - 0 - 0 - 0 - 0 - 0% - 0% - 0% - 0% - 0% - 0% - 0 - 0 - 0 - 0 - 0% - 0 - 0 - 0% - - - 0 - 0 - 0 - 0 - 0 - 0% - 0% - 0% - 0% - 0% - 0% - 0 - 0 - 0 - 0 - 0% - 0 - 0 - 0% - - \ No newline at end of file diff --git a/dspace-api/src/test/resources/piwik/issue_917_view_report.xml b/dspace-api/src/test/resources/piwik/issue_917_view_report.xml deleted file mode 100644 index 3b15f1f7d7f0..000000000000 --- a/dspace-api/src/test/resources/piwik/issue_917_view_report.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - 0 - 0 - 0 - 0 - 0 - 0% - 0% - 0% - 0% - 0% - 0% - 0 - 0 - 0 - 0 - 0% - 0 - 0 - 0% - - - 0 - 0 - 0 - 0 - 0 - 0% - 0% - 0% - 0% - 0% - 0% - 0 - 0 - 0 - 0 - 0% - 0 - 0 - 0% - - \ No newline at end of file diff --git a/dspace-api/src/test/resources/piwik/view_report.json b/dspace-api/src/test/resources/piwik/view_report.json deleted file mode 100644 index da8d8603214b..000000000000 --- a/dspace-api/src/test/resources/piwik/view_report.json +++ /dev/null @@ -1 +0,0 @@ -{"2015-07-27":{"nb_uniq_visitors":3,"nb_visits":3,"nb_users":0,"nb_actions":8,"max_actions":4,"bounce_count":1,"sum_visit_length":545,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_pageviews":3,"nb_uniq_pageviews":3,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"nb_hits_with_time_generation":3,"avg_time_generation":1.16,"bounce_rate":"33%","nb_actions_per_visit":2.7,"avg_time_on_site":182,"avg_time_on_site_returning":0,"nb_actions_per_visit_returning":0,"bounce_rate_returning":"0%"},"2015-07-28":{"nb_uniq_visitors":7,"nb_visits":9,"nb_users":0,"nb_actions":43,"max_actions":24,"bounce_count":4,"sum_visit_length":1949,"nb_visits_returning":5,"nb_actions_returning":13,"nb_uniq_visitors_returning":3,"nb_users_returning":0,"max_actions_returning":8,"bounce_rate_returning":"60%","nb_actions_per_visit_returning":2.6,"avg_time_on_site_returning":142,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_conversions_returning_visit":0,"nb_visits_converted_returning_visit":0,"revenue_returning_visit":0,"conversion_rate_returning_visit":"0%","nb_pageviews":10,"nb_uniq_pageviews":9,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"nb_hits_with_time_generation":10,"avg_time_generation":0.443,"bounce_rate":"44%","nb_actions_per_visit":4.8,"avg_time_on_site":217},"2015-07-29":{"nb_uniq_visitors":6,"nb_visits":6,"nb_users":0,"nb_actions":10,"max_actions":3,"bounce_count":3,"sum_visit_length":63,"nb_visits_returning":2,"nb_actions_returning":3,"nb_uniq_visitors_returning":2,"nb_users_returning":0,"max_actions_returning":2,"bounce_rate_returning":"50%","nb_actions_per_visit_returning":1.5,"avg_time_on_site_returning":5,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_conversions_returning_visit":0,"nb_visits_converted_returning_visit":0,"revenue_returning_visit":0,"conversion_rate_returning_visit":"0%","nb_pageviews":7,"nb_uniq_pageviews":6,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"nb_hits_with_time_generation":5,"avg_time_generation":0.416,"bounce_rate":"50%","nb_actions_per_visit":1.7,"avg_time_on_site":11},"2015-07-30":{"nb_uniq_visitors":7,"nb_visits":10,"nb_users":0,"nb_actions":55,"max_actions":28,"bounce_count":3,"sum_visit_length":3324,"nb_visits_returning":5,"nb_actions_returning":16,"nb_uniq_visitors_returning":3,"nb_users_returning":0,"max_actions_returning":4,"bounce_rate_returning":"20%","nb_actions_per_visit_returning":3.2,"avg_time_on_site_returning":299,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_conversions_returning_visit":0,"nb_visits_converted_returning_visit":0,"revenue_returning_visit":0,"conversion_rate_returning_visit":"0%","nb_pageviews":17,"nb_uniq_pageviews":10,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"nb_hits_with_time_generation":17,"avg_time_generation":0.708,"bounce_rate":"30%","nb_actions_per_visit":5.5,"avg_time_on_site":332},"2015-07-31":{"nb_uniq_visitors":4,"nb_visits":4,"nb_users":0,"nb_actions":6,"max_actions":2,"bounce_count":2,"sum_visit_length":25,"nb_visits_returning":1,"nb_actions_returning":2,"nb_uniq_visitors_returning":1,"nb_users_returning":0,"max_actions_returning":2,"bounce_rate_returning":"0%","nb_actions_per_visit_returning":2,"avg_time_on_site_returning":7,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_conversions_returning_visit":0,"nb_visits_converted_returning_visit":0,"revenue_returning_visit":0,"conversion_rate_returning_visit":"0%","nb_pageviews":4,"nb_uniq_pageviews":4,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"nb_hits_with_time_generation":4,"avg_time_generation":1.323,"bounce_rate":"50%","nb_actions_per_visit":1.5,"avg_time_on_site":6},"2015-08-01":[],"2015-08-02":{"nb_uniq_visitors":2,"nb_visits":2,"nb_users":0,"nb_actions":6,"max_actions":5,"bounce_count":1,"sum_visit_length":63,"nb_visits_returning":1,"nb_actions_returning":5,"nb_uniq_visitors_returning":1,"nb_users_returning":0,"max_actions_returning":5,"bounce_rate_returning":"0%","nb_actions_per_visit_returning":5,"avg_time_on_site_returning":63,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_conversions_returning_visit":0,"nb_visits_converted_returning_visit":0,"revenue_returning_visit":0,"conversion_rate_returning_visit":"0%","nb_pageviews":2,"nb_uniq_pageviews":2,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"nb_hits_with_time_generation":2,"avg_time_generation":0.375,"bounce_rate":"50%","nb_actions_per_visit":3,"avg_time_on_site":32},"2015-08-03":{"nb_uniq_visitors":8,"nb_visits":8,"nb_users":0,"nb_actions":98,"max_actions":73,"bounce_count":2,"sum_visit_length":5258,"nb_visits_returning":5,"nb_actions_returning":92,"nb_uniq_visitors_returning":5,"nb_users_returning":0,"max_actions_returning":73,"bounce_rate_returning":"20%","nb_actions_per_visit_returning":18.4,"avg_time_on_site_returning":939,"nb_conversions":0,"nb_visits_converted":0,"revenue":0,"conversion_rate":"0%","nb_conversions_new_visit":0,"nb_visits_converted_new_visit":0,"revenue_new_visit":0,"conversion_rate_new_visit":"0%","nb_conversions_returning_visit":0,"nb_visits_converted_returning_visit":0,"revenue_returning_visit":0,"conversion_rate_returning_visit":"0%","nb_pageviews":21,"nb_uniq_pageviews":11,"nb_downloads":0,"nb_uniq_downloads":0,"nb_outlinks":0,"nb_uniq_outlinks":0,"nb_searches":0,"nb_keywords":0,"nb_hits_with_time_generation":18,"avg_time_generation":0.337,"bounce_rate":"25%","nb_actions_per_visit":12.3,"avg_time_on_site":657}} \ No newline at end of file diff --git a/dspace-api/src/test/resources/piwik/view_report.xml b/dspace-api/src/test/resources/piwik/view_report.xml deleted file mode 100644 index 6a0a97044b6a..000000000000 --- a/dspace-api/src/test/resources/piwik/view_report.xml +++ /dev/null @@ -1,289 +0,0 @@ - - - - 3 - 3 - 0 - 8 - 4 - 1 - 545 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 3 - 3 - 0 - 0 - 0 - 0 - 0 - 0 - 3 - 1.16 - 33% - 2.7 - 182 - 0 - 0 - 0% - - - 7 - 9 - 0 - 43 - 24 - 4 - 1949 - 5 - 13 - 3 - 0 - 8 - 60% - 2.6 - 142 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 10 - 9 - 0 - 0 - 0 - 0 - 0 - 0 - 10 - 0.443 - 44% - 4.8 - 217 - - - 6 - 6 - 0 - 10 - 3 - 3 - 63 - 2 - 3 - 2 - 0 - 2 - 50% - 1.5 - 5 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 7 - 6 - 0 - 0 - 0 - 0 - 0 - 0 - 5 - 0.416 - 50% - 1.7 - 11 - - - 7 - 10 - 0 - 55 - 28 - 3 - 3324 - 5 - 16 - 3 - 0 - 4 - 20% - 3.2 - 299 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 17 - 10 - 0 - 0 - 0 - 0 - 0 - 0 - 17 - 0.708 - 30% - 5.5 - 332 - - - 4 - 4 - 0 - 6 - 2 - 2 - 25 - 1 - 2 - 1 - 0 - 2 - 0% - 2 - 7 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 4 - 4 - 0 - 0 - 0 - 0 - 0 - 0 - 4 - 1.323 - 50% - 1.5 - 6 - - - - 2 - 2 - 0 - 6 - 5 - 1 - 63 - 1 - 5 - 1 - 0 - 5 - 0% - 5 - 63 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 2 - 2 - 0 - 0 - 0 - 0 - 0 - 0 - 2 - 0.375 - 50% - 3 - 32 - - - 8 - 8 - 0 - 98 - 73 - 2 - 5258 - 5 - 92 - 5 - 0 - 73 - 20% - 18.4 - 939 - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 0 - 0 - 0 - 0% - 21 - 11 - 0 - 0 - 0 - 0 - 0 - 0 - 18 - 0.337 - 25% - 12.3 - 657 - - \ No newline at end of file diff --git a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java index 5b151633744f..ae7ba3ad7b92 100644 --- a/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java +++ b/dspace-xmlui/src/main/java/cz/cuni/mff/ufal/PiwikStatisticsReader.java @@ -44,16 +44,6 @@ public class PiwikStatisticsReader extends AbstractReader { private String rest = ""; - private String dspaceURL = ConfigurationManager.getProperty("dspace.url"); - - /** Piwik configurations */ - private static final String PIWIK_API_MODE = ConfigurationManager.getProperty("lr", "lr.statistics.api.mode"); - private static final String PIWIK_API_URL = ConfigurationManager.getProperty("lr", "lr.statistics.api.url"); - private static final String PIWIK_API_URL_CACHED = ConfigurationManager.getProperty("lr", "lr.statistics.api.cached.url"); - private static final String PIWIK_AUTH_TOKEN = ConfigurationManager.getProperty("lr", "lr.statistics.api.auth.token"); - private static final String PIWIK_SITE_ID = ConfigurationManager.getProperty("lr", "lr.statistics.api.site_id"); - private static final String PIWIK_DOWNLOAD_SITE_ID = ConfigurationManager.getProperty("lr", "lr.tracker.bitstream.site_id"); - /** * Set up the PiwikStatisticsReader * @@ -101,15 +91,12 @@ public void setup(SourceResolver resolver, Map objectModel, String src, Paramete @Override public void generate() throws ProcessingException { try { - String mergedResult = ""; - if(PIWIK_API_MODE.equals("cached")) { - log.debug("========CACHED MODE"); - mergedResult = getDataFromLindatPiwikCacheServer(); - } else { - // direct mode as default - log.debug("========DIRECT MODE"); - mergedResult = getDataFromPiwikServer(); - } + String period = request.getParameter("period"); + String date = request.getParameter("date"); + + PiwikHelper piwikHelper = new PiwikHelper(period, date, item, rest); + + String mergedResult = piwikHelper.getDataAsJsonString(); out.write(mergedResult.getBytes()); out.flush(); } catch (Exception e) { @@ -117,156 +104,6 @@ public void generate() throws ProcessingException { } } - private String getDataFromPiwikServer() throws Exception { - SortedMap urls = buildViewsURL(); - urls.put("downloads", buildDownloadsURL()); - String bulkApiGetRequestURL = buildBulkApiGetRequestURL(urls); - - log.debug(String.format("Fetching data from piwik server; requesting \"%s\"", bulkApiGetRequestURL)); - - String report = PiwikHelper.readFromURL(bulkApiGetRequestURL); - return PiwikHelper.transformJSONResults(urls.keySet(), report); - } - - private String buildBulkApiGetRequestURL(SortedMap urls){ - String piwikBulkApiGetQuery = "module=API&method=API.getBulkRequest&format=JSON" - + "&token_auth=" + PIWIK_AUTH_TOKEN; - StringBuilder sb = new StringBuilder(); - sb.append(PIWIK_API_URL) - .append(rest) - .append("?") - .append(piwikBulkApiGetQuery); - int i = 0; - for(String url : urls.values()){ - sb.append("&urls[") - .append(i++) - .append("]=") - .append(url); - } - return sb.toString(); - } - - private SortedMap buildViewsURL() throws UnsupportedEncodingException, ParseException { - // use Actions.getPageUrl; call it twice; once with ?show=full - String paramsFmt = "method=Actions.getPageUrl&pageUrl=%s"; - String summaryItemView = buildURL(PIWIK_SITE_ID, - String.format(paramsFmt, URLEncoder.encode(dspaceURL + "/handle/" + item.getHandle(), "UTF-8"))); - String fullItemView = buildURL(PIWIK_SITE_ID, - String.format(paramsFmt, URLEncoder.encode( - dspaceURL + "/handle" + "/" + item.getHandle() + "?show=full", "UTF-8"))); - SortedMap ret = new TreeMap<>(); - ret.put("summaryItemView", summaryItemView); - ret.put("fullItemView", fullItemView); - return ret; - } - - private String buildDownloadsURL() throws UnsupportedEncodingException, ParseException { - String filterPattern = URLEncoder.encode(dspaceURL + "/bitstream/handle/" + item.getHandle(), "UTF-8"); - String params = - "method=Actions.getPageUrls" + - "&expanded=1&flat=1" + - "&filter_column=url" + - "&filter_pattern=" + filterPattern; - return buildURL(PIWIK_DOWNLOAD_SITE_ID, params); - } - - private String buildURL(String siteID, String specificParams) throws UnsupportedEncodingException, ParseException { - // should contain the period - String period = request.getParameter("period"); - String dateRange = DateRange.fromDateString(request.getParameter("date")).toString(); - - /* - The Actions API lets you request reports for all your Visitor Actions: Page URLs, Page titles, Events, - Content Tracking, File Downloads and Clicks on external websites. - Actions.getPageUrls: - - stats(nb_visits, nb_hits, etc) per url, in given date broken down by period - expanded: - - some API functions have a parameter 'expanded'. If 'expanded' is set to 1, the returned data will contain the first level results, as well as all sub-tables. - - basically fetches subtable (if present as idsubdatatable) - - eg. urls broken down by directory structure: - - lvl 1: pageUrl=^https%253A%252F%252Fdivezone.net%252Fdiving - - lvl 2: pageUrl==https%253A%252F%252Fdivezone.net%252Fdiving%252Fbali; pageUrl==https%253A%252F%252Fdivezone.net%252Fdiving%252Fthailand - - lvl 1 has no url, ie. it's cummulative for the "underlying" urls - flat: - - some API functions have a parameter 'expanded', which means that the data is hierarchical. For such API function, if 'flat' is set to 1, the returned data will contain the flattened view of the table data set. The children of all first level rows will be aggregated under one row. This is useful for example to see all Custom Variables names and values at once, for example, Matomo forum user status, or to see the full URLs not broken down by directory or structure. - - this will remove the cummulative results; all rows will be of the same type (having url field) - */ - String params = - specificParams - + "&date=" + dateRange - + "&period=" + period - + "&token_auth=" + PIWIK_AUTH_TOKEN - + "&showColumns=label,url,nb_visits,nb_hits" - // don't want to handle "paging" (summary views) - + "&filter_limit=-1" - + "&idSite=" + siteID; - return URLEncoder.encode(params, "UTF-8"); - } - - private String getDataFromLindatPiwikCacheServer() throws IOException { - String period = request.getParameter("period"); - String url = PIWIK_API_URL_CACHED + "handle?h=" + item.getHandle() + "&period=" + period; - - if(request.getParameter("date")!=null) { - String date = request.getParameter("date"); - url += "&date=" + date; - } - - return PiwikHelper.readFromURL(url); - } - - private static class DateRange { - private static final SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); - - private final Date startDate; - private final Date endDate; - - private DateRange(Date startDate, Date endDate){ - this.startDate = startDate; - this.endDate = endDate; - } - - private static DateRange fromDateString(String date) throws ParseException { - Date startDate; - Date endDate; - if(date!=null) { - String sdate = date; - String edate = date; - if(sdate.length()==4) { - sdate += "-01-01"; - edate += "-12-31"; - } else if(sdate.length()==7) { - Calendar cal = Calendar.getInstance(); - cal.set(Calendar.YEAR, Integer.parseInt(sdate.substring(0,4))); - cal.set(Calendar.MONTH, Integer.parseInt(sdate.substring(5,7))-1); - cal.set(Calendar.DATE, cal.getActualMaximum(Calendar.DATE)); - sdate += "-01"; - edate = df.format(cal.getTime()); - } - startDate = df.parse(sdate); - endDate = df.parse(edate); - } else { - // default start and end data - startDate = df.parse("2014-01-01"); - endDate = Calendar.getInstance().getTime(); - } - return new DateRange(startDate, endDate); - } - - public String getFormattedStartDate(){ - return df.format(startDate); - } - - public String getFormattedEndDate(){ - return df.format(endDate); - } - - @Override - public String toString(){ - return getFormattedStartDate() + "," + getFormattedEndDate(); - } - - } } From 4d808a7057d0454bc65edae79f81d802bfd3f3b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Fri, 28 Jan 2022 16:11:27 +0100 Subject: [PATCH 07/19] Need Long not Integer --- dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java | 2 +- .../src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java index 3154f90bc26e..e6bf53429c59 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java @@ -410,7 +410,7 @@ private List getCountryDataFromLindatPiwikCacheServer() throws Excepti if(country instanceof JSONObject){ JSONObject c = (JSONObject) country; String label = (String) c.get("label"); - String count = ((Integer) c.get("nb_visits")).toString(); + String count = ((Long) c.get("nb_visits")).toString(); result.add(new String[]{label, count}); } } diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index a086020460de..1c725b95686e 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -212,13 +212,13 @@ private static Map extractStats(JSONObject statsObject, TimeSer if(dayo instanceof String){ String day = (String) dayo; JSONObject stats = (JSONObject) days.get(day); - int valueForSeries = (Integer)stats.get(keyForSeries); + int valueForSeries = ((Long)stats.get(keyForSeries)).intValue(); if(max < valueForSeries){ max = valueForSeries; } series.add(Day.parseDay(String.format("%s-%s-%s", year, month, day)), valueForSeries); for(String key : keysForTotals){ - int valueForTotal = (Integer)stats.get(key); + int valueForTotal = ((Long)stats.get(key)).intValue(); int number = totals.get(key); totals.put(key, number + valueForTotal); } From 3107821eefea303a12728a3d0040068ff16cbc60 Mon Sep 17 00:00:00 2001 From: Ondrej Kosarko Date: Fri, 28 Jan 2022 16:46:40 +0100 Subject: [PATCH 08/19] Handle 404 to a degree (for PDFreports) if there are no downloads/views just don't send the report. if there are no country data (possible when only downloads) return empty list. --- .../java/cz/cuni/mff/ufal/PiwikHelper.java | 19 ++++++++++++------- .../cz/cuni/mff/ufal/PiwikPDFExporter.java | 4 ++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java index e6bf53429c59..b8df6ae7ccbf 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java @@ -381,13 +381,18 @@ public String getDataAsJsonString() throws Exception { } public List getCountryData() throws Exception{ - if(PIWIK_API_MODE.equals("cached")) { - log.debug("========CACHED MODE"); - return getCountryDataFromLindatPiwikCacheServer(); - } else { - // direct mode as default - log.debug("========DIRECT MODE"); - return getCountryDataFromPiwik(); + try { + if (PIWIK_API_MODE.equals("cached")) { + log.debug("========CACHED MODE"); + return getCountryDataFromLindatPiwikCacheServer(); + } else { + // direct mode as default + log.debug("========DIRECT MODE"); + return getCountryDataFromPiwik(); + } + } catch (FileNotFoundException e){ + log.info(String.format("No country data for '%s'", e.getMessage())); + return new ArrayList<>(); } } diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index 1c725b95686e..d9477b65b295 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -7,6 +7,7 @@ import java.awt.geom.Ellipse2D; import java.awt.geom.Rectangle2D; import java.io.File; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.net.URL; @@ -125,6 +126,9 @@ public static void generateReports() throws SQLException { log.info("Processing Item : " + item.getHandle()); generateItemReport(item); done.add(item); + } catch(FileNotFoundException e){ + log.info(String.format("404 '%s' probably nothing logged for that date", e.getMessage())); + continue; } catch(Exception e) { log.error("Unable to generate report.", e); continue; From 30535fadc1a43007bce8fc7e5e28b630ce51dfd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Fri, 28 Jan 2022 17:12:33 +0100 Subject: [PATCH 09/19] plot unique visitors instead of summing them --- .../cz/cuni/mff/ufal/PiwikPDFExporter.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index d9477b65b295..17e6f9216aed 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -242,29 +242,29 @@ public static JFreeChart createViewsChart(JSONObject report, Map viewsTotals = extractStats(views, viewsSeries, "nb_hits", new String[]{"nb_hits", - "nb_uniq_pageviews", "nb_visits", "nb_uniq_visitors"}); + "nb_uniq_pageviews", "nb_visits"}); Map downloadsTotals = extractStats(downloads, downloadsSeries, "nb_hits", new String[]{ "nb_hits", "nb_uniq_pageviews"}); + extractStats(views, downloadsSeries, "nb_uniq_visitors", new String[]{ }); int maxPageViews = viewsTotals.get("max"); summary.put("pageviews", viewsTotals.get("nb_hits")); summary.put("unique pageviews", viewsTotals.get("nb_uniq_pageviews")); summary.put("visits", viewsTotals.get("nb_visits")); - // XXX IMO this number makes no sense (if there are 10 uniq visitors on 1st and 10 on 2nd, the actual number - // of uniq visitors is any number between 10 and 20, inclusive). - summary.put("unique visitors", viewsTotals.get("nb_uniq_visitors")); summary.put("downloads", downloadsTotals.get("nb_hits")); summary.put("unique downloads", downloadsTotals.get("nb_uniq_pageviews")); TimeSeriesCollection dataset = new TimeSeriesCollection(); dataset.addSeries(viewsSeries); - dataset.addSeries(downloadsSeries); + dataset.addSeries(downloadsSeries); + dataset.addSeries(visitorsSeries); lineChart = ChartFactory.createTimeSeriesChart("Views Over Time", "", "", dataset); lineChart.setBackgroundPaint(Color.WHITE); @@ -289,10 +289,13 @@ public static JFreeChart createViewsChart(JSONObject report, Map Date: Fri, 28 Jan 2022 17:18:44 +0100 Subject: [PATCH 10/19] correct series --- dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index 17e6f9216aed..a3601317c98a 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -251,7 +251,7 @@ public static JFreeChart createViewsChart(JSONObject report, Map viewsTotals = extractStats(views, viewsSeries, "nb_hits", new String[]{"nb_hits", "nb_uniq_pageviews", "nb_visits"}); Map downloadsTotals = extractStats(downloads, downloadsSeries, "nb_hits", new String[]{ "nb_hits", "nb_uniq_pageviews"}); - extractStats(views, downloadsSeries, "nb_uniq_visitors", new String[]{ }); + extractStats(views, visitorsSeries, "nb_uniq_visitors", new String[]{ }); int maxPageViews = viewsTotals.get("max"); From b0c35b4ba8cceb1854379afd0687f272f5d489a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Mon, 31 Jan 2022 14:42:21 +0100 Subject: [PATCH 11/19] zero should be plotted if no value for given day --- .../java/cz/cuni/mff/ufal/PiwikPDFExporter.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index a3601317c98a..552f98d283f6 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -207,20 +207,31 @@ private static Map extractStats(JSONObject statsObject, TimeSer for(Object yo : statsObject.keySet()){ if(yo instanceof String){ String year = (String)yo; + int y = Integer.parseInt(year); JSONObject months = (JSONObject) statsObject.get(year); for(Object mo : months.keySet()){ if(mo instanceof String){ String month = (String)mo; + int m = Integer.parseInt(month); + Calendar c = Calendar.getInstance(); + // months are zero based, 0 january + c.set(y, m-1, 1); + // get the valid days in month and fill the series with zeros + for(int d = 1; d <= c.getActualMaximum(Calendar.DATE); d++){ + // here month is 1-12 + series.add(new Day(d, m ,y), 0); + } JSONObject days = (JSONObject) months.get(month); for(Object dayo: days.keySet()){ if(dayo instanceof String){ String day = (String) dayo; + int d = Integer.parseInt(day); JSONObject stats = (JSONObject) days.get(day); int valueForSeries = ((Long)stats.get(keyForSeries)).intValue(); if(max < valueForSeries){ max = valueForSeries; } - series.add(Day.parseDay(String.format("%s-%s-%s", year, month, day)), valueForSeries); + series.addOrUpdate( new Day(d, m, y), valueForSeries); for(String key : keysForTotals){ int valueForTotal = ((Long)stats.get(key)).intValue(); int number = totals.get(key); From 0b1ca5c3c63cda8ddbc21fd48667d688bf9dcb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Mon, 31 Jan 2022 15:06:51 +0100 Subject: [PATCH 12/19] change the shapes --- .../src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index 552f98d283f6..10c1cfe07631 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -45,6 +45,7 @@ import org.jfree.data.time.TimeSeriesCollection; import org.jfree.ui.RectangleEdge; import org.jfree.ui.RectangleInsets; +import org.jfree.util.ShapeUtilities; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -79,7 +80,7 @@ public class PiwikPDFExporter { private static SimpleDateFormat inputDateFormat = new SimpleDateFormat("yyyy-MM-dd"); private static SimpleDateFormat outputDateFormat = new SimpleDateFormat("MMM-dd"); - + private static Logger log = cz.cuni.mff.ufal.Logger.getLogger(PiwikPDFExporter.class); @@ -299,8 +300,8 @@ public static JFreeChart createViewsChart(JSONObject report, Map Date: Mon, 31 Jan 2022 15:23:34 +0100 Subject: [PATCH 13/19] prettier crosses --- .../src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index 10c1cfe07631..65459863b57a 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -300,8 +300,8 @@ public static JFreeChart createViewsChart(JSONObject report, Map Date: Mon, 31 Jan 2022 15:24:32 +0100 Subject: [PATCH 14/19] reorder the summary --- .../cz/cuni/mff/ufal/PiwikPDFExporter.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index 65459863b57a..fb570a16ba64 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -453,10 +453,14 @@ public static void geneartePDF(Item item, Date date, JFreeChart viewsChart, Map< text = new Paragraph(); text.setFont(FONT[7]); - text.add("" + summary.get("visits")); + text.add("" + summary.get("downloads")); text.setFont(FONT[6]); - text.add(" visits, "); - + text.add(" downloads, "); + text.setFont(FONT[7]); + text.add("" + summary.get("unique downloads")); + text.setFont(FONT[6]); + text.add(" unique downloads"); + srow = new PdfPCell(); srow.setBorder(0); srow.addElement(text); @@ -465,14 +469,10 @@ public static void geneartePDF(Item item, Date date, JFreeChart viewsChart, Map< text = new Paragraph(); text.setFont(FONT[7]); - text.add("" + summary.get("downloads")); - text.setFont(FONT[6]); - text.add(" downloads, "); - text.setFont(FONT[7]); - text.add("" + summary.get("unique downloads")); + text.add("" + summary.get("visits")); text.setFont(FONT[6]); - text.add(" unique downloads"); - + text.add(" visits "); + srow = new PdfPCell(); srow.setBorder(0); srow.addElement(text); From 6c2824a1eb109f5f2ac6ccdbeb9b8417a59d8a51 Mon Sep 17 00:00:00 2001 From: Ondra Kosarko Date: Fri, 4 Feb 2022 13:20:25 +0100 Subject: [PATCH 15/19] handle dates with one digit month when we get 2018-5 instead of 2018-05 --- dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java index b8df6ae7ccbf..a5698759bb54 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java @@ -579,6 +579,7 @@ private String buildURL(String siteID, String specificParams) throws Unsupported private static class DateRange { private static final SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); + private static final SimpleDateFormat odf = new SimpleDateFormat("yyyy-MM"); private final Date startDate; private final Date endDate; @@ -597,12 +598,12 @@ private static DateRange fromDateString(String date) throws ParseException { if(sdate.length()==4) { sdate += "-01-01"; edate += "-12-31"; - } else if(sdate.length()==7) { + } else if(sdate.length()==6 || sdate.length()==7) { Calendar cal = Calendar.getInstance(); cal.set(Calendar.YEAR, Integer.parseInt(sdate.substring(0,4))); - cal.set(Calendar.MONTH, Integer.parseInt(sdate.substring(5,7))-1); + cal.set(Calendar.MONTH, Integer.parseInt(sdate.substring(5))-1); cal.set(Calendar.DATE, cal.getActualMaximum(Calendar.DATE)); - sdate += "-01"; + sdate = odf.format(cal.getTime()) + "-01"; edate = df.format(cal.getTime()); } startDate = df.parse(sdate); From 644a8b669d668ff78a50fb3914695d8b247c4cb2 Mon Sep 17 00:00:00 2001 From: Ondra Kosarko Date: Fri, 4 Feb 2022 16:11:10 +0100 Subject: [PATCH 16/19] Shouldn't be strings we are reusing the json elsewhere and are expecting ints (or long). Moreover the frontend seems to behave better this way. There might be some unnecessary parseInt/toString left over. --- .../java/cz/cuni/mff/ufal/PiwikHelper.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java index a5698759bb54..029f5a418396 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java @@ -122,8 +122,8 @@ public static JSONObject transformJSON(JSONObject views) { v = new JSONObject(); nb_visits = nb_visits + Integer.parseInt(row.get("nb_visits").toString()); nb_hits = nb_hits + Integer.parseInt(row.get("nb_hits").toString()); - v.put("nb_hits", "" + nb_hits); - v.put("nb_visits", "" + nb_visits); + v.put("nb_hits", nb_hits); + v.put("nb_visits", nb_visits); year.put(url, v); int total_nb_visits = 0; @@ -137,8 +137,8 @@ public static JSONObject transformJSON(JSONObject views) { v = new JSONObject(); total_nb_visits = total_nb_visits + Integer.parseInt(row.get("nb_visits").toString()); total_nb_hits = total_nb_hits + Integer.parseInt(row.get("nb_hits").toString()); - v.put("nb_hits", "" + total_nb_hits); - v.put("nb_visits", "" + total_nb_visits); + v.put("nb_hits", total_nb_hits); + v.put("nb_visits", total_nb_visits); total.put(y, v); total_nb_visits = 0; @@ -190,8 +190,8 @@ public static JSONObject transformJSON(JSONObject views) { v = new JSONObject(); nb_visits = nb_visits + Integer.parseInt(row.get("nb_visits").toString()); nb_hits = nb_hits + Integer.parseInt(row.get("nb_hits").toString()); - v.put("nb_hits", "" + nb_hits); - v.put("nb_visits", "" + nb_visits); + v.put("nb_hits", nb_hits); + v.put("nb_visits", nb_visits); month.put(url, v); JSONObject tyear = null; @@ -225,8 +225,8 @@ public static JSONObject transformJSON(JSONObject views) { v = new JSONObject(); total_nb_visits = total_nb_visits + Integer.parseInt(row.get("nb_visits").toString()); total_nb_hits = total_nb_hits + Integer.parseInt(row.get("nb_hits").toString()); - v.put("nb_hits", "" + total_nb_hits); - v.put("nb_visits", "" + total_nb_visits); + v.put("nb_hits", total_nb_hits); + v.put("nb_visits", total_nb_visits); tyear.put(m, v); } @@ -271,8 +271,8 @@ public static JSONObject transformJSON(JSONObject views) { v = new JSONObject(); nb_visits = nb_visits + Integer.parseInt(row.get("nb_visits").toString()); nb_hits = nb_hits + Integer.parseInt(row.get("nb_hits").toString()); - v.put("nb_hits", "" + nb_hits); - v.put("nb_visits", "" + nb_visits); + v.put("nb_hits", nb_hits); + v.put("nb_visits", nb_visits); day.put(url, v); JSONObject tyear = null; @@ -314,8 +314,8 @@ public static JSONObject transformJSON(JSONObject views) { v = new JSONObject(); total_nb_visits = total_nb_visits + Integer.parseInt(row.get("nb_visits").toString()); total_nb_hits = total_nb_hits + Integer.parseInt(row.get("nb_hits").toString()); - v.put("nb_hits", "" + total_nb_hits); - v.put("nb_visits", "" + total_nb_visits); + v.put("nb_hits", total_nb_hits); + v.put("nb_visits", total_nb_visits); tmonth.put(d, v); } From abca3e39a24c7c408b6b4ebc2f3a599cb7a35168 Mon Sep 17 00:00:00 2001 From: Ondra Kosarko Date: Fri, 4 Feb 2022 16:38:53 +0100 Subject: [PATCH 17/19] limit the country response to this instance only --- dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java index 029f5a418396..c415ff766aa8 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikHelper.java @@ -438,7 +438,7 @@ private List getCountryDataFromPiwik() throws Exception { + "&token_auth=" + PIWIK_AUTH_TOKEN + "&filter_limit=10" + "&format=xml" - + "&segment=pageUrl=@" + item.getHandle(); + + "&segment=pageUrl=@" + URLEncoder.encode(dspaceURL + "/handle/" + item.getHandle(), "UTF-8"); String xml = PiwikHelper.readFromURL(countryReportURL); From f78c8f7cde315afdb2a62f43729fc6b8b1865ab6 Mon Sep 17 00:00:00 2001 From: Ondra Kosarko Date: Fri, 4 Feb 2022 16:42:20 +0100 Subject: [PATCH 18/19] if using direct mode issue a warning transformJSON does not implement all the things the XML version did. There are other issues with the direct mode; so this might never get fixed. --- .../src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index fb570a16ba64..74a3183398bc 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -77,6 +77,8 @@ public class PiwikPDFExporter { private static boolean PIWIK_KEEP_REPORTS; private static String LINDAT_LOGO; + + private static String PIWIK_API_MODE; private static SimpleDateFormat inputDateFormat = new SimpleDateFormat("yyyy-MM-dd"); private static SimpleDateFormat outputDateFormat = new SimpleDateFormat("MMM-dd"); @@ -87,6 +89,9 @@ public class PiwikPDFExporter { public static void main(String args[]) throws Exception { log.info("Generating PIWIK pdf reports ...."); initialize(); + if (!PIWIK_API_MODE.equals("cached")) { + log.warn("Not using the cached mode: the reports will be missing uniq stats for pageviews, downloads and visitors as this is not implemented in transformJSON."); + } generateReports(); log.info("PIWIK pdf reports generation finished."); } @@ -95,6 +100,7 @@ public static void initialize() { PIWIK_REPORTS_OUTPUT_PATH = ConfigurationManager.getProperty("lr", "lr.statistics.report.path"); PIWIK_KEEP_REPORTS = ConfigurationManager.getBooleanProperty("lr", "lr.statistics.keep.reports", true); LINDAT_LOGO = ConfigurationManager.getProperty("lr", "lr.lindat.logo.mono"); + PIWIK_API_MODE = ConfigurationManager.getProperty("lr", "lr.statistics.api.mode"); } public static void generateReports() throws SQLException { From 62fc678b89f81bc5a97ed558887df172224b2c2b Mon Sep 17 00:00:00 2001 From: Ondra Kosarko Date: Fri, 4 Feb 2022 16:45:16 +0100 Subject: [PATCH 19/19] NPE guards and 0 is more reasonable max. --- .../cz/cuni/mff/ufal/PiwikPDFExporter.java | 74 ++++++++++--------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java index 74a3183398bc..6634815bbfec 100644 --- a/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java +++ b/dspace-api/src/main/java/cz/cuni/mff/ufal/PiwikPDFExporter.java @@ -209,40 +209,46 @@ private static Map extractStats(JSONObject statsObject, TimeSer totals.put(key, 0); } - int max = Integer.MIN_VALUE; - - for(Object yo : statsObject.keySet()){ - if(yo instanceof String){ - String year = (String)yo; - int y = Integer.parseInt(year); - JSONObject months = (JSONObject) statsObject.get(year); - for(Object mo : months.keySet()){ - if(mo instanceof String){ - String month = (String)mo; - int m = Integer.parseInt(month); - Calendar c = Calendar.getInstance(); - // months are zero based, 0 january - c.set(y, m-1, 1); - // get the valid days in month and fill the series with zeros - for(int d = 1; d <= c.getActualMaximum(Calendar.DATE); d++){ - // here month is 1-12 - series.add(new Day(d, m ,y), 0); - } - JSONObject days = (JSONObject) months.get(month); - for(Object dayo: days.keySet()){ - if(dayo instanceof String){ - String day = (String) dayo; - int d = Integer.parseInt(day); - JSONObject stats = (JSONObject) days.get(day); - int valueForSeries = ((Long)stats.get(keyForSeries)).intValue(); - if(max < valueForSeries){ - max = valueForSeries; - } - series.addOrUpdate( new Day(d, m, y), valueForSeries); - for(String key : keysForTotals){ - int valueForTotal = ((Long)stats.get(key)).intValue(); - int number = totals.get(key); - totals.put(key, number + valueForTotal); + int max = 0; + + if(statsObject != null) { + for (Object yo : statsObject.keySet()) { + if (yo instanceof String) { + String year = (String) yo; + int y = Integer.parseInt(year); + JSONObject months = (JSONObject) statsObject.get(year); + for (Object mo : months.keySet()) { + if (mo instanceof String) { + String month = (String) mo; + int m = Integer.parseInt(month); + Calendar c = Calendar.getInstance(); + // months are zero based, 0 january + c.set(y, m - 1, 1); + // get the valid days in month and fill the series with zeros + for (int d = 1; d <= c.getActualMaximum(Calendar.DATE); d++) { + // here month is 1-12 + series.add(new Day(d, m, y), 0); + } + JSONObject days = (JSONObject) months.get(month); + for (Object dayo : days.keySet()) { + if (dayo instanceof String) { + String day = (String) dayo; + int d = Integer.parseInt(day); + JSONObject stats = (JSONObject) days.get(day); + if(stats.containsKey(keyForSeries)) { + int valueForSeries = ((Long) stats.get(keyForSeries)).intValue(); + if (max < valueForSeries) { + max = valueForSeries; + } + series.addOrUpdate(new Day(d, m, y), valueForSeries); + } + for (String key : keysForTotals) { + if(stats.containsKey(key)) { + int valueForTotal = ((Long) stats.get(key)).intValue(); + int number = totals.get(key); + totals.put(key, number + valueForTotal); + } + } } } }