From bab09f6fe46885cb1c367ed6c8fce31cb0c3df67 Mon Sep 17 00:00:00 2001 From: Sebastian Scholze <4063307+schlotze@users.noreply.github.com> Date: Mon, 11 Apr 2022 15:58:39 +0200 Subject: [PATCH 01/34] prepapre structure --- .../analyser/GitlabCommitAnalyser.java | 67 +++++++++++ .../models/GitlabCommitDataModel.java | 111 ++++++++++++++++++ .../monitors/GitlabCommitMonitor.java | 38 ++++++ .../monitoring/parser/GitlabCommitParser.java | 37 ++++++ 4 files changed, 253 insertions(+) create mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java create mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java create mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java create mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitlabCommitParser.java diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java new file mode 100644 index 00000000..2c83879a --- /dev/null +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java @@ -0,0 +1,67 @@ +package de.atb.context.monitoring.analyser; + +/* + * #%L + * SmartCLIDE Monitoring + * %% + * Copyright (C) 2022 ATB – Institut für angewandte Systemtechnik Bremen GmbH + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import com.google.gson.Gson; +import de.atb.context.monitoring.analyser.webservice.WebServiceAnalyser; +import de.atb.context.monitoring.config.models.DataSource; +import de.atb.context.monitoring.config.models.InterpreterConfiguration; +import de.atb.context.monitoring.index.Indexer; +import de.atb.context.monitoring.models.GitMessage; +import de.atb.context.monitoring.models.GitlabCommitDataModel; +import de.atb.context.monitoring.models.IWebService; +import de.atb.context.tools.ontology.AmIMonitoringConfiguration; +import org.apache.lucene.document.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.util.Date; +import java.util.List; + +public class GitlabCommitAnalyser extends WebServiceAnalyser { + + private static final Logger logger = LoggerFactory.getLogger(GitlabCommitAnalyser.class); + + private static final Gson GSON = new Gson(); + + public GitlabCommitAnalyser(final DataSource dataSource, + final InterpreterConfiguration interpreterConfiguration, + final Indexer indexer, + final Document document, + final AmIMonitoringConfiguration amiConfiguration) { + super(dataSource, interpreterConfiguration, indexer, document, amiConfiguration); + } + + @Override + public List analyseObject(IWebService service) { + try { + final GitlabCommitDataModel model = new GitlabCommitDataModel(); + URI gitlabEndpoint = service.getURI(); + + // TODO "NEED TO BE IMPLEMENTED" + // * get list of projects where CH user has access to + // * iterate through projects and get last commits + // * for each identified "new" commit add a GitNessage below + model.addGitMessage(new GitMessage()); + + model.setMonitoredAt(new Date()); + return List.of(model); + } catch (Exception e) { + logger.error("Error analysing service: {}", service); + return List.of(); + } + } +} diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java new file mode 100644 index 00000000..3fa8ee4a --- /dev/null +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java @@ -0,0 +1,111 @@ +package de.atb.context.monitoring.models; + +/* + * #%L + * SmartCLIDE Monitoring + * %% + * Copyright (C) 2022 ATB – Institut für angewandte Systemtechnik Bremen GmbH + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import de.atb.context.common.Version; +import de.atb.context.common.util.ApplicationScenario; +import de.atb.context.common.util.BusinessCase; +import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; +import de.atb.context.monitoring.config.models.datasources.WebServiceDataSource; +import de.atb.context.monitoring.rdf.RdfHelper; +import de.atb.context.persistence.ModelOutputLanguage; +import lombok.Getter; +import lombok.Setter; +import org.apache.jena.rdf.model.Model; +import org.simpleframework.xml.Root; +import thewebsemantic.Id; +import thewebsemantic.Namespace; +import thewebsemantic.RdfType; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +@RdfType("GitlabCommitDataModel") +@Namespace(BusinessCase.NS_DUMMY_URL) +@Root +@Getter +@Setter +public class GitlabCommitDataModel implements IMonitoringDataModel { + private Date monitoredAt; + private String documentIndexId = "index/broker"; + private String documentUri; + @Id + private String identifier; + private MessageBrokerDataSource dataSource; + private String implementingClassName = GitlabCommitDataModel.class.getName(); + private String monitoringDataVersion = Version.MONITORING_DATA.getVersionString(); + private List gitMessages = new ArrayList<>(); + + public GitlabCommitDataModel() { + this.identifier = UUID.randomUUID().toString(); + } + + public void addGitMessage(GitMessage gitMessage) { + if (!this.gitMessages.contains(gitMessage)) { + this.gitMessages.add(gitMessage); + } + } + + @Override + public GitlabCommitDataModel fromRdfModel(String rdfModel) { + return RdfHelper.createMonitoringData(rdfModel, GitlabCommitDataModel.class); + } + + @Override + public GitlabCommitDataModel fromRdfModel(Model model) { + return RdfHelper.createMonitoringData(model, GitlabCommitDataModel.class); + } + + @Override + public String toRdfString() { + return ModelOutputLanguage.RDFXML.getModelAsString(this.toRdfModel()); + } + + @Override + public Model toRdfModel() { + return RdfHelper.createRdfModel(this); + } + + @Override + public String getContextIdentifierClassName() { + return null; + } + + @Override + public void setDataSource(WebServiceDataSource dataSource) { + } + + @Override + public BusinessCase getBusinessCase() { + return BusinessCase.getInstance(BusinessCase.NS_DUMMY_ID, BusinessCase.NS_DUMMY_URL); + } + + @Override + public ApplicationScenario getApplicationScenario() { + return ApplicationScenario.getInstance(getBusinessCase()); + } + + @Override + public boolean triggersContextChange() { + return false; + } + + @Override + public void initialize() { + + } +} diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java new file mode 100644 index 00000000..a942d728 --- /dev/null +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java @@ -0,0 +1,38 @@ +package de.atb.context.monitoring.monitors; + +/* + * #%L + * SmartCLIDE Monitoring + * %% + * Copyright (C) 2022 ATB – Institut für angewandte Systemtechnik Bremen GmbH + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import de.atb.context.monitoring.config.models.DataSource; +import de.atb.context.monitoring.config.models.Interpreter; +import de.atb.context.monitoring.config.models.Monitor; +import de.atb.context.monitoring.index.Indexer; +import de.atb.context.monitoring.monitors.webservice.WebServiceMonitor; +import de.atb.context.tools.ontology.AmIMonitoringConfiguration; + +import java.io.IOException; +import java.util.concurrent.TimeoutException; + +public class GitlabCommitMonitor extends WebServiceMonitor { + public GitlabCommitMonitor(final DataSource dataSource, + final Interpreter interpreter, + final Monitor monitor, + final Indexer indexer, + final AmIMonitoringConfiguration configuration) throws IOException, TimeoutException { + super(dataSource, interpreter, monitor, indexer, configuration); + + // FIXME: this is a temporary workaround and should be removed! + //addProgressListener(new DleGitMonitorProgressListener((WebServiceDataSource) dataSource)); + } +} diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitlabCommitParser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitlabCommitParser.java new file mode 100644 index 00000000..3c350844 --- /dev/null +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitlabCommitParser.java @@ -0,0 +1,37 @@ +package de.atb.context.monitoring.parser; + +/* + * #%L + * SmartCLIDE Monitoring + * %% + * Copyright (C) 2021 ATB – Institut für angewandte Systemtechnik Bremen GmbH + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import de.atb.context.monitoring.config.models.DataSource; +import de.atb.context.monitoring.config.models.InterpreterConfiguration; +import de.atb.context.monitoring.index.Indexer; +import de.atb.context.monitoring.models.IWebService; +import de.atb.context.monitoring.parser.webservice.WebServiceParser; +import de.atb.context.tools.ontology.AmIMonitoringConfiguration; +import org.apache.lucene.document.Document; + +public class GitlabCommitParser extends WebServiceParser { + public GitlabCommitParser(final DataSource dataSource, + final InterpreterConfiguration interpreterConfiguration, + final Indexer indexer, + final AmIMonitoringConfiguration amiConfiguration) { + super(dataSource, interpreterConfiguration, indexer, amiConfiguration); + } + + @Override + protected boolean parseObject(IWebService service, Document document) { + return true; + } +} From 1aa61aa305bc8805a8ff7c7466db9e510fa52a84 Mon Sep 17 00:00:00 2001 From: Sebastian Scholze <4063307+schlotze@users.noreply.github.com> Date: Mon, 11 Apr 2022 16:15:20 +0200 Subject: [PATCH 02/34] fi xtypo --- .../atb/context/monitoring/models/GitlabCommitDataModel.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java index 3fa8ee4a..883925d0 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java @@ -17,7 +17,6 @@ import de.atb.context.common.Version; import de.atb.context.common.util.ApplicationScenario; import de.atb.context.common.util.BusinessCase; -import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; import de.atb.context.monitoring.config.models.datasources.WebServiceDataSource; import de.atb.context.monitoring.rdf.RdfHelper; import de.atb.context.persistence.ModelOutputLanguage; @@ -45,7 +44,7 @@ public class GitlabCommitDataModel implements IMonitoringDataModel gitMessages = new ArrayList<>(); From cbd367274c8529b3be6d002bac3c642febbe17a8 Mon Sep 17 00:00:00 2001 From: Sebastian Scholze <4063307+schlotze@users.noreply.github.com> Date: Wed, 13 Apr 2022 14:02:46 +0200 Subject: [PATCH 03/34] update --- .../config/MonitoringConfiguration.java | 80 +++++++++---------- .../datasources/MessageBrokerDataSource.java | 2 - .../file/TestFilePairSystemMonitor.java | 5 +- .../monitoring-config.xml} | 4 +- .../analyser/GitlabCommitAnalyser.java | 8 +- .../models/datasources/GitlabDataSource.java | 78 ++++++++++++++++++ .../monitoring/monitors/GitMonitor.java | 3 +- .../monitors/GitlabCommitMonitor.java | 3 +- .../DleGitMonitorProgressListener.java | 12 +++ .../datasources/GitlabDataSourceTest.java | 54 +++++++++++++ .../gitlab-monitoring/monitoring-config.xml | 31 +++++++ 11 files changed, 227 insertions(+), 53 deletions(-) rename context-monitoring/src/test/resources/{monitoring-filepair-config.xml => filepairmonitor/monitoring-config.xml} (87%) create mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java create mode 100644 smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java create mode 100644 smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml diff --git a/context-monitoring/src/main/java/de/atb/context/monitoring/config/MonitoringConfiguration.java b/context-monitoring/src/main/java/de/atb/context/monitoring/config/MonitoringConfiguration.java index 9a3d99a0..32cf3645 100644 --- a/context-monitoring/src/main/java/de/atb/context/monitoring/config/MonitoringConfiguration.java +++ b/context-monitoring/src/main/java/de/atb/context/monitoring/config/MonitoringConfiguration.java @@ -15,6 +15,12 @@ */ +import de.atb.context.common.Configuration; +import de.atb.context.common.exceptions.ConfigurationException; +import de.atb.context.monitoring.config.models.*; +import de.atb.context.tools.ontology.AmIMonitoringConfiguration; +import org.simpleframework.xml.core.Persister; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; @@ -22,18 +28,6 @@ import java.util.List; import java.util.Map; -import de.atb.context.monitoring.config.models.IMonitoringConfiguration; -import de.atb.context.monitoring.config.models.Index; -import org.simpleframework.xml.Serializer; -import org.simpleframework.xml.core.Persister; -import de.atb.context.tools.ontology.AmIMonitoringConfiguration; -import de.atb.context.common.Configuration; -import de.atb.context.common.exceptions.ConfigurationException; -import de.atb.context.monitoring.config.models.Config; -import de.atb.context.monitoring.config.models.DataSource; -import de.atb.context.monitoring.config.models.Interpreter; -import de.atb.context.monitoring.config.models.Monitor; - /** * Settings * @@ -42,35 +36,35 @@ */ public final class MonitoringConfiguration extends Configuration implements IMonitoringConfiguration { - private static volatile Map settings = new HashMap<>(); - private static final String DefaultFileName = "monitoring-config.xml"; + private static final Map SETTINGS = new HashMap<>(); + private static final String DEFAULT_FILE_NAME = "monitoring-config.xml"; public static MonitoringConfiguration getInstance() { - if (settings.get(DefaultFileName) == null) { - settings.put(DefaultFileName, new MonitoringConfiguration(DefaultFileName)); + if (SETTINGS.get(DEFAULT_FILE_NAME) == null) { + SETTINGS.put(DEFAULT_FILE_NAME, new MonitoringConfiguration(DEFAULT_FILE_NAME)); } - return settings.get(DefaultFileName); + return SETTINGS.get(DEFAULT_FILE_NAME); } public static MonitoringConfiguration getInstance(final AmIMonitoringConfiguration config) { - if (settings.get(config) == null) { - settings.put(config.getId(), new MonitoringConfiguration(config)); + if (SETTINGS.get(config) == null) { + SETTINGS.put(config.getId(), new MonitoringConfiguration(config)); } - return settings.get(config.getId()); + return SETTINGS.get(config.getId()); } public static MonitoringConfiguration getInstance(final String configFileName) { - if (settings.get(configFileName) == null) { - settings.put(configFileName, new MonitoringConfiguration(configFileName)); + if (SETTINGS.get(configFileName) == null) { + SETTINGS.put(configFileName, new MonitoringConfiguration(configFileName)); } - return settings.get(configFileName); + return SETTINGS.get(configFileName); } public static MonitoringConfiguration getInstance(final String configFileName, final String configFilePath) { - if (settings.get(configFileName) == null) { - settings.put(configFileName, new MonitoringConfiguration(configFileName, configFilePath)); + if (SETTINGS.get(configFileName) == null) { + SETTINGS.put(configFileName, new MonitoringConfiguration(configFileName, configFilePath)); } - return settings.get(configFileName); + return SETTINGS.get(configFileName); } private MonitoringConfiguration(final String givenName, final String givenPath) { @@ -88,23 +82,19 @@ private MonitoringConfiguration(final AmIMonitoringConfiguration config) { protected void readConfigurationFile() { InputStream is = null; try { - final Serializer serializer = new Persister(); - - String drmHandle = sysCaller.openDRMobject("monitoring-config.xml", configurationLookupPath,"read"); + final String drmHandle = sysCaller.openDRMobject(configurationFileName, configurationLookupPath, "read"); if (drmHandle != null) { - byte[] readConfig = sysCaller.getDRMobject("monitoring-config.xml", configurationLookupPath); + final byte[] readConfig = sysCaller.getDRMobject(configurationFileName, configurationLookupPath); if (readConfig != null) { is = new ByteArrayInputStream(readConfig); - this.configurationBean = serializer.read( - this.configurationClass, is); + this.configurationBean = new Persister().read(this.configurationClass, is); is.close(); - logger.info("" + this.configurationFileName + " loaded!"); + logger.info("{} loaded!", configurationFileName); } sysCaller.closeDRMobject(drmHandle); } } catch (final Exception e) { - logger.error("Could not serialize the " + configurationName - + " file: " + this.configurationFileName, e); + logger.error("Could not serialize the {} file {}", configurationName, configurationFileName, e); } finally { if (is != null) { try { @@ -170,15 +160,25 @@ public void checkConsistency() throws ConfigurationException { } if (getDataSource(monitor.getDataSourceId()) == null) { - throw new ConfigurationException("DataSource '%s' for Monitor '%s' is not configured", monitor.getDataSourceId(), - monitor.getId()); + throw new ConfigurationException( + "DataSource '%s' for Monitor '%s' is not configured", + monitor.getDataSourceId(), + monitor.getId() + ); } if (getInterpreter(monitor.getInterpreterId()) == null) { - throw new ConfigurationException("Interpreter '%s' for Monitor '%s' is not configured", monitor.getInterpreterId(), - monitor.getId()); + throw new ConfigurationException( + "Interpreter '%s' for Monitor '%s' is not configured", + monitor.getInterpreterId(), + monitor.getId() + ); } if (getIndex(monitor.getIndexId()) == null) { - throw new ConfigurationException("Index '%s' for Monitor '%s' is not configured", monitor.getIndexId(), monitor.getId()); + throw new ConfigurationException( + "Index '%s' for Monitor '%s' is not configured", + monitor.getIndexId(), + monitor.getId() + ); } } } diff --git a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java index 0ac35321..526d7951 100644 --- a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java +++ b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java @@ -35,8 +35,6 @@ @Namespace("http://atb-bremen.de/") public class MessageBrokerDataSource extends DataSource { - private static final long serialVersionUID = 3490943354053238739L; - public MessageBrokerDataSource() { } diff --git a/context-monitoring/src/test/java/de/atb/context/monitoring/monitors/file/TestFilePairSystemMonitor.java b/context-monitoring/src/test/java/de/atb/context/monitoring/monitors/file/TestFilePairSystemMonitor.java index 0dd7c62d..6e60f1fd 100644 --- a/context-monitoring/src/test/java/de/atb/context/monitoring/monitors/file/TestFilePairSystemMonitor.java +++ b/context-monitoring/src/test/java/de/atb/context/monitoring/monitors/file/TestFilePairSystemMonitor.java @@ -39,9 +39,10 @@ public class TestFilePairSystemMonitor { @BeforeClass public static void beforeClass() throws ConfigurationException { - String absolutefilePath = Path.of("src", "test", "resources").toAbsolutePath().toString(); + String absoluteFilePath = + Path.of("src", "test", "resources", "filepairmonitor").toAbsolutePath().toString(); - config = MonitoringConfiguration.getInstance("monitoring-filepair-config.xml", absolutefilePath); + config = MonitoringConfiguration.getInstance("monitoring-config.xml", absoluteFilePath); monitor = config.getMonitor("monitor-dummy"); Assert.assertTrue("No monitors 'monitor-dummy' specified!", config.getMonitor("monitor-dummy") != null); diff --git a/context-monitoring/src/test/resources/monitoring-filepair-config.xml b/context-monitoring/src/test/resources/filepairmonitor/monitoring-config.xml similarity index 87% rename from context-monitoring/src/test/resources/monitoring-filepair-config.xml rename to context-monitoring/src/test/resources/filepairmonitor/monitoring-config.xml index 47228d83..71d82581 100644 --- a/context-monitoring/src/test/resources/monitoring-filepair-config.xml +++ b/context-monitoring/src/test/resources/filepairmonitor/monitoring-config.xml @@ -1,10 +1,10 @@ + xsi:schemaLocation="http://www.atb-bremen.de ../monitoring-config.xsd"> - + diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java index 2c83879a..b1c21b9a 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java @@ -19,8 +19,8 @@ import de.atb.context.monitoring.config.models.DataSource; import de.atb.context.monitoring.config.models.InterpreterConfiguration; import de.atb.context.monitoring.index.Indexer; +import de.atb.context.monitoring.models.GitDataModel; import de.atb.context.monitoring.models.GitMessage; -import de.atb.context.monitoring.models.GitlabCommitDataModel; import de.atb.context.monitoring.models.IWebService; import de.atb.context.tools.ontology.AmIMonitoringConfiguration; import org.apache.lucene.document.Document; @@ -31,7 +31,7 @@ import java.util.Date; import java.util.List; -public class GitlabCommitAnalyser extends WebServiceAnalyser { +public class GitlabCommitAnalyser extends WebServiceAnalyser { private static final Logger logger = LoggerFactory.getLogger(GitlabCommitAnalyser.class); @@ -46,9 +46,9 @@ public GitlabCommitAnalyser(final DataSource dataSource, } @Override - public List analyseObject(IWebService service) { + public List analyseObject(IWebService service) { try { - final GitlabCommitDataModel model = new GitlabCommitDataModel(); + final GitDataModel model = new GitDataModel(); URI gitlabEndpoint = service.getURI(); // TODO "NEED TO BE IMPLEMENTED" diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java new file mode 100644 index 00000000..148024f3 --- /dev/null +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java @@ -0,0 +1,78 @@ +package de.atb.context.monitoring.config.models.datasources; + +/*- + * #%L + * SmartCLIDE Monitoring + * %% + * Copyright (C) 2015 - 2022 ATB – Institut für angewandte Systemtechnik Bremen GmbH + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import de.atb.context.monitoring.config.models.DataSource; +import lombok.NoArgsConstructor; +import thewebsemantic.Namespace; +import thewebsemantic.RdfType; + +import java.io.Serializable; + +/** + * GitLabDataSource + * + * @author scholze + * @version $LastChangedRevision: 156 $ + */ +@RdfType("GitlabDataSource") +@Namespace("http://atb-bremen.de/") +@NoArgsConstructor +public class GitlabDataSource extends WebServiceDataSource { + + private static final IDataSourceOptionValue ACCESS_TOKEN_OPTION = new IDataSourceOptionValue() { + @Override + public String getKeyName() { + return "token"; + } + + @Override + public Class getValueType() { + return String.class; + } + }; + + public GitlabDataSource(final DataSource base) { + this.id = base.getId(); + this.monitor = base.getMonitor(); + this.options = base.getOptions(); + this.type = base.getType().toString(); + this.uri = base.getUri(); + } + + public final String getGitLabAccessToken() { + return this.getOptionValue(ACCESS_TOKEN_OPTION, true); + } + + public final String getMessageBrokerServer() { + return this.getOptionValue(MessageBrokerDataSourceOptions.MessageBrokerServer, true); + } + + public final Integer getMessageBrokerPort() { return this.getOptionValue(MessageBrokerDataSourceOptions.MessageBrokerPort, true); + } + + public final String getExchange() { + return this.getOptionValue(MessageBrokerDataSourceOptions.Exchange, true); + } + + public final String getTopic() { + return this.getOptionValue(MessageBrokerDataSourceOptions.Topic, true); + } + + public final String getDleTopic() { + return this.getOptionValue(MessageBrokerDataSourceOptions.DleTopic, true); + } + +} diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java index 443f9231..d0fe3219 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java @@ -20,6 +20,7 @@ import de.atb.context.monitoring.config.models.DataSource; import de.atb.context.monitoring.config.models.Interpreter; import de.atb.context.monitoring.config.models.Monitor; +import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; import de.atb.context.monitoring.index.Indexer; import de.atb.context.monitoring.monitors.messagebroker.MessageBrokerMonitor; @@ -35,6 +36,6 @@ public GitMonitor(final DataSource dataSource, super(dataSource, interpreter, monitor, indexer, configuration); // FIXME: this is a temporary workaround and should be removed! - addProgressListener(new DleGitMonitorProgressListener((MessageBrokerDataSource) dataSource)); + addProgressListener(new DleGitMonitorProgressListener((GitlabDataSource) dataSource)); } } diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java index a942d728..15873e32 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java @@ -32,7 +32,6 @@ public GitlabCommitMonitor(final DataSource dataSource, final AmIMonitoringConfiguration configuration) throws IOException, TimeoutException { super(dataSource, interpreter, monitor, indexer, configuration); - // FIXME: this is a temporary workaround and should be removed! - //addProgressListener(new DleGitMonitorProgressListener((WebServiceDataSource) dataSource)); + //addProgressListener(new DleGitMonitorProgressListener((GitlabDataSource) dataSource)); } } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java index 0b1ae8c4..71d54ce2 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java @@ -19,6 +19,7 @@ import java.util.concurrent.TimeoutException; import com.rabbitmq.client.Channel; +import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; import de.atb.context.monitoring.events.MonitoringProgressListener; import de.atb.context.monitoring.models.GitDataModel; @@ -46,6 +47,17 @@ public DleGitMonitorProgressListener(final MessageBrokerDataSource messageBroker channel = MessageBrokerUtil.connectToTopicExchange(messageBrokerDataSource); } + public DleGitMonitorProgressListener(final GitlabDataSource gitlabDataSource) + throws IOException, TimeoutException { + exchange = gitlabDataSource.getExchange(); + topic = gitlabDataSource.getDleTopic(); + channel = MessageBrokerUtil.connectToTopicExchange(gitlabDataSource.getMessageBrokerServer(), + gitlabDataSource.getMessageBrokerPort(), + gitlabDataSource.getUserName(), + gitlabDataSource.getPassword(), + gitlabDataSource.getExchange()); + } + @Override public void documentIndexed(final String indexId, final Document document) { // noop diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java new file mode 100644 index 00000000..cede148c --- /dev/null +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java @@ -0,0 +1,54 @@ +package de.atb.context.monitoring.config.models.datasources; + +import de.atb.context.monitoring.config.MonitoringConfiguration; +import de.atb.context.monitoring.config.models.DataSource; +import org.hamcrest.core.IsInstanceOf; +import org.junit.Test; + +import java.util.Objects; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.IsNull.notNullValue; + +public class GitlabDataSourceTest { + + @Test + public void configFileShouldBeDeserializedToCorrectGitlabDataSource() { + final String expectedId = "datasource-gitlab"; + final String expectedMonitor = "de.atb.context.monitoring.monitors.GitMonitor"; + final String expectedUri = "https://gitlab.example.com/api/v4"; + final String expectedAccessToken = "foobar"; + final String expectedMessageBrokerHost = "localhost"; + final int expectedMessageBrokerPort = 5672; + final String expectedUsername = "username"; + final String expectedPassword = "password"; + final String expectedMessageBrokerExchange = "smartclide-monitoring"; + final String expectedMessageBrokerTopicReceive = "monitoring.git.*"; + final String expectedMessageBrokerTopicSend = "dle.git.commits"; + final String configDirPath = + Objects.requireNonNull(this.getClass().getResource("/config/gitlab-monitoring")).getPath(); + final String configFileName = "monitoring-config.xml"; + + final MonitoringConfiguration config = MonitoringConfiguration.getInstance(configFileName, configDirPath); + + assertThat(config, is(notNullValue())); + assertThat(config.getDataSources(), is(notNullValue())); + assertThat(config.getDataSources().size(), equalTo(1)); + final DataSource dataSource = config.getDataSources().get(0); + assertThat(dataSource, is(IsInstanceOf.instanceOf(GitlabDataSource.class))); + final GitlabDataSource gitlabDataSource = (GitlabDataSource) dataSource; + assertThat(gitlabDataSource.getId(), equalTo(expectedId)); + assertThat(gitlabDataSource.getMonitor(), equalTo(expectedMonitor)); + assertThat(gitlabDataSource.getUri(), equalTo(expectedUri)); + assertThat(gitlabDataSource.getGitLabAccessToken(), equalTo(expectedAccessToken)); + assertThat(gitlabDataSource.getMessageBrokerServer(), equalTo(expectedMessageBrokerHost)); + assertThat(gitlabDataSource.getMessageBrokerPort(), equalTo(expectedMessageBrokerPort)); + assertThat(gitlabDataSource.getUserName(), equalTo(expectedUsername)); + assertThat(gitlabDataSource.getPassword(), equalTo(expectedPassword)); + assertThat(gitlabDataSource.getExchange(), equalTo(expectedMessageBrokerExchange)); + assertThat(gitlabDataSource.getTopic(), equalTo(expectedMessageBrokerTopicReceive)); + assertThat(gitlabDataSource.getDleTopic(), equalTo(expectedMessageBrokerTopicSend)); + } +} diff --git a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml new file mode 100644 index 00000000..1f371f7d --- /dev/null +++ b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + From f579de55633fd71567ff595f563263afe274ebbf Mon Sep 17 00:00:00 2001 From: Sebastian Scholze <4063307+schlotze@users.noreply.github.com> Date: Wed, 13 Apr 2022 14:39:23 +0200 Subject: [PATCH 04/34] fix test --- .../models/datasources/GitlabDataSourceTest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java index cede148c..c76bfc7f 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java @@ -5,6 +5,9 @@ import org.hamcrest.core.IsInstanceOf; import org.junit.Test; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Paths; import java.util.Objects; import static org.hamcrest.MatcherAssert.assertThat; @@ -27,8 +30,14 @@ public void configFileShouldBeDeserializedToCorrectGitlabDataSource() { final String expectedMessageBrokerExchange = "smartclide-monitoring"; final String expectedMessageBrokerTopicReceive = "monitoring.git.*"; final String expectedMessageBrokerTopicSend = "dle.git.commits"; + URI temp = null; + try { + temp = this.getClass().getResource("/config/gitlab-monitoring").toURI(); + } catch (URISyntaxException e) { + e.printStackTrace(); + } final String configDirPath = - Objects.requireNonNull(this.getClass().getResource("/config/gitlab-monitoring")).getPath(); + Objects.requireNonNull(Paths.get(temp).toString()); final String configFileName = "monitoring-config.xml"; final MonitoringConfiguration config = MonitoringConfiguration.getInstance(configFileName, configDirPath); From 299eaba2345161f703712308b9c8ecb72b13e6d0 Mon Sep 17 00:00:00 2001 From: Sebastian Scholze <4063307+schlotze@users.noreply.github.com> Date: Wed, 13 Apr 2022 14:49:52 +0200 Subject: [PATCH 05/34] fix typo --- .../src/test/resources/config/monitoring-config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml index 78990e25..8a2e0bf3 100644 --- a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml @@ -12,7 +12,7 @@ monitor="de.atb.context.monitoring.monitors.GitMonitor" uri="" options="server=localhost&port=5672&exchange=smartclide-monitoring&topic=monitoring.git.*&dle-topic=dle.git.commits" - class="de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource"/> + class="de.atb.context.monitoring.config.models.datasources.GitlabDataSource"/> From 7f8665613a3679d49aa53324d354df6aa818c015 Mon Sep 17 00:00:00 2001 From: Sebastian Scholze <4063307+schlotze@users.noreply.github.com> Date: Wed, 13 Apr 2022 15:29:42 +0200 Subject: [PATCH 06/34] fix any error --- .../context/monitoring/monitors/GitMonitor.java | 3 +-- .../datasources/GitlabDataSourceTest.java | 2 +- .../gitlab-monitoring/monitoring-config.xml | 8 ++++---- .../test/resources/config/monitoring-config.xml | 17 +++++++++++++++-- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java index d0fe3219..443f9231 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java @@ -20,7 +20,6 @@ import de.atb.context.monitoring.config.models.DataSource; import de.atb.context.monitoring.config.models.Interpreter; import de.atb.context.monitoring.config.models.Monitor; -import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; import de.atb.context.monitoring.index.Indexer; import de.atb.context.monitoring.monitors.messagebroker.MessageBrokerMonitor; @@ -36,6 +35,6 @@ public GitMonitor(final DataSource dataSource, super(dataSource, interpreter, monitor, indexer, configuration); // FIXME: this is a temporary workaround and should be removed! - addProgressListener(new DleGitMonitorProgressListener((GitlabDataSource) dataSource)); + addProgressListener(new DleGitMonitorProgressListener((MessageBrokerDataSource) dataSource)); } } diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java index c76bfc7f..e1cb7de6 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java @@ -20,7 +20,7 @@ public class GitlabDataSourceTest { @Test public void configFileShouldBeDeserializedToCorrectGitlabDataSource() { final String expectedId = "datasource-gitlab"; - final String expectedMonitor = "de.atb.context.monitoring.monitors.GitMonitor"; + final String expectedMonitor = "de.atb.context.monitoring.monitors.GitlabCommitMonitor"; final String expectedUri = "https://gitlab.example.com/api/v4"; final String expectedAccessToken = "foobar"; final String expectedMessageBrokerHost = "localhost"; diff --git a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml index 1f371f7d..16504255 100644 --- a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml @@ -4,13 +4,13 @@ xsi:schemaLocation="http://www.atb-bremen.de ../monitoring-config.xsd"> - + @@ -19,8 +19,8 @@ + parser="de.atb.context.monitoring.parser.GitlabCommitParser" + analyser="de.atb.context.monitoring.analyser.GitlabCommitAnalyser"/> diff --git a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml index 8a2e0bf3..d9e691d2 100644 --- a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml @@ -1,13 +1,20 @@ + xsi:schemaLocation="http://www.atb-bremen.de ../monitoring-config.xsd"> - + + + + + + + From 6e9f78c7f955e5f98f0efcc40a628e8f2c445ab6 Mon Sep 17 00:00:00 2001 From: Philip Reimer <5737222+philipreimer@users.noreply.github.com> Date: Wed, 13 Apr 2022 23:35:58 +0200 Subject: [PATCH 07/34] fix test --- .../datasources/MessageBrokerDataSource.java | 87 +++++++++---------- .../MessageBrokerDataSourceOptions.java | 4 +- .../messagebroker/util/MessageBrokerUtil.java | 37 ++++---- .../models/datasources/GitlabDataSource.java | 12 +-- .../DleGitMonitorProgressListener.java | 27 +++--- .../context/monitoring/TestDataRetrieval.java | 32 ++++--- .../datasources/GitlabDataSourceTest.java | 15 +--- .../gitlab-monitoring/monitoring-config.xml | 2 +- .../resources/config/monitoring-config.xml | 8 +- 9 files changed, 108 insertions(+), 116 deletions(-) diff --git a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java index 526d7951..6a1844a1 100644 --- a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java +++ b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java @@ -15,8 +15,6 @@ */ -import java.net.URI; - import de.atb.context.common.authentication.Credentials; import de.atb.context.monitoring.config.models.DataSource; import de.atb.context.monitoring.config.models.DataSourceType; @@ -24,81 +22,82 @@ import thewebsemantic.Namespace; import thewebsemantic.RdfType; +import java.net.URI; + /** * MessageBrokerDataSource * * @author scholze * @version $LastChangedRevision: 156 $ - * */ @RdfType("MessageBrokerDataSource") @Namespace("http://atb-bremen.de/") public class MessageBrokerDataSource extends DataSource { - public MessageBrokerDataSource() { - } + public MessageBrokerDataSource() { + } - public MessageBrokerDataSource(final DataSource base) { - this.id = base.getId(); - this.monitor = base.getMonitor(); - this.options = base.getOptions(); - this.type = base.getType().toString(); - this.uri = base.getUri(); - } + public MessageBrokerDataSource(final DataSource base) { + this.id = base.getId(); + this.monitor = base.getMonitor(); + this.options = base.getOptions(); + this.type = base.getType().toString(); + this.uri = base.getUri(); + } - @Override - public final DataSourceType getType() { - return DataSourceType.MessageBroker; - } + @Override + public final DataSourceType getType() { + return DataSourceType.MessageBroker; + } - public final String getMessageBrokerServer() { - return this.getOptionValue(MessageBrokerDataSourceOptions.MessageBrokerServer, true); - } + public final String getMessageBrokerServer() { + return this.getOptionValue(MessageBrokerDataSourceOptions.MessageBrokerServer, true); + } - public final Integer getMessageBrokerPort() { - return this.getOptionValue(MessageBrokerDataSourceOptions.MessageBrokerPort, true); + public final Integer getMessageBrokerPort() { + return this.getOptionValue(MessageBrokerDataSourceOptions.MessageBrokerPort, true); } - public final String getUserName() { - return this.getOptionValue(MessageBrokerDataSourceOptions.UserName, true); - } + public final String getUserName() { + return this.getOptionValue(MessageBrokerDataSourceOptions.UserName, true); + } - public final String getPassword() { - return this.getOptionValue(MessageBrokerDataSourceOptions.Password, true); - } + public final String getPassword() { + return this.getOptionValue(MessageBrokerDataSourceOptions.Password, true); + } public final String getExchange() { return this.getOptionValue(MessageBrokerDataSourceOptions.Exchange, true); } - public final String getTopic() { - return this.getOptionValue(MessageBrokerDataSourceOptions.Topic, true); + public final String getInTopic() { + return this.getOptionValue(MessageBrokerDataSourceOptions.InTopic, true); } - public final String getDleTopic() { - return this.getOptionValue(MessageBrokerDataSourceOptions.DleTopic, true); + public final String getOutTopic() { + return this.getOptionValue(MessageBrokerDataSourceOptions.OutTopic, true); } public final Credentials getCredentials() { - String userName = this.getUserName(); - String password = this.getPassword(); - return new Credentials(userName, password); - } + String userName = this.getUserName(); + String password = this.getPassword(); + return new Credentials(userName, password); + } - public final IMessageBroker toMessageBroker() { - final URI myUri = URI.create(uri); + public final IMessageBroker toMessageBroker() { + final URI myUri = URI.create(uri); final Credentials myCredentials = getCredentials(); - return new IMessageBroker() { + return new IMessageBroker() { - @Override - public URI getURI() { - return myUri; - } + @Override + public URI getURI() { + return myUri; + } @Override public Credentials getCredentials() { return myCredentials; } - }; - } + }; + } } diff --git a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSourceOptions.java b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSourceOptions.java index e50fa48e..9d234edb 100644 --- a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSourceOptions.java +++ b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSourceOptions.java @@ -36,9 +36,9 @@ public enum MessageBrokerDataSourceOptions implements IDataSourceOptionValue { Exchange("exchange", String.class), - Topic("topic", String.class), + InTopic("incoming-topic", String.class), - DleTopic("dle-topic", String.class); + OutTopic("outgoing-topic", String.class); private final String key; private final Class valueType; diff --git a/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/util/MessageBrokerUtil.java b/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/util/MessageBrokerUtil.java index 6ba3f4b8..664d44d5 100644 --- a/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/util/MessageBrokerUtil.java +++ b/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/util/MessageBrokerUtil.java @@ -14,23 +14,18 @@ * #L% */ -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.UUID; -import java.util.concurrent.TimeoutException; - import com.google.gson.Gson; -import com.rabbitmq.client.BuiltinExchangeType; -import com.rabbitmq.client.CancelCallback; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; -import com.rabbitmq.client.DeliverCallback; +import com.rabbitmq.client.*; import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import java.util.concurrent.TimeoutException; + /** * Helper class wrapping methods for interacting with message broker. */ @@ -112,12 +107,12 @@ public static Channel connectToTopicExchange(final String host, * @see MessageBrokerUtil#connectToTopicExchange(String, int, String, String, String) */ public static Channel connectToTopicExchange(final MessageBrokerDataSource dataSource) - throws IOException, TimeoutException { + throws IOException, TimeoutException { return connectToTopicExchange(dataSource.getMessageBrokerServer(), - dataSource.getMessageBrokerPort(), - dataSource.getUserName(), - dataSource.getPassword(), - dataSource.getExchange()); + dataSource.getMessageBrokerPort(), + dataSource.getUserName(), + dataSource.getPassword(), + dataSource.getExchange()); } /** @@ -166,11 +161,11 @@ public static void registerListenerOnTopic(final Channel channel, final DeliverCallback deliverCallback, final CancelCallback cancelCallback) throws IOException { registerListenerOnTopic(channel, - dataSource.getExchange(), - dataSource.getTopic(), - dataSource.getId(), - deliverCallback, - cancelCallback); + dataSource.getExchange(), + dataSource.getInTopic(), + dataSource.getId(), + deliverCallback, + cancelCallback); } /** diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java index 148024f3..594e0bed 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java @@ -60,19 +60,15 @@ public final String getMessageBrokerServer() { return this.getOptionValue(MessageBrokerDataSourceOptions.MessageBrokerServer, true); } - public final Integer getMessageBrokerPort() { return this.getOptionValue(MessageBrokerDataSourceOptions.MessageBrokerPort, true); + public final Integer getMessageBrokerPort() { + return this.getOptionValue(MessageBrokerDataSourceOptions.MessageBrokerPort, true); } public final String getExchange() { return this.getOptionValue(MessageBrokerDataSourceOptions.Exchange, true); } - public final String getTopic() { - return this.getOptionValue(MessageBrokerDataSourceOptions.Topic, true); + public final String getOutTopic() { + return this.getOptionValue(MessageBrokerDataSourceOptions.OutTopic, true); } - - public final String getDleTopic() { - return this.getOptionValue(MessageBrokerDataSourceOptions.DleTopic, true); - } - } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java index 71d54ce2..adb10da8 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java @@ -43,19 +43,20 @@ public class DleGitMonitorProgressListener implements MonitoringProgressListener public DleGitMonitorProgressListener(final MessageBrokerDataSource messageBrokerDataSource) throws IOException, TimeoutException { exchange = messageBrokerDataSource.getExchange(); - topic = messageBrokerDataSource.getDleTopic(); + topic = messageBrokerDataSource.getOutTopic(); channel = MessageBrokerUtil.connectToTopicExchange(messageBrokerDataSource); } - public DleGitMonitorProgressListener(final GitlabDataSource gitlabDataSource) - throws IOException, TimeoutException { + public DleGitMonitorProgressListener(final GitlabDataSource gitlabDataSource) throws IOException, TimeoutException { exchange = gitlabDataSource.getExchange(); - topic = gitlabDataSource.getDleTopic(); - channel = MessageBrokerUtil.connectToTopicExchange(gitlabDataSource.getMessageBrokerServer(), - gitlabDataSource.getMessageBrokerPort(), - gitlabDataSource.getUserName(), - gitlabDataSource.getPassword(), - gitlabDataSource.getExchange()); + topic = gitlabDataSource.getOutTopic(); + channel = MessageBrokerUtil.connectToTopicExchange( + gitlabDataSource.getMessageBrokerServer(), + gitlabDataSource.getMessageBrokerPort(), + gitlabDataSource.getUserName(), + gitlabDataSource.getPassword(), + gitlabDataSource.getExchange() + ); } @Override @@ -83,10 +84,10 @@ public void documentAnalysed(final List> analysed, private DleMessage convertToDleMessage(final GitMessage gitMessage) { return DleMessage.builder() .monitor(CommitMessage.builder() - .user(gitMessage.getUser()) - .branch(gitMessage.getBranch()) - .files(gitMessage.getNoOfModifiedFiles()) - .build()) + .user(gitMessage.getUser()) + .branch(gitMessage.getBranch()) + .files(gitMessage.getNoOfModifiedFiles()) + .build()) .build(); } diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java index 7947667f..1dfde95c 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java @@ -49,6 +49,7 @@ public class TestDataRetrieval { private static final String ROUTING_KEY_DLE = "dle.git.commits"; private static final String QUEUE_PREFIX_DLE = "Fake-DLE"; private static final String DATASOURCE_GIT = "datasource-git"; + private static final String DATASOURCE_GITLAB = "datasource-gitlab"; private static final String MONITORING_CONFIG_FILE_NAME = "monitoring-config.xml"; private static final String AMI_REPOSITORY_ID = "AmI-repository"; @@ -73,7 +74,8 @@ public void setup() throws Exception { // write dynamically allocated message broker host and port to monitoring config file final Path monitoringConfigFilePath = ContextPathUtils.getConfigDirPath().resolve(MONITORING_CONFIG_FILE_NAME); - updateMessageBrokerDataSource(monitoringConfigFilePath, rabbitMQContainerHost, rabbitMQContainerAmqpPort); + updateMessageBrokerDataSource(monitoringConfigFilePath, DATASOURCE_GIT, rabbitMQContainerHost, rabbitMQContainerAmqpPort); + updateMessageBrokerDataSource(monitoringConfigFilePath, DATASOURCE_GITLAB, rabbitMQContainerHost, rabbitMQContainerAmqpPort); // start service ServiceMain.startService(); @@ -117,12 +119,20 @@ public void testDataRetrieval() throws IOException, InterruptedException { Thread.sleep(10000); - // get the monitored data from the repository (the latest registry) + // get the latest two entries of monitored data from the repository, one from each Analyser: + // * GitLabCommitAnalyser + // * GitAnalyser final List data = - monitoringDataRepository.getMonitoringData(ApplicationScenario.getInstance(), GitDataModel.class, 1); - - assertEquals(1, data.size()); - final List gitMessages = data.get(0).getGitMessages(); + monitoringDataRepository.getMonitoringData(ApplicationScenario.getInstance(), GitDataModel.class, 2); + + assertEquals(2, data.size()); + // TODO: GitLabCommitAnalyser stores empty GitMessage in repository. + // This is a hack to get the supposedly correct GitMessage, which was stored by GitAnalyser + final List gitMessages = data.stream() + .filter(gdm -> gdm.getGitMessages().stream().anyMatch(gm -> gm.getTimestamp() != null)) + .findAny() + .map(GitDataModel::getGitMessages) + .orElseThrow(() -> new AssertionError("received data does not contain expected GitMessages")); assertEquals(1, gitMessages.size()); final GitMessage fromRepo = gitMessages.get(0); assertEquals(gitMessage.getTimestamp(), fromRepo.getTimestamp()); @@ -134,14 +144,16 @@ public void testDataRetrieval() throws IOException, InterruptedException { assertEquals(gitMessage.getNoOfPushesInBranch(), fromRepo.getNoOfPushesInBranch()); } - private void updateMessageBrokerDataSource(final Path monitoringConfig, final String host, final Integer port) - throws Exception { + private void updateMessageBrokerDataSource(final Path monitoringConfig, + final String dataSourceId, + final String host, + final Integer port) throws Exception { final Persister persister = new Persister(); final Config config = persister.read(Config.class, new File(monitoringConfig.toString())); - final Map optionsMap = config.getDataSource(DATASOURCE_GIT).getOptionsMap(); + final Map optionsMap = config.getDataSource(dataSourceId).getOptionsMap(); optionsMap.put(MessageBrokerDataSourceOptions.MessageBrokerServer.getKeyName(), host); optionsMap.put(MessageBrokerDataSourceOptions.MessageBrokerPort.getKeyName(), port.toString()); - config.getDataSource(DATASOURCE_GIT).setOptions(optionsMap); + config.getDataSource(dataSourceId).setOptions(optionsMap); persister.write(config, new File(monitoringConfig.toString())); } diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java index e1cb7de6..f2b20922 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java @@ -5,9 +5,6 @@ import org.hamcrest.core.IsInstanceOf; import org.junit.Test; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Paths; import java.util.Objects; import static org.hamcrest.MatcherAssert.assertThat; @@ -28,16 +25,9 @@ public void configFileShouldBeDeserializedToCorrectGitlabDataSource() { final String expectedUsername = "username"; final String expectedPassword = "password"; final String expectedMessageBrokerExchange = "smartclide-monitoring"; - final String expectedMessageBrokerTopicReceive = "monitoring.git.*"; final String expectedMessageBrokerTopicSend = "dle.git.commits"; - URI temp = null; - try { - temp = this.getClass().getResource("/config/gitlab-monitoring").toURI(); - } catch (URISyntaxException e) { - e.printStackTrace(); - } final String configDirPath = - Objects.requireNonNull(Paths.get(temp).toString()); + Objects.requireNonNull(this.getClass().getResource("/config/gitlab-monitoring")).getPath(); final String configFileName = "monitoring-config.xml"; final MonitoringConfiguration config = MonitoringConfiguration.getInstance(configFileName, configDirPath); @@ -57,7 +47,6 @@ public void configFileShouldBeDeserializedToCorrectGitlabDataSource() { assertThat(gitlabDataSource.getUserName(), equalTo(expectedUsername)); assertThat(gitlabDataSource.getPassword(), equalTo(expectedPassword)); assertThat(gitlabDataSource.getExchange(), equalTo(expectedMessageBrokerExchange)); - assertThat(gitlabDataSource.getTopic(), equalTo(expectedMessageBrokerTopicReceive)); - assertThat(gitlabDataSource.getDleTopic(), equalTo(expectedMessageBrokerTopicSend)); + assertThat(gitlabDataSource.getOutTopic(), equalTo(expectedMessageBrokerTopicSend)); } } diff --git a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml index 16504255..b53531d5 100644 --- a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml @@ -12,7 +12,7 @@ type="gitlab" monitor="de.atb.context.monitoring.monitors.GitlabCommitMonitor" uri="https://gitlab.example.com/api/v4" - options="token=foobar&server=localhost&port=5672&username=username&password=password&exchange=smartclide-monitoring&topic=monitoring.git.*&dle-topic=dle.git.commits" + options="token=foobar&server=localhost&port=5672&username=username&password=password&exchange=smartclide-monitoring&outgoing-topic=dle.git.commits" class="de.atb.context.monitoring.config.models.datasources.GitlabDataSource"/> diff --git a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml index d9e691d2..e25e630f 100644 --- a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml @@ -1,7 +1,7 @@ + xsi:schemaLocation="http://www.atb-bremen.de monitoring-config.xsd"> @@ -13,13 +13,13 @@ type="gitlab" monitor="de.atb.context.monitoring.monitors.GitlabCommitMonitor" uri="https://gitlab.example.com/api/v4" - options="token=foobar&server=localhost&port=5672&username=username&password=password&exchange=smartclide-monitoring&topic=monitoring.git.*&dle-topic=dle.git.commits" + options="token=foobar&server=localhost&port=5672&username=username&password=password&exchange=smartclide-monitoring&outgoing-topic=dle.git.commits" class="de.atb.context.monitoring.config.models.datasources.GitlabDataSource"/> + options="server=localhost&port=5672&exchange=smartclide-monitoring&incoming-topic=monitoring.git.*&outgoing-topic=dle.git.commits" + class="de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource"/> From b0798ac097084d2481a1a542759427d47875d43f Mon Sep 17 00:00:00 2001 From: Philip Reimer <5737222+philipreimer@users.noreply.github.com> Date: Mon, 25 Apr 2022 20:14:34 +0200 Subject: [PATCH 08/34] enable DleGitMonitorProgressListener to send to topics or queues --- .../datasources/MessageBrokerDataSource.java | 31 +++- .../MessageBrokerDataSourceOptions.java | 14 +- .../messagebroker/MessageBrokerMonitor.java | 30 +++- .../messagebroker/util/MessageBrokerUtil.java | 164 +++++++++--------- .../models/datasources/GitlabDataSource.java | 17 +- .../monitors/GitlabCommitMonitor.java | 7 +- .../DleGitMonitorProgressListener.java | 77 ++++++-- .../context/monitoring/TestDataRetrieval.java | 127 ++++++++------ .../datasources/GitlabDataSourceTest.java | 7 +- .../gitlab-monitoring/monitoring-config.xml | 2 +- .../resources/config/monitoring-config.xml | 15 +- 11 files changed, 291 insertions(+), 200 deletions(-) diff --git a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java index 6a1844a1..3d532d2b 100644 --- a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java +++ b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSource.java @@ -22,6 +22,7 @@ import thewebsemantic.Namespace; import thewebsemantic.RdfType; +import java.io.Serializable; import java.net.URI; /** @@ -66,16 +67,34 @@ public final String getPassword() { return this.getOptionValue(MessageBrokerDataSourceOptions.Password, true); } - public final String getExchange() { - return this.getOptionValue(MessageBrokerDataSourceOptions.Exchange, true); + public final String getIncomingExchange() { + return this.getOptionValue(MessageBrokerDataSourceOptions.IncomingExchange, true); } - public final String getInTopic() { - return this.getOptionValue(MessageBrokerDataSourceOptions.InTopic, true); + public final String getIncomingTopic() { + return this.getOptionValue(MessageBrokerDataSourceOptions.IncomingTopic, true); } - public final String getOutTopic() { - return this.getOptionValue(MessageBrokerDataSourceOptions.OutTopic, true); + public final boolean isIncomingDurable() { + final Serializable value = this.getOptionValue(MessageBrokerDataSourceOptions.IncomingDurable, false); + return value != null && (boolean) value; + } + + public final String getOutgoingExchange() { + return this.getOptionValue(MessageBrokerDataSourceOptions.OutgoingExchange, true); + } + + public final String getOutgoingTopic() { + return this.getOptionValue(MessageBrokerDataSourceOptions.OutgoingTopic, true); + } + + public final String getOutgoingQueue() { + return this.getOptionValue(MessageBrokerDataSourceOptions.OutgoingQueue, true); + } + + public final boolean isOutgoingDurable() { + final Serializable value = this.getOptionValue(MessageBrokerDataSourceOptions.OutgoingDurable, false); + return value != null && (boolean) value; } public final Credentials getCredentials() { diff --git a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSourceOptions.java b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSourceOptions.java index 9d234edb..83867d66 100644 --- a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSourceOptions.java +++ b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/MessageBrokerDataSourceOptions.java @@ -34,11 +34,19 @@ public enum MessageBrokerDataSourceOptions implements IDataSourceOptionValue { Password("password", String.class), - Exchange("exchange", String.class), + IncomingExchange("incoming-exchange", String.class), - InTopic("incoming-topic", String.class), + IncomingTopic("incoming-topic", String.class), - OutTopic("outgoing-topic", String.class); + IncomingDurable("incoming-durable", Boolean.class), + + OutgoingExchange("outgoing-exchange", String.class), + + OutgoingTopic("outgoing-topic", String.class), + + OutgoingQueue("outgoing-queue", String.class), + + OutgoingDurable("outgoing-durable", Boolean.class); private final String key; private final Class valueType; diff --git a/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/MessageBrokerMonitor.java b/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/MessageBrokerMonitor.java index 0c7688fb..113c548d 100644 --- a/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/MessageBrokerMonitor.java +++ b/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/MessageBrokerMonitor.java @@ -14,8 +14,6 @@ * #L% */ -import java.nio.charset.StandardCharsets; - import com.rabbitmq.client.CancelCallback; import com.rabbitmq.client.Channel; import com.rabbitmq.client.DeliverCallback; @@ -34,6 +32,8 @@ import de.atb.context.monitoring.parser.messagebroker.MessageBrokerParser; import de.atb.context.tools.ontology.AmIMonitoringConfiguration; +import java.nio.charset.StandardCharsets; + /** * WebServiceMonitor * @@ -89,16 +89,32 @@ protected void doMonitor(final InterpreterConfiguration setting) throws Exceptio this.parser = getParser(setting); } - final Channel channel = MessageBrokerUtil.connectToTopicExchange(dataSource); - MessageBrokerUtil.registerListenerOnTopic(channel, dataSource, deliverCallback, cancelCallback); + final Channel channel = MessageBrokerUtil.connectToTopicExchange( + dataSource.getMessageBrokerServer(), + dataSource.getMessageBrokerPort(), + dataSource.getUserName(), + dataSource.getPassword(), + dataSource.getIncomingExchange(), + dataSource.isIncomingDurable() + ); + MessageBrokerUtil.registerListenerOnTopic( + channel, + dataSource.getIncomingExchange(), + dataSource.getIncomingTopic(), + dataSource.getId(), + deliverCallback, + cancelCallback + ); } } // FIXME: envelope is never used protected final void handleMessage(Envelope envelope, String message) { - this.logger.info("Handling message from exchange \"{}\" with routing key \"{}\" ...", - envelope.getExchange(), - envelope.getRoutingKey()); + this.logger.info( + "Handling message from exchange \"{}\" with routing key \"{}\" ...", + envelope.getExchange(), + envelope.getRoutingKey() + ); try { if ((this.dataSource.getUri() != null)) { MessageBrokerAnalyser analyser = (MessageBrokerAnalyser) parser.getAnalyser(); diff --git a/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/util/MessageBrokerUtil.java b/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/util/MessageBrokerUtil.java index 664d44d5..f4106a35 100644 --- a/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/util/MessageBrokerUtil.java +++ b/context-monitoring/src/main/java/de/atb/context/monitoring/monitors/messagebroker/util/MessageBrokerUtil.java @@ -15,8 +15,12 @@ */ import com.google.gson.Gson; -import com.rabbitmq.client.*; -import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; +import com.rabbitmq.client.BuiltinExchangeType; +import com.rabbitmq.client.CancelCallback; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; +import com.rabbitmq.client.DeliverCallback; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +48,7 @@ public class MessageBrokerUtil { * @param userName the username to use when connecting to message broker - optional * @param password the password to use when connecting to message broker - optional * @param exchange the topic exchange's name + * @param durable whether the topic should be durable or not * @return a {@link Channel} object representing the established connection to the message broker * @throws IOException in case of error * @throws TimeoutException in case of error @@ -52,73 +57,53 @@ public static Channel connectToTopicExchange(final String host, final int port, final String userName, final String password, - final String exchange) throws IOException, TimeoutException { - LOGGER.info("Connecting to messagebroker {}:{} with user {}", host, port, userName != null ? userName : ""); - final ConnectionFactory factory = new ConnectionFactory(); - - factory.setHost(host); - factory.setPort(port); - - if (StringUtils.isNotBlank(userName)) { - factory.setUsername(userName); - } - if (StringUtils.isNotBlank(password)) { - factory.setPassword(password); - } - - factory.setAutomaticRecoveryEnabled(true); - - final Connection connection = factory.newConnection(); + final String exchange, + final boolean durable) throws IOException, TimeoutException { + final Connection connection = getConnection(host, port, userName, password); final Channel channel = connection.createChannel(); LOGGER.info("Creating topic exchange {}", exchange); - channel.exchangeDeclare(exchange, BuiltinExchangeType.TOPIC, true); + channel.exchangeDeclare(exchange, BuiltinExchangeType.TOPIC, durable); return channel; } /** - * Connect to the message broker specified by {@code host} and {@code port}. - * Create the given {@code exchange} if it does not exist yet. + * Connect to the message broker specified by {@code host} and {@code port} + * with the credentials specified by {@code userName} and {@code password}. + * Create the given {@code queue} if it does not exist yet. * * @param host the host where the message broker is running * @param port the port where the message broker is listening - * @param exchange the topic exchange's name + * @param userName the username to use when connecting to message broker - optional + * @param password the password to use when connecting to message broker - optional + * @param queue the queue's name + * @param durable whether the queue should be durable or not * @return a {@link Channel} object representing the established connection to the message broker * @throws IOException in case of error * @throws TimeoutException in case of error - * @see MessageBrokerUtil#connectToTopicExchange(String, int, String, String, String) */ - public static Channel connectToTopicExchange(final String host, - final int port, - final String exchange) throws IOException, TimeoutException { - return connectToTopicExchange(host, port, null, null, exchange); - } + public static Channel connectToQueue(final String host, + final int port, + final String userName, + final String password, + final String queue, + final boolean durable) throws IOException, TimeoutException { + final Connection connection = getConnection(host, port, userName, password); - /** - * Connect to message broker. Message broker details and credentials are specified in the given {@code dataSource}. - * Create the given {@code exchange} if it does not exist yet. - * - * @param dataSource the {@link MessageBrokerDataSource} containing the message broker connection details - * @return a {@link Channel} object representing the established connection to the message broker - * @throws IOException in case of error - * @throws TimeoutException in case of error - * @see MessageBrokerUtil#connectToTopicExchange(String, int, String, String, String) - */ - public static Channel connectToTopicExchange(final MessageBrokerDataSource dataSource) - throws IOException, TimeoutException { - return connectToTopicExchange(dataSource.getMessageBrokerServer(), - dataSource.getMessageBrokerPort(), - dataSource.getUserName(), - dataSource.getPassword(), - dataSource.getExchange()); + final Channel channel = connection.createChannel(); + + LOGGER.info("Creating queue {}", queue); + channel.queueDeclare(queue, durable, false, false, null); + + return channel; } /** * Register the given callback functions to consume messages on the given {@code exchange} for the given {@code topic}. *

- * Use {@link MessageBrokerUtil#connectToTopicExchange(String, int, String, String, String)} or one of its overloads + * Use {@link MessageBrokerUtil#connectToTopicExchange(String, int, String, String, String, boolean)} * to create {@link Channel}. * * @param channel the {@link Channel} object representing the established connection to the message broker @@ -143,57 +128,70 @@ public static void registerListenerOnTopic(final Channel channel, channel.basicConsume(queue, true, deliverCallback, cancelCallback); } - /** - * Register the given callback functions to consume messages. - * The exchange and topic to register for are specified in the given {@code dataSource}. - *

- * Use {@link MessageBrokerUtil#connectToTopicExchange(String, int, String, String, String)} or one of its overloads - * to create {@link Channel}. - * - * @param channel the {@link Channel} object representing the established connection to the message broker - * @param dataSource the {@link MessageBrokerDataSource} containing the exchange and topic details - * @param deliverCallback callback function to handle received messages - * @param cancelCallback callback function to handle cancellation of the listener - * @throws IOException in case of error - */ - public static void registerListenerOnTopic(final Channel channel, - final MessageBrokerDataSource dataSource, - final DeliverCallback deliverCallback, - final CancelCallback cancelCallback) throws IOException { - registerListenerOnTopic(channel, - dataSource.getExchange(), - dataSource.getInTopic(), - dataSource.getId(), - deliverCallback, - cancelCallback); - } - /** * Converts the given {@code payload} object to a JSON string * and sends it to the given {@code topic} on the given {@code exchange}. *

- * Use {@link MessageBrokerUtil#connectToTopicExchange(String, int, String, String, String)} or one of its overloads + * Use {@link MessageBrokerUtil#connectToTopicExchange(String, int, String, String, String, boolean)} * to create {@link Channel}. * * @param channel the {@link Channel} object representing the established connection to the message broker * @param exchange the topic exchange's name * @param topic the topic's name * @param payload the object to send - * @throws IOException in case of error */ public static void convertAndSendToTopic(final Channel channel, final String exchange, final String topic, - final Object payload) throws IOException { - final String jsonMessage = GSON.toJson(payload); - sendToTopic(channel, exchange, topic, jsonMessage); + final Object payload) { + try { + final String jsonMessage = GSON.toJson(payload); + LOGGER.info("Publishing message to topic {}/{}: {}", exchange, topic, jsonMessage); + channel.basicPublish(exchange, topic, null, jsonMessage.getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + LOGGER.error("Failed to send {} to topic {}/{}", payload, exchange, topic, e); + } } - private static void sendToTopic(final Channel channel, - final String exchange, - final String topic, - final String jsonMessage) throws IOException { - LOGGER.info("Publishing message to topic {}/{}: {}", exchange, topic, jsonMessage); - channel.basicPublish(exchange, topic, null, jsonMessage.getBytes(StandardCharsets.UTF_8)); + /** + * Converts the given {@code payload} object to a JSON string and sends it to the given {@code queue}. + *

+ * Use {@link MessageBrokerUtil#connectToQueue(String, int, String, String, String, boolean)} + * to create {@link Channel}. + * + * @param channel the {@link Channel} object representing the established connection to the message broker + * @param queue the queue's name + * @param payload the object to send + */ + public static void convertAndSendToQueue(final Channel channel, final String queue, final Object payload) { + try { + final String jsonMessage = GSON.toJson(payload); + LOGGER.info("Publishing message to queue {}: {}", queue, jsonMessage); + channel.basicPublish("", queue, null, jsonMessage.getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + LOGGER.error("Failed to send {} to queue {}", payload, queue, e); + } + } + + private static Connection getConnection(final String host, + final int port, + final String userName, + final String password) throws IOException, TimeoutException { + LOGGER.info("Connecting to messagebroker {}:{} with user {}", host, port, userName != null ? userName : ""); + final ConnectionFactory factory = new ConnectionFactory(); + + factory.setHost(host); + factory.setPort(port); + + if (StringUtils.isNotBlank(userName)) { + factory.setUsername(userName); + } + if (StringUtils.isNotBlank(password)) { + factory.setPassword(password); + } + + factory.setAutomaticRecoveryEnabled(true); + + return factory.newConnection(); } } diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java index 594e0bed..9d7e5d18 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java @@ -64,11 +64,20 @@ public final Integer getMessageBrokerPort() { return this.getOptionValue(MessageBrokerDataSourceOptions.MessageBrokerPort, true); } - public final String getExchange() { - return this.getOptionValue(MessageBrokerDataSourceOptions.Exchange, true); + public final String getOutgoingExchange() { + return this.getOptionValue(MessageBrokerDataSourceOptions.OutgoingExchange, true); } - public final String getOutTopic() { - return this.getOptionValue(MessageBrokerDataSourceOptions.OutTopic, true); + public final String getOutgoingTopic() { + return this.getOptionValue(MessageBrokerDataSourceOptions.OutgoingTopic, true); + } + + public final String getOutgoingQueue() { + return this.getOptionValue(MessageBrokerDataSourceOptions.OutgoingQueue, true); + } + + public final boolean isOutgoingDurable() { + final Serializable value = this.getOptionValue(MessageBrokerDataSourceOptions.OutgoingDurable, false); + return value != null && (boolean) value; } } diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java index 15873e32..9c1b3bb2 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java @@ -21,17 +21,12 @@ import de.atb.context.monitoring.monitors.webservice.WebServiceMonitor; import de.atb.context.tools.ontology.AmIMonitoringConfiguration; -import java.io.IOException; -import java.util.concurrent.TimeoutException; - public class GitlabCommitMonitor extends WebServiceMonitor { public GitlabCommitMonitor(final DataSource dataSource, final Interpreter interpreter, final Monitor monitor, final Indexer indexer, - final AmIMonitoringConfiguration configuration) throws IOException, TimeoutException { + final AmIMonitoringConfiguration configuration) { super(dataSource, interpreter, monitor, indexer, configuration); - - //addProgressListener(new DleGitMonitorProgressListener((GitlabDataSource) dataSource)); } } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java index adb10da8..51e4ba92 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java @@ -14,13 +14,10 @@ * #L% */ -import java.io.IOException; -import java.util.List; -import java.util.concurrent.TimeoutException; - import com.rabbitmq.client.Channel; import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; +import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSourceOptions; import de.atb.context.monitoring.events.MonitoringProgressListener; import de.atb.context.monitoring.models.GitDataModel; import de.atb.context.monitoring.models.GitMessage; @@ -28,34 +25,52 @@ import de.atb.context.monitoring.monitors.messagebroker.util.MessageBrokerUtil; import eu.smartclide.contexthandling.dle.model.CommitMessage; import eu.smartclide.contexthandling.dle.model.DleMessage; +import org.apache.commons.lang3.StringUtils; import org.apache.lucene.document.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeoutException; + public class DleGitMonitorProgressListener implements MonitoringProgressListener> { private static final Logger LOGGER = LoggerFactory.getLogger(DleGitMonitorProgressListener.class); - private final String topic; - private final String exchange; private final Channel channel; + private final String exchange; + private final String topic; + private final String queue; + + private boolean useTopic = false; public DleGitMonitorProgressListener(final MessageBrokerDataSource messageBrokerDataSource) throws IOException, TimeoutException { - exchange = messageBrokerDataSource.getExchange(); - topic = messageBrokerDataSource.getOutTopic(); - channel = MessageBrokerUtil.connectToTopicExchange(messageBrokerDataSource); + exchange = messageBrokerDataSource.getOutgoingExchange(); + topic = messageBrokerDataSource.getOutgoingTopic(); + queue = messageBrokerDataSource.getOutgoingQueue(); + + channel = connectToMessageBroker( + messageBrokerDataSource.getMessageBrokerServer(), + messageBrokerDataSource.getMessageBrokerPort(), + messageBrokerDataSource.getUserName(), + messageBrokerDataSource.getPassword(), + messageBrokerDataSource.isOutgoingDurable() + ); } public DleGitMonitorProgressListener(final GitlabDataSource gitlabDataSource) throws IOException, TimeoutException { - exchange = gitlabDataSource.getExchange(); - topic = gitlabDataSource.getOutTopic(); - channel = MessageBrokerUtil.connectToTopicExchange( + exchange = gitlabDataSource.getOutgoingExchange(); + topic = gitlabDataSource.getOutgoingTopic(); + queue = gitlabDataSource.getOutgoingQueue(); + + channel = connectToMessageBroker( gitlabDataSource.getMessageBrokerServer(), gitlabDataSource.getMessageBrokerPort(), gitlabDataSource.getUserName(), gitlabDataSource.getPassword(), - gitlabDataSource.getExchange() + gitlabDataSource.isOutgoingDurable() ); } @@ -91,13 +106,39 @@ private DleMessage convertToDleMessage(final GitMessage gitMessage) { .build(); } + private Channel connectToMessageBroker(final String host, + final int port, + final String userName, + final String password, + final boolean durable) throws IOException, TimeoutException { + if (StringUtils.isBlank(topic) && StringUtils.isBlank(queue)) { + throw new IllegalArgumentException(String.format( + "Must specify either %s or %s!", + MessageBrokerDataSourceOptions.OutgoingTopic.getKeyName(), + MessageBrokerDataSourceOptions.OutgoingTopic.getKeyName() + )); + } + if (StringUtils.isNotBlank(topic)) { + if (StringUtils.isBlank(exchange)) { + throw new IllegalArgumentException(String.format( + "Must specify %s when connecting to topic %s!", + MessageBrokerDataSourceOptions.OutgoingExchange.getKeyName(), + MessageBrokerDataSourceOptions.OutgoingTopic.getKeyName() + )); + } + useTopic = true; + return MessageBrokerUtil.connectToTopicExchange(host, port, userName, password, exchange, durable); + } else { + useTopic = false; + return MessageBrokerUtil.connectToQueue(host, port, userName, password, queue, durable); + } + } + private void send(final DleMessage dleMessage) { - try { - // simulate that actual context-extraction will take some time - Thread.sleep(1000); + if (useTopic) { MessageBrokerUtil.convertAndSendToTopic(channel, exchange, topic, dleMessage); - } catch (Exception e) { - LOGGER.error("Failed to send {} to {}/{}", dleMessage, exchange, topic, e); + } else { + MessageBrokerUtil.convertAndSendToQueue(channel, queue, dleMessage); } } } diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java index 1dfde95c..6f1eecb0 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java @@ -1,15 +1,5 @@ package de.atb.context.monitoring; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeoutException; - import com.rabbitmq.client.Channel; import de.atb.context.common.ContextPathUtils; import de.atb.context.common.util.ApplicationScenario; @@ -31,6 +21,16 @@ import org.slf4j.LoggerFactory; import org.testcontainers.containers.RabbitMQContainer; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeoutException; + import static org.junit.Assert.assertEquals; /** @@ -44,18 +44,17 @@ public class TestDataRetrieval { private static final Logger logger = LoggerFactory.getLogger(TestDataRetrieval.class); private static final String RABBITMQ_3_ALPINE = "rabbitmq:3-alpine"; - private static final String EXCHANGE_NAME = "smartclide-monitoring"; + private static final String EXCHANGE_NAME = "mom"; private static final String ROUTING_KEY_MONITORING = "monitoring.git.commits"; - private static final String ROUTING_KEY_DLE = "dle.git.commits"; - private static final String QUEUE_PREFIX_DLE = "Fake-DLE"; + private static final String QUEUE_NAME_DLE = "code_repo_recommendation_queue"; private static final String DATASOURCE_GIT = "datasource-git"; - private static final String DATASOURCE_GITLAB = "datasource-gitlab"; private static final String MONITORING_CONFIG_FILE_NAME = "monitoring-config.xml"; private static final String AMI_REPOSITORY_ID = "AmI-repository"; private AmIMonitoringDataRepositoryServiceWrapper monitoringDataRepository; - private Channel channel; + private Channel fakeRmvChannel; + private Channel fakeDleChannel; // starts a new rabbitmq message broker in a docker container. // @Rule must be final. @@ -67,15 +66,16 @@ public void setup() throws Exception { // setup message broker final String rabbitMQContainerHost = container.getHost(); final Integer rabbitMQContainerAmqpPort = container.getAmqpPort(); - channel = MessageBrokerUtil.connectToTopicExchange(rabbitMQContainerHost, rabbitMQContainerAmqpPort, EXCHANGE_NAME); + + // setup fake RMV + fakeRmvChannel = createFakeRmvPublisher(rabbitMQContainerHost, rabbitMQContainerAmqpPort); // setup fake DLE - createFakeDleListener(); + fakeDleChannel = createFakeDleListener(rabbitMQContainerHost, rabbitMQContainerAmqpPort); // write dynamically allocated message broker host and port to monitoring config file final Path monitoringConfigFilePath = ContextPathUtils.getConfigDirPath().resolve(MONITORING_CONFIG_FILE_NAME); - updateMessageBrokerDataSource(monitoringConfigFilePath, DATASOURCE_GIT, rabbitMQContainerHost, rabbitMQContainerAmqpPort); - updateMessageBrokerDataSource(monitoringConfigFilePath, DATASOURCE_GITLAB, rabbitMQContainerHost, rabbitMQContainerAmqpPort); + updateMessageBrokerDataSource(monitoringConfigFilePath, rabbitMQContainerHost, rabbitMQContainerAmqpPort); // start service ServiceMain.startService(); @@ -96,43 +96,29 @@ public void setup() throws Exception { public void tearDown() throws IOException, TimeoutException { monitoringDataRepository.shutdown(); - if (channel != null) { - channel.close(); + if (fakeDleChannel != null) { + fakeDleChannel.close(); + } + if (fakeRmvChannel != null) { + fakeRmvChannel.close(); } } @Test - public void testDataRetrieval() throws IOException, InterruptedException { + public void testDataRetrieval() throws InterruptedException { // wait for services to start Thread.sleep(10000); - final GitMessage gitMessage = GitMessage.builder() - .timestamp(ZonedDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)) - .user("user@smartclide.eu") - .repository("git@github.com:eclipse-researchlabs/smartclide-context.git") - .branch("branch") - .noOfCommitsInBranch(42) - .noOfModifiedFiles(3) - .noOfPushesInBranch(17) - .build(); - MessageBrokerUtil.convertAndSendToTopic(channel, EXCHANGE_NAME, ROUTING_KEY_MONITORING, gitMessage); + final GitMessage gitMessage = sendFakeRmvMessage(); Thread.sleep(10000); - // get the latest two entries of monitored data from the repository, one from each Analyser: - // * GitLabCommitAnalyser - // * GitAnalyser + // get the latest entry of monitored data from the repository final List data = - monitoringDataRepository.getMonitoringData(ApplicationScenario.getInstance(), GitDataModel.class, 2); - - assertEquals(2, data.size()); - // TODO: GitLabCommitAnalyser stores empty GitMessage in repository. - // This is a hack to get the supposedly correct GitMessage, which was stored by GitAnalyser - final List gitMessages = data.stream() - .filter(gdm -> gdm.getGitMessages().stream().anyMatch(gm -> gm.getTimestamp() != null)) - .findAny() - .map(GitDataModel::getGitMessages) - .orElseThrow(() -> new AssertionError("received data does not contain expected GitMessages")); + monitoringDataRepository.getMonitoringData(ApplicationScenario.getInstance(), GitDataModel.class, 1); + + assertEquals(1, data.size()); + final List gitMessages = data.get(0).getGitMessages(); assertEquals(1, gitMessages.size()); final GitMessage fromRepo = gitMessages.get(0); assertEquals(gitMessage.getTimestamp(), fromRepo.getTimestamp()); @@ -144,28 +130,61 @@ public void testDataRetrieval() throws IOException, InterruptedException { assertEquals(gitMessage.getNoOfPushesInBranch(), fromRepo.getNoOfPushesInBranch()); } + private GitMessage sendFakeRmvMessage() { + final GitMessage gitMessage = GitMessage.builder() + .timestamp(ZonedDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)) + .user("user@smartclide.eu") + .repository("git@github.com:eclipse-researchlabs/smartclide-context.git") + .branch("branch") + .noOfCommitsInBranch(42) + .noOfModifiedFiles(3) + .noOfPushesInBranch(17) + .build(); + MessageBrokerUtil.convertAndSendToTopic(fakeRmvChannel, EXCHANGE_NAME, ROUTING_KEY_MONITORING, gitMessage); + logger.info("RMV sent message: {}", gitMessage); + return gitMessage; + } + private void updateMessageBrokerDataSource(final Path monitoringConfig, - final String dataSourceId, final String host, final Integer port) throws Exception { final Persister persister = new Persister(); final Config config = persister.read(Config.class, new File(monitoringConfig.toString())); - final Map optionsMap = config.getDataSource(dataSourceId).getOptionsMap(); + final Map optionsMap = config.getDataSource(DATASOURCE_GIT).getOptionsMap(); optionsMap.put(MessageBrokerDataSourceOptions.MessageBrokerServer.getKeyName(), host); optionsMap.put(MessageBrokerDataSourceOptions.MessageBrokerPort.getKeyName(), port.toString()); - config.getDataSource(dataSourceId).setOptions(optionsMap); + config.getDataSource(DATASOURCE_GIT).setOptions(optionsMap); persister.write(config, new File(monitoringConfig.toString())); } - private void createFakeDleListener() throws IOException { - MessageBrokerUtil.registerListenerOnTopic( - channel, + private Channel createFakeRmvPublisher(final String rabbitMQContainerHost, final Integer rabbitMQContainerAmqpPort) + throws IOException, TimeoutException { + return MessageBrokerUtil.connectToTopicExchange( + rabbitMQContainerHost, + rabbitMQContainerAmqpPort, + null, + null, EXCHANGE_NAME, - ROUTING_KEY_DLE, - QUEUE_PREFIX_DLE, + true + ); + } + + private Channel createFakeDleListener(final String rabbitMQContainerHost, final Integer rabbitMQContainerAmqpPort) + throws IOException, TimeoutException { + final Channel channel = MessageBrokerUtil.connectToQueue( + rabbitMQContainerHost, + rabbitMQContainerAmqpPort, + null, + null, + QUEUE_NAME_DLE, + false + ); + channel.basicConsume( + QUEUE_NAME_DLE, + true, (t, m) -> logger.info("DLE received message: {}", new String(m.getBody(), StandardCharsets.UTF_8)), (t) -> logger.info("cancelled!") ); + return channel; } - } diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java index f2b20922..ed5e5976 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java @@ -24,8 +24,7 @@ public void configFileShouldBeDeserializedToCorrectGitlabDataSource() { final int expectedMessageBrokerPort = 5672; final String expectedUsername = "username"; final String expectedPassword = "password"; - final String expectedMessageBrokerExchange = "smartclide-monitoring"; - final String expectedMessageBrokerTopicSend = "dle.git.commits"; + final String expectedOutgoingQueue = "code_repo_recommendation_queue"; final String configDirPath = Objects.requireNonNull(this.getClass().getResource("/config/gitlab-monitoring")).getPath(); final String configFileName = "monitoring-config.xml"; @@ -46,7 +45,7 @@ public void configFileShouldBeDeserializedToCorrectGitlabDataSource() { assertThat(gitlabDataSource.getMessageBrokerPort(), equalTo(expectedMessageBrokerPort)); assertThat(gitlabDataSource.getUserName(), equalTo(expectedUsername)); assertThat(gitlabDataSource.getPassword(), equalTo(expectedPassword)); - assertThat(gitlabDataSource.getExchange(), equalTo(expectedMessageBrokerExchange)); - assertThat(gitlabDataSource.getOutTopic(), equalTo(expectedMessageBrokerTopicSend)); + assertThat(gitlabDataSource.getOutgoingQueue(), equalTo(expectedOutgoingQueue)); + assertThat(gitlabDataSource.isOutgoingDurable(), equalTo(false)); } } diff --git a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml index b53531d5..fb94fa27 100644 --- a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml @@ -12,7 +12,7 @@ type="gitlab" monitor="de.atb.context.monitoring.monitors.GitlabCommitMonitor" uri="https://gitlab.example.com/api/v4" - options="token=foobar&server=localhost&port=5672&username=username&password=password&exchange=smartclide-monitoring&outgoing-topic=dle.git.commits" + options="token=foobar&server=localhost&port=5672&username=username&password=password&outgoing-queue=code_repo_recommendation_queue" class="de.atb.context.monitoring.config.models.datasources.GitlabDataSource"/> diff --git a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml index e25e630f..b5ad516f 100644 --- a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml @@ -4,30 +4,18 @@ xsi:schemaLocation="http://www.atb-bremen.de monitoring-config.xsd"> - - - - - - From e6f3e65d3716c29b1f70aeb2afc878aadb5f0ec8 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Mon, 25 Apr 2022 23:46:26 +0200 Subject: [PATCH 09/34] get calls to Gitlab API for getting projects, commit information for given user --- smartclide-monitoring/pom.xml | 9 ++ .../services/GitRestCallService.java | 141 ++++++++++++++++++ .../datasources/GitlabDataSourceTest.java | 26 +++- 3 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java diff --git a/smartclide-monitoring/pom.xml b/smartclide-monitoring/pom.xml index 67aef472..0a888e12 100644 --- a/smartclide-monitoring/pom.xml +++ b/smartclide-monitoring/pom.xml @@ -31,6 +31,15 @@ rabbitmq test + + + + com.google.code.gson + gson + 2.9.0 + + + diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java new file mode 100644 index 00000000..dd9c5e32 --- /dev/null +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -0,0 +1,141 @@ +package eu.smartclide.contexthandling.services; + +import com.google.gson.*; +import de.atb.context.monitoring.models.GitMessage; +import org.apache.http.ParseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.InvalidPathException; +import java.time.Duration; +import java.util.LinkedList; +import java.util.List; + +import com.google.gson.JsonArray; + +public class GitRestCallService { + + private static final Logger logger = LoggerFactory.getLogger(GitRestCallService.class); + + final String baseUri = "https://gitlab.atb-bremen.de/api/v4/projects/"; + final String accessToken = "?private_token=YFTwy2E727PbGHaiNx4e"; + final String membership = "&membership=true"; + final String uriCommitStatesComponent = "with_stats=yes"; + final String uriPagination = "&per_page=100"; + final String uriNormalEndPart = accessToken + uriPagination + membership; + + final String uriProjectGetCall = baseUri + uriNormalEndPart; + + final String uriBranchEndPart = "/refs/" + + accessToken + uriPagination + membership + "&type=all"; + + /** + * this method makes get call to gitlab server with given uri + * @param uri, string as uri for get api endpoint + * @return, JsonArray as response + */ + private JsonArray makeGetCallToGitlab(String uri) { + final HttpClient httpClient = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .connectTimeout(Duration.ofSeconds(10)) + .build(); + + HttpRequest request = null; + JsonArray returnJsonArray = new JsonArray(); + + try { + // create a GET request + request = HttpRequest.newBuilder() + .GET() + .uri(URI.create(uri)) + .build(); + } catch (InvalidPathException e) { + logger.error("HttpRequest exception", e); + } + + HttpResponse response = null; + try { + // receive response from Gitlab + response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + // parse response to JsonArray + JsonParser parser = new JsonParser(); + try { + Object object = (Object) parser.parse(response.body()); + returnJsonArray = (JsonArray) object; + } catch (ParseException e) { + logger.error("JSON Parse exception", e); + } + + } catch (IOException | InterruptedException | InvalidPathException e) { + logger.error("HTTP Client connection interruption exception", e); + } + return returnJsonArray; + } + + /** + * get user projects from Gitlab + * @param + * @return + */ + public JsonArray getUserProjects() { + return makeGetCallToGitlab(uriProjectGetCall); + } + + /** + * generates gitMessages for given user + * creates separate message for the commit in all the branches + * @return + */ + public List getGitMessages() { + // first get all user projects + JsonArray projects = getUserProjects(); + JsonArray allCommits = new JsonArray(); + List gitMessages = new LinkedList<>(); + // for each project, get commit messages for all branches + for (JsonElement project : projects) { + // get project id + String projectId = project.getAsJsonObject().get("id").getAsString(); + // get all commits for given project with uri + allCommits = makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + uriNormalEndPart); + for (JsonElement commit : allCommits) { + String commitId = commit.getAsJsonObject().get("id").getAsString(); + // get all branches for given commit, create a new GitMessage + JsonArray branches = getAllBranchesForGivenCommit(projectId, commitId); + for (JsonElement branch : branches) { + // create a new GitMessage + GitMessage gitMessage = new GitMessage(); + // set message time stamp as commit created_at + gitMessage.setTimestamp(commit.getAsJsonObject().get("created_at").getAsString()); + // set user as commit author_name + gitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); + // set repository for git message as path_with_namespace, e.g. "smartclide/keycloak-client-ng" + gitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); + // set branch as commit author_name + gitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); + // set branch name + gitMessage.setBranch(branch.getAsJsonObject().get("name").getAsString()); + // TODO: imrove this: get difference as file changed for commit + //gitMessage.setNoOfModifiedFiles(commit.getAsJsonObject().get("stats").getAsJsonObject().get("total").getAsInt()); + // add GitMessage to list of messages + gitMessages.add(gitMessage); + } + + + } + } + return gitMessages; + } + + public JsonArray getAllBranchesForGivenCommit(String projectId, String commitId) { + return makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + commitId + uriBranchEndPart); + } + + public JsonArray getCommitDiff(String projectId, String commitId) { + return makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + commitId + "/" + "diff"); + } +} diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java index f2b20922..0c8d187e 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java @@ -1,10 +1,19 @@ package de.atb.context.monitoring.config.models.datasources; +import com.google.gson.JsonArray; import de.atb.context.monitoring.config.MonitoringConfiguration; import de.atb.context.monitoring.config.models.DataSource; +import de.atb.context.monitoring.models.GitMessage; +import eu.smartclide.contexthandling.services.GitRestCallService; import org.hamcrest.core.IsInstanceOf; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.List; import java.util.Objects; import static org.hamcrest.MatcherAssert.assertThat; @@ -14,8 +23,12 @@ public class GitlabDataSourceTest { + private static final GitRestCallService gitRestCallService = new GitRestCallService(); + private static final Logger logger = LoggerFactory.getLogger(GitlabDataSourceTest.class); + + @Test - public void configFileShouldBeDeserializedToCorrectGitlabDataSource() { + public void configFileShouldBeDeserializedToCorrectGitlabDataSource() throws URISyntaxException { final String expectedId = "datasource-gitlab"; final String expectedMonitor = "de.atb.context.monitoring.monitors.GitlabCommitMonitor"; final String expectedUri = "https://gitlab.example.com/api/v4"; @@ -26,8 +39,9 @@ public void configFileShouldBeDeserializedToCorrectGitlabDataSource() { final String expectedPassword = "password"; final String expectedMessageBrokerExchange = "smartclide-monitoring"; final String expectedMessageBrokerTopicSend = "dle.git.commits"; - final String configDirPath = - Objects.requireNonNull(this.getClass().getResource("/config/gitlab-monitoring")).getPath(); + final URI uri = Objects.requireNonNull(this.getClass().getResource("/config/gitlab-monitoring")).toURI(); + final String configDirPath = Path.of(uri).toAbsolutePath().toString(); + final String configFileName = "monitoring-config.xml"; final MonitoringConfiguration config = MonitoringConfiguration.getInstance(configFileName, configDirPath); @@ -49,4 +63,10 @@ public void configFileShouldBeDeserializedToCorrectGitlabDataSource() { assertThat(gitlabDataSource.getExchange(), equalTo(expectedMessageBrokerExchange)); assertThat(gitlabDataSource.getOutTopic(), equalTo(expectedMessageBrokerTopicSend)); } + + @Test + public void getUserProjects() { + List response = gitRestCallService.getGitMessages(); + logger.info("response: " + response.size()); + } } From f497c39ede745fca197b040fafd9f6a952f45689 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Thu, 28 Apr 2022 15:08:00 +0200 Subject: [PATCH 10/34] call for retrieving number of files modified in a commit --- .../contexthandling/services/GitRestCallService.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index dd9c5e32..dbbd95ae 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -115,17 +115,14 @@ public List getGitMessages() { gitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); // set repository for git message as path_with_namespace, e.g. "smartclide/keycloak-client-ng" gitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); - // set branch as commit author_name - gitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); // set branch name gitMessage.setBranch(branch.getAsJsonObject().get("name").getAsString()); - // TODO: imrove this: get difference as file changed for commit - //gitMessage.setNoOfModifiedFiles(commit.getAsJsonObject().get("stats").getAsJsonObject().get("total").getAsInt()); + // get difference as file changed for commit + JsonArray commitDiff = getCommitDiff(projectId, commitId); + gitMessage.setNoOfModifiedFiles(commitDiff.size()); // add GitMessage to list of messages gitMessages.add(gitMessage); } - - } } return gitMessages; @@ -136,6 +133,6 @@ public JsonArray getAllBranchesForGivenCommit(String projectId, String commitId) } public JsonArray getCommitDiff(String projectId, String commitId) { - return makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + commitId + "/" + "diff"); + return makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + commitId + "/diff/" + uriNormalEndPart); } } From a642a0050cb69c7a8a26157bd3fb481c9a7cff51 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Fri, 29 Apr 2022 11:43:54 +0200 Subject: [PATCH 11/34] call for retrieving number of commits in branch --- .../contexthandling/services/GitRestCallService.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index dbbd95ae..fe3b548d 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -116,7 +116,11 @@ public List getGitMessages() { // set repository for git message as path_with_namespace, e.g. "smartclide/keycloak-client-ng" gitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); // set branch name - gitMessage.setBranch(branch.getAsJsonObject().get("name").getAsString()); + String branchName = branch.getAsJsonObject().get("name").getAsString(); + gitMessage.setBranch(branchName); + // get number of commits in a branch + JsonArray noOfCommitsInBranch = getNoOfCommitsInBranch(projectId, branchName); + gitMessage.setNoOfCommitsInBranch(noOfCommitsInBranch.size()); // get difference as file changed for commit JsonArray commitDiff = getCommitDiff(projectId, commitId); gitMessage.setNoOfModifiedFiles(commitDiff.size()); @@ -135,4 +139,8 @@ public JsonArray getAllBranchesForGivenCommit(String projectId, String commitId) public JsonArray getCommitDiff(String projectId, String commitId) { return makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + commitId + "/diff/" + uriNormalEndPart); } + + public JsonArray getNoOfCommitsInBranch(String projectId, String branchName) { + return makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + uriNormalEndPart + "&ref_name=" + branchName); + } } From 501ad2c449493ab5272538446064e65bbabd3414 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Fri, 29 Apr 2022 21:08:42 +0200 Subject: [PATCH 12/34] code cleanup --- .../services/GitRestCallService.java | 163 +++++++++--------- 1 file changed, 78 insertions(+), 85 deletions(-) diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index fe3b548d..2eefd614 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -22,31 +22,85 @@ public class GitRestCallService { private static final Logger logger = LoggerFactory.getLogger(GitRestCallService.class); - final String baseUri = "https://gitlab.atb-bremen.de/api/v4/projects/"; - final String accessToken = "?private_token=YFTwy2E727PbGHaiNx4e"; - final String membership = "&membership=true"; - final String uriCommitStatesComponent = "with_stats=yes"; - final String uriPagination = "&per_page=100"; - final String uriNormalEndPart = accessToken + uriPagination + membership; + private final String accessToken = "private_token=YFTwy2E727PbGHaiNx4e"; + private final String baseUri = "https://gitlab.atb-bremen.de/api/v4/projects/"; + private final String membership = "&membership=true"; + private final String pagination = "&per_page=100"; + private final String uriNormalEndPart = accessToken + membership + pagination; - final String uriProjectGetCall = baseUri + uriNormalEndPart; + /** + * generates gitMessages for given user + * creates separate message for the commit in all the branches + * + * @return + */ + public List getGitMessages() { + // first get all user projects + JsonArray projects = getUserProjects(); + List gitMessages = new LinkedList<>(); + for (JsonElement project : projects) { + // get project id + String projectId = project.getAsJsonObject().get("id").getAsString(); + // get all branches for given project, create a new GitMessage + JsonArray branches = getAllBranchesForGivenProject(projectId); + for (JsonElement branch : branches) { + String branchName = branch.getAsJsonObject().get("name").getAsString(); + // get all commits for given branch + JsonArray allCommitsInBranch = getAllCommitsForGivenBranch(projectId, branchName); + Integer noOfCommitsInBranch = allCommitsInBranch.size(); + for (JsonElement commit : allCommitsInBranch) { + String commitId = commit.getAsJsonObject().get("id").getAsString(); + + GitMessage gitMessage = new GitMessage(); + gitMessage.setTimestamp(commit.getAsJsonObject().get("created_at").getAsString()); + gitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); + gitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); + gitMessage.setBranch(branchName); + gitMessage.setNoOfCommitsInBranch(noOfCommitsInBranch); + + JsonArray commitDiff = getCommitDiff(projectId, commitId); + gitMessage.setNoOfModifiedFiles(commitDiff.size()); + + gitMessages.add(gitMessage); + } + } + } + return gitMessages; + } - final String uriBranchEndPart = "/refs/" + - accessToken + uriPagination + membership + "&type=all"; + private JsonArray getAllBranchesForGivenProject(String projectId) { + return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + + "/repository/branches/?" + uriNormalEndPart)); + } + + private JsonArray getAllCommitsForGivenBranch(String projectId, String branchName) { + return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + + "/repository/commits/" + "?ref_name=" + branchName + "&" + uriNormalEndPart)); + } + + private JsonArray getCommitDiff(String projectId, String commitId) { + return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + + "/repository/commits/" + commitId + "/diff/?" + uriNormalEndPart)); + } + + private JsonArray getUserProjects() { + return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + "?" + uriNormalEndPart)); + } /** * this method makes get call to gitlab server with given uri + * * @param uri, string as uri for get api endpoint * @return, JsonArray as response */ - private JsonArray makeGetCallToGitlab(String uri) { + private HttpResponse makeGetCallToGitlab(String uri) { + final HttpClient httpClient = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_1_1) - .connectTimeout(Duration.ofSeconds(10)) + .version(HttpClient.Version.HTTP_2) + .connectTimeout(Duration.ofSeconds(5 * 60)) .build(); HttpRequest request = null; - JsonArray returnJsonArray = new JsonArray(); try { // create a GET request @@ -62,85 +116,24 @@ private JsonArray makeGetCallToGitlab(String uri) { try { // receive response from Gitlab response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - // parse response to JsonArray - JsonParser parser = new JsonParser(); - try { - Object object = (Object) parser.parse(response.body()); - returnJsonArray = (JsonArray) object; - } catch (ParseException e) { - logger.error("JSON Parse exception", e); - } + } catch (IOException | InterruptedException | InvalidPathException e) { logger.error("HTTP Client connection interruption exception", e); } - return returnJsonArray; + return response; } - /** - * get user projects from Gitlab - * @param - * @return - */ - public JsonArray getUserProjects() { - return makeGetCallToGitlab(uriProjectGetCall); - } + private JsonArray parseHttpResponseToJsonArray(HttpResponse response) { + JsonArray returnJsonArray = new JsonArray(); - /** - * generates gitMessages for given user - * creates separate message for the commit in all the branches - * @return - */ - public List getGitMessages() { - // first get all user projects - JsonArray projects = getUserProjects(); - JsonArray allCommits = new JsonArray(); - List gitMessages = new LinkedList<>(); - // for each project, get commit messages for all branches - for (JsonElement project : projects) { - // get project id - String projectId = project.getAsJsonObject().get("id").getAsString(); - // get all commits for given project with uri - allCommits = makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + uriNormalEndPart); - for (JsonElement commit : allCommits) { - String commitId = commit.getAsJsonObject().get("id").getAsString(); - // get all branches for given commit, create a new GitMessage - JsonArray branches = getAllBranchesForGivenCommit(projectId, commitId); - for (JsonElement branch : branches) { - // create a new GitMessage - GitMessage gitMessage = new GitMessage(); - // set message time stamp as commit created_at - gitMessage.setTimestamp(commit.getAsJsonObject().get("created_at").getAsString()); - // set user as commit author_name - gitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); - // set repository for git message as path_with_namespace, e.g. "smartclide/keycloak-client-ng" - gitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); - // set branch name - String branchName = branch.getAsJsonObject().get("name").getAsString(); - gitMessage.setBranch(branchName); - // get number of commits in a branch - JsonArray noOfCommitsInBranch = getNoOfCommitsInBranch(projectId, branchName); - gitMessage.setNoOfCommitsInBranch(noOfCommitsInBranch.size()); - // get difference as file changed for commit - JsonArray commitDiff = getCommitDiff(projectId, commitId); - gitMessage.setNoOfModifiedFiles(commitDiff.size()); - // add GitMessage to list of messages - gitMessages.add(gitMessage); - } - } + JsonParser parser = new JsonParser(); + try { + Object object = (Object) parser.parse(response.body()); + returnJsonArray = (JsonArray) object; + } catch (ParseException e) { + logger.error("JSON Parse exception", e); } - return gitMessages; - } - - public JsonArray getAllBranchesForGivenCommit(String projectId, String commitId) { - return makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + commitId + uriBranchEndPart); - } - - public JsonArray getCommitDiff(String projectId, String commitId) { - return makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + commitId + "/diff/" + uriNormalEndPart); - } - - public JsonArray getNoOfCommitsInBranch(String projectId, String branchName) { - return makeGetCallToGitlab(baseUri + projectId + "/repository/commits/" + uriNormalEndPart + "&ref_name=" + branchName); + return returnJsonArray; } } From a0361bd409883a5379f313b41d6edf2adeb55c8f Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Mon, 2 May 2022 17:50:54 +0200 Subject: [PATCH 13/34] code cleanup --- .../services/GitRestCallService.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index 2eefd614..f6da78c4 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -22,11 +22,15 @@ public class GitRestCallService { private static final Logger logger = LoggerFactory.getLogger(GitRestCallService.class); - private final String accessToken = "private_token=YFTwy2E727PbGHaiNx4e"; + private final String accessTokenParam = "private_token=YFTwy2E727PbGHaiNx4e"; private final String baseUri = "https://gitlab.atb-bremen.de/api/v4/projects/"; - private final String membership = "&membership=true"; - private final String pagination = "&per_page=100"; - private final String uriNormalEndPart = accessToken + membership + pagination; + private final String membershipParam = "&membership=true"; + private final String paginationParam = "&per_page=100"; + private final String refNameParam = "&ref_name="; + private final String uriParams = "?" + accessTokenParam + membershipParam + paginationParam; + private final String uriPartForBranches = "/repository/branches/"; + private final String uriPartForCommits = "/repository/commits/"; + private final String uriPartForDiff = "/diff/"; /** * generates gitMessages for given user @@ -69,22 +73,21 @@ public List getGitMessages() { } private JsonArray getAllBranchesForGivenProject(String projectId) { - return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId - + "/repository/branches/?" + uriNormalEndPart)); + return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForBranches + uriParams)); } private JsonArray getAllCommitsForGivenBranch(String projectId, String branchName) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId - + "/repository/commits/" + "?ref_name=" + branchName + "&" + uriNormalEndPart)); + + uriPartForCommits + uriParams + refNameParam + branchName)); } private JsonArray getCommitDiff(String projectId, String commitId) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId - + "/repository/commits/" + commitId + "/diff/?" + uriNormalEndPart)); + + uriPartForCommits + commitId + uriPartForDiff + uriParams)); } private JsonArray getUserProjects() { - return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + "?" + uriNormalEndPart)); + return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + uriParams)); } /** From eed0cc5596023a641d088131168f8dc417fbd374 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Tue, 3 May 2022 13:01:28 +0200 Subject: [PATCH 14/34] calculate time since last commit --- .../context/monitoring/models/GitMessage.java | 3 + .../monitoring/models/GitMessageHeader.java | 17 +++++ .../services/GitRestCallService.java | 69 +++++++++++++++---- 3 files changed, 74 insertions(+), 15 deletions(-) create mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessage.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessage.java index b4deda2b..25c0c90e 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessage.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessage.java @@ -37,7 +37,10 @@ public class GitMessage { String user; String repository; String branch; + String header; + String state; Integer noOfCommitsInBranch; Integer noOfPushesInBranch; Integer noOfModifiedFiles; + Integer timeSinceLastCommit; } diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java new file mode 100644 index 00000000..b628c3a2 --- /dev/null +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java @@ -0,0 +1,17 @@ +package de.atb.context.monitoring.models; + +public enum GitMessageHeader { + NEW_COMMIT("new commit"), + + NEW_FILE_CHANGED("new file changed"); + + private final String header; + + GitMessageHeader(final String header) { + this.header = header; + } + + public String getHeader() { + return header; + } +} diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index f6da78c4..a5841afe 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -2,6 +2,7 @@ import com.google.gson.*; import de.atb.context.monitoring.models.GitMessage; +import de.atb.context.monitoring.models.GitMessageHeader; import org.apache.http.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,7 +13,9 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.file.InvalidPathException; +import java.text.SimpleDateFormat; import java.time.Duration; +import java.util.Date; import java.util.LinkedList; import java.util.List; @@ -27,6 +30,7 @@ public class GitRestCallService { private final String membershipParam = "&membership=true"; private final String paginationParam = "&per_page=100"; private final String refNameParam = "&ref_name="; + private final String sinceParam = "&since=2022-04-25T13:05:00"; private final String uriParams = "?" + accessTokenParam + membershipParam + paginationParam; private final String uriPartForBranches = "/repository/branches/"; private final String uriPartForCommits = "/repository/commits/"; @@ -49,21 +53,25 @@ public List getGitMessages() { JsonArray branches = getAllBranchesForGivenProject(projectId); for (JsonElement branch : branches) { String branchName = branch.getAsJsonObject().get("name").getAsString(); - // get all commits for given branch - JsonArray allCommitsInBranch = getAllCommitsForGivenBranch(projectId, branchName); - Integer noOfCommitsInBranch = allCommitsInBranch.size(); - for (JsonElement commit : allCommitsInBranch) { - String commitId = commit.getAsJsonObject().get("id").getAsString(); + // get all new commits for given branch + JsonArray newCommitsInBranch = getNewCommitsForGivenBranch(projectId, branchName); + for (JsonElement newCommit : newCommitsInBranch) { + String newCommitId = newCommit.getAsJsonObject().get("id").getAsString(); GitMessage gitMessage = new GitMessage(); - gitMessage.setTimestamp(commit.getAsJsonObject().get("created_at").getAsString()); - gitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); + gitMessage.setHeader(GitMessageHeader.NEW_COMMIT.getHeader()); + gitMessage.setState("info"); + gitMessage.setUser(newCommit.getAsJsonObject().get("author_name").getAsString()); gitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); gitMessage.setBranch(branchName); - gitMessage.setNoOfCommitsInBranch(noOfCommitsInBranch); - JsonArray commitDiff = getCommitDiff(projectId, commitId); - gitMessage.setNoOfModifiedFiles(commitDiff.size()); + String lastCommitId = newCommit.getAsJsonObject().get("parent_ids").getAsString(); + String newCommitCreation = newCommit.getAsJsonObject().get("created_at").getAsString(); + String lastCommitCreation = getLastCommitsCreationDate(projectId, lastCommitId); + gitMessage.setTimeSinceLastCommit(calculateTimeSinceLastCommit(newCommitCreation, lastCommitCreation)); + + JsonArray newCommitDiff = getCommitDiff(projectId, newCommitId); + gitMessage.setNoOfModifiedFiles(newCommitDiff.size()); gitMessages.add(gitMessage); } @@ -72,18 +80,38 @@ public List getGitMessages() { return gitMessages; } + private Integer calculateTimeSinceLastCommit(String newCommitDateStr, String lastCommitDateStr) { + int difference = 0; + try { + Date newCommitDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").parse(newCommitDateStr); + Date lastCommitDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").parse(lastCommitDateStr); + difference = Math.toIntExact(Math.abs(newCommitDate.getTime() - lastCommitDate.getTime())); + } catch (java.text.ParseException e) { + logger.error("date to string parse error", e); + } + return difference /1000; + } + private JsonArray getAllBranchesForGivenProject(String projectId) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForBranches + uriParams)); } - private JsonArray getAllCommitsForGivenBranch(String projectId, String branchName) { + private JsonArray getCommitDiff(String projectId, String commitId) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId - + uriPartForCommits + uriParams + refNameParam + branchName)); + + uriPartForCommits + commitId + uriPartForDiff + uriParams)); } - private JsonArray getCommitDiff(String projectId, String commitId) { + private String getLastCommitsCreationDate(String projectId, String commitId) { + String lastCommitsCreationDate; + JsonObject lastCommit = parseHttpResponseToJsonObject(makeGetCallToGitlab(baseUri + projectId + + uriPartForCommits + commitId + uriParams)); + lastCommitsCreationDate = lastCommit.getAsJsonObject().get("created_at").getAsString(); + return lastCommitsCreationDate; + } + + private JsonArray getNewCommitsForGivenBranch(String projectId, String branchName) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId - + uriPartForCommits + commitId + uriPartForDiff + uriParams)); + + uriPartForCommits + uriParams + refNameParam + branchName + sinceParam)); } private JsonArray getUserProjects() { @@ -129,7 +157,6 @@ private HttpResponse makeGetCallToGitlab(String uri) { private JsonArray parseHttpResponseToJsonArray(HttpResponse response) { JsonArray returnJsonArray = new JsonArray(); - JsonParser parser = new JsonParser(); try { Object object = (Object) parser.parse(response.body()); @@ -139,4 +166,16 @@ private JsonArray parseHttpResponseToJsonArray(HttpResponse response) { } return returnJsonArray; } + + private JsonObject parseHttpResponseToJsonObject(HttpResponse response) { + JsonObject returnJsonObject = new JsonObject(); + JsonParser parser = new JsonParser(); + try { + Object object = (Object) parser.parse(response.body()); + returnJsonObject = (JsonObject) object; + } catch (ParseException e) { + logger.error("JSON Parse exception", e); + } + return returnJsonObject; + } } From fc48d8e79e45284099e8ae1ae2c6c11beae19d62 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Tue, 3 May 2022 13:03:56 +0200 Subject: [PATCH 15/34] add todo --- .../smartclide/contexthandling/services/GitRestCallService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index a5841afe..0b505ad7 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -30,7 +30,7 @@ public class GitRestCallService { private final String membershipParam = "&membership=true"; private final String paginationParam = "&per_page=100"; private final String refNameParam = "&ref_name="; - private final String sinceParam = "&since=2022-04-25T13:05:00"; + private final String sinceParam = "&since=2022-04-25T13:05:00"; // TODO change this based on requirement private final String uriParams = "?" + accessTokenParam + membershipParam + paginationParam; private final String uriPartForBranches = "/repository/branches/"; private final String uriPartForCommits = "/repository/commits/"; From 97ab85a345cf734632d57d816349bb1e5f0bdb26 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Tue, 3 May 2022 17:34:33 +0200 Subject: [PATCH 16/34] code refactoring --- .../services/GitRestCallService.java | 77 ++++++------------- 1 file changed, 24 insertions(+), 53 deletions(-) diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index 0b505ad7..e1cc361b 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -3,7 +3,6 @@ import com.google.gson.*; import de.atb.context.monitoring.models.GitMessage; import de.atb.context.monitoring.models.GitMessageHeader; -import org.apache.http.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,6 +12,7 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.file.InvalidPathException; +import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Duration; import java.util.Date; @@ -40,7 +40,7 @@ public class GitRestCallService { * generates gitMessages for given user * creates separate message for the commit in all the branches * - * @return + * @return List */ public List getGitMessages() { // first get all user projects @@ -64,11 +64,7 @@ public List getGitMessages() { gitMessage.setUser(newCommit.getAsJsonObject().get("author_name").getAsString()); gitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); gitMessage.setBranch(branchName); - - String lastCommitId = newCommit.getAsJsonObject().get("parent_ids").getAsString(); - String newCommitCreation = newCommit.getAsJsonObject().get("created_at").getAsString(); - String lastCommitCreation = getLastCommitsCreationDate(projectId, lastCommitId); - gitMessage.setTimeSinceLastCommit(calculateTimeSinceLastCommit(newCommitCreation, lastCommitCreation)); + gitMessage.setTimeSinceLastCommit(calculateTimeSinceLastCommit(projectId, newCommit.getAsJsonObject())); JsonArray newCommitDiff = getCommitDiff(projectId, newCommitId); gitMessage.setNoOfModifiedFiles(newCommitDiff.size()); @@ -80,35 +76,38 @@ public List getGitMessages() { return gitMessages; } - private Integer calculateTimeSinceLastCommit(String newCommitDateStr, String lastCommitDateStr) { + private Integer calculateTimeSinceLastCommit(String projectId, JsonObject newCommit) { int difference = 0; + String lastCommitId = newCommit.get("parent_ids").getAsString(); + String newCommitCreationDateStr = newCommit.get("created_at").getAsString(); + JsonObject lastCommit = getCommitById(projectId, lastCommitId); + String lastCommitCreationDateStr = lastCommit.getAsJsonObject().get("created_at").getAsString(); + + DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); try { - Date newCommitDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").parse(newCommitDateStr); - Date lastCommitDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX").parse(lastCommitDateStr); - difference = Math.toIntExact(Math.abs(newCommitDate.getTime() - lastCommitDate.getTime())); + Date newCommitCreationDate = formatter.parse(newCommitCreationDateStr); + Date lastCommitCreationDate = formatter.parse(lastCommitCreationDateStr); + difference = Math.toIntExact(Math.abs(newCommitCreationDate.getTime() - lastCommitCreationDate.getTime()) / 1000); } catch (java.text.ParseException e) { logger.error("date to string parse error", e); } - return difference /1000; + return difference; } private JsonArray getAllBranchesForGivenProject(String projectId) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForBranches + uriParams)); } + private JsonObject getCommitById(String projectId, String commitId) { + return parseHttpResponseToJsonObject(makeGetCallToGitlab(baseUri + projectId + + uriPartForCommits + commitId + uriParams)); + } + private JsonArray getCommitDiff(String projectId, String commitId) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForCommits + commitId + uriPartForDiff + uriParams)); } - private String getLastCommitsCreationDate(String projectId, String commitId) { - String lastCommitsCreationDate; - JsonObject lastCommit = parseHttpResponseToJsonObject(makeGetCallToGitlab(baseUri + projectId - + uriPartForCommits + commitId + uriParams)); - lastCommitsCreationDate = lastCommit.getAsJsonObject().get("created_at").getAsString(); - return lastCommitsCreationDate; - } - private JsonArray getNewCommitsForGivenBranch(String projectId, String branchName) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForCommits + uriParams + refNameParam + branchName + sinceParam)); @@ -121,8 +120,8 @@ private JsonArray getUserProjects() { /** * this method makes get call to gitlab server with given uri * - * @param uri, string as uri for get api endpoint - * @return, JsonArray as response + * @param uri as string + * @return HttpResponse as response */ private HttpResponse makeGetCallToGitlab(String uri) { @@ -131,24 +130,12 @@ private HttpResponse makeGetCallToGitlab(String uri) { .connectTimeout(Duration.ofSeconds(5 * 60)) .build(); - HttpRequest request = null; - - try { - // create a GET request - request = HttpRequest.newBuilder() - .GET() - .uri(URI.create(uri)) - .build(); - } catch (InvalidPathException e) { - logger.error("HttpRequest exception", e); - } + HttpRequest request = HttpRequest.newBuilder().GET().uri(URI.create(uri)).build(); HttpResponse response = null; try { // receive response from Gitlab response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - - } catch (IOException | InterruptedException | InvalidPathException e) { logger.error("HTTP Client connection interruption exception", e); } @@ -156,26 +143,10 @@ private HttpResponse makeGetCallToGitlab(String uri) { } private JsonArray parseHttpResponseToJsonArray(HttpResponse response) { - JsonArray returnJsonArray = new JsonArray(); - JsonParser parser = new JsonParser(); - try { - Object object = (Object) parser.parse(response.body()); - returnJsonArray = (JsonArray) object; - } catch (ParseException e) { - logger.error("JSON Parse exception", e); - } - return returnJsonArray; + return JsonParser.parseString(response.body()).getAsJsonArray(); } private JsonObject parseHttpResponseToJsonObject(HttpResponse response) { - JsonObject returnJsonObject = new JsonObject(); - JsonParser parser = new JsonParser(); - try { - Object object = (Object) parser.parse(response.body()); - returnJsonObject = (JsonObject) object; - } catch (ParseException e) { - logger.error("JSON Parse exception", e); - } - return returnJsonObject; + return JsonParser.parseString(response.body()).getAsJsonObject(); } } From b53f109e232ef2d87ecbbabe2d1b3c1f2e4b322f Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Thu, 5 May 2022 15:58:44 +0200 Subject: [PATCH 17/34] new test file for git rest call is created. Access token is retrieved from environment variable. --- .../services/GitRestCallService.java | 9 +++-- .../monitoring/GitRestCallServiceTest.java | 33 +++++++++++++++++++ .../datasources/GitlabDataSourceTest.java | 15 --------- 3 files changed, 40 insertions(+), 17 deletions(-) create mode 100644 smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index e1cc361b..551a3868 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -25,17 +25,22 @@ public class GitRestCallService { private static final Logger logger = LoggerFactory.getLogger(GitRestCallService.class); - private final String accessTokenParam = "private_token=YFTwy2E727PbGHaiNx4e"; + private final String accessTokenParam; private final String baseUri = "https://gitlab.atb-bremen.de/api/v4/projects/"; private final String membershipParam = "&membership=true"; private final String paginationParam = "&per_page=100"; private final String refNameParam = "&ref_name="; private final String sinceParam = "&since=2022-04-25T13:05:00"; // TODO change this based on requirement - private final String uriParams = "?" + accessTokenParam + membershipParam + paginationParam; + private final String uriParams; private final String uriPartForBranches = "/repository/branches/"; private final String uriPartForCommits = "/repository/commits/"; private final String uriPartForDiff = "/diff/"; + public GitRestCallService(String accessTokenParam) { + this.accessTokenParam = "private_token=" + accessTokenParam; + this.uriParams = "?" + this.accessTokenParam + this.membershipParam + this.paginationParam; + } + /** * generates gitMessages for given user * creates separate message for the commit in all the branches diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java new file mode 100644 index 00000000..707a45aa --- /dev/null +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java @@ -0,0 +1,33 @@ +package de.atb.context.monitoring; + +import de.atb.context.monitoring.config.models.datasources.GitlabDataSourceTest; +import de.atb.context.monitoring.models.GitMessage; +import eu.smartclide.contexthandling.services.GitRestCallService; +import org.apache.commons.lang3.StringUtils; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +public class GitRestCallServiceTest { + + private GitRestCallService gitRestCallService; + private static final Logger logger = LoggerFactory.getLogger(GitlabDataSourceTest.class); + + @Before + public void setup() { + final String gitlabApiToken = System.getenv("SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN"); + if (StringUtils.isBlank(gitlabApiToken)) { + throw new IllegalStateException("Did not find valid GitLab API token in \"SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN\" environment variable!"); + } + gitRestCallService = new GitRestCallService(gitlabApiToken); + } + + @Test + public void getUserProjects() { + List response = gitRestCallService.getGitMessages(); + logger.info("response: " + response.size()); + } +} diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java index 6b351ee9..c3c428ad 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java @@ -1,19 +1,13 @@ package de.atb.context.monitoring.config.models.datasources; -import com.google.gson.JsonArray; import de.atb.context.monitoring.config.MonitoringConfiguration; import de.atb.context.monitoring.config.models.DataSource; -import de.atb.context.monitoring.models.GitMessage; -import eu.smartclide.contexthandling.services.GitRestCallService; import org.hamcrest.core.IsInstanceOf; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; -import java.util.List; import java.util.Objects; import static org.hamcrest.MatcherAssert.assertThat; @@ -23,9 +17,6 @@ public class GitlabDataSourceTest { - private static final GitRestCallService gitRestCallService = new GitRestCallService(); - private static final Logger logger = LoggerFactory.getLogger(GitlabDataSourceTest.class); - @Test public void configFileShouldBeDeserializedToCorrectGitlabDataSource() throws URISyntaxException { final String expectedId = "datasource-gitlab"; @@ -61,10 +52,4 @@ public void configFileShouldBeDeserializedToCorrectGitlabDataSource() throws URI assertThat(gitlabDataSource.getOutgoingQueue(), equalTo(expectedOutgoingQueue)); assertThat(gitlabDataSource.isOutgoingDurable(), equalTo(false)); } - - @Test - public void getUserProjects() { - List response = gitRestCallService.getGitMessages(); - logger.info("response: " + response.size()); - } } From 6cd7fb510edf6c5a24cdbcaaf2e99ef470a5429b Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Thu, 5 May 2022 21:33:55 +0200 Subject: [PATCH 18/34] checking for exceptions --- .../services/GitRestCallService.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index 551a3868..902056b2 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -3,6 +3,7 @@ import com.google.gson.*; import de.atb.context.monitoring.models.GitMessage; import de.atb.context.monitoring.models.GitMessageHeader; +import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,6 +142,10 @@ private HttpResponse makeGetCallToGitlab(String uri) { try { // receive response from Gitlab response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() != HttpStatus.SC_OK) { + logger.error("Http response error:" + response.statusCode() + response.body()); + } } catch (IOException | InterruptedException | InvalidPathException e) { logger.error("HTTP Client connection interruption exception", e); } @@ -148,10 +153,16 @@ private HttpResponse makeGetCallToGitlab(String uri) { } private JsonArray parseHttpResponseToJsonArray(HttpResponse response) { - return JsonParser.parseString(response.body()).getAsJsonArray(); + if (response != null && response.statusCode() == 200) { + return JsonParser.parseString(response.body()).getAsJsonArray(); + } + return new JsonArray(); } private JsonObject parseHttpResponseToJsonObject(HttpResponse response) { - return JsonParser.parseString(response.body()).getAsJsonObject(); + if (response != null && response.statusCode() == 200) { + return JsonParser.parseString(response.body()).getAsJsonObject(); + } + return new JsonObject(); } } From e97d7b3bfda7c8c04bb6fffacb4348c99f189c9a Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Mon, 9 May 2022 09:54:58 +0200 Subject: [PATCH 19/34] applied suggestions --- .github/workflows/run-tests.yml | 2 ++ smartclide-monitoring/pom.xml | 9 --------- .../contexthandling/services/GitRestCallService.java | 5 +++-- .../atb/context/monitoring/GitRestCallServiceTest.java | 3 +++ 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 2750a8e4..39392746 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -29,3 +29,5 @@ jobs: - name: Run Tests run: mvn --file pom.xml --batch-mode test + env: + SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN: ${{ secrets.SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN }} diff --git a/smartclide-monitoring/pom.xml b/smartclide-monitoring/pom.xml index 0a888e12..67aef472 100644 --- a/smartclide-monitoring/pom.xml +++ b/smartclide-monitoring/pom.xml @@ -31,15 +31,6 @@ rabbitmq test - - - - com.google.code.gson - gson - 2.9.0 - - - diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index 902056b2..2bd72de2 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -145,6 +145,7 @@ private HttpResponse makeGetCallToGitlab(String uri) { if (response.statusCode() != HttpStatus.SC_OK) { logger.error("Http response error:" + response.statusCode() + response.body()); + return null; } } catch (IOException | InterruptedException | InvalidPathException e) { logger.error("HTTP Client connection interruption exception", e); @@ -153,14 +154,14 @@ private HttpResponse makeGetCallToGitlab(String uri) { } private JsonArray parseHttpResponseToJsonArray(HttpResponse response) { - if (response != null && response.statusCode() == 200) { + if (response != null) { return JsonParser.parseString(response.body()).getAsJsonArray(); } return new JsonArray(); } private JsonObject parseHttpResponseToJsonObject(HttpResponse response) { - if (response != null && response.statusCode() == 200) { + if (response != null) { return JsonParser.parseString(response.body()).getAsJsonObject(); } return new JsonObject(); diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java index 707a45aa..4863a534 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java @@ -27,7 +27,10 @@ public void setup() { @Test public void getUserProjects() { + List response = gitRestCallService.getGitMessages(); + // TODO: some assertions here.. + logger.info("response: " + response.size()); } } From 8c7266b31fc237d4d8527fa5aa18a9edf2b772c5 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Mon, 9 May 2022 22:17:34 +0200 Subject: [PATCH 20/34] assertions for git rest services --- .../services/GitRestCallService.java | 26 ++++++------- .../monitoring/GitRestCallServiceTest.java | 38 ++++++++++++++----- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index 2bd72de2..9a5fbe38 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -59,20 +59,20 @@ public List getGitMessages() { JsonArray branches = getAllBranchesForGivenProject(projectId); for (JsonElement branch : branches) { String branchName = branch.getAsJsonObject().get("name").getAsString(); - // get all new commits for given branch - JsonArray newCommitsInBranch = getNewCommitsForGivenBranch(projectId, branchName); - for (JsonElement newCommit : newCommitsInBranch) { - String newCommitId = newCommit.getAsJsonObject().get("id").getAsString(); + // get all commits for given branch and since + JsonArray commitsInBranch = getCommitsForGivenBranchAndSince(projectId, branchName, sinceParam); + for (JsonElement commit : commitsInBranch) { + String commitId = commit.getAsJsonObject().get("id").getAsString(); GitMessage gitMessage = new GitMessage(); gitMessage.setHeader(GitMessageHeader.NEW_COMMIT.getHeader()); gitMessage.setState("info"); - gitMessage.setUser(newCommit.getAsJsonObject().get("author_name").getAsString()); + gitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); gitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); gitMessage.setBranch(branchName); - gitMessage.setTimeSinceLastCommit(calculateTimeSinceLastCommit(projectId, newCommit.getAsJsonObject())); + gitMessage.setTimeSinceLastCommit(calculateTimeSinceLastCommit(projectId, commit.getAsJsonObject())); - JsonArray newCommitDiff = getCommitDiff(projectId, newCommitId); + JsonArray newCommitDiff = getCommitDiff(projectId, commitId); gitMessage.setNoOfModifiedFiles(newCommitDiff.size()); gitMessages.add(gitMessage); @@ -82,7 +82,7 @@ public List getGitMessages() { return gitMessages; } - private Integer calculateTimeSinceLastCommit(String projectId, JsonObject newCommit) { + public Integer calculateTimeSinceLastCommit(String projectId, JsonObject newCommit) { int difference = 0; String lastCommitId = newCommit.get("parent_ids").getAsString(); String newCommitCreationDateStr = newCommit.get("created_at").getAsString(); @@ -100,26 +100,26 @@ private Integer calculateTimeSinceLastCommit(String projectId, JsonObject newCom return difference; } - private JsonArray getAllBranchesForGivenProject(String projectId) { + public JsonArray getAllBranchesForGivenProject(String projectId) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForBranches + uriParams)); } - private JsonObject getCommitById(String projectId, String commitId) { + public JsonObject getCommitById(String projectId, String commitId) { return parseHttpResponseToJsonObject(makeGetCallToGitlab(baseUri + projectId + uriPartForCommits + commitId + uriParams)); } - private JsonArray getCommitDiff(String projectId, String commitId) { + public JsonArray getCommitDiff(String projectId, String commitId) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForCommits + commitId + uriPartForDiff + uriParams)); } - private JsonArray getNewCommitsForGivenBranch(String projectId, String branchName) { + public JsonArray getCommitsForGivenBranchAndSince(String projectId, String branchName, String sinceParam) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForCommits + uriParams + refNameParam + branchName + sinceParam)); } - private JsonArray getUserProjects() { + public JsonArray getUserProjects() { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + uriParams)); } diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java index 4863a534..29137ff7 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java @@ -1,20 +1,19 @@ package de.atb.context.monitoring; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; import de.atb.context.monitoring.config.models.datasources.GitlabDataSourceTest; -import de.atb.context.monitoring.models.GitMessage; import eu.smartclide.contexthandling.services.GitRestCallService; import org.apache.commons.lang3.StringUtils; import org.junit.Before; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.List; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; public class GitRestCallServiceTest { private GitRestCallService gitRestCallService; - private static final Logger logger = LoggerFactory.getLogger(GitlabDataSourceTest.class); @Before public void setup() { @@ -26,11 +25,32 @@ public void setup() { } @Test - public void getUserProjects() { + public void testGitServices() { + final int expectedNumberOfProjects = 5; + final String todoProjectId = "233"; + final int expectedNumberOfBranches = 2; + final int expectedNumberOfCommits = 2; - List response = gitRestCallService.getGitMessages(); - // TODO: some assertions here.. + final String expectedBranchName = "main"; + final String expectedCommitId = "dc5b9dedf4d1c83ed6ce162196d411aa32e5e08b"; + final Integer expectedCommitTimeSinceLastCommit = 41996279; + final Integer expectedCommitNoOfFilesChanged = 1; + final String sinceParam = "&since=2020-01-26T13:05:00"; - logger.info("response: " + response.size()); + JsonArray projects = gitRestCallService.getUserProjects(); + assertThat(projects.size(), equalTo(expectedNumberOfProjects)); + + JsonArray branches = gitRestCallService.getAllBranchesForGivenProject(todoProjectId); + assertThat(branches.size(), equalTo(expectedNumberOfBranches)); + + JsonArray commits = gitRestCallService.getCommitsForGivenBranchAndSince(todoProjectId, expectedBranchName, sinceParam); + assertThat(commits.size(), equalTo(expectedNumberOfCommits)); + + JsonObject commit = gitRestCallService.getCommitById(todoProjectId, expectedCommitId); + Integer timeSinceLastCommit = gitRestCallService.calculateTimeSinceLastCommit(todoProjectId, commit); + assertThat(timeSinceLastCommit, equalTo(expectedCommitTimeSinceLastCommit)); + + JsonArray newCommitDiff = gitRestCallService.getCommitDiff(todoProjectId, expectedCommitId); + assertThat(newCommitDiff.size(), equalTo(expectedCommitNoOfFilesChanged)); } } From b461c1e3e980509f170004bc10d48395aaecf5bb Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Tue, 10 May 2022 10:50:41 +0200 Subject: [PATCH 21/34] gitlab_api_token env is added to publish-container-image.yml file --- .github/workflows/publish-container-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-container-image.yml b/.github/workflows/publish-container-image.yml index 03ced126..7eeadcc9 100644 --- a/.github/workflows/publish-container-image.yml +++ b/.github/workflows/publish-container-image.yml @@ -15,6 +15,7 @@ jobs: env: CONTAINER_REGISTRY_USERNAME: ${GITHUB_ACTOR} CONTAINER_REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN: ${{ secrets.SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN }} steps: - name: Checkout From 569fe75c90d92ebcd42d639955cf537bef4c83ad Mon Sep 17 00:00:00 2001 From: Philip Reimer <5737222+philipreimer@users.noreply.github.com> Date: Tue, 10 May 2022 16:54:39 +0200 Subject: [PATCH 22/34] remove obsolete GitMonitor and all related classes update tests for new GitlabCommitMonitor --- .../datasources/WebServiceDataSource.java | 14 +- .../monitoring/analyser/GitAnalyser.java | 59 ------- .../analyser/GitlabCommitAnalyser.java | 29 ++-- .../models/datasources/GitlabDataSource.java | 2 +- .../monitoring/models/GitDataModel.java | 110 ------------- .../monitoring/models/GitMessageHeader.java | 14 ++ .../monitoring/monitors/GitMonitor.java | 40 ----- .../monitors/GitlabCommitMonitor.java | 5 + .../context/monitoring/parser/GitParser.java | 36 ----- .../monitoring/parser/GitlabCommitParser.java | 4 + .../DleGitMonitorProgressListener.java | 144 ------------------ .../dle/model/CommitMessage.java | 8 +- .../services/GitRestCallService.java | 14 ++ ...eval.java => GitlabCommitMonitorTest.java} | 118 ++++++-------- .../datasources/GitlabDataSourceTest.java | 2 +- .../gitlab-monitoring/monitoring-config.xml | 9 +- .../resources/config/monitoring-config.xml | 24 +-- 17 files changed, 134 insertions(+), 498 deletions(-) delete mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitAnalyser.java delete mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitDataModel.java delete mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java delete mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitParser.java delete mode 100644 smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java rename smartclide-monitoring/src/test/java/de/atb/context/monitoring/{TestDataRetrieval.java => GitlabCommitMonitorTest.java} (57%) diff --git a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/WebServiceDataSource.java b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/WebServiceDataSource.java index e8042093..61efca4b 100644 --- a/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/WebServiceDataSource.java +++ b/context-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/WebServiceDataSource.java @@ -15,14 +15,14 @@ */ -import java.net.URI; - +import de.atb.context.common.authentication.Credentials; import de.atb.context.monitoring.config.models.DataSource; import de.atb.context.monitoring.config.models.DataSourceType; import de.atb.context.monitoring.models.IWebService; import thewebsemantic.Namespace; import thewebsemantic.RdfType; -import de.atb.context.common.authentication.Credentials; + +import java.net.URI; /** * WebServiceDataSource @@ -68,6 +68,10 @@ public final String getPassword() { return this.getOptionValue(WebServiceDataSourceOptions.Password, true); } + public final Long getStartDelay() { + return this.getOptionValue(WebServiceDataSourceOptions.StartDelay, true); + } + public final String getMachineId() { return this.getOptionValue(WebServiceDataSourceOptions.MachineId, true); } @@ -78,10 +82,6 @@ public final Credentials getCredentials() { return new Credentials(userName, password); } - public final Long getStartDelay() { - return this.getOptionValue(WebServiceDataSourceOptions.StartDelay, true); - } - public final IWebService toWebService() { final URI myUri = URI.create(uri); final Credentials myCredentials = getCredentials(); diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitAnalyser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitAnalyser.java deleted file mode 100644 index 75a48db2..00000000 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitAnalyser.java +++ /dev/null @@ -1,59 +0,0 @@ -package de.atb.context.monitoring.analyser; - -/* - * #%L - * SmartCLIDE Monitoring - * %% - * Copyright (C) 2021 ATB – Institut für angewandte Systemtechnik Bremen GmbH - * %% - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * #L% - */ - -import java.util.Date; -import java.util.List; - -import com.google.gson.Gson; -import de.atb.context.monitoring.analyser.messagebroker.MessageBrokerAnalyser; -import de.atb.context.monitoring.config.models.DataSource; -import de.atb.context.monitoring.config.models.InterpreterConfiguration; -import de.atb.context.monitoring.index.Indexer; -import de.atb.context.monitoring.models.GitDataModel; -import de.atb.context.monitoring.models.GitMessage; -import de.atb.context.tools.ontology.AmIMonitoringConfiguration; -import org.apache.lucene.document.Document; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class GitAnalyser extends MessageBrokerAnalyser { - - private static final Logger logger = LoggerFactory.getLogger(GitAnalyser.class); - - private static final Gson GSON = new Gson(); - - public GitAnalyser(final DataSource dataSource, - final InterpreterConfiguration interpreterConfiguration, - final Indexer indexer, - final Document document, - final AmIMonitoringConfiguration amiConfiguration) { - super(dataSource, interpreterConfiguration, indexer, document, amiConfiguration); - } - - @Override - public List analyseObject(final String input) { - try { - final GitMessage gitMessage = GSON.fromJson(input, GitMessage.class); - final GitDataModel model = new GitDataModel(); - model.addGitMessage(gitMessage); - model.setMonitoredAt(new Date()); - return List.of(model); - } catch (Exception e) { - logger.error("Error analysing input: {}", input); - return List.of(); - } - } -} diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java index b1c21b9a..734a0382 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java @@ -14,28 +14,28 @@ * #L% */ -import com.google.gson.Gson; import de.atb.context.monitoring.analyser.webservice.WebServiceAnalyser; import de.atb.context.monitoring.config.models.DataSource; import de.atb.context.monitoring.config.models.InterpreterConfiguration; +import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.index.Indexer; -import de.atb.context.monitoring.models.GitDataModel; import de.atb.context.monitoring.models.GitMessage; +import de.atb.context.monitoring.models.GitlabCommitDataModel; import de.atb.context.monitoring.models.IWebService; import de.atb.context.tools.ontology.AmIMonitoringConfiguration; +import eu.smartclide.contexthandling.services.GitRestCallService; import org.apache.lucene.document.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URI; import java.util.Date; import java.util.List; -public class GitlabCommitAnalyser extends WebServiceAnalyser { +public class GitlabCommitAnalyser extends WebServiceAnalyser { private static final Logger logger = LoggerFactory.getLogger(GitlabCommitAnalyser.class); - private static final Gson GSON = new Gson(); + private final GitRestCallService gitRestCallService; public GitlabCommitAnalyser(final DataSource dataSource, final InterpreterConfiguration interpreterConfiguration, @@ -43,20 +43,23 @@ public GitlabCommitAnalyser(final DataSource dataSource, final Document document, final AmIMonitoringConfiguration amiConfiguration) { super(dataSource, interpreterConfiguration, indexer, document, amiConfiguration); + if (!(dataSource instanceof GitlabDataSource)) { + throw new IllegalArgumentException("Given dataSource must be of type GitlabDataSource!"); + } + gitRestCallService = new GitRestCallService(((GitlabDataSource) dataSource).getGitLabAccessToken()); } @Override - public List analyseObject(IWebService service) { + public List analyseObject(IWebService service) { try { - final GitDataModel model = new GitDataModel(); - URI gitlabEndpoint = service.getURI(); + final List gitMessages = gitRestCallService.getGitMessages(); - // TODO "NEED TO BE IMPLEMENTED" - // * get list of projects where CH user has access to - // * iterate through projects and get last commits - // * for each identified "new" commit add a GitNessage below - model.addGitMessage(new GitMessage()); + if (gitMessages.isEmpty()) { + return List.of(); + } + final GitlabCommitDataModel model = new GitlabCommitDataModel(); + model.setGitMessages(gitMessages); model.setMonitoredAt(new Date()); return List.of(model); } catch (Exception e) { diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java index 9d7e5d18..967ff246 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSource.java @@ -32,7 +32,7 @@ @NoArgsConstructor public class GitlabDataSource extends WebServiceDataSource { - private static final IDataSourceOptionValue ACCESS_TOKEN_OPTION = new IDataSourceOptionValue() { + public static final IDataSourceOptionValue ACCESS_TOKEN_OPTION = new IDataSourceOptionValue() { @Override public String getKeyName() { return "token"; diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitDataModel.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitDataModel.java deleted file mode 100644 index 1f7c0e57..00000000 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitDataModel.java +++ /dev/null @@ -1,110 +0,0 @@ -package de.atb.context.monitoring.models; - -/* - * #%L - * SmartCLIDE Monitoring - * %% - * Copyright (C) 2021 ATB – Institut für angewandte Systemtechnik Bremen GmbH - * %% - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * #L% - */ - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.UUID; - -import org.apache.jena.rdf.model.Model; -import de.atb.context.common.Version; -import de.atb.context.common.util.ApplicationScenario; -import de.atb.context.common.util.BusinessCase; -import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; -import de.atb.context.monitoring.rdf.RdfHelper; -import de.atb.context.persistence.ModelOutputLanguage; -import lombok.Getter; -import lombok.Setter; -import org.simpleframework.xml.Root; -import thewebsemantic.Namespace; -import thewebsemantic.RdfType; -import thewebsemantic.Id; - -@RdfType("GitDataModel") -@Namespace(BusinessCase.NS_DUMMY_URL) -@Root -@Getter -@Setter -public class GitDataModel implements IMonitoringDataModel { - private Date monitoredAt; - private String documentIndexId = "index/broker"; - private String documentUri; - @Id - private String identifier; - private MessageBrokerDataSource dataSource; - private String implementingClassName = GitDataModel.class.getName(); - private String monitoringDataVersion = Version.MONITORING_DATA.getVersionString(); - private List gitMessages = new ArrayList<>(); - - public GitDataModel() { - this.identifier = UUID.randomUUID().toString(); - } - - @Override - public GitDataModel fromRdfModel(String rdfModel) { - return RdfHelper.createMonitoringData(rdfModel, GitDataModel.class); - } - - public void addGitMessage(GitMessage gitMessage) { - if (!this.gitMessages.contains(gitMessage)) { - this.gitMessages.add(gitMessage); - } - } - - @Override - public GitDataModel fromRdfModel(Model model) { - return RdfHelper.createMonitoringData(model, GitDataModel.class); - } - - @Override - public String toRdfString() { - return ModelOutputLanguage.RDFXML.getModelAsString(this.toRdfModel()); - } - - @Override - public Model toRdfModel() { - return RdfHelper.createRdfModel(this); - } - - @Override - public String getContextIdentifierClassName() { - return null; - } - - @Override - public void setDataSource(MessageBrokerDataSource dataSource) { - } - - @Override - public BusinessCase getBusinessCase() { - return BusinessCase.getInstance(BusinessCase.NS_DUMMY_ID, BusinessCase.NS_DUMMY_URL); - } - - @Override - public ApplicationScenario getApplicationScenario() { - return ApplicationScenario.getInstance(getBusinessCase()); - } - - @Override - public boolean triggersContextChange() { - return false; - } - - @Override - public void initialize() { - - } -} diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java index b628c3a2..083c79f0 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java @@ -1,5 +1,19 @@ package de.atb.context.monitoring.models; +/*- + * #%L + * SmartCLIDE Monitoring + * %% + * Copyright (C) 2015 - 2022 ATB – Institut für angewandte Systemtechnik Bremen GmbH + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + public enum GitMessageHeader { NEW_COMMIT("new commit"), diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java deleted file mode 100644 index 443f9231..00000000 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitMonitor.java +++ /dev/null @@ -1,40 +0,0 @@ -package de.atb.context.monitoring.monitors; - -/* - * #%L - * SmartCLIDE Monitoring - * %% - * Copyright (C) 2021 ATB – Institut für angewandte Systemtechnik Bremen GmbH - * %% - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * #L% - */ - -import java.io.IOException; -import java.util.concurrent.TimeoutException; - -import de.atb.context.monitoring.config.models.DataSource; -import de.atb.context.monitoring.config.models.Interpreter; -import de.atb.context.monitoring.config.models.Monitor; -import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; -import de.atb.context.monitoring.index.Indexer; -import de.atb.context.monitoring.monitors.messagebroker.MessageBrokerMonitor; -import de.atb.context.tools.ontology.AmIMonitoringConfiguration; -import eu.smartclide.contexthandling.dle.listener.DleGitMonitorProgressListener; - -public class GitMonitor extends MessageBrokerMonitor { - public GitMonitor(final DataSource dataSource, - final Interpreter interpreter, - final Monitor monitor, - final Indexer indexer, - final AmIMonitoringConfiguration configuration) throws IOException, TimeoutException { - super(dataSource, interpreter, monitor, indexer, configuration); - - // FIXME: this is a temporary workaround and should be removed! - addProgressListener(new DleGitMonitorProgressListener((MessageBrokerDataSource) dataSource)); - } -} diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java index 9c1b3bb2..e98bdee0 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java @@ -17,6 +17,7 @@ import de.atb.context.monitoring.config.models.DataSource; import de.atb.context.monitoring.config.models.Interpreter; import de.atb.context.monitoring.config.models.Monitor; +import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.index.Indexer; import de.atb.context.monitoring.monitors.webservice.WebServiceMonitor; import de.atb.context.tools.ontology.AmIMonitoringConfiguration; @@ -28,5 +29,9 @@ public GitlabCommitMonitor(final DataSource dataSource, final Indexer indexer, final AmIMonitoringConfiguration configuration) { super(dataSource, interpreter, monitor, indexer, configuration); + if (!(dataSource instanceof GitlabDataSource)) { + throw new IllegalArgumentException("Given dataSource must be of type GitlabDataSource!"); + } + this.logger.info("Initialized GitlabCommitMonitor for uri: " + dataSource.getUri()); } } diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitParser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitParser.java deleted file mode 100644 index d0f2c252..00000000 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitParser.java +++ /dev/null @@ -1,36 +0,0 @@ -package de.atb.context.monitoring.parser; - -/* - * #%L - * SmartCLIDE Monitoring - * %% - * Copyright (C) 2021 ATB – Institut für angewandte Systemtechnik Bremen GmbH - * %% - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * #L% - */ - -import de.atb.context.monitoring.config.models.DataSource; -import de.atb.context.monitoring.config.models.InterpreterConfiguration; -import de.atb.context.monitoring.index.Indexer; -import de.atb.context.monitoring.parser.messagebroker.MessageBrokerParser; -import de.atb.context.tools.ontology.AmIMonitoringConfiguration; -import org.apache.lucene.document.Document; - -public class GitParser extends MessageBrokerParser { - public GitParser(final DataSource dataSource, - final InterpreterConfiguration interpreterConfiguration, - final Indexer indexer, - final AmIMonitoringConfiguration amiConfiguration) { - super(dataSource, interpreterConfiguration, indexer, amiConfiguration); - } - - @Override - protected boolean parseObject(final String message, final Document document) { - return true; - } -} diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitlabCommitParser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitlabCommitParser.java index 3c350844..4d5ea56b 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitlabCommitParser.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/parser/GitlabCommitParser.java @@ -16,6 +16,7 @@ import de.atb.context.monitoring.config.models.DataSource; import de.atb.context.monitoring.config.models.InterpreterConfiguration; +import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.index.Indexer; import de.atb.context.monitoring.models.IWebService; import de.atb.context.monitoring.parser.webservice.WebServiceParser; @@ -28,6 +29,9 @@ public GitlabCommitParser(final DataSource dataSource, final Indexer indexer, final AmIMonitoringConfiguration amiConfiguration) { super(dataSource, interpreterConfiguration, indexer, amiConfiguration); + if (!(dataSource instanceof GitlabDataSource)) { + throw new IllegalArgumentException("Given dataSource must be of type GitlabDataSource!"); + } } @Override diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java deleted file mode 100644 index 51e4ba92..00000000 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitMonitorProgressListener.java +++ /dev/null @@ -1,144 +0,0 @@ -package eu.smartclide.contexthandling.dle.listener; - -/*- - * #%L - * SmartCLIDE Monitoring - * %% - * Copyright (C) 2015 - 2021 ATB – Institut für angewandte Systemtechnik Bremen GmbH - * %% - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * #L% - */ - -import com.rabbitmq.client.Channel; -import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; -import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource; -import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSourceOptions; -import de.atb.context.monitoring.events.MonitoringProgressListener; -import de.atb.context.monitoring.models.GitDataModel; -import de.atb.context.monitoring.models.GitMessage; -import de.atb.context.monitoring.models.IMonitoringDataModel; -import de.atb.context.monitoring.monitors.messagebroker.util.MessageBrokerUtil; -import eu.smartclide.contexthandling.dle.model.CommitMessage; -import eu.smartclide.contexthandling.dle.model.DleMessage; -import org.apache.commons.lang3.StringUtils; -import org.apache.lucene.document.Document; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.TimeoutException; - -public class DleGitMonitorProgressListener implements MonitoringProgressListener> { - - private static final Logger LOGGER = LoggerFactory.getLogger(DleGitMonitorProgressListener.class); - - private final Channel channel; - private final String exchange; - private final String topic; - private final String queue; - - private boolean useTopic = false; - - public DleGitMonitorProgressListener(final MessageBrokerDataSource messageBrokerDataSource) - throws IOException, TimeoutException { - exchange = messageBrokerDataSource.getOutgoingExchange(); - topic = messageBrokerDataSource.getOutgoingTopic(); - queue = messageBrokerDataSource.getOutgoingQueue(); - - channel = connectToMessageBroker( - messageBrokerDataSource.getMessageBrokerServer(), - messageBrokerDataSource.getMessageBrokerPort(), - messageBrokerDataSource.getUserName(), - messageBrokerDataSource.getPassword(), - messageBrokerDataSource.isOutgoingDurable() - ); - } - - public DleGitMonitorProgressListener(final GitlabDataSource gitlabDataSource) throws IOException, TimeoutException { - exchange = gitlabDataSource.getOutgoingExchange(); - topic = gitlabDataSource.getOutgoingTopic(); - queue = gitlabDataSource.getOutgoingQueue(); - - channel = connectToMessageBroker( - gitlabDataSource.getMessageBrokerServer(), - gitlabDataSource.getMessageBrokerPort(), - gitlabDataSource.getUserName(), - gitlabDataSource.getPassword(), - gitlabDataSource.isOutgoingDurable() - ); - } - - @Override - public void documentIndexed(final String indexId, final Document document) { - // noop - } - - @Override - public void documentParsed(final String parsed, final Document document) { - // noop - } - - @Override - public void documentAnalysed(final List> analysed, - final String parsed, - final Document document) { - analysed.stream() - .filter(iMonitoringDataModel -> iMonitoringDataModel instanceof GitDataModel) - .map(iMonitoringDataModel -> (GitDataModel) iMonitoringDataModel) - .flatMap(gitDataModel -> gitDataModel.getGitMessages().stream()) - .map(this::convertToDleMessage) - .forEach(this::send); - } - - private DleMessage convertToDleMessage(final GitMessage gitMessage) { - return DleMessage.builder() - .monitor(CommitMessage.builder() - .user(gitMessage.getUser()) - .branch(gitMessage.getBranch()) - .files(gitMessage.getNoOfModifiedFiles()) - .build()) - .build(); - } - - private Channel connectToMessageBroker(final String host, - final int port, - final String userName, - final String password, - final boolean durable) throws IOException, TimeoutException { - if (StringUtils.isBlank(topic) && StringUtils.isBlank(queue)) { - throw new IllegalArgumentException(String.format( - "Must specify either %s or %s!", - MessageBrokerDataSourceOptions.OutgoingTopic.getKeyName(), - MessageBrokerDataSourceOptions.OutgoingTopic.getKeyName() - )); - } - if (StringUtils.isNotBlank(topic)) { - if (StringUtils.isBlank(exchange)) { - throw new IllegalArgumentException(String.format( - "Must specify %s when connecting to topic %s!", - MessageBrokerDataSourceOptions.OutgoingExchange.getKeyName(), - MessageBrokerDataSourceOptions.OutgoingTopic.getKeyName() - )); - } - useTopic = true; - return MessageBrokerUtil.connectToTopicExchange(host, port, userName, password, exchange, durable); - } else { - useTopic = false; - return MessageBrokerUtil.connectToQueue(host, port, userName, password, queue, durable); - } - } - - private void send(final DleMessage dleMessage) { - if (useTopic) { - MessageBrokerUtil.convertAndSendToTopic(channel, exchange, topic, dleMessage); - } else { - MessageBrokerUtil.convertAndSendToQueue(channel, queue, dleMessage); - } - } -} diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java index f13552be..5ba45455 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java @@ -14,6 +14,7 @@ * #L% */ +import com.google.gson.annotations.SerializedName; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; @@ -32,7 +33,12 @@ public class CommitMessage { final String header = "new commit"; @Builder.Default final String state = "info"; + @SerializedName("repo_id") + String repoId; String user; String branch; - Integer files; + @SerializedName("time_since_last_commit") + Integer timeSinceLastCommit; + @SerializedName("number_of_files_modified") + Integer numberOfFilesModified; } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java index 9a5fbe38..662b260e 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java @@ -1,5 +1,19 @@ package eu.smartclide.contexthandling.services; +/*- + * #%L + * SmartCLIDE Monitoring + * %% + * Copyright (C) 2015 - 2022 ATB – Institut für angewandte Systemtechnik Bremen GmbH + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + import com.google.gson.*; import de.atb.context.monitoring.models.GitMessage; import de.atb.context.monitoring.models.GitMessageHeader; diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java similarity index 57% rename from smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java rename to smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java index 6f1eecb0..46cb5dac 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/TestDataRetrieval.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java @@ -4,14 +4,16 @@ import de.atb.context.common.ContextPathUtils; import de.atb.context.common.util.ApplicationScenario; import de.atb.context.monitoring.config.models.Config; +import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSourceOptions; -import de.atb.context.monitoring.models.GitDataModel; import de.atb.context.monitoring.models.GitMessage; +import de.atb.context.monitoring.models.GitlabCommitDataModel; import de.atb.context.monitoring.monitors.messagebroker.util.MessageBrokerUtil; import de.atb.context.services.IAmIMonitoringDataRepositoryService; import de.atb.context.services.manager.ServiceManager; import de.atb.context.services.wrapper.AmIMonitoringDataRepositoryServiceWrapper; import eu.smartclide.contexthandling.ServiceMain; +import org.apache.commons.lang3.StringUtils; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -25,13 +27,12 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Map; import java.util.concurrent.TimeoutException; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * TestMonitoringService @@ -39,21 +40,17 @@ * @author scholze * @version $LastChangedRevision: 577 $ */ -public class TestDataRetrieval { +public class GitlabCommitMonitorTest { - private static final Logger logger = LoggerFactory.getLogger(TestDataRetrieval.class); + private static final Logger logger = LoggerFactory.getLogger(GitlabCommitMonitorTest.class); private static final String RABBITMQ_3_ALPINE = "rabbitmq:3-alpine"; - private static final String EXCHANGE_NAME = "mom"; - private static final String ROUTING_KEY_MONITORING = "monitoring.git.commits"; private static final String QUEUE_NAME_DLE = "code_repo_recommendation_queue"; - private static final String DATASOURCE_GIT = "datasource-git"; + private static final String DATASOURCE_GITLAB = "datasource-gitlab"; private static final String MONITORING_CONFIG_FILE_NAME = "monitoring-config.xml"; private static final String AMI_REPOSITORY_ID = "AmI-repository"; - private AmIMonitoringDataRepositoryServiceWrapper monitoringDataRepository; - - private Channel fakeRmvChannel; + private AmIMonitoringDataRepositoryServiceWrapper monitoringDataRepository; private Channel fakeDleChannel; // starts a new rabbitmq message broker in a docker container. @@ -63,19 +60,21 @@ public class TestDataRetrieval { @Before public void setup() throws Exception { + final String gitlabApiToken = System.getenv("SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN"); + if (StringUtils.isBlank(gitlabApiToken)) { + throw new IllegalStateException("Did not find valid GitLab API token in \"SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN\" environment variable!"); + } + // setup message broker final String rabbitMQContainerHost = container.getHost(); final Integer rabbitMQContainerAmqpPort = container.getAmqpPort(); - // setup fake RMV - fakeRmvChannel = createFakeRmvPublisher(rabbitMQContainerHost, rabbitMQContainerAmqpPort); - // setup fake DLE fakeDleChannel = createFakeDleListener(rabbitMQContainerHost, rabbitMQContainerAmqpPort); // write dynamically allocated message broker host and port to monitoring config file final Path monitoringConfigFilePath = ContextPathUtils.getConfigDirPath().resolve(MONITORING_CONFIG_FILE_NAME); - updateMessageBrokerDataSource(monitoringConfigFilePath, rabbitMQContainerHost, rabbitMQContainerAmqpPort); + updateDataSource(monitoringConfigFilePath, rabbitMQContainerHost, rabbitMQContainerAmqpPort, gitlabApiToken); // start service ServiceMain.startService(); @@ -85,7 +84,7 @@ public void setup() throws Exception { .filter(container -> container.getId().equals(AMI_REPOSITORY_ID)) .findFirst() .map(container -> { - final IAmIMonitoringDataRepositoryService repositoryService = + final IAmIMonitoringDataRepositoryService repositoryService = ServiceManager.getWebservice(container); return new AmIMonitoringDataRepositoryServiceWrapper<>(repositoryService); }) @@ -94,81 +93,54 @@ public void setup() throws Exception { @After public void tearDown() throws IOException, TimeoutException { - monitoringDataRepository.shutdown(); + if (monitoringDataRepository != null) { + monitoringDataRepository.shutdown(); + } if (fakeDleChannel != null) { fakeDleChannel.close(); } - if (fakeRmvChannel != null) { - fakeRmvChannel.close(); - } } @Test - public void testDataRetrieval() throws InterruptedException { - // wait for services to start - Thread.sleep(10000); - - final GitMessage gitMessage = sendFakeRmvMessage(); - - Thread.sleep(10000); - + public void testDoMonitor() throws InterruptedException { // get the latest entry of monitored data from the repository - final List data = - monitoringDataRepository.getMonitoringData(ApplicationScenario.getInstance(), GitDataModel.class, 1); + List data = List.of(); + int counter = 0; + while (counter <= 5 && data.isEmpty()) { + counter++; + //noinspection BusyWait + Thread.sleep(2000); + data = monitoringDataRepository.getMonitoringData(ApplicationScenario.getInstance(), GitlabCommitDataModel.class, 1); + } assertEquals(1, data.size()); final List gitMessages = data.get(0).getGitMessages(); - assertEquals(1, gitMessages.size()); - final GitMessage fromRepo = gitMessages.get(0); - assertEquals(gitMessage.getTimestamp(), fromRepo.getTimestamp()); - assertEquals(gitMessage.getUser(), fromRepo.getUser()); - assertEquals(gitMessage.getRepository(), fromRepo.getRepository()); - assertEquals(gitMessage.getBranch(), fromRepo.getBranch()); - assertEquals(gitMessage.getNoOfCommitsInBranch(), fromRepo.getNoOfCommitsInBranch()); - assertEquals(gitMessage.getNoOfModifiedFiles(), fromRepo.getNoOfModifiedFiles()); - assertEquals(gitMessage.getNoOfPushesInBranch(), fromRepo.getNoOfPushesInBranch()); + gitMessages.forEach(gitMessage -> { + assertEquals("new commit", gitMessage.getHeader()); + assertEquals("info", gitMessage.getState()); + assertTrue(StringUtils.isNotBlank(gitMessage.getUser())); + assertTrue(StringUtils.isNotBlank(gitMessage.getRepository())); + assertTrue(StringUtils.isNotBlank(gitMessage.getBranch())); + assertTrue(gitMessage.getTimeSinceLastCommit() > 0); + assertTrue(gitMessage.getNoOfModifiedFiles() > 0); + }); } - private GitMessage sendFakeRmvMessage() { - final GitMessage gitMessage = GitMessage.builder() - .timestamp(ZonedDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)) - .user("user@smartclide.eu") - .repository("git@github.com:eclipse-researchlabs/smartclide-context.git") - .branch("branch") - .noOfCommitsInBranch(42) - .noOfModifiedFiles(3) - .noOfPushesInBranch(17) - .build(); - MessageBrokerUtil.convertAndSendToTopic(fakeRmvChannel, EXCHANGE_NAME, ROUTING_KEY_MONITORING, gitMessage); - logger.info("RMV sent message: {}", gitMessage); - return gitMessage; - } - - private void updateMessageBrokerDataSource(final Path monitoringConfig, - final String host, - final Integer port) throws Exception { + private void updateDataSource(final Path monitoringConfig, + final String messageBrokerHost, + final Integer messageBrokerPort, + final String gitlabApiToken) throws Exception { final Persister persister = new Persister(); final Config config = persister.read(Config.class, new File(monitoringConfig.toString())); - final Map optionsMap = config.getDataSource(DATASOURCE_GIT).getOptionsMap(); - optionsMap.put(MessageBrokerDataSourceOptions.MessageBrokerServer.getKeyName(), host); - optionsMap.put(MessageBrokerDataSourceOptions.MessageBrokerPort.getKeyName(), port.toString()); - config.getDataSource(DATASOURCE_GIT).setOptions(optionsMap); + final Map optionsMap = config.getDataSource(DATASOURCE_GITLAB).getOptionsMap(); + optionsMap.put(MessageBrokerDataSourceOptions.MessageBrokerServer.getKeyName(), messageBrokerHost); + optionsMap.put(MessageBrokerDataSourceOptions.MessageBrokerPort.getKeyName(), messageBrokerPort.toString()); + optionsMap.put(GitlabDataSource.ACCESS_TOKEN_OPTION.getKeyName(), gitlabApiToken); + config.getDataSource(DATASOURCE_GITLAB).setOptions(optionsMap); persister.write(config, new File(monitoringConfig.toString())); } - private Channel createFakeRmvPublisher(final String rabbitMQContainerHost, final Integer rabbitMQContainerAmqpPort) - throws IOException, TimeoutException { - return MessageBrokerUtil.connectToTopicExchange( - rabbitMQContainerHost, - rabbitMQContainerAmqpPort, - null, - null, - EXCHANGE_NAME, - true - ); - } - private Channel createFakeDleListener(final String rabbitMQContainerHost, final Integer rabbitMQContainerAmqpPort) throws IOException, TimeoutException { final Channel channel = MessageBrokerUtil.connectToQueue( diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java index c3c428ad..b8a85596 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java @@ -22,7 +22,7 @@ public void configFileShouldBeDeserializedToCorrectGitlabDataSource() throws URI final String expectedId = "datasource-gitlab"; final String expectedMonitor = "de.atb.context.monitoring.monitors.GitlabCommitMonitor"; final String expectedUri = "https://gitlab.example.com/api/v4"; - final String expectedAccessToken = "foobar"; + final String expectedAccessToken = "s3cr3t"; final String expectedMessageBrokerHost = "localhost"; final int expectedMessageBrokerPort = 5672; final String expectedUsername = "username"; diff --git a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml index fb94fa27..11fc7d48 100644 --- a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml @@ -9,10 +9,10 @@ @@ -25,7 +25,10 @@ - + diff --git a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml index b5ad516f..7b623543 100644 --- a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml @@ -4,27 +4,31 @@ xsi:schemaLocation="http://www.atb-bremen.de monitoring-config.xsd"> - + - + - + + parser="de.atb.context.monitoring.parser.GitlabCommitParser" + analyser="de.atb.context.monitoring.analyser.GitlabCommitAnalyser"/> - + From 821086de29ce60377bc5592e1ce1c6f6197b8bbb Mon Sep 17 00:00:00 2001 From: Philip Reimer <5737222+philipreimer@users.noreply.github.com> Date: Tue, 10 May 2022 17:25:56 +0200 Subject: [PATCH 23/34] increase wait time for test --- .../java/de/atb/context/monitoring/GitlabCommitMonitorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java index 46cb5dac..8a7eb0c6 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java @@ -107,7 +107,7 @@ public void testDoMonitor() throws InterruptedException { // get the latest entry of monitored data from the repository List data = List.of(); int counter = 0; - while (counter <= 5 && data.isEmpty()) { + while (counter <= 10 && data.isEmpty()) { counter++; //noinspection BusyWait Thread.sleep(2000); From 1ae3aa18a845031918397551aca5a221a5690fe9 Mon Sep 17 00:00:00 2001 From: Philip Reimer <5737222+philipreimer@users.noreply.github.com> Date: Tue, 10 May 2022 17:30:00 +0200 Subject: [PATCH 24/34] increase wait time for test even more because calls to GitLab seem to be slow on GH actions --- .../java/de/atb/context/monitoring/GitlabCommitMonitorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java index 8a7eb0c6..552fc01c 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java @@ -107,7 +107,7 @@ public void testDoMonitor() throws InterruptedException { // get the latest entry of monitored data from the repository List data = List.of(); int counter = 0; - while (counter <= 10 && data.isEmpty()) { + while (counter <= 30 && data.isEmpty()) { counter++; //noinspection BusyWait Thread.sleep(2000); From 1756567ef0af462055a0504f678b594a5e551b21 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Thu, 12 May 2022 10:17:38 +0200 Subject: [PATCH 25/34] applied suggestions --- .../analyser/GitlabCommitAnalyser.java | 15 ++--- .../monitoring/models/GitMessageHeader.java | 31 ---------- .../models/GitlabCommitDataModel.java | 8 +-- ...tMessage.java => GitlabCommitMessage.java} | 7 +-- ...tCallService.java => GitlabApiClient.java} | 58 +++++++++---------- ...viceTest.java => GitlabApiClientTest.java} | 25 ++++---- .../monitoring/GitlabCommitMonitorTest.java | 18 +++--- 7 files changed, 63 insertions(+), 99 deletions(-) delete mode 100644 smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java rename smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/{GitMessage.java => GitlabCommitMessage.java} (86%) rename smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/{GitRestCallService.java => GitlabApiClient.java} (74%) rename smartclide-monitoring/src/test/java/de/atb/context/monitoring/{GitRestCallServiceTest.java => GitlabApiClientTest.java} (61%) diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java index 734a0382..15ad635a 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java @@ -19,11 +19,11 @@ import de.atb.context.monitoring.config.models.InterpreterConfiguration; import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.index.Indexer; -import de.atb.context.monitoring.models.GitMessage; +import de.atb.context.monitoring.models.GitlabCommitMessage; import de.atb.context.monitoring.models.GitlabCommitDataModel; import de.atb.context.monitoring.models.IWebService; import de.atb.context.tools.ontology.AmIMonitoringConfiguration; -import eu.smartclide.contexthandling.services.GitRestCallService; +import eu.smartclide.contexthandling.services.GitlabApiClient; import org.apache.lucene.document.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,9 +33,10 @@ public class GitlabCommitAnalyser extends WebServiceAnalyser { + private static final String gitlabBaseUri = "https://gitlab.atb-bremen.de"; private static final Logger logger = LoggerFactory.getLogger(GitlabCommitAnalyser.class); - private final GitRestCallService gitRestCallService; + private final GitlabApiClient gitlabApiClient; public GitlabCommitAnalyser(final DataSource dataSource, final InterpreterConfiguration interpreterConfiguration, @@ -46,20 +47,20 @@ public GitlabCommitAnalyser(final DataSource dataSource, if (!(dataSource instanceof GitlabDataSource)) { throw new IllegalArgumentException("Given dataSource must be of type GitlabDataSource!"); } - gitRestCallService = new GitRestCallService(((GitlabDataSource) dataSource).getGitLabAccessToken()); + gitlabApiClient = new GitlabApiClient(((GitlabDataSource) dataSource).getGitLabAccessToken(), gitlabBaseUri); } @Override public List analyseObject(IWebService service) { try { - final List gitMessages = gitRestCallService.getGitMessages(); + final List gitlabCommitMessages = gitlabApiClient.getGitlabCommitMessages(); - if (gitMessages.isEmpty()) { + if (gitlabCommitMessages.isEmpty()) { return List.of(); } final GitlabCommitDataModel model = new GitlabCommitDataModel(); - model.setGitMessages(gitMessages); + model.setGitlabCommitMessages(gitlabCommitMessages); model.setMonitoredAt(new Date()); return List.of(model); } catch (Exception e) { diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java deleted file mode 100644 index 083c79f0..00000000 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessageHeader.java +++ /dev/null @@ -1,31 +0,0 @@ -package de.atb.context.monitoring.models; - -/*- - * #%L - * SmartCLIDE Monitoring - * %% - * Copyright (C) 2015 - 2022 ATB – Institut für angewandte Systemtechnik Bremen GmbH - * %% - * This program and the accompanying materials are made - * available under the terms of the Eclipse Public License 2.0 - * which is available at https://www.eclipse.org/legal/epl-2.0/ - * - * SPDX-License-Identifier: EPL-2.0 - * #L% - */ - -public enum GitMessageHeader { - NEW_COMMIT("new commit"), - - NEW_FILE_CHANGED("new file changed"); - - private final String header; - - GitMessageHeader(final String header) { - this.header = header; - } - - public String getHeader() { - return header; - } -} diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java index 883925d0..969221f1 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitDataModel.java @@ -47,15 +47,15 @@ public class GitlabCommitDataModel implements IMonitoringDataModel gitMessages = new ArrayList<>(); + private List gitlabCommitMessages = new ArrayList<>(); public GitlabCommitDataModel() { this.identifier = UUID.randomUUID().toString(); } - public void addGitMessage(GitMessage gitMessage) { - if (!this.gitMessages.contains(gitMessage)) { - this.gitMessages.add(gitMessage); + public void addGitMessage(GitlabCommitMessage gitlabCommitMessage) { + if (!this.gitlabCommitMessages.contains(gitlabCommitMessage)) { + this.gitlabCommitMessages.add(gitlabCommitMessage); } } diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessage.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java similarity index 86% rename from smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessage.java rename to smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java index 25c0c90e..3159fce7 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitMessage.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java @@ -32,15 +32,10 @@ @NoArgsConstructor @AllArgsConstructor @ToString -public class GitMessage { - String timestamp; +public class GitlabCommitMessage { String user; String repository; String branch; - String header; - String state; - Integer noOfCommitsInBranch; - Integer noOfPushesInBranch; Integer noOfModifiedFiles; Integer timeSinceLastCommit; } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java similarity index 74% rename from smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java rename to smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java index 662b260e..0370ebce 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitRestCallService.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java @@ -14,9 +14,10 @@ * #L% */ -import com.google.gson.*; -import de.atb.context.monitoring.models.GitMessage; -import de.atb.context.monitoring.models.GitMessageHeader; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import de.atb.context.monitoring.models.GitlabCommitMessage; import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,24 +37,25 @@ import com.google.gson.JsonArray; -public class GitRestCallService { +public class GitlabApiClient { - private static final Logger logger = LoggerFactory.getLogger(GitRestCallService.class); + private static final Logger logger = LoggerFactory.getLogger(GitlabApiClient.class); - private final String accessTokenParam; - private final String baseUri = "https://gitlab.atb-bremen.de/api/v4/projects/"; - private final String membershipParam = "&membership=true"; - private final String paginationParam = "&per_page=100"; - private final String refNameParam = "&ref_name="; - private final String sinceParam = "&since=2022-04-25T13:05:00"; // TODO change this based on requirement + private static final String membershipParam = "&membership=true"; + private static final String paginationParam = "&per_page=100"; + private static final String refNameParam = "&ref_name="; + private static final String sinceParam = "&since=2022-04-25T13:05:00"; // TODO change this based on requirement + private static final String uriPartForBranches = "/repository/branches/"; + private static final String uriPartForCommits = "/repository/commits/"; + private static final String uriPartForDiff = "/diff/"; + private static final String uriPartForProjects = "/api/v4/projects/"; + private final String baseUri; private final String uriParams; - private final String uriPartForBranches = "/repository/branches/"; - private final String uriPartForCommits = "/repository/commits/"; - private final String uriPartForDiff = "/diff/"; - public GitRestCallService(String accessTokenParam) { - this.accessTokenParam = "private_token=" + accessTokenParam; - this.uriParams = "?" + this.accessTokenParam + this.membershipParam + this.paginationParam; + public GitlabApiClient(String accessToken, String gitlabBaseUri) { + String accessTokenParam = "private_token=" + accessToken; + this.uriParams = "?" + accessTokenParam + membershipParam + paginationParam; + this.baseUri = gitlabBaseUri + uriPartForProjects; } /** @@ -62,10 +64,10 @@ public GitRestCallService(String accessTokenParam) { * * @return List */ - public List getGitMessages() { + public List getGitlabCommitMessages() { // first get all user projects JsonArray projects = getUserProjects(); - List gitMessages = new LinkedList<>(); + List gitlabCommitMessages = new LinkedList<>(); for (JsonElement project : projects) { // get project id String projectId = project.getAsJsonObject().get("id").getAsString(); @@ -78,22 +80,20 @@ public List getGitMessages() { for (JsonElement commit : commitsInBranch) { String commitId = commit.getAsJsonObject().get("id").getAsString(); - GitMessage gitMessage = new GitMessage(); - gitMessage.setHeader(GitMessageHeader.NEW_COMMIT.getHeader()); - gitMessage.setState("info"); - gitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); - gitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); - gitMessage.setBranch(branchName); - gitMessage.setTimeSinceLastCommit(calculateTimeSinceLastCommit(projectId, commit.getAsJsonObject())); + GitlabCommitMessage gitlabCommitMessage = new GitlabCommitMessage(); + gitlabCommitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); + gitlabCommitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); + gitlabCommitMessage.setBranch(branchName); + gitlabCommitMessage.setTimeSinceLastCommit(calculateTimeSinceLastCommit(projectId, commit.getAsJsonObject())); JsonArray newCommitDiff = getCommitDiff(projectId, commitId); - gitMessage.setNoOfModifiedFiles(newCommitDiff.size()); + gitlabCommitMessage.setNoOfModifiedFiles(newCommitDiff.size()); - gitMessages.add(gitMessage); + gitlabCommitMessages.add(gitlabCommitMessage); } } } - return gitMessages; + return gitlabCommitMessages; } public Integer calculateTimeSinceLastCommit(String projectId, JsonObject newCommit) { diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java similarity index 61% rename from smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java rename to smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java index 29137ff7..daed28a1 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitRestCallServiceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java @@ -2,18 +2,19 @@ import com.google.gson.JsonArray; import com.google.gson.JsonObject; -import de.atb.context.monitoring.config.models.datasources.GitlabDataSourceTest; -import eu.smartclide.contexthandling.services.GitRestCallService; +import eu.smartclide.contexthandling.services.GitlabApiClient; import org.apache.commons.lang3.StringUtils; import org.junit.Before; import org.junit.Test; import static org.hamcrest.core.IsEqual.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertTrue; -public class GitRestCallServiceTest { +public class GitlabApiClientTest { - private GitRestCallService gitRestCallService; + private static final String gitlabBaseUri = "https://gitlab.atb-bremen.de"; + private GitlabApiClient gitlabApiClient; @Before public void setup() { @@ -21,7 +22,7 @@ public void setup() { if (StringUtils.isBlank(gitlabApiToken)) { throw new IllegalStateException("Did not find valid GitLab API token in \"SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN\" environment variable!"); } - gitRestCallService = new GitRestCallService(gitlabApiToken); + gitlabApiClient = new GitlabApiClient(gitlabApiToken, gitlabBaseUri); } @Test @@ -37,20 +38,20 @@ public void testGitServices() { final Integer expectedCommitNoOfFilesChanged = 1; final String sinceParam = "&since=2020-01-26T13:05:00"; - JsonArray projects = gitRestCallService.getUserProjects(); + JsonArray projects = gitlabApiClient.getUserProjects(); assertThat(projects.size(), equalTo(expectedNumberOfProjects)); - JsonArray branches = gitRestCallService.getAllBranchesForGivenProject(todoProjectId); - assertThat(branches.size(), equalTo(expectedNumberOfBranches)); + JsonArray branches = gitlabApiClient.getAllBranchesForGivenProject(todoProjectId); + assertTrue(branches.size() > 0); - JsonArray commits = gitRestCallService.getCommitsForGivenBranchAndSince(todoProjectId, expectedBranchName, sinceParam); + JsonArray commits = gitlabApiClient.getCommitsForGivenBranchAndSince(todoProjectId, expectedBranchName, sinceParam); assertThat(commits.size(), equalTo(expectedNumberOfCommits)); - JsonObject commit = gitRestCallService.getCommitById(todoProjectId, expectedCommitId); - Integer timeSinceLastCommit = gitRestCallService.calculateTimeSinceLastCommit(todoProjectId, commit); + JsonObject commit = gitlabApiClient.getCommitById(todoProjectId, expectedCommitId); + Integer timeSinceLastCommit = gitlabApiClient.calculateTimeSinceLastCommit(todoProjectId, commit); assertThat(timeSinceLastCommit, equalTo(expectedCommitTimeSinceLastCommit)); - JsonArray newCommitDiff = gitRestCallService.getCommitDiff(todoProjectId, expectedCommitId); + JsonArray newCommitDiff = gitlabApiClient.getCommitDiff(todoProjectId, expectedCommitId); assertThat(newCommitDiff.size(), equalTo(expectedCommitNoOfFilesChanged)); } } diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java index 552fc01c..7ab0e6f2 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java @@ -6,7 +6,7 @@ import de.atb.context.monitoring.config.models.Config; import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSourceOptions; -import de.atb.context.monitoring.models.GitMessage; +import de.atb.context.monitoring.models.GitlabCommitMessage; import de.atb.context.monitoring.models.GitlabCommitDataModel; import de.atb.context.monitoring.monitors.messagebroker.util.MessageBrokerUtil; import de.atb.context.services.IAmIMonitoringDataRepositoryService; @@ -115,15 +115,13 @@ public void testDoMonitor() throws InterruptedException { } assertEquals(1, data.size()); - final List gitMessages = data.get(0).getGitMessages(); - gitMessages.forEach(gitMessage -> { - assertEquals("new commit", gitMessage.getHeader()); - assertEquals("info", gitMessage.getState()); - assertTrue(StringUtils.isNotBlank(gitMessage.getUser())); - assertTrue(StringUtils.isNotBlank(gitMessage.getRepository())); - assertTrue(StringUtils.isNotBlank(gitMessage.getBranch())); - assertTrue(gitMessage.getTimeSinceLastCommit() > 0); - assertTrue(gitMessage.getNoOfModifiedFiles() > 0); + final List gitlabCommitMessages = data.get(0).getGitlabCommitMessages(); + gitlabCommitMessages.forEach(gitlabCommitMessage -> { + assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getUser())); + assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getRepository())); + assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getBranch())); + assertTrue(gitlabCommitMessage.getTimeSinceLastCommit() > 0); + assertTrue(gitlabCommitMessage.getNoOfModifiedFiles() > 0); }); } From baaba8d1251b624b53e3faa28ed60db11d7941c4 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Thu, 12 May 2022 10:18:56 +0200 Subject: [PATCH 26/34] clean up --- .../test/java/de/atb/context/monitoring/GitlabApiClientTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java index daed28a1..9306e289 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java @@ -29,7 +29,6 @@ public void setup() { public void testGitServices() { final int expectedNumberOfProjects = 5; final String todoProjectId = "233"; - final int expectedNumberOfBranches = 2; final int expectedNumberOfCommits = 2; final String expectedBranchName = "main"; From a3c361ef51a73dae2adc91dcc4e583a1259aa214 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Thu, 12 May 2022 12:53:20 +0200 Subject: [PATCH 27/34] applied suggestions --- .../analyser/GitlabCommitAnalyser.java | 6 +-- .../services/GitlabApiClient.java | 16 ++++---- .../monitoring/GitlabApiClientTest.java | 40 ++++++------------- 3 files changed, 23 insertions(+), 39 deletions(-) diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java index 15ad635a..6bfb1285 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java @@ -9,7 +9,7 @@ * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - * + * * SPDX-License-Identifier: EPL-2.0 * #L% */ @@ -33,7 +33,6 @@ public class GitlabCommitAnalyser extends WebServiceAnalyser { - private static final String gitlabBaseUri = "https://gitlab.atb-bremen.de"; private static final Logger logger = LoggerFactory.getLogger(GitlabCommitAnalyser.class); private final GitlabApiClient gitlabApiClient; @@ -47,7 +46,8 @@ public GitlabCommitAnalyser(final DataSource dataSource, if (!(dataSource instanceof GitlabDataSource)) { throw new IllegalArgumentException("Given dataSource must be of type GitlabDataSource!"); } - gitlabApiClient = new GitlabApiClient(((GitlabDataSource) dataSource).getGitLabAccessToken(), gitlabBaseUri); + gitlabApiClient = new GitlabApiClient(((GitlabDataSource) dataSource).getGitLabAccessToken(), + dataSource.getUri()); } @Override diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java index 0370ebce..8aab300d 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java @@ -76,7 +76,7 @@ public List getGitlabCommitMessages() { for (JsonElement branch : branches) { String branchName = branch.getAsJsonObject().get("name").getAsString(); // get all commits for given branch and since - JsonArray commitsInBranch = getCommitsForGivenBranchAndSince(projectId, branchName, sinceParam); + JsonArray commitsInBranch = getCommitsForGivenBranch(projectId, branchName); for (JsonElement commit : commitsInBranch) { String commitId = commit.getAsJsonObject().get("id").getAsString(); @@ -96,7 +96,7 @@ public List getGitlabCommitMessages() { return gitlabCommitMessages; } - public Integer calculateTimeSinceLastCommit(String projectId, JsonObject newCommit) { + private Integer calculateTimeSinceLastCommit(String projectId, JsonObject newCommit) { int difference = 0; String lastCommitId = newCommit.get("parent_ids").getAsString(); String newCommitCreationDateStr = newCommit.get("created_at").getAsString(); @@ -114,26 +114,26 @@ public Integer calculateTimeSinceLastCommit(String projectId, JsonObject newComm return difference; } - public JsonArray getAllBranchesForGivenProject(String projectId) { + private JsonArray getAllBranchesForGivenProject(String projectId) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForBranches + uriParams)); } - public JsonObject getCommitById(String projectId, String commitId) { + private JsonObject getCommitById(String projectId, String commitId) { return parseHttpResponseToJsonObject(makeGetCallToGitlab(baseUri + projectId + uriPartForCommits + commitId + uriParams)); } - public JsonArray getCommitDiff(String projectId, String commitId) { + private JsonArray getCommitDiff(String projectId, String commitId) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForCommits + commitId + uriPartForDiff + uriParams)); } - public JsonArray getCommitsForGivenBranchAndSince(String projectId, String branchName, String sinceParam) { + private JsonArray getCommitsForGivenBranch(String projectId, String branchName) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId + uriPartForCommits + uriParams + refNameParam + branchName + sinceParam)); } - public JsonArray getUserProjects() { + private JsonArray getUserProjects() { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + uriParams)); } @@ -147,7 +147,7 @@ private HttpResponse makeGetCallToGitlab(String uri) { final HttpClient httpClient = HttpClient.newBuilder() .version(HttpClient.Version.HTTP_2) - .connectTimeout(Duration.ofSeconds(5 * 60)) + .connectTimeout(Duration.ofMinutes(5)) .build(); HttpRequest request = HttpRequest.newBuilder().GET().uri(URI.create(uri)).build(); diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java index 9306e289..e4b664aa 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java @@ -1,14 +1,13 @@ package de.atb.context.monitoring; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; +import de.atb.context.monitoring.models.GitlabCommitMessage; import eu.smartclide.contexthandling.services.GitlabApiClient; import org.apache.commons.lang3.StringUtils; import org.junit.Before; import org.junit.Test; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; +import java.util.List; + import static org.junit.Assert.assertTrue; public class GitlabApiClientTest { @@ -27,30 +26,15 @@ public void setup() { @Test public void testGitServices() { - final int expectedNumberOfProjects = 5; - final String todoProjectId = "233"; - final int expectedNumberOfCommits = 2; - - final String expectedBranchName = "main"; - final String expectedCommitId = "dc5b9dedf4d1c83ed6ce162196d411aa32e5e08b"; - final Integer expectedCommitTimeSinceLastCommit = 41996279; - final Integer expectedCommitNoOfFilesChanged = 1; - final String sinceParam = "&since=2020-01-26T13:05:00"; - - JsonArray projects = gitlabApiClient.getUserProjects(); - assertThat(projects.size(), equalTo(expectedNumberOfProjects)); - - JsonArray branches = gitlabApiClient.getAllBranchesForGivenProject(todoProjectId); - assertTrue(branches.size() > 0); - - JsonArray commits = gitlabApiClient.getCommitsForGivenBranchAndSince(todoProjectId, expectedBranchName, sinceParam); - assertThat(commits.size(), equalTo(expectedNumberOfCommits)); - - JsonObject commit = gitlabApiClient.getCommitById(todoProjectId, expectedCommitId); - Integer timeSinceLastCommit = gitlabApiClient.calculateTimeSinceLastCommit(todoProjectId, commit); - assertThat(timeSinceLastCommit, equalTo(expectedCommitTimeSinceLastCommit)); - JsonArray newCommitDiff = gitlabApiClient.getCommitDiff(todoProjectId, expectedCommitId); - assertThat(newCommitDiff.size(), equalTo(expectedCommitNoOfFilesChanged)); + List gitlabCommitMessages = gitlabApiClient.getGitlabCommitMessages(); + assertTrue(gitlabCommitMessages.size() > 0); + gitlabCommitMessages.forEach(gitlabCommitMessage -> { + assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getUser())); + assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getRepository())); + assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getBranch())); + assertTrue(gitlabCommitMessage.getNoOfModifiedFiles() > 0); + assertTrue(gitlabCommitMessage.getTimeSinceLastCommit() > 0); + }); } } From c835206ddca588fdc367d04d0d06432168aabb18 Mon Sep 17 00:00:00 2001 From: Philip Reimer <5737222+philipreimer@users.noreply.github.com> Date: Thu, 12 May 2022 14:24:32 +0200 Subject: [PATCH 28/34] add progress listener to send analysed GitlabCommitMessages to DLE --- .../analyser/GitlabCommitAnalyser.java | 11 +- .../monitors/GitlabCommitMonitor.java | 7 +- ...leGitlabCommitMonitorProgressListener.java | 132 ++++++++++++++++++ .../services/GitlabApiClient.java | 2 +- .../monitoring/GitlabCommitMonitorTest.java | 14 +- .../datasources/GitlabDataSourceTest.java | 2 +- .../gitlab-monitoring/monitoring-config.xml | 2 +- .../resources/config/monitoring-config.xml | 2 +- 8 files changed, 161 insertions(+), 11 deletions(-) create mode 100644 smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitlabCommitMonitorProgressListener.java diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java index 6bfb1285..873a5a74 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java @@ -9,7 +9,7 @@ * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - * + * * SPDX-License-Identifier: EPL-2.0 * #L% */ @@ -19,8 +19,8 @@ import de.atb.context.monitoring.config.models.InterpreterConfiguration; import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.index.Indexer; -import de.atb.context.monitoring.models.GitlabCommitMessage; import de.atb.context.monitoring.models.GitlabCommitDataModel; +import de.atb.context.monitoring.models.GitlabCommitMessage; import de.atb.context.monitoring.models.IWebService; import de.atb.context.tools.ontology.AmIMonitoringConfiguration; import eu.smartclide.contexthandling.services.GitlabApiClient; @@ -46,8 +46,10 @@ public GitlabCommitAnalyser(final DataSource dataSource, if (!(dataSource instanceof GitlabDataSource)) { throw new IllegalArgumentException("Given dataSource must be of type GitlabDataSource!"); } - gitlabApiClient = new GitlabApiClient(((GitlabDataSource) dataSource).getGitLabAccessToken(), - dataSource.getUri()); + gitlabApiClient = new GitlabApiClient( + ((GitlabDataSource) dataSource).getGitLabAccessToken(), + dataSource.getUri() + ); } @Override @@ -62,6 +64,7 @@ public List analyseObject(IWebService service) { final GitlabCommitDataModel model = new GitlabCommitDataModel(); model.setGitlabCommitMessages(gitlabCommitMessages); model.setMonitoredAt(new Date()); + logger.info("Analysed {} GitlabCommitMessages", gitlabCommitMessages.size()); return List.of(model); } catch (Exception e) { logger.error("Error analysing service: {}", service); diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java index e98bdee0..75ccf904 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/monitors/GitlabCommitMonitor.java @@ -21,17 +21,22 @@ import de.atb.context.monitoring.index.Indexer; import de.atb.context.monitoring.monitors.webservice.WebServiceMonitor; import de.atb.context.tools.ontology.AmIMonitoringConfiguration; +import eu.smartclide.contexthandling.dle.listener.DleGitlabCommitMonitorProgressListener; + +import java.io.IOException; +import java.util.concurrent.TimeoutException; public class GitlabCommitMonitor extends WebServiceMonitor { public GitlabCommitMonitor(final DataSource dataSource, final Interpreter interpreter, final Monitor monitor, final Indexer indexer, - final AmIMonitoringConfiguration configuration) { + final AmIMonitoringConfiguration configuration) throws IOException, TimeoutException { super(dataSource, interpreter, monitor, indexer, configuration); if (!(dataSource instanceof GitlabDataSource)) { throw new IllegalArgumentException("Given dataSource must be of type GitlabDataSource!"); } + this.addProgressListener(new DleGitlabCommitMonitorProgressListener((GitlabDataSource) dataSource)); this.logger.info("Initialized GitlabCommitMonitor for uri: " + dataSource.getUri()); } } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitlabCommitMonitorProgressListener.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitlabCommitMonitorProgressListener.java new file mode 100644 index 00000000..bed10013 --- /dev/null +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/listener/DleGitlabCommitMonitorProgressListener.java @@ -0,0 +1,132 @@ +package eu.smartclide.contexthandling.dle.listener; + +/*- + * #%L + * SmartCLIDE Monitoring + * %% + * Copyright (C) 2015 - 2022 ATB – Institut für angewandte Systemtechnik Bremen GmbH + * %% + * This program and the accompanying materials are made + * available under the terms of the Eclipse Public License 2.0 + * which is available at https://www.eclipse.org/legal/epl-2.0/ + * + * SPDX-License-Identifier: EPL-2.0 + * #L% + */ + +import com.rabbitmq.client.Channel; +import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; +import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSourceOptions; +import de.atb.context.monitoring.events.MonitoringProgressListener; +import de.atb.context.monitoring.models.GitlabCommitDataModel; +import de.atb.context.monitoring.models.GitlabCommitMessage; +import de.atb.context.monitoring.models.IMonitoringDataModel; +import de.atb.context.monitoring.models.IWebService; +import de.atb.context.monitoring.monitors.messagebroker.util.MessageBrokerUtil; +import eu.smartclide.contexthandling.dle.model.CommitMessage; +import eu.smartclide.contexthandling.dle.model.DleMessage; +import org.apache.commons.lang3.StringUtils; +import org.apache.lucene.document.Document; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeoutException; + +public class DleGitlabCommitMonitorProgressListener + implements MonitoringProgressListener> { + + private final Channel channel; + private final String exchange; + private final String topic; + private final String queue; + private final boolean useTopic; + + public DleGitlabCommitMonitorProgressListener(final GitlabDataSource dataSource) + throws IOException, TimeoutException { + exchange = dataSource.getOutgoingExchange(); + topic = dataSource.getOutgoingTopic(); + queue = dataSource.getOutgoingQueue(); + useTopic = isOutgoingTopic(); + channel = connectToMessageBroker( + dataSource.getMessageBrokerServer(), + dataSource.getMessageBrokerPort(), + dataSource.getUserName(), + dataSource.getPassword(), + dataSource.isOutgoingDurable() + ); + } + + @Override + public void documentIndexed(final String indexId, final Document document) { + // noop + } + + @Override + public void documentParsed(final IWebService parsed, final Document document) { + // noop + } + + @Override + public void documentAnalysed(final List> analysed, + final IWebService parsed, + final Document document) { + analysed.stream() + .filter(iMonitoringDataModel -> iMonitoringDataModel instanceof GitlabCommitDataModel) + .map(iMonitoringDataModel -> (GitlabCommitDataModel) iMonitoringDataModel) + .flatMap(gitlabCommitDataModel -> gitlabCommitDataModel.getGitlabCommitMessages().stream()) + .map(this::convertToDleMessage) + .forEach(this::send); + } + + private DleMessage convertToDleMessage(final GitlabCommitMessage gitlabCommitMessage) { + return DleMessage.builder() + .monitor(CommitMessage.builder() + .repoId(gitlabCommitMessage.getRepository()) + .user(gitlabCommitMessage.getUser()) + .branch(gitlabCommitMessage.getBranch()) + .timeSinceLastCommit(gitlabCommitMessage.getTimeSinceLastCommit()) + .numberOfFilesModified(gitlabCommitMessage.getNoOfModifiedFiles()) + .build()) + .build(); + } + + private void send(final DleMessage dleMessage) { + if (useTopic) { + MessageBrokerUtil.convertAndSendToTopic(channel, exchange, topic, dleMessage); + } else { + MessageBrokerUtil.convertAndSendToQueue(channel, queue, dleMessage); + } + } + + private Channel connectToMessageBroker(final String host, + final Integer port, + final String userName, + final String password, + final boolean durable) throws IOException, TimeoutException { + return useTopic + ? MessageBrokerUtil.connectToTopicExchange(host, port, userName, password, exchange, durable) + : MessageBrokerUtil.connectToQueue(host, port, userName, password, queue, durable); + } + + private boolean isOutgoingTopic() { + if (StringUtils.isBlank(topic) && StringUtils.isBlank(queue)) { + throw new IllegalArgumentException(String.format( + "Must specify either %s or %s!", + MessageBrokerDataSourceOptions.OutgoingTopic.getKeyName(), + MessageBrokerDataSourceOptions.OutgoingQueue.getKeyName() + )); + } + if (StringUtils.isNotBlank(topic)) { + if (StringUtils.isBlank(exchange)) { + throw new IllegalArgumentException(String.format( + "Must specify %s when connecting to topic %s!", + MessageBrokerDataSourceOptions.OutgoingExchange.getKeyName(), + MessageBrokerDataSourceOptions.OutgoingTopic.getKeyName() + )); + } + return true; + } else { + return false; + } + } +} diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java index 8aab300d..b8d6e42e 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java @@ -9,7 +9,7 @@ * This program and the accompanying materials are made * available under the terms of the Eclipse Public License 2.0 * which is available at https://www.eclipse.org/legal/epl-2.0/ - * + * * SPDX-License-Identifier: EPL-2.0 * #L% */ diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java index 7ab0e6f2..4370b764 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java @@ -53,6 +53,8 @@ public class GitlabCommitMonitorTest { private AmIMonitoringDataRepositoryServiceWrapper monitoringDataRepository; private Channel fakeDleChannel; + private int fakeDleNumberOfReceivedMessages; + // starts a new rabbitmq message broker in a docker container. // @Rule must be final. @Rule @@ -70,6 +72,7 @@ public void setup() throws Exception { final Integer rabbitMQContainerAmqpPort = container.getAmqpPort(); // setup fake DLE + fakeDleNumberOfReceivedMessages = 0; fakeDleChannel = createFakeDleListener(rabbitMQContainerHost, rabbitMQContainerAmqpPort); // write dynamically allocated message broker host and port to monitoring config file @@ -100,6 +103,7 @@ public void tearDown() throws IOException, TimeoutException { if (fakeDleChannel != null) { fakeDleChannel.close(); } + fakeDleNumberOfReceivedMessages = 0; } @Test @@ -121,8 +125,11 @@ public void testDoMonitor() throws InterruptedException { assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getRepository())); assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getBranch())); assertTrue(gitlabCommitMessage.getTimeSinceLastCommit() > 0); - assertTrue(gitlabCommitMessage.getNoOfModifiedFiles() > 0); + assertTrue(gitlabCommitMessage.getNoOfModifiedFiles() >= 0); }); + // assert that all GitLabCommitMessages have been received by fake DLE + Thread.sleep(10000); + assertEquals(gitlabCommitMessages.size(), fakeDleNumberOfReceivedMessages); } private void updateDataSource(final Path monitoringConfig, @@ -152,7 +159,10 @@ private Channel createFakeDleListener(final String rabbitMQContainerHost, final channel.basicConsume( QUEUE_NAME_DLE, true, - (t, m) -> logger.info("DLE received message: {}", new String(m.getBody(), StandardCharsets.UTF_8)), + (t, m) -> { + logger.info("DLE received message: {}", new String(m.getBody(), StandardCharsets.UTF_8)); + fakeDleNumberOfReceivedMessages++; + }, (t) -> logger.info("cancelled!") ); return channel; diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java index b8a85596..3161d710 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/config/models/datasources/GitlabDataSourceTest.java @@ -21,7 +21,7 @@ public class GitlabDataSourceTest { public void configFileShouldBeDeserializedToCorrectGitlabDataSource() throws URISyntaxException { final String expectedId = "datasource-gitlab"; final String expectedMonitor = "de.atb.context.monitoring.monitors.GitlabCommitMonitor"; - final String expectedUri = "https://gitlab.example.com/api/v4"; + final String expectedUri = "https://gitlab.example.com"; final String expectedAccessToken = "s3cr3t"; final String expectedMessageBrokerHost = "localhost"; final int expectedMessageBrokerPort = 5672; diff --git a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml index 11fc7d48..c09465bc 100644 --- a/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/gitlab-monitoring/monitoring-config.xml @@ -11,7 +11,7 @@ diff --git a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml index 7b623543..ffc09d41 100644 --- a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml @@ -11,7 +11,7 @@ From 68eb292915cd5a912299a5ffc3891f0f83d7f1e6 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Fri, 13 May 2022 14:19:08 +0200 Subject: [PATCH 29/34] applied suggestions --- .../services/GitlabApiClient.java | 75 +++++++++++-------- .../monitoring/GitlabApiClientTest.java | 2 +- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java index 8aab300d..872e357e 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java @@ -27,7 +27,6 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.nio.file.InvalidPathException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Duration; @@ -39,12 +38,14 @@ public class GitlabApiClient { + private static final DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); private static final Logger logger = LoggerFactory.getLogger(GitlabApiClient.class); private static final String membershipParam = "&membership=true"; private static final String paginationParam = "&per_page=100"; private static final String refNameParam = "&ref_name="; - private static final String sinceParam = "&since=2022-04-25T13:05:00"; // TODO change this based on requirement + private static final String sinceDate = "2022-04-25T13:05:00"; // TODO change this based on requirement + private static final String sinceParam = "&since=" + sinceDate; // TODO change this based on requirement private static final String uriPartForBranches = "/repository/branches/"; private static final String uriPartForCommits = "/repository/commits/"; private static final String uriPartForDiff = "/diff/"; @@ -67,49 +68,61 @@ public GitlabApiClient(String accessToken, String gitlabBaseUri) { public List getGitlabCommitMessages() { // first get all user projects JsonArray projects = getUserProjects(); + logger.info("Total {} user projects are found", projects.size()); List gitlabCommitMessages = new LinkedList<>(); for (JsonElement project : projects) { + JsonObject projectJsonObject = project.getAsJsonObject(); // get project id - String projectId = project.getAsJsonObject().get("id").getAsString(); + String projectId = projectJsonObject.get("id").getAsString(); // get all branches for given project, create a new GitMessage JsonArray branches = getAllBranchesForGivenProject(projectId); + logger.info("Total {} branches for project with given ID {} are found", branches.size(), projectId); + for (JsonElement branch : branches) { String branchName = branch.getAsJsonObject().get("name").getAsString(); - // get all commits for given branch and since + // get all commits for given branch JsonArray commitsInBranch = getCommitsForGivenBranch(projectId, branchName); + logger.info("Total {} commits are found in a given branch with name '{}' since {}", + commitsInBranch.size(), branchName, sinceDate); + for (JsonElement commit : commitsInBranch) { - String commitId = commit.getAsJsonObject().get("id").getAsString(); + JsonObject commitJsonObject = commit.getAsJsonObject(); + String commitId = commitJsonObject.get("id").getAsString(); GitlabCommitMessage gitlabCommitMessage = new GitlabCommitMessage(); - gitlabCommitMessage.setUser(commit.getAsJsonObject().get("author_name").getAsString()); - gitlabCommitMessage.setRepository(project.getAsJsonObject().get("path_with_namespace").getAsString()); + gitlabCommitMessage.setUser(commitJsonObject.get("author_name").getAsString()); + gitlabCommitMessage.setRepository(projectJsonObject.get("path_with_namespace").getAsString()); gitlabCommitMessage.setBranch(branchName); - gitlabCommitMessage.setTimeSinceLastCommit(calculateTimeSinceLastCommit(projectId, commit.getAsJsonObject())); - + gitlabCommitMessage.setTimeSinceLastCommit(calculateTimeSinceLastCommit(projectId, commitJsonObject)); JsonArray newCommitDiff = getCommitDiff(projectId, commitId); gitlabCommitMessage.setNoOfModifiedFiles(newCommitDiff.size()); - gitlabCommitMessages.add(gitlabCommitMessage); + logger.info("Git commit message: " + gitlabCommitMessage); } } } return gitlabCommitMessages; } - private Integer calculateTimeSinceLastCommit(String projectId, JsonObject newCommit) { + private Integer calculateTimeSinceLastCommit(String projectId, JsonObject commit) { int difference = 0; - String lastCommitId = newCommit.get("parent_ids").getAsString(); - String newCommitCreationDateStr = newCommit.get("created_at").getAsString(); - JsonObject lastCommit = getCommitById(projectId, lastCommitId); - String lastCommitCreationDateStr = lastCommit.getAsJsonObject().get("created_at").getAsString(); - - DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); - try { - Date newCommitCreationDate = formatter.parse(newCommitCreationDateStr); - Date lastCommitCreationDate = formatter.parse(lastCommitCreationDateStr); - difference = Math.toIntExact(Math.abs(newCommitCreationDate.getTime() - lastCommitCreationDate.getTime()) / 1000); - } catch (java.text.ParseException e) { - logger.error("date to string parse error", e); + // check if parent id exists for given commit + if (commit.get("parent_ids").getAsJsonArray().size() > 0) { + String parentCommitId = commit.get("parent_ids").getAsString(); + String commitCreationDateStr = commit.get("created_at").getAsString(); + JsonObject parentCommit = getCommitById(projectId, parentCommitId); + String parentCommitCreationDateStr = parentCommit.getAsJsonObject().get("created_at").getAsString(); + + try { + Date commitCreationDate = formatter.parse(commitCreationDateStr); + Date parentCommitCreationDate = formatter.parse(parentCommitCreationDateStr); + difference = Math.toIntExact( + Math.abs(commitCreationDate.getTime() - parentCommitCreationDate.getTime()) / 1000); + } catch (java.text.ParseException e) { + logger.error("date to string parse error", e); + } + } else { + logger.info("No previous commit exist for given commit with ID: " + commit.get("id").getAsString()); } return difference; } @@ -145,15 +158,15 @@ private JsonArray getUserProjects() { */ private HttpResponse makeGetCallToGitlab(String uri) { - final HttpClient httpClient = HttpClient.newBuilder() - .version(HttpClient.Version.HTTP_2) - .connectTimeout(Duration.ofMinutes(5)) - .build(); - - HttpRequest request = HttpRequest.newBuilder().GET().uri(URI.create(uri)).build(); - HttpResponse response = null; try { + final HttpClient httpClient = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_2) + .connectTimeout(Duration.ofMinutes(5)) + .build(); + + HttpRequest request = HttpRequest.newBuilder().GET().uri(URI.create(uri)).build(); + // receive response from Gitlab response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); @@ -161,7 +174,7 @@ private HttpResponse makeGetCallToGitlab(String uri) { logger.error("Http response error:" + response.statusCode() + response.body()); return null; } - } catch (IOException | InterruptedException | InvalidPathException e) { + } catch (IOException | InterruptedException e) { logger.error("HTTP Client connection interruption exception", e); } return response; diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java index e4b664aa..e92c113e 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java @@ -34,7 +34,7 @@ public void testGitServices() { assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getRepository())); assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getBranch())); assertTrue(gitlabCommitMessage.getNoOfModifiedFiles() > 0); - assertTrue(gitlabCommitMessage.getTimeSinceLastCommit() > 0); + assertTrue(gitlabCommitMessage.getTimeSinceLastCommit() >= 0); }); } } From 16f221d682a7d97ccd0a99e48865d008db0d077d Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Mon, 16 May 2022 17:41:08 +0200 Subject: [PATCH 30/34] set sinceDate to every interval and check for NullPointerException at getCommitById method --- .../analyser/GitlabCommitAnalyser.java | 2 +- .../services/GitlabApiClient.java | 47 ++++++++++++++----- .../monitoring/GitlabApiClientTest.java | 3 +- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java index 873a5a74..3d041efb 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java @@ -48,7 +48,7 @@ public GitlabCommitAnalyser(final DataSource dataSource, } gitlabApiClient = new GitlabApiClient( ((GitlabDataSource) dataSource).getGitLabAccessToken(), - dataSource.getUri() + dataSource.getUri(), ((GitlabDataSource) dataSource).getInterval() ); } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java index dce04df7..39d0ea52 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java @@ -44,19 +44,22 @@ public class GitlabApiClient { private static final String membershipParam = "&membership=true"; private static final String paginationParam = "&per_page=100"; private static final String refNameParam = "&ref_name="; - private static final String sinceDate = "2022-04-25T13:05:00"; // TODO change this based on requirement - private static final String sinceParam = "&since=" + sinceDate; // TODO change this based on requirement + private static final String sinceParam = "&since="; private static final String uriPartForBranches = "/repository/branches/"; private static final String uriPartForCommits = "/repository/commits/"; private static final String uriPartForDiff = "/diff/"; private static final String uriPartForProjects = "/api/v4/projects/"; private final String baseUri; + private final Long interval; private final String uriParams; + private boolean isRunningFirstTimeFlag = true; + private String sinceDate = "2022-01-01T00:00:00"; - public GitlabApiClient(String accessToken, String gitlabBaseUri) { + public GitlabApiClient(String accessToken, String gitlabBaseUri, Long interval) { String accessTokenParam = "private_token=" + accessToken; this.uriParams = "?" + accessTokenParam + membershipParam + paginationParam; this.baseUri = gitlabBaseUri + uriPartForProjects; + this.interval = interval; } /** @@ -66,6 +69,9 @@ public GitlabApiClient(String accessToken, String gitlabBaseUri) { * @return List */ public List getGitlabCommitMessages() { + // set a sinceDate for retrieving commits from repository + setSinceDate(); + // first get all user projects JsonArray projects = getUserProjects(); logger.info("Total {} user projects are found", projects.size()); @@ -111,15 +117,19 @@ private Integer calculateTimeSinceLastCommit(String projectId, JsonObject commit String parentCommitId = commit.get("parent_ids").getAsString(); String commitCreationDateStr = commit.get("created_at").getAsString(); JsonObject parentCommit = getCommitById(projectId, parentCommitId); - String parentCommitCreationDateStr = parentCommit.getAsJsonObject().get("created_at").getAsString(); - - try { - Date commitCreationDate = formatter.parse(commitCreationDateStr); - Date parentCommitCreationDate = formatter.parse(parentCommitCreationDateStr); - difference = Math.toIntExact( - Math.abs(commitCreationDate.getTime() - parentCommitCreationDate.getTime()) / 1000); - } catch (java.text.ParseException e) { - logger.error("date to string parse error", e); + if (parentCommit.getAsJsonObject().has("created_at")) { + String parentCommitCreationDateStr = parentCommit.getAsJsonObject().get("created_at").getAsString(); + + try { + Date commitCreationDate = formatter.parse(commitCreationDateStr); + Date parentCommitCreationDate = formatter.parse(parentCommitCreationDateStr); + difference = Math.toIntExact( + Math.abs(commitCreationDate.getTime() - parentCommitCreationDate.getTime()) / 1000); + } catch (java.text.ParseException e) { + logger.error("date to string parse error", e); + } + } else { + logger.info("Can not find previous commit at the moment!"); } } else { logger.info("No previous commit exist for given commit with ID: " + commit.get("id").getAsString()); @@ -143,7 +153,7 @@ private JsonArray getCommitDiff(String projectId, String commitId) { private JsonArray getCommitsForGivenBranch(String projectId, String branchName) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId - + uriPartForCommits + uriParams + refNameParam + branchName + sinceParam)); + + uriPartForCommits + uriParams + refNameParam + branchName + sinceParam + sinceDate)); } private JsonArray getUserProjects() { @@ -193,4 +203,15 @@ private JsonObject parseHttpResponseToJsonObject(HttpResponse response) } return new JsonObject(); } + + private void setSinceDate() { + if (isRunningFirstTimeFlag) { + // at the start of API calls we assume that the flag is set to true + isRunningFirstTimeFlag = false; + } else { + // after the flag has been set as false, set sinceDate to every interval + Date date = new Date(System.currentTimeMillis() - interval); + sinceDate = formatter.format(date); + } + } } diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java index e92c113e..aa530611 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java @@ -13,6 +13,7 @@ public class GitlabApiClientTest { private static final String gitlabBaseUri = "https://gitlab.atb-bremen.de"; + private static final Long interval = 60000L; private GitlabApiClient gitlabApiClient; @Before @@ -21,7 +22,7 @@ public void setup() { if (StringUtils.isBlank(gitlabApiToken)) { throw new IllegalStateException("Did not find valid GitLab API token in \"SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN\" environment variable!"); } - gitlabApiClient = new GitlabApiClient(gitlabApiToken, gitlabBaseUri); + gitlabApiClient = new GitlabApiClient(gitlabApiToken, gitlabBaseUri, interval); } @Test From a2e457c48148a7c5ab3a04f6f1991f91562766c1 Mon Sep 17 00:00:00 2001 From: Purva Vyas Date: Mon, 16 May 2022 17:52:56 +0200 Subject: [PATCH 31/34] fix test --- .../java/de/atb/context/monitoring/GitlabCommitMonitorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java index 4370b764..0e39a986 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java @@ -124,7 +124,7 @@ public void testDoMonitor() throws InterruptedException { assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getUser())); assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getRepository())); assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getBranch())); - assertTrue(gitlabCommitMessage.getTimeSinceLastCommit() > 0); + assertTrue(gitlabCommitMessage.getTimeSinceLastCommit() >= 0); assertTrue(gitlabCommitMessage.getNoOfModifiedFiles() >= 0); }); // assert that all GitLabCommitMessages have been received by fake DLE From 3f94b463a02e6089c19863cd5bf2f2dc3993d5c3 Mon Sep 17 00:00:00 2001 From: Philip Reimer <5737222+philipreimer@users.noreply.github.com> Date: Tue, 17 May 2022 12:28:37 +0200 Subject: [PATCH 32/34] calculate sinceDateTime based on last run use java.time.* instead of java.util.Date --- .../analyser/GitlabCommitAnalyser.java | 2 +- .../models/GitlabCommitMessage.java | 2 +- .../dle/model/CommitMessage.java | 2 +- .../services/GitlabApiClient.java | 70 ++++++++----------- .../monitoring/GitlabApiClientTest.java | 14 ++-- 5 files changed, 43 insertions(+), 47 deletions(-) diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java index 3d041efb..873a5a74 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/analyser/GitlabCommitAnalyser.java @@ -48,7 +48,7 @@ public GitlabCommitAnalyser(final DataSource dataSource, } gitlabApiClient = new GitlabApiClient( ((GitlabDataSource) dataSource).getGitLabAccessToken(), - dataSource.getUri(), ((GitlabDataSource) dataSource).getInterval() + dataSource.getUri() ); } diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java index 3159fce7..8c6fe8e2 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java @@ -37,5 +37,5 @@ public class GitlabCommitMessage { String repository; String branch; Integer noOfModifiedFiles; - Integer timeSinceLastCommit; + Long timeSinceLastCommit; } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java index 5ba45455..013158b1 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java @@ -38,7 +38,7 @@ public class CommitMessage { String user; String branch; @SerializedName("time_since_last_commit") - Integer timeSinceLastCommit; + Long timeSinceLastCommit; @SerializedName("number_of_files_modified") Integer numberOfFilesModified; } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java index 39d0ea52..f6a76d11 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java @@ -14,6 +14,7 @@ * #L% */ +import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; @@ -27,39 +28,35 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.time.Duration; -import java.util.Date; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import java.util.LinkedList; import java.util.List; -import com.google.gson.JsonArray; - public class GitlabApiClient { - private static final DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); private static final Logger logger = LoggerFactory.getLogger(GitlabApiClient.class); - + private static final DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; private static final String membershipParam = "&membership=true"; private static final String paginationParam = "&per_page=100"; private static final String refNameParam = "&ref_name="; private static final String sinceParam = "&since="; + private static final ZonedDateTime initialSinceDate = ZonedDateTime.of(2022, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); private static final String uriPartForBranches = "/repository/branches/"; private static final String uriPartForCommits = "/repository/commits/"; private static final String uriPartForDiff = "/diff/"; private static final String uriPartForProjects = "/api/v4/projects/"; private final String baseUri; - private final Long interval; private final String uriParams; - private boolean isRunningFirstTimeFlag = true; - private String sinceDate = "2022-01-01T00:00:00"; + private ZonedDateTime lastRun = null; - public GitlabApiClient(String accessToken, String gitlabBaseUri, Long interval) { + public GitlabApiClient(String accessToken, String gitlabBaseUri) { String accessTokenParam = "private_token=" + accessToken; this.uriParams = "?" + accessTokenParam + membershipParam + paginationParam; this.baseUri = gitlabBaseUri + uriPartForProjects; - this.interval = interval; } /** @@ -69,8 +66,14 @@ public GitlabApiClient(String accessToken, String gitlabBaseUri, Long interval) * @return List */ public List getGitlabCommitMessages() { - // set a sinceDate for retrieving commits from repository - setSinceDate(); + // if we are running for the first time get all commits since `initialSinceDate` + // otherwise get all commits since last run + final ZonedDateTime nowAtUtc = ZonedDateTime.now(ZoneOffset.UTC); + final String sinceDateTime = (lastRun == null) + ? initialSinceDate.format(formatter) + : lastRun.format(formatter); + // adjust the time of last run + lastRun = nowAtUtc; // first get all user projects JsonArray projects = getUserProjects(); @@ -87,9 +90,9 @@ public List getGitlabCommitMessages() { for (JsonElement branch : branches) { String branchName = branch.getAsJsonObject().get("name").getAsString(); // get all commits for given branch - JsonArray commitsInBranch = getCommitsForGivenBranch(projectId, branchName); + JsonArray commitsInBranch = getCommitsForGivenBranch(projectId, branchName, sinceDateTime); logger.info("Total {} commits are found in a given branch with name '{}' since {}", - commitsInBranch.size(), branchName, sinceDate); + commitsInBranch.size(), branchName, sinceDateTime); for (JsonElement commit : commitsInBranch) { JsonObject commitJsonObject = commit.getAsJsonObject(); @@ -110,8 +113,8 @@ public List getGitlabCommitMessages() { return gitlabCommitMessages; } - private Integer calculateTimeSinceLastCommit(String projectId, JsonObject commit) { - int difference = 0; + private long calculateTimeSinceLastCommit(String projectId, JsonObject commit) { + long difference = 0; // check if parent id exists for given commit if (commit.get("parent_ids").getAsJsonArray().size() > 0) { String parentCommitId = commit.get("parent_ids").getAsString(); @@ -121,18 +124,18 @@ private Integer calculateTimeSinceLastCommit(String projectId, JsonObject commit String parentCommitCreationDateStr = parentCommit.getAsJsonObject().get("created_at").getAsString(); try { - Date commitCreationDate = formatter.parse(commitCreationDateStr); - Date parentCommitCreationDate = formatter.parse(parentCommitCreationDateStr); - difference = Math.toIntExact( - Math.abs(commitCreationDate.getTime() - parentCommitCreationDate.getTime()) / 1000); - } catch (java.text.ParseException e) { - logger.error("date to string parse error", e); + ZonedDateTime commitCreationDate = ZonedDateTime.parse(commitCreationDateStr, formatter); + ZonedDateTime parentCommitCreationDate = ZonedDateTime.parse(parentCommitCreationDateStr, formatter); + difference = commitCreationDate.toInstant().getEpochSecond() - + parentCommitCreationDate.toInstant().getEpochSecond(); + } catch (DateTimeParseException e) { + logger.error("Failed to parse commit creation date", e); } } else { - logger.info("Can not find previous commit at the moment!"); + logger.info("Could not get parent commit with ID: {}", parentCommitId); } } else { - logger.info("No previous commit exist for given commit with ID: " + commit.get("id").getAsString()); + logger.info("No parent commit exist for commit with ID: {}", commit.get("id").getAsString()); } return difference; } @@ -151,9 +154,9 @@ private JsonArray getCommitDiff(String projectId, String commitId) { + uriPartForCommits + commitId + uriPartForDiff + uriParams)); } - private JsonArray getCommitsForGivenBranch(String projectId, String branchName) { + private JsonArray getCommitsForGivenBranch(String projectId, String branchName, String sinceDateTime) { return parseHttpResponseToJsonArray(makeGetCallToGitlab(baseUri + projectId - + uriPartForCommits + uriParams + refNameParam + branchName + sinceParam + sinceDate)); + + uriPartForCommits + uriParams + refNameParam + branchName + sinceParam + sinceDateTime)); } private JsonArray getUserProjects() { @@ -203,15 +206,4 @@ private JsonObject parseHttpResponseToJsonObject(HttpResponse response) } return new JsonObject(); } - - private void setSinceDate() { - if (isRunningFirstTimeFlag) { - // at the start of API calls we assume that the flag is set to true - isRunningFirstTimeFlag = false; - } else { - // after the flag has been set as false, set sinceDate to every interval - Date date = new Date(System.currentTimeMillis() - interval); - sinceDate = formatter.format(date); - } - } } diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java index aa530611..bfd666a2 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabApiClientTest.java @@ -8,12 +8,12 @@ import java.util.List; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class GitlabApiClientTest { private static final String gitlabBaseUri = "https://gitlab.atb-bremen.de"; - private static final Long interval = 60000L; private GitlabApiClient gitlabApiClient; @Before @@ -22,14 +22,14 @@ public void setup() { if (StringUtils.isBlank(gitlabApiToken)) { throw new IllegalStateException("Did not find valid GitLab API token in \"SMARTCLIDE_CONTEXT_GITLAB_API_TOKEN\" environment variable!"); } - gitlabApiClient = new GitlabApiClient(gitlabApiToken, gitlabBaseUri, interval); + gitlabApiClient = new GitlabApiClient(gitlabApiToken, gitlabBaseUri); } @Test - public void testGitServices() { - + public void testGetGitlabCommitMessages() { List gitlabCommitMessages = gitlabApiClient.getGitlabCommitMessages(); - assertTrue(gitlabCommitMessages.size() > 0); + + assertFalse(gitlabCommitMessages.isEmpty()); gitlabCommitMessages.forEach(gitlabCommitMessage -> { assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getUser())); assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getRepository())); @@ -37,5 +37,9 @@ public void testGitServices() { assertTrue(gitlabCommitMessage.getNoOfModifiedFiles() > 0); assertTrue(gitlabCommitMessage.getTimeSinceLastCommit() >= 0); }); + + // run a second time, which should not produce any results + gitlabCommitMessages = gitlabApiClient.getGitlabCommitMessages(); + assertTrue(gitlabCommitMessages.isEmpty()); } } From 0351a78f1166564764cec43712f99e19ba20efe1 Mon Sep 17 00:00:00 2001 From: Philip Reimer <5737222+philipreimer@users.noreply.github.com> Date: Tue, 17 May 2022 13:49:55 +0200 Subject: [PATCH 33/34] revert back to Integer type for timeSinceLastCommit RDF persist does not work with long values for some reason --- .../models/GitlabCommitMessage.java | 7 +++++-- .../dle/model/CommitMessage.java | 2 +- .../services/GitlabApiClient.java | 12 +++++++---- .../monitoring/GitlabCommitMonitorTest.java | 21 +++++++++---------- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java index 8c6fe8e2..98e8c089 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java @@ -24,7 +24,7 @@ import thewebsemantic.Namespace; import thewebsemantic.RdfType; -@RdfType("GitMessage") +@RdfType("GitlabCommitMessage") @Namespace(BusinessCase.NS_DUMMY_URL) @Getter @Setter @@ -37,5 +37,8 @@ public class GitlabCommitMessage { String repository; String branch; Integer noOfModifiedFiles; - Long timeSinceLastCommit; + // Has to be Integer because + // de.atb.context.persistence.monitoring.MonitoringDataRepository.persist(java.lang.String, java.lang.Class) + // cannot persist long values for some reason. + Integer timeSinceLastCommit; } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java index 013158b1..5ba45455 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/dle/model/CommitMessage.java @@ -38,7 +38,7 @@ public class CommitMessage { String user; String branch; @SerializedName("time_since_last_commit") - Long timeSinceLastCommit; + Integer timeSinceLastCommit; @SerializedName("number_of_files_modified") Integer numberOfFilesModified; } diff --git a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java index f6a76d11..d8f3f2fc 100644 --- a/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java +++ b/smartclide-monitoring/src/main/java/eu/smartclide/contexthandling/services/GitlabApiClient.java @@ -102,7 +102,8 @@ public List getGitlabCommitMessages() { gitlabCommitMessage.setUser(commitJsonObject.get("author_name").getAsString()); gitlabCommitMessage.setRepository(projectJsonObject.get("path_with_namespace").getAsString()); gitlabCommitMessage.setBranch(branchName); - gitlabCommitMessage.setTimeSinceLastCommit(calculateTimeSinceLastCommit(projectId, commitJsonObject)); + final int timeSinceLastCommit = calculateTimeSinceLastCommit(projectId, commitJsonObject); + gitlabCommitMessage.setTimeSinceLastCommit(timeSinceLastCommit); JsonArray newCommitDiff = getCommitDiff(projectId, commitId); gitlabCommitMessage.setNoOfModifiedFiles(newCommitDiff.size()); gitlabCommitMessages.add(gitlabCommitMessage); @@ -113,8 +114,8 @@ public List getGitlabCommitMessages() { return gitlabCommitMessages; } - private long calculateTimeSinceLastCommit(String projectId, JsonObject commit) { - long difference = 0; + private int calculateTimeSinceLastCommit(String projectId, JsonObject commit) { + int difference = 0; // check if parent id exists for given commit if (commit.get("parent_ids").getAsJsonArray().size() > 0) { String parentCommitId = commit.get("parent_ids").getAsString(); @@ -126,8 +127,11 @@ private long calculateTimeSinceLastCommit(String projectId, JsonObject commit) { try { ZonedDateTime commitCreationDate = ZonedDateTime.parse(commitCreationDateStr, formatter); ZonedDateTime parentCommitCreationDate = ZonedDateTime.parse(parentCommitCreationDateStr, formatter); - difference = commitCreationDate.toInstant().getEpochSecond() - + long longDifference = commitCreationDate.toInstant().getEpochSecond() - parentCommitCreationDate.toInstant().getEpochSecond(); + if (longDifference <= (long) Integer.MAX_VALUE) { + difference = (int) longDifference; + } } catch (DateTimeParseException e) { logger.error("Failed to parse commit creation date", e); } diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java index 0e39a986..03ce57ea 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java @@ -6,8 +6,8 @@ import de.atb.context.monitoring.config.models.Config; import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSourceOptions; -import de.atb.context.monitoring.models.GitlabCommitMessage; import de.atb.context.monitoring.models.GitlabCommitDataModel; +import de.atb.context.monitoring.models.GitlabCommitMessage; import de.atb.context.monitoring.monitors.messagebroker.util.MessageBrokerUtil; import de.atb.context.services.IAmIMonitoringDataRepositoryService; import de.atb.context.services.manager.ServiceManager; @@ -108,17 +108,16 @@ public void tearDown() throws IOException, TimeoutException { @Test public void testDoMonitor() throws InterruptedException { - // get the latest entry of monitored data from the repository - List data = List.of(); - int counter = 0; - while (counter <= 30 && data.isEmpty()) { - counter++; - //noinspection BusyWait - Thread.sleep(2000); - data = monitoringDataRepository.getMonitoringData(ApplicationScenario.getInstance(), GitlabCommitDataModel.class, 1); - } + Thread.sleep(30000); + // get the latest entry of monitored data from the repository + List data = monitoringDataRepository.getMonitoringData( + ApplicationScenario.getInstance(), + GitlabCommitDataModel.class, + 1 + ); assertEquals(1, data.size()); + final List gitlabCommitMessages = data.get(0).getGitlabCommitMessages(); gitlabCommitMessages.forEach(gitlabCommitMessage -> { assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getUser())); @@ -127,8 +126,8 @@ public void testDoMonitor() throws InterruptedException { assertTrue(gitlabCommitMessage.getTimeSinceLastCommit() >= 0); assertTrue(gitlabCommitMessage.getNoOfModifiedFiles() >= 0); }); + // assert that all GitLabCommitMessages have been received by fake DLE - Thread.sleep(10000); assertEquals(gitlabCommitMessages.size(), fakeDleNumberOfReceivedMessages); } From 7c3c3890f030ab0cf57a1a1fde95f0caefe7540a Mon Sep 17 00:00:00 2001 From: Philip Reimer <5737222+philipreimer@users.noreply.github.com> Date: Tue, 17 May 2022 17:24:25 +0200 Subject: [PATCH 34/34] use fake GitlabCommitAnalyser for test --- .../models/GitlabCommitMessage.java | 2 + .../monitoring/GitlabCommitMonitorTest.java | 18 +++--- .../analyser/FakeGitlabCommitAnalyser.java | 62 +++++++++++++++++++ .../resources/config/monitoring-config.xml | 6 +- 4 files changed, 75 insertions(+), 13 deletions(-) create mode 100644 smartclide-monitoring/src/test/java/de/atb/context/monitoring/analyser/FakeGitlabCommitAnalyser.java diff --git a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java index 98e8c089..b9cff37d 100644 --- a/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java +++ b/smartclide-monitoring/src/main/java/de/atb/context/monitoring/models/GitlabCommitMessage.java @@ -17,6 +17,7 @@ import de.atb.context.common.util.BusinessCase; import lombok.AllArgsConstructor; import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -32,6 +33,7 @@ @NoArgsConstructor @AllArgsConstructor @ToString +@EqualsAndHashCode public class GitlabCommitMessage { String user; String repository; diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java index 03ce57ea..4aa43415 100644 --- a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/GitlabCommitMonitorTest.java @@ -3,6 +3,7 @@ import com.rabbitmq.client.Channel; import de.atb.context.common.ContextPathUtils; import de.atb.context.common.util.ApplicationScenario; +import de.atb.context.monitoring.analyser.FakeGitlabCommitAnalyser; import de.atb.context.monitoring.config.models.Config; import de.atb.context.monitoring.config.models.datasources.GitlabDataSource; import de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSourceOptions; @@ -31,8 +32,7 @@ import java.util.Map; import java.util.concurrent.TimeoutException; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /** * TestMonitoringService @@ -108,7 +108,8 @@ public void tearDown() throws IOException, TimeoutException { @Test public void testDoMonitor() throws InterruptedException { - Thread.sleep(30000); + // give services some time to start up and run monitor once + Thread.sleep(15000); // get the latest entry of monitored data from the repository List data = monitoringDataRepository.getMonitoringData( @@ -119,13 +120,10 @@ public void testDoMonitor() throws InterruptedException { assertEquals(1, data.size()); final List gitlabCommitMessages = data.get(0).getGitlabCommitMessages(); - gitlabCommitMessages.forEach(gitlabCommitMessage -> { - assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getUser())); - assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getRepository())); - assertTrue(StringUtils.isNotBlank(gitlabCommitMessage.getBranch())); - assertTrue(gitlabCommitMessage.getTimeSinceLastCommit() >= 0); - assertTrue(gitlabCommitMessage.getNoOfModifiedFiles() >= 0); - }); + assertArrayEquals( + FakeGitlabCommitAnalyser.FAKE_GITLAB_COMMIT_MESSAGES.toArray(), + gitlabCommitMessages.toArray() + ); // assert that all GitLabCommitMessages have been received by fake DLE assertEquals(gitlabCommitMessages.size(), fakeDleNumberOfReceivedMessages); diff --git a/smartclide-monitoring/src/test/java/de/atb/context/monitoring/analyser/FakeGitlabCommitAnalyser.java b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/analyser/FakeGitlabCommitAnalyser.java new file mode 100644 index 00000000..8fbd5b21 --- /dev/null +++ b/smartclide-monitoring/src/test/java/de/atb/context/monitoring/analyser/FakeGitlabCommitAnalyser.java @@ -0,0 +1,62 @@ +package de.atb.context.monitoring.analyser; + +import de.atb.context.monitoring.config.models.DataSource; +import de.atb.context.monitoring.config.models.InterpreterConfiguration; +import de.atb.context.monitoring.index.Indexer; +import de.atb.context.monitoring.models.GitlabCommitDataModel; +import de.atb.context.monitoring.models.GitlabCommitMessage; +import de.atb.context.monitoring.models.IWebService; +import de.atb.context.tools.ontology.AmIMonitoringConfiguration; +import org.apache.lucene.document.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Date; +import java.util.List; + +public class FakeGitlabCommitAnalyser extends GitlabCommitAnalyser { + + public static final List FAKE_GITLAB_COMMIT_MESSAGES = List.of( + GitlabCommitMessage.builder() + .user("John Doe") + .repository("smartclide/foo") + .branch("main") + .noOfModifiedFiles(29) + .timeSinceLastCommit(0) + .build(), + GitlabCommitMessage.builder() + .user("John Doe") + .repository("smartclide/bar") + .branch("fix/4711/npe-in-constructor") + .noOfModifiedFiles(4) + .timeSinceLastCommit(766) + .build(), + GitlabCommitMessage.builder() + .user("Jane Doe") + .repository("smartclide/bar") + .branch("main") + .noOfModifiedFiles(17) + .timeSinceLastCommit(98438) + .build() + ); + + private static final Logger logger = LoggerFactory.getLogger(GitlabCommitAnalyser.class); + + public FakeGitlabCommitAnalyser(final DataSource dataSource, + final InterpreterConfiguration interpreterConfiguration, + final Indexer indexer, + final Document document, + final AmIMonitoringConfiguration amiConfiguration) { + super(dataSource, interpreterConfiguration, indexer, document, amiConfiguration); + } + + @Override + public List analyseObject(final IWebService service) { + final List gitlabCommitMessages = FAKE_GITLAB_COMMIT_MESSAGES; + final GitlabCommitDataModel model = new GitlabCommitDataModel(); + model.setGitlabCommitMessages(gitlabCommitMessages); + model.setMonitoredAt(new Date()); + logger.info("Analysed {} GitlabCommitMessages", gitlabCommitMessages.size()); + return List.of(model); + } +} diff --git a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml index ffc09d41..dd3cc40e 100644 --- a/smartclide-monitoring/src/test/resources/config/monitoring-config.xml +++ b/smartclide-monitoring/src/test/resources/config/monitoring-config.xml @@ -11,8 +11,8 @@ @@ -20,7 +20,7 @@ + analyser="de.atb.context.monitoring.analyser.FakeGitlabCommitAnalyser"/>