Merge branch 'main' into GitlabCommitMonitor

eclipse-opensmartclide · May 18, 2022 · a58fba3 · a58fba3
2 parents 7c3c389 + 61b8ad3
commit a58fba3
Show file tree

Hide file tree

Showing 12 changed files with 514 additions and 8 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -43,7 +43,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -54,7 +54,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,4 +68,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2
diff --git a/README.md b/README.md
@@ -1,2 +1,119 @@
 # smartclide-context
 SmartCLIDE Context Handling Component
+
+## Preconditions to build and run Context Handling
+
+To build and run Context Handling, the following software is required:
+
+- Java (at least version 11)
+- Apache Maven (at least version 3.5.4)
+- Docker (for running tests and deploying Context Handling on the SmartCLIDE cluster)
+- docker-compose (for running local sample instance only)
+
+## How to build Context Handling
+
+Context Handling can be built using maven with the following command:
+
+  ```shell
+  mvn install
+  ```
+
+In order to build and push a container image that can be deployed, the following command can be used:
+
+  ```shell
+  mvn install
+  mvn jib:build -pl smartclide-monitoring -Djib.to.image="${IMAGE_NAME:IMAGE_TAG}" -Djib.to.auth.username="${CONTAINER_REGISTRY_USERNAME}" -Djib.to.auth.password="${CONTAINER_REGISTRY_TOKEN}"
+  ```
+
+## How to run Context Handling
+
+A sample configuration and docker-compose file can be found in the [samples folder](samples).
+
+You can run the sample with the following command: 
+
+   ```shell
+   docker-compose -f samples/docker-compose.yml up
+   ```
+
+## How to configure Context Handling
+
+### Monitoring Config
+
+**monitoring-config.xml**
+
+An example monitoring configuration can be found here: [monitoring-config.xml](samples/config/monitoring-config.xml)
+
+**monitoring-config.xsd**
+
+The corresponding XSD file can be found here: [monitoring-config.xsd](samples/config/monitoring-config.xsd)
+
+### Description
+
+#### indexes
+
+Each index entry has the following mandatory attributes
+
+- id: The unique name of the index
+- location: The URI of the location the index is stored
+
+#### datasources
+
+Each datasource entry has the following mandatory attributes
+
+- id:The unique name of the datasource
+- type:The type of the datasource. Possible values are: filesystem, webservice, database, messageBroker
+- monitor:The class of the monitor to be used. Possible values are:
+    - package de.atb.context.monitoring.monitors.database.DatabaseMonitor
+    - package de.atb.context.monitoring.monitors.file.FileSystemMonitor
+    - package de.atb.context.monitoring.monitors.file.FilePairSystemMonitor
+    - package de.atb.context.monitoring.monitors.file.FileTripletSystemMonitor
+    - package de.atb.context.monitoring.monitors.webservice.MessageBrokerMonitor
+    - package de.atb.context.monitoring.monitors.webservice.WebServiceMonitor
+    - package de.atb.context.monitoring.monitors.GitlabCommitMonitor
+    - package de.atb.context.monitoring.monitors.GitMonitor
+- options: Options for the datasource can be entered using this value. The options are dependent on the datasource to be used
+- uri:The uri of the data source to be monitored
+- class:The following datasource implementations are available
+    - package de.atb.context.monitoring.config.models.datasources.DatabaseDataSource
+    - package de.atb.context.monitoring.config.models.datasources.FilePairSystemDataSource
+    - package de.atb.context.monitoring.config.models.datasources.FileSystemDataSource
+    - package de.atb.context.monitoring.config.models.datasources.FileTripletSystemDataSource
+    - package de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource
+    - package de.atb.context.monitoring.config.models.datasources.WebServiceDataSource
+    - package de.atb.context.monitoring.config.models.datasources.GitlabDataSource
+
+#### interpreters
+
+Each interpreter entry has the following mandatory attributes
+
+- id: The unique name of the interpreter
+- configuration
+    - analyser: The analyser class to be used. The following implementations are available:
+        - package de.atb.context.monitoring.analyser.database.DatabaseAnalyser
+        - package de.atb.context.monitoring.analyser.file.FileAnalyser
+        - package de.atb.context.monitoring.analyser.file.FilePairAnalyser
+        - package de.atb.context.monitoring.analyser.file.FileTripletAnalyser
+        - package de.atb.context.monitoring.analyser.webservice.MessageBrokerAnalyser
+        - package de.atb.context.monitoring.analyser.webservice.WebServiceAnalyser
+        - package de.atb.context.monitoring.analyser.webserviceGitAnalyser
+        - package de.atb.context.monitoring.analyser.webservice.GitlabCommitAnalyser
+    - parser: The parser class to be used. The following implementations are available:
+        - package de.atb.context.monitoring.parser.database.DatabaseParser
+        - package de.atb.context.monitoring.parser.file.FileParser
+        - package de.atb.context.monitoring.parser.file.FilePairParser
+        - package de.atb.context.monitoring.parser.file.FileTripletParser
+        - package de.atb.context.monitoring.parser.webservice.MessageBrokerParser
+        - package de.atb.context.monitoring.parser.webservice.WebServiceParser
+        - package de.atb.context.monitoring.parser.GitlabCommitParser
+        - package de.atb.context.monitoring.parser.GitParser
+
+    - type: Currently only used for File analyser and parser. Defines the file extensions to be used.
+
+#### monitors
+
+Each monitor entry has the following mandatory attributes
+
+- id: The unique name of the monitor
+- datasource: The id of one previously defined datasource (see above)
+- interpreter: The id of one previously defined interpreter (see above)
+- index: The id of one previously defined index (see above)
diff --git a/context-core/pom.xml b/context-core/pom.xml
@@ -16,8 +16,8 @@
     <name>ATB Context Extraction Core Lib</name>
 
     <properties>
-        <cxf.version>3.5.1</cxf.version>
-        <jena.version>4.4.0</jena.version>
+        <cxf.version>3.5.2</cxf.version>
+        <jena.version>4.5.0</jena.version>
     </properties>
 
     <dependencies>
@@ -107,7 +107,7 @@
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>
-            <version>1.18.22</version>
+            <version>1.18.24</version>
         </dependency>
 
         <!-- API, java.xml.bind module -->

diff --git a/pom.xml b/pom.xml
@@ -24,7 +24,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.version>3.5.4</maven.version>
         <java.version>11</java.version>
-        <testcontainers.version>1.16.3</testcontainers.version>
+        <testcontainers.version>1.17.1</testcontainers.version>
     </properties>
 
     <build>

diff --git a/samples/config/monitoring-config.xml b/samples/config/monitoring-config.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<config xmlns="http://www.atb-bremen.de"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.atb-bremen.de monitoring-config.xsd">
+
+    <indexes>
+        <index id="index-git" location="target/indexes/git"/>
+    </indexes>
+
+    <datasources>
+        <datasource id="datasource-git" type="messagebroker"
+                    monitor="de.atb.context.monitoring.monitors.GitMonitor"
+                    uri=""
+                    options="server=rabbitmq&amp;port=5672&amp;exchange=smartclide-monitoring&amp;topic=monitoring.git.*&amp;dle-topic=dle.git.commits"
+                    class="de.atb.context.monitoring.config.models.datasources.MessageBrokerDataSource"/>
+    </datasources>
+
+    <interpreters>
+        <interpreter id="interpreter-git">
+            <configuration type="*"
+                           parser="de.atb.context.monitoring.parser.GitParser"
+                           analyser="de.atb.context.monitoring.analyser.GitAnalyser"/>
+        </interpreter>
+    </interpreters>
+
+    <monitors>
+        <monitor id="monitor-git" datasource="datasource-git" interpreter="interpreter-git" index="index-git"/>
+    </monitors>
+</config>
diff --git a/samples/config/monitoring-config.xsd b/samples/config/monitoring-config.xsd
@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" targetNamespace="http://www.atb-bremen.de" xmlns="http://www.atb-bremen.de">
+    <xs:element name="config">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element ref="indexes"/>
+                <xs:element ref="datasources"/>
+                <xs:element ref="interpreters"/>
+                <xs:element ref="monitors"/>
+            </xs:sequence>
+        </xs:complexType>
+    </xs:element>
+    <xs:element name="indexes">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element maxOccurs="unbounded" minOccurs="1" ref="index"/>
+            </xs:sequence>
+        </xs:complexType>
+    </xs:element>
+    <xs:element name="index">
+        <xs:complexType>
+            <xs:attribute name="id" use="required" type="xs:ID"/>
+            <xs:attribute name="location" use="required" type="xs:anyURI"/>
+        </xs:complexType>
+    </xs:element>
+    <xs:element name="monitors">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element maxOccurs="unbounded" minOccurs="1" ref="monitor"/>
+            </xs:sequence>
+        </xs:complexType>
+    </xs:element>
+    <xs:element name="monitor">
+        <xs:complexType>
+            <xs:attribute name="id" use="required" type="xs:ID"/>
+            <xs:attribute name="datasource" use="required" type="xs:IDREF"/>
+            <xs:attribute name="index" use="required" type="xs:IDREF"/>
+            <xs:attribute name="interpreter" use="required" type="xs:IDREF"/>
+        </xs:complexType>
+    </xs:element>
+    <xs:element name="datasources">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element maxOccurs="unbounded" minOccurs="1" ref="datasource"/>
+            </xs:sequence>
+        </xs:complexType>
+    </xs:element>
+    <xs:element name="datasource">
+        <xs:complexType>
+            <xs:attribute name="class" use="required"/>
+            <xs:attribute name="id" use="required" type="xs:ID"/>
+            <xs:attribute name="monitor" use="required"/>
+            <xs:attribute name="options" use="required"/>
+            <xs:attribute name="type" use="required" type="xs:NCName"/>
+            <xs:attribute name="uri" use="required" type="xs:anyURI"/>
+        </xs:complexType>
+    </xs:element>
+    <xs:element name="interpreters">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element maxOccurs="unbounded" minOccurs="1" ref="interpreter"/>
+            </xs:sequence>
+        </xs:complexType>
+    </xs:element>
+    <xs:element name="interpreter">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element maxOccurs="unbounded" minOccurs="1" ref="configuration"/>
+            </xs:sequence>
+            <xs:attribute name="id" use="required" type="xs:ID"/>
+        </xs:complexType>
+    </xs:element>
+    <xs:element name="configuration">
+        <xs:complexType>
+            <xs:attribute name="analyser" use="required"/>
+            <xs:attribute name="parser" use="required"/>
+            <xs:attribute name="type" use="required"/>
+        </xs:complexType>
+    </xs:element>
+</xs:schema>
diff --git a/samples/config/services-config.xml b/samples/config/services-config.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="utf-8"?>
+<config xmlns="http://www.atb-bremen.de"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.atb-bremen.de services-config.xsd">
+
+    <services>
+        <service id="AmIMonitoring">
+            <host>localhost</host>
+            <location>http://localhost:19001</location>
+            <name>AmIMonitoringService</name>
+            <server>de.atb.context.services.AmIMonitoringService</server>
+            <proxy>de.atb.context.services.IAmIMonitoringService</proxy>
+        </service>
+        <service id="AmI-repository">
+            <host>localhost</host>
+            <location>http://localhost:19002</location>
+            <name>AmIMonitoringDataRepositoryService</name>
+            <server>de.atb.context.services.AmIMonitoringDataRepositoryService</server>
+            <proxy>de.atb.context.services.IAmIMonitoringDataRepositoryService</proxy>
+        </service>
+        <service id="PersistenceUnitService">
+            <host>localhost</host>
+            <location>http://localhost:19004</location>
+            <name>PersistenceUnitService</name>
+            <server>de.atb.context.services.PersistenceUnitService</server>
+            <proxy>de.atb.core.services.IPersistenceUnitService</proxy>
+        </service>
+    </services>
+</config>
diff --git a/samples/config/services-config.xsd b/samples/config/services-config.xsd
@@ -0,0 +1,26 @@
+<xs:schema attributeFormDefault="unqualified" elementFormDefault="qualified" targetNamespace="http://www.atb-bremen.de" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+    <xs:element name="config">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element name="services">
+                    <xs:complexType>
+                        <xs:sequence>
+                            <xs:element name="service" maxOccurs="unbounded" minOccurs="0">
+                                <xs:complexType>
+                                    <xs:sequence>
+                                        <xs:element type="xs:string" name="host"/>
+                                        <xs:element type="xs:anyURI" name="location"/>
+                                        <xs:element type="xs:string" name="name"/>
+                                        <xs:element type="xs:string" name="server"/>
+                                        <xs:element type="xs:string" name="proxy"/>
+                                    </xs:sequence>
+                                    <xs:attribute type="xs:string" name="id" use="optional"/>
+                                </xs:complexType>
+                            </xs:element>
+                        </xs:sequence>
+                    </xs:complexType>
+                </xs:element>
+            </xs:sequence>
+        </xs:complexType>
+    </xs:element>
+</xs:schema>
diff --git a/samples/docker-compose.yml b/samples/docker-compose.yml
@@ -0,0 +1,41 @@
+version: "3.8"
+
+services:
+  # rabbitmq - this is the datasource configured in config/monitoring-config.xml
+  rabbitmq:
+    image: rabbitmq:3.10.0-alpine
+    hostname: rabbitmq
+    ports:
+      - "5672:5672"
+    networks:
+      - smartclide-monitoring
+  # smartclide-monitoring
+  smartclide-monitoring:
+    image: ghcr.io/eclipse-researchlabs/smartclide/smartclide-monitoring:latest
+    depends_on:
+      - rabbitmq
+    volumes:
+      # configuration files are mounted to '/opt/smartclide/config'
+      # see eu.smartclide.contexthandling.ServiceMain for other options
+      - ./config:/opt/smartclide/config
+    # waits for rabbitmq datasource to be available before starting smartclide-monitoring
+    entrypoint:
+      [
+        "/entrypoint.d/wait-for-it.sh",
+        "rabbitmq:5672",
+        "--strict",
+        "-t",
+        "30",
+        "--",
+        "java",
+        "-Djava.security.egd=file:/dev/./urandom",
+        "-cp",
+        "app/libs/*:app/resources:app/classes",
+        "eu.smartclide.contexthandling.ServiceMain",
+      ]
+    networks:
+      - smartclide-monitoring
+
+networks:
+  smartclide-monitoring:
+    name: smartclide-monitoring
diff --git a/smartclide-monitoring/pom.xml b/smartclide-monitoring/pom.xml
@@ -46,6 +46,14 @@
                     <to>
                         <image>ghcr.io/eclipse-researchlabs/smartclide/${project.artifactId}:latest</image>
                     </to>
+                    <extraDirectories>
+                        <permissions>
+                            <permission>
+                                <file>/entrypoint.d/wait-for-it.sh</file>
+                                <mode>755</mode>
+                            </permission>
+                        </permissions>
+                    </extraDirectories>
                     <container>
                         <ports>
                             <port>19001</port>