diff --git a/deploy/db-server/Dockerfile b/deploy/db-server/Dockerfile new file mode 100644 index 0000000..2140a02 --- /dev/null +++ b/deploy/db-server/Dockerfile @@ -0,0 +1,33 @@ +FROM ontotext/graphdb:10.6.3 + +# Override parent entrypoint +ENTRYPOINT [] + +ENV GRAPHDB_HOME=/opt/graphdb/home +ENV GRAPHDB_INSTALL_DIR=/opt/graphdb/dist + +WORKDIR ${GRAPHDB_HOME} + +# Install libs related to RDF processing +### for arm64 +RUN if command -v apt >/dev/null; then \ + apt update && \ + apt install -y python3-rdflib && \ + apt install -y liburi-perl; \ +fi + +### for amd64 +RUN if command -v apk >/dev/null; then \ + apk add py3-rdflib && \ + apk add perl-uri; \ +fi + +# Copy scripts +COPY bin/* ${GRAPHDB_INSTALL_DIR}/bin/ + +EXPOSE 7200 + +# Assuming following input directories: +# - /repo-config and data -- configuration ttl files to create repositories +# - /root/graphdb-import -- files to import data to specific repositories +CMD ${GRAPHDB_INSTALL_DIR}/bin/repo-init.sh /repo-config ${GRAPHDB_HOME} & ${GRAPHDB_INSTALL_DIR}/bin/graphdb -Dgraphdb.home=${GRAPHDB_HOME} -Dgraphdb.logback=${GRAPHDB_INSTALL_DIR}/conf/logback.xml diff --git a/deploy/db-server/bin/get-value-of-rdf-property.py b/deploy/db-server/bin/get-value-of-rdf-property.py new file mode 100755 index 0000000..9f9a8c4 --- /dev/null +++ b/deploy/db-server/bin/get-value-of-rdf-property.py @@ -0,0 +1,73 @@ +#!/usr/bin/python3 + +import sys +from rdflib import Graph, URIRef + +def log(message): + print("ERROR: " + message, file=sys.stderr) + +def check_params(): + if len(sys.argv) != 3: + log(f"""Illegal number of parameters. + +Script returns single value of from file specified by . + +Usage: {sys.argv[0]} + +Example: {sys.argv[0]} "./init-config/repo-config.ttl" "http://www.openrdf.org/config/repository#repositoryID" +""") + sys.exit(1) + + +def check_property_has_single_value(results, rdf_property): + if len(results) == 0: + log(f"No values found for the specified property {rdf_property}.") + sys.exit(2) + elif len(results) > 1: + error_message = f"Multiple values found for the property {rdf_property}. Triple that match pattern '?s <{rdf_property}> ?o' are:\n" + for row in results: + subject, value = row + error_message += f" {subject} {rdf_property} {value} .\n" + log(error_message) + sys.exit(3) + +def load_rdf_graph(file_path): + # Load RDF file into an RDFLib graph + g = Graph() + + # Explicitly specify the format based on the file extension + if file_path.endswith(".ttl"): + g.parse(file_path, format="turtle") + elif file_path.endswith(".rdf"): + g.parse(file_path, format="xml") + else: + log(f"Unsupported RDF file format of {file_path}.") + sys.exit(1) + return g + +def main(): + check_params() + + file_path = sys.argv[1] + rdf_property = URIRef(sys.argv[2]) + + g = load_rdf_graph(file_path) + + # Query for subjects with the specified property + query = f""" + SELECT ?subject ?value + WHERE {{ + ?subject <{rdf_property}> ?value. + }} + """ + results = g.query(query) + + check_property_has_single_value(results, rdf_property) + + for row in results: + subject, value = row + print(f"{value}") + +if __name__ == "__main__": + main() + diff --git a/deploy/db-server/bin/repo-init.sh b/deploy/db-server/bin/repo-init.sh new file mode 100755 index 0000000..27462e2 --- /dev/null +++ b/deploy/db-server/bin/repo-init.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +# +# Initializes GraphDB repositories (the repositories are created if they do not exist yet and some of the data are replaced) +# + +SOURCE_DIR=$1 +GRAPHDB_HOME=$2 + +SCRIPT_DIR="`dirname $0`" + +echo "INFO: Running initializer for GraphDB repositories ..." + +# Wait for GraphDB to start up +echo "INFO: Waiting for GraphDB to start up..." +sleep 15s + +ls ${SOURCE_DIR}/*-config.ttl | while read REPO_CONFIG_FILE; do + + REPO_NAME=`$SCRIPT_DIR/get-value-of-rdf-property.py $REPO_CONFIG_FILE 'http://www.openrdf.org/config/repository#repositoryID'` + + if [ -z "$REPO_NAME" ]; then + echo "ERROR: Could not parse repository name from file $REPO_CONFIG_FILE" + exit 1 + fi + + if [ ! -d ${GRAPHDB_HOME}/data/repositories/${REPO_NAME} ] || [ -z "$(ls -A ${GRAPHDB_HOME})/data/repositories/${REPO_NAME}" ]; then + echo "INFO: Initializing repository $REPO_NAME..." + + # Create repository based on configuration + echo "INFO: Creating repository $REPO_NAME..." + curl -X POST --header "Content-Type: multipart/form-data" -F "config=@${REPO_CONFIG_FILE}" "http://localhost:7200/rest/repositories" + echo "INFO: Repository $REPO_NAME successfully initialized." + else + echo "INFO: Repository $REPO_NAME already exists. Skipping initialization..." + fi +done diff --git a/deploy/db-server/conf/logback.xml b/deploy/db-server/conf/logback.xml new file mode 100644 index 0000000..2b55f0d --- /dev/null +++ b/deploy/db-server/conf/logback.xml @@ -0,0 +1,188 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${logDestinationDirectory}/audit-log-%d{yyyy-MM-dd}.log + + + ${defaultPattern} + ${encoding} + + + + + + ${logDestinationDirectory}/main-%d{yyyy-MM-dd}.log + + + ${defaultPattern} + ${encoding} + + + + + + ${logDestinationDirectory}/error-%d{yyyy-MM-dd}.log + + + ${defaultPattern} + ${encoding} + + + ERROR + + + + + + ${logDestinationDirectory}/query-log-%d{yyyy-MM-dd}.log + + + ${defaultPattern} + ${encoding} + + + + + + ${logDestinationDirectory}/slow-query-log-%d{yyyy-MM-dd}.log + + + ${defaultPattern} + ${encoding} + + + + + + + + ${defaultPattern} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deploy/db-server/init-config/s-pipes-hello-world-config.ttl b/deploy/db-server/init-config/s-pipes-hello-world-config.ttl new file mode 100644 index 0000000..e286e67 --- /dev/null +++ b/deploy/db-server/init-config/s-pipes-hello-world-config.ttl @@ -0,0 +1,33 @@ +@prefix rdfs: . +@prefix rep: . +@prefix sail: . +@prefix xsd: . +@prefix graphdb: . + +<#s-pipes-hello-world> a rep:Repository; + rep:repositoryID "s-pipes-hello-world"; + rep:repositoryImpl [ + rep:repositoryType "graphdb:SailRepository"; + [ + graphdb:base-URL "http://example.org/owlim#"; + graphdb:check-for-inconsistencies "false"; + graphdb:defaultNS ""; + graphdb:disable-sameAs "true"; + graphdb:enable-context-index "true"; + graphdb:enable-literal-index "true"; + graphdb:enablePredicateList "true"; + graphdb:entity-id-size "32"; + graphdb:entity-index-size "10000000"; + graphdb:imports ""; + graphdb:in-memory-literal-properties "true"; + graphdb:owlim-license ""; + graphdb:query-limit-results "0"; + graphdb:query-timeout "0"; + graphdb:read-only "false"; + graphdb:repository-type "file-repository"; + graphdb:storage-folder "storage"; + graphdb:throw-QueryEvaluationException-on-timeout "false"; + sail:sailType "graphdb:Sail" + ] + ]; + rdfs:label "SPipes Hello World repository" . diff --git a/docker-compose.yml b/docker-compose.yml index 2a975c4..2255855 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,8 +6,6 @@ services: container_name: s-pipes-editor-ui ports: - '3000:80' - networks: - - overlay depends_on: - s-pipes-editor-rest environment: @@ -18,16 +16,14 @@ services: container_name: s-pipes-editor-rest expose: - "18115" - networks: - - overlay depends_on: - s-pipes-engine - - rdf4j + - db-server environment: - SCRIPTPATHS=${SCRIPTPATHS:-${PWD}/../s-pipes-modules;${CUSTOM_SCRIPT_PATHS:-${PWD}/../s-pipes/doc/examples}} - SCRIPTRULES=${SCRIPTRULES:-${PWD}/../s-pipes-editor/src/main/resources/rules} - ENGINEURL=${SPIPES_ENGINE:-http://s-pipes-engine:8080/s-pipes}/ - - RDF4J_REPOSITORYURL=${RDF4J_SERVER_URL:-http://rdf4j:8080/rdf4j-server}/repositories + - RDF4J_REPOSITORYURL=${RDF4J_SERVER_URL:-http://db-server:7200}/repositories - RDF4J_REPOSITORYNAME=${RDF4J_REPOSITORYNAME:-s-pipes-hello-world} - RDF4J_PCONFIGURL=${RDF4J_PCONFIGURL:-../s-pipes/doc/examples/hello-world/config.ttl} volumes: @@ -41,10 +37,8 @@ services: # container_name: s-pipes-engine expose: - "8080" - networks: - - overlay depends_on: - - rdf4j + - db-server environment: - CONTEXTS_SCRIPTPATHS=${SCRIPTPATHS:-${PWD}/../s-pipes-modules;${CUSTOM_SCRIPT_PATHS:-${PWD}/../s-pipes/doc/examples}} volumes: @@ -52,22 +46,16 @@ services: - ${SHARED_ROOT:-/home}:${SHARED_ROOT:-/home} - /usr/local/tomcat/temp/:/usr/local/tomcat/temp/ - rdf4j: - image: 'eclipse/rdf4j-workbench:3.7.7' - container_name: rdf4j + db-server: + build: + context: ./deploy/db-server expose: - - "8080" - networks: - - overlay - environment: - - JAVA_OPTS=-Xms1g -Xmx4g + - "7200" volumes: - - data:/var/rdf4j - - logs:/usr/local/tomcat/logs + - ./deploy/db-server/init-config:/repo-config:ro + - db-server:/opt/graphdb/home volumes: data: logs: - -networks: - overlay: + db-server: