Skip to content

Commit ae4ba13

Browse files
Lorenz BuehmannAklakan
Lorenz Buehmann
authored and
Aklakan
committed
GH-3026: spatial index per graph and kryo serialization
1 parent a3983b2 commit ae4ba13

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+3266
-155
lines changed

‎jena-arq/src/main/java/org/apache/jena/system/TxnCtl.java

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.jena.system;
19+
20+
import java.util.Objects;
21+
22+
import org.apache.jena.query.ReadWrite;
23+
import org.apache.jena.query.TxnType;
24+
import org.apache.jena.sparql.core.Transactional;
25+
26+
/**
27+
* Txn variant for use with try-with-resources. Allows raising
28+
* checked exceptions in an idiomatic way. Closing the TxnCtl
29+
* instance will abort the transaction unless it
30+
* has been manually committed.
31+
* <p>
32+
*
33+
* Usage example:
34+
* <pre>
35+
* public void myMethod() throws IOException {
36+
* try (TxnCtl txn = TxnCtl.begin(dataset, TxnType.WRITE)) {
37+
* // Do work
38+
* if (someError) {
39+
* throw new IOException();
40+
* }
41+
* // Must manually call commit on success.
42+
* txn.commit();
43+
* }
44+
* }
45+
* </pre>
46+
*/
47+
public class TxnCtl
48+
implements AutoCloseable
49+
{
50+
private Transactional txn;
51+
private boolean b;
52+
53+
private TxnCtl(Transactional txn, boolean b) {
54+
super();
55+
this.txn = txn;
56+
this.b = b;
57+
}
58+
59+
public static TxnCtl begin(Transactional txn, ReadWrite readWrite) {
60+
return begin(txn, TxnType.convert(readWrite));
61+
}
62+
63+
public static TxnCtl begin(Transactional txn, TxnType txnType) {
64+
Objects.requireNonNull(txn);
65+
Objects.requireNonNull(txnType);
66+
boolean b = txn.isInTransaction();
67+
if ( b )
68+
TxnOp.compatibleWithPromote(txnType, txn);
69+
else
70+
txn.begin(txnType);
71+
return new TxnCtl(txn, b);
72+
}
73+
74+
public void commit() {
75+
if ( txn.isInTransaction() ) {
76+
77+
// May have been explicit commit or abort.
78+
txn.commit();
79+
}
80+
}
81+
82+
@Override
83+
public void close() {
84+
if ( !b ) {
85+
if ( txn.isInTransaction() )
86+
// May have been explicit commit or abort.
87+
txn.abort();
88+
txn.end();
89+
}
90+
}
91+
}

‎jena-fuseki2/jena-fuseki-geosparql/src/main/java/org/apache/jena/fuseki/geosparql/DatasetOperations.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -228,10 +228,10 @@ private static void prepareSpatialExtension(Dataset dataset, ArgsConfig argsConf
228228
if (!isEmpty) {
229229
if (argsConfig.getSpatialIndexFile() != null) {
230230
File spatialIndexFile = argsConfig.getSpatialIndexFile();
231-
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile);
231+
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile.toPath());
232232
} else if (argsConfig.isTDBFileSetup()) {
233233
File spatialIndexFile = new File(argsConfig.getTdbFile(), SPATIAL_INDEX_FILE);
234-
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile);
234+
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile.toPath());
235235
} else {
236236
GeoSPARQLConfig.setupSpatialIndex(dataset);
237237
}

‎jena-geosparql/pom.xml

+101-16
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@
1414
limitations under the License.
1515
-->
1616

17-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
17+
<project xmlns="http://maven.apache.org/POM/4.0.0"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
1820
<modelVersion>4.0.0</modelVersion>
1921
<artifactId>jena-geosparql</artifactId>
2022
<packaging>jar</packaging>
@@ -34,6 +36,16 @@
3436

3537
<dependencies>
3638

39+
<dependency>
40+
<groupId>io.github.galbiston</groupId>
41+
<artifactId>expiring-map</artifactId>
42+
</dependency>
43+
44+
<dependency>
45+
<groupId>javax.xml.bind</groupId>
46+
<artifactId>jaxb-api</artifactId>
47+
</dependency>
48+
3749
<dependency>
3850
<groupId>org.apache.jena</groupId>
3951
<artifactId>jena-arq</artifactId>
@@ -44,18 +56,6 @@
4456
<groupId>org.apache.sis.core</groupId>
4557
<artifactId>sis-referencing</artifactId>
4658
</dependency>
47-
48-
<!-- Evntually this wil be:
49-
<dependency>
50-
<groupId>jakarta.xml.bind</groupId>
51-
<artifactId>jakarta.xml.bind-api</artifactId>
52-
</dependency>
53-
-->
54-
55-
<dependency>
56-
<groupId>javax.xml.bind</groupId>
57-
<artifactId>jaxb-api</artifactId>
58-
</dependency>
5959

6060
<dependency>
6161
<groupId>org.slf4j</groupId>
@@ -82,6 +82,90 @@
8282
<artifactId>commons-collections4</artifactId>
8383
</dependency>
8484

85+
<!-- Kryo serialization via Apache Sedona -->
86+
87+
<!--
88+
<dependency>
89+
<groupId>org.apache.sedona</groupId>
90+
<artifactId>sedona-spark-common-3.5_2.13</artifactId>
91+
<version>1.6.0</version>
92+
<exclusions>
93+
<exclusion>
94+
<groupId>org.apache.spark</groupId>
95+
<artifactId>spark-core_2.12</artifactId>
96+
</exclusion>
97+
<exclusion>
98+
<groupId>org.apache.hadoop</groupId>
99+
<artifactId>hadoop-minicluster</artifactId>
100+
</exclusion>
101+
<exclusion>
102+
<groupId>org.apache.hadoop</groupId>
103+
<artifactId>hadoop-client</artifactId>
104+
</exclusion>
105+
<exclusion>
106+
<groupId>org.geotools</groupId>
107+
<artifactId>gt-geotiff</artifactId>
108+
</exclusion>
109+
<exclusion>
110+
<groupId>org.geotools</groupId>
111+
<artifactId>gt-coverage</artifactId>
112+
</exclusion>
113+
</exclusions>
114+
</dependency>
115+
-->
116+
117+
<dependency>
118+
<groupId>org.apache.sedona</groupId>
119+
<artifactId>sedona-common</artifactId>
120+
<version>1.7.0</version>
121+
<exclusions>
122+
<exclusion>
123+
<groupId>org.apache.commons</groupId>
124+
<artifactId>commons-math3</artifactId>
125+
</exclusion>
126+
<exclusion>
127+
<groupId>org.wololo</groupId>
128+
<artifactId>jts2geojson</artifactId>
129+
</exclusion>
130+
<exclusion>
131+
<groupId>com.google.geometry</groupId>
132+
<artifactId>s2-geometry</artifactId>
133+
</exclusion>
134+
<exclusion>
135+
<groupId>com.uber</groupId>
136+
<artifactId>h3</artifactId>
137+
</exclusion>
138+
<exclusion>
139+
<groupId>net.sf.geographiclib</groupId>
140+
<artifactId>GeographicLib-Java</artifactId>
141+
</exclusion>
142+
</exclusions>
143+
</dependency>
144+
145+
<!-- H3Utils only exists in newer sedona-common versions -->
146+
<!--
147+
<dependency>
148+
<groupId>org.apache.sedona</groupId>
149+
<artifactId>sedona-common</artifactId>
150+
<version>1.6.0</version>
151+
</dependency>
152+
-->
153+
154+
<dependency>
155+
<groupId>com.esotericsoftware</groupId>
156+
<artifactId>kryo</artifactId>
157+
<version>4.0.2</version>
158+
</dependency>
159+
160+
<!-- Resolve conflicts between jenax/tika/nashorn/hadoop/kryo-->
161+
<!--
162+
<dependency>
163+
<groupId>org.ow2.asm</groupId>
164+
<artifactId>asm</artifactId>
165+
<version>8.0.1</version>
166+
</dependency>
167+
-->
168+
85169
<!-- Non-free; testing only -->
86170
<dependency>
87171
<groupId>org.apache.sis.non-free</groupId>
@@ -102,6 +186,7 @@
102186
<scope>test</scope>
103187
</dependency>
104188

189+
105190
</dependencies>
106191

107192
<build>
@@ -133,11 +218,11 @@
133218
<plugin>
134219
<groupId>org.apache.maven.plugins</groupId>
135220
<artifactId>maven-source-plugin</artifactId>
136-
<executions>
221+
<executions>
137222
<execution>
138-
<id>attach-sources-test</id>
223+
<id>attach-sources-test</id>
139224
<goals>
140-
<goal>test-jar-no-fork</goal>
225+
<goal>test-jar-no-fork</goal>
141226
</goals>
142227
</execution>
143228
</executions>

‎jena-geosparql/src/main/java/org/apache/jena/geosparql/InitGeoSPARQL.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public static void init() {
4242
if ( initialized )
4343
return ;
4444
synchronized (initLock) {
45-
if ( initialized ) {
45+
if ( initialized || System.getProperty("jena.geosparql.skip", "false").equalsIgnoreCase("true") ) {
4646
JenaSystem.logLifecycle("InitGeoSPARQL - skip") ;
4747
return ;
4848
}

‎jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/GeoAssembler.java

+31-9
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,16 @@
1818

1919
package org.apache.jena.geosparql.assembler;
2020

21-
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.*;
21+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pApplyDefaultGeometry;
22+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pDataset;
23+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexEnabled;
24+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexExpiries;
25+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexSizes;
26+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pInference;
27+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pQueryRewrite;
28+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSpatialIndexFile;
29+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSpatialIndexPerGraph;
30+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSrsUri;
2231
import static org.apache.jena.sparql.util.graph.GraphUtils.getBooleanValue;
2332

2433
import java.io.IOException;
@@ -34,6 +43,7 @@
3443
import org.apache.jena.geosparql.configuration.GeoSPARQLOperations;
3544
import org.apache.jena.geosparql.configuration.SrsException;
3645
import org.apache.jena.geosparql.spatial.SpatialIndexException;
46+
import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexUtils;
3747
import org.apache.jena.graph.Graph;
3848
import org.apache.jena.graph.Node;
3949
import org.apache.jena.query.Dataset;
@@ -117,9 +127,21 @@ public DatasetGraph createDataset(Assembler a, Resource root) {
117127
if (root.hasProperty(pSpatialIndexFile) )
118128
spatialIndexFilename = GraphUtils.getStringValue(root, pSpatialIndexFile);
119129

120-
// ---- Build
130+
// spatial index per named graph option
131+
boolean spatialIndexPerGraph = false;
132+
if (root.hasProperty(pSpatialIndexPerGraph) )
133+
spatialIndexPerGraph = getBooleanValue(root, pSpatialIndexPerGraph);
134+
135+
// SRS URI
136+
String srsURI = null;
137+
if (root.hasProperty(pSrsUri) )
138+
srsURI = GraphUtils.getStringValue(root, pSrsUri);
121139

140+
141+
// ---- Build
122142
Dataset dataset = DatasetFactory.wrap(base);
143+
dataset.getContext().set(SpatialIndexUtils.symSpatialIndexPerGraph, spatialIndexPerGraph);
144+
dataset.getContext().set(SpatialIndexUtils.symSrsUri, srsURI);
123145

124146
// Conversion of data. Startup-only.
125147
// needed for w3c:geo/wgs84_pos#lat/log.
@@ -142,13 +164,13 @@ public DatasetGraph createDataset(Assembler a, Resource root) {
142164
//Setup GeoSPARQL
143165
if (indexEnabled) {
144166
GeoSPARQLConfig.setupMemoryIndex(indexSizes.get(0), indexSizes.get(1), indexSizes.get(2),
145-
(long)indexExpiries.get(0), (long)indexExpiries.get(1), (long)indexExpiries.get(2),
146-
queryRewrite);
167+
(long)indexExpiries.get(0), (long)indexExpiries.get(1), (long)indexExpiries.get(2),
168+
queryRewrite);
147169
} else {
148170
GeoSPARQLConfig.setupNoIndex(queryRewrite);
149171
}
150172

151-
prepareSpatialExtension(dataset, spatialIndexFilename);
173+
prepareSpatialExtension(dataset, spatialIndexFilename, spatialIndexPerGraph);
152174
return base;
153175
}
154176

@@ -165,8 +187,8 @@ private static List<Integer> getListInteger(Resource r, Property p, int len) {
165187
return integerList;
166188
}
167189

168-
private static void prepareSpatialExtension(Dataset dataset, String spatialIndex){
169-
boolean isEmpty = dataset.calculateRead(()->dataset.isEmpty());
190+
private static void prepareSpatialExtension(Dataset dataset, String spatialIndex, boolean spatialIndexPerGraph){
191+
boolean isEmpty = dataset.calculateRead(dataset::isEmpty);
170192
if ( isEmpty && spatialIndex != null ) {
171193
LOG.warn("Dataset empty. Spatial Index not constructed. Server will require restarting after adding data and any updates to build Spatial Index.");
172194
return;
@@ -185,12 +207,12 @@ private static void prepareSpatialExtension(Dataset dataset, String spatialIndex
185207
// file given but empty -> compute and serialize index
186208
Path spatialIndexPath = Path.of(spatialIndex);
187209
if ( ! Files.exists(spatialIndexPath) || Files.size(spatialIndexPath) == 0 ) {
188-
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexPath.toFile());
210+
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexPath, spatialIndexPerGraph);
189211
return;
190212
}
191213

192214
// load and setup the precomputed index
193-
GeoSPARQLConfig.setupPrecomputedSpatialIndex(dataset, spatialIndexPath.toFile());
215+
GeoSPARQLConfig.setupPrecomputedSpatialIndex(dataset, spatialIndexPath);
194216
}
195217
catch (SrsException ex) {
196218
// Data but no spatial data.

‎jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/VocabGeoSPARQL.java

+4
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ private static Property property(String shortName) {
7373
// "File to load or store the spatial index. Default to " + SPATIAL_INDEX_FILE + " in TDB folder if using TDB and not set. Otherwise spatial index is not stored.
7474
public static final Property pSpatialIndexFile = property("spatialIndexFile");
7575

76+
public static final Property pSpatialIndexPerGraph = property("spatialIndexPerGraph");
77+
78+
public static final Property pSrsUri = property("srsUri");
79+
7680
// Dataset
7781
public static final Property pDataset = property("dataset");
7882
}

0 commit comments

Comments
 (0)