Skip to content

Commit 52317c9

Browse files
Lorenz BuehmannAklakan
Lorenz Buehmann
authored and
Aklakan
committed
GH-3026: spatial index per graph and kryo serialization
1 parent 9fa2fe0 commit 52317c9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+3336
-187
lines changed

‎jena-arq/src/main/java/org/apache/jena/system/TxnCtl.java

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.jena.system;
19+
20+
import java.util.Objects;
21+
22+
import org.apache.jena.query.ReadWrite;
23+
import org.apache.jena.query.TxnType;
24+
import org.apache.jena.sparql.core.Transactional;
25+
26+
/**
27+
* Txn variant for use with try-with-resources. Allows raising
28+
* checked exceptions in an idiomatic way. Closing the TxnCtl
29+
* instance will abort the transaction unless it
30+
* has been manually committed.
31+
* <p>
32+
*
33+
* Usage example:
34+
* <pre>
35+
* public void myMethod() throws IOException {
36+
* try (TxnCtl txn = TxnCtl.begin(dataset, TxnType.WRITE)) {
37+
* // Do work
38+
* if (someError) {
39+
* throw new IOException();
40+
* }
41+
* // Must manually call commit on success.
42+
* txn.commit();
43+
* }
44+
* }
45+
* </pre>
46+
*/
47+
public class TxnCtl
48+
implements AutoCloseable
49+
{
50+
private Transactional txn;
51+
private boolean b;
52+
53+
private TxnCtl(Transactional txn, boolean b) {
54+
super();
55+
this.txn = txn;
56+
this.b = b;
57+
}
58+
59+
public static TxnCtl begin(Transactional txn, ReadWrite readWrite) {
60+
return begin(txn, TxnType.convert(readWrite));
61+
}
62+
63+
public static TxnCtl begin(Transactional txn, TxnType txnType) {
64+
Objects.requireNonNull(txn);
65+
Objects.requireNonNull(txnType);
66+
boolean b = txn.isInTransaction();
67+
if ( b )
68+
TxnOp.compatibleWithPromote(txnType, txn);
69+
else
70+
txn.begin(txnType);
71+
return new TxnCtl(txn, b);
72+
}
73+
74+
public void commit() {
75+
if ( txn.isInTransaction() ) {
76+
77+
// May have been explicit commit or abort.
78+
txn.commit();
79+
}
80+
}
81+
82+
@Override
83+
public void close() {
84+
if ( !b ) {
85+
if ( txn.isInTransaction() )
86+
// May have been explicit commit or abort.
87+
txn.abort();
88+
txn.end();
89+
}
90+
}
91+
}

‎jena-fuseki2/jena-fuseki-geosparql/src/main/java/org/apache/jena/fuseki/geosparql/DatasetOperations.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -228,10 +228,10 @@ private static void prepareSpatialExtension(Dataset dataset, ArgsConfig argsConf
228228
if (!isEmpty) {
229229
if (argsConfig.getSpatialIndexFile() != null) {
230230
File spatialIndexFile = argsConfig.getSpatialIndexFile();
231-
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile);
231+
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile.toPath());
232232
} else if (argsConfig.isTDBFileSetup()) {
233233
File spatialIndexFile = new File(argsConfig.getTdbFile(), SPATIAL_INDEX_FILE);
234-
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile);
234+
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexFile.toPath());
235235
} else {
236236
GeoSPARQLConfig.setupSpatialIndex(dataset);
237237
}

‎jena-geosparql/pom.xml

+20-17
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@
1414
limitations under the License.
1515
-->
1616

17-
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
17+
<project xmlns="http://maven.apache.org/POM/4.0.0"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
1820
<modelVersion>4.0.0</modelVersion>
1921
<artifactId>jena-geosparql</artifactId>
2022
<packaging>jar</packaging>
@@ -33,6 +35,15 @@
3335
</properties>
3436

3537
<dependencies>
38+
<dependency>
39+
<groupId>io.github.galbiston</groupId>
40+
<artifactId>expiring-map</artifactId>
41+
</dependency>
42+
43+
<dependency>
44+
<groupId>javax.xml.bind</groupId>
45+
<artifactId>jaxb-api</artifactId>
46+
</dependency>
3647

3748
<dependency>
3849
<groupId>org.apache.jena</groupId>
@@ -44,18 +55,6 @@
4455
<groupId>org.apache.sis.core</groupId>
4556
<artifactId>sis-referencing</artifactId>
4657
</dependency>
47-
48-
<!-- Evntually this wil be:
49-
<dependency>
50-
<groupId>jakarta.xml.bind</groupId>
51-
<artifactId>jakarta.xml.bind-api</artifactId>
52-
</dependency>
53-
-->
54-
55-
<dependency>
56-
<groupId>javax.xml.bind</groupId>
57-
<artifactId>jaxb-api</artifactId>
58-
</dependency>
5958

6059
<dependency>
6160
<groupId>org.slf4j</groupId>
@@ -82,6 +81,11 @@
8281
<artifactId>commons-collections4</artifactId>
8382
</dependency>
8483

84+
<dependency>
85+
<groupId>com.esotericsoftware</groupId>
86+
<artifactId>kryo</artifactId>
87+
</dependency>
88+
8589
<!-- Non-free; testing only -->
8690
<dependency>
8791
<groupId>org.apache.sis.non-free</groupId>
@@ -101,7 +105,6 @@
101105
<artifactId>log4j-slf4j2-impl</artifactId>
102106
<scope>test</scope>
103107
</dependency>
104-
105108
</dependencies>
106109

107110
<build>
@@ -133,11 +136,11 @@
133136
<plugin>
134137
<groupId>org.apache.maven.plugins</groupId>
135138
<artifactId>maven-source-plugin</artifactId>
136-
<executions>
139+
<executions>
137140
<execution>
138-
<id>attach-sources-test</id>
141+
<id>attach-sources-test</id>
139142
<goals>
140-
<goal>test-jar-no-fork</goal>
143+
<goal>test-jar-no-fork</goal>
141144
</goals>
142145
</execution>
143146
</executions>

‎jena-geosparql/src/main/java/org/apache/jena/geosparql/InitGeoSPARQL.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public static void init() {
4242
if ( initialized )
4343
return ;
4444
synchronized (initLock) {
45-
if ( initialized ) {
45+
if ( initialized || System.getProperty("jena.geosparql.skip", "false").equalsIgnoreCase("true") ) {
4646
JenaSystem.logLifecycle("InitGeoSPARQL - skip") ;
4747
return ;
4848
}

‎jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/GeoAssembler.java

+31-9
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,16 @@
1818

1919
package org.apache.jena.geosparql.assembler;
2020

21-
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.*;
21+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pApplyDefaultGeometry;
22+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pDataset;
23+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexEnabled;
24+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexExpiries;
25+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pIndexSizes;
26+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pInference;
27+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pQueryRewrite;
28+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSpatialIndexFile;
29+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSpatialIndexPerGraph;
30+
import static org.apache.jena.geosparql.assembler.VocabGeoSPARQL.pSrsUri;
2231
import static org.apache.jena.sparql.util.graph.GraphUtils.getBooleanValue;
2332

2433
import java.io.IOException;
@@ -34,6 +43,7 @@
3443
import org.apache.jena.geosparql.configuration.GeoSPARQLOperations;
3544
import org.apache.jena.geosparql.configuration.SrsException;
3645
import org.apache.jena.geosparql.spatial.SpatialIndexException;
46+
import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexUtils;
3747
import org.apache.jena.graph.Graph;
3848
import org.apache.jena.graph.Node;
3949
import org.apache.jena.query.Dataset;
@@ -117,9 +127,21 @@ public DatasetGraph createDataset(Assembler a, Resource root) {
117127
if (root.hasProperty(pSpatialIndexFile) )
118128
spatialIndexFilename = GraphUtils.getStringValue(root, pSpatialIndexFile);
119129

120-
// ---- Build
130+
// spatial index per named graph option
131+
boolean spatialIndexPerGraph = false;
132+
if (root.hasProperty(pSpatialIndexPerGraph) )
133+
spatialIndexPerGraph = getBooleanValue(root, pSpatialIndexPerGraph);
134+
135+
// SRS URI
136+
String srsURI = null;
137+
if (root.hasProperty(pSrsUri) )
138+
srsURI = GraphUtils.getStringValue(root, pSrsUri);
121139

140+
141+
// ---- Build
122142
Dataset dataset = DatasetFactory.wrap(base);
143+
dataset.getContext().set(SpatialIndexUtils.symSpatialIndexPerGraph, spatialIndexPerGraph);
144+
dataset.getContext().set(SpatialIndexUtils.symSrsUri, srsURI);
123145

124146
// Conversion of data. Startup-only.
125147
// needed for w3c:geo/wgs84_pos#lat/log.
@@ -142,13 +164,13 @@ public DatasetGraph createDataset(Assembler a, Resource root) {
142164
//Setup GeoSPARQL
143165
if (indexEnabled) {
144166
GeoSPARQLConfig.setupMemoryIndex(indexSizes.get(0), indexSizes.get(1), indexSizes.get(2),
145-
(long)indexExpiries.get(0), (long)indexExpiries.get(1), (long)indexExpiries.get(2),
146-
queryRewrite);
167+
(long)indexExpiries.get(0), (long)indexExpiries.get(1), (long)indexExpiries.get(2),
168+
queryRewrite);
147169
} else {
148170
GeoSPARQLConfig.setupNoIndex(queryRewrite);
149171
}
150172

151-
prepareSpatialExtension(dataset, spatialIndexFilename);
173+
prepareSpatialExtension(dataset, spatialIndexFilename, spatialIndexPerGraph);
152174
return base;
153175
}
154176

@@ -165,8 +187,8 @@ private static List<Integer> getListInteger(Resource r, Property p, int len) {
165187
return integerList;
166188
}
167189

168-
private static void prepareSpatialExtension(Dataset dataset, String spatialIndex){
169-
boolean isEmpty = dataset.calculateRead(()->dataset.isEmpty());
190+
private static void prepareSpatialExtension(Dataset dataset, String spatialIndex, boolean spatialIndexPerGraph){
191+
boolean isEmpty = dataset.calculateRead(dataset::isEmpty);
170192
if ( isEmpty && spatialIndex != null ) {
171193
LOG.warn("Dataset empty. Spatial Index not constructed. Server will require restarting after adding data and any updates to build Spatial Index.");
172194
return;
@@ -185,12 +207,12 @@ private static void prepareSpatialExtension(Dataset dataset, String spatialIndex
185207
// file given but empty -> compute and serialize index
186208
Path spatialIndexPath = Path.of(spatialIndex);
187209
if ( ! Files.exists(spatialIndexPath) || Files.size(spatialIndexPath) == 0 ) {
188-
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexPath.toFile());
210+
GeoSPARQLConfig.setupSpatialIndex(dataset, spatialIndexPath, spatialIndexPerGraph);
189211
return;
190212
}
191213

192214
// load and setup the precomputed index
193-
GeoSPARQLConfig.setupPrecomputedSpatialIndex(dataset, spatialIndexPath.toFile());
215+
GeoSPARQLConfig.setupPrecomputedSpatialIndex(dataset, spatialIndexPath);
194216
}
195217
catch (SrsException ex) {
196218
// Data but no spatial data.

‎jena-geosparql/src/main/java/org/apache/jena/geosparql/assembler/VocabGeoSPARQL.java

+4
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,10 @@ private static Property property(String shortName) {
7373
// "File to load or store the spatial index. Default to " + SPATIAL_INDEX_FILE + " in TDB folder if using TDB and not set. Otherwise spatial index is not stored.
7474
public static final Property pSpatialIndexFile = property("spatialIndexFile");
7575

76+
public static final Property pSpatialIndexPerGraph = property("spatialIndexPerGraph");
77+
78+
public static final Property pSrsUri = property("srsUri");
79+
7680
// Dataset
7781
public static final Property pDataset = property("dataset");
7882
}

‎jena-geosparql/src/main/java/org/apache/jena/geosparql/configuration/GeoSPARQLConfig.java

+24-11
Original file line numberDiff line numberDiff line change
@@ -17,20 +17,29 @@
1717
*/
1818
package org.apache.jena.geosparql.configuration;
1919

20-
import java.io.File;
20+
import java.nio.file.Path;
2121

2222
import org.apache.jena.geosparql.geof.topological.RelateFF;
2323
import org.apache.jena.geosparql.implementation.datatype.GeometryDatatype;
24-
import org.apache.jena.geosparql.implementation.function_registration.*;
24+
import org.apache.jena.geosparql.implementation.function_registration.Egenhofer;
25+
import org.apache.jena.geosparql.implementation.function_registration.GeometryProperty;
26+
import org.apache.jena.geosparql.implementation.function_registration.NonTopological;
27+
import org.apache.jena.geosparql.implementation.function_registration.RCC8;
28+
import org.apache.jena.geosparql.implementation.function_registration.Relate;
29+
import org.apache.jena.geosparql.implementation.function_registration.SimpleFeatures;
30+
import org.apache.jena.geosparql.implementation.function_registration.Spatial;
2531
import org.apache.jena.geosparql.implementation.index.GeometryLiteralIndex;
2632
import org.apache.jena.geosparql.implementation.index.GeometryTransformIndex;
2733
import org.apache.jena.geosparql.implementation.index.IndexConfiguration;
2834
import org.apache.jena.geosparql.implementation.index.IndexConfiguration.IndexOption;
2935
import org.apache.jena.geosparql.implementation.index.QueryRewriteIndex;
3036
import org.apache.jena.geosparql.implementation.registry.SRSRegistry;
3137
import org.apache.jena.geosparql.implementation.vocabulary.Geo;
32-
import org.apache.jena.geosparql.spatial.SpatialIndex;
3338
import org.apache.jena.geosparql.spatial.SpatialIndexException;
39+
import org.apache.jena.geosparql.spatial.index.compat.SpatialIndexIo;
40+
import org.apache.jena.geosparql.spatial.index.v2.SpatialIndex;
41+
import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexIoKryo;
42+
import org.apache.jena.geosparql.spatial.index.v2.SpatialIndexUtils;
3443
import org.apache.jena.query.Dataset;
3544
import org.apache.jena.sparql.function.FunctionRegistry;
3645
import org.apache.jena.sparql.pfunction.PropertyFunctionRegistry;
@@ -249,7 +258,7 @@ public static final void setupQueryRewriteIndex(Dataset dataset, String queryRew
249258
* @throws SpatialIndexException
250259
*/
251260
public static final void setupSpatialIndex(Dataset dataset) throws SpatialIndexException {
252-
SpatialIndex.buildSpatialIndex(dataset);
261+
SpatialIndexUtils.buildSpatialIndex(dataset.asDatasetGraph());
253262
}
254263

255264
/**
@@ -260,9 +269,9 @@ public static final void setupSpatialIndex(Dataset dataset) throws SpatialIndexE
260269
* @param spatialIndexFile the file containing the serialized spatial index
261270
* @throws SpatialIndexException
262271
*/
263-
public static final void setupPrecomputedSpatialIndex(Dataset dataset, File spatialIndexFile) throws SpatialIndexException {
264-
SpatialIndex si = SpatialIndex.load(spatialIndexFile);
265-
SpatialIndex.setSpatialIndex(dataset, si);
272+
public static final void setupPrecomputedSpatialIndex(Dataset dataset, Path spatialIndexFile) throws SpatialIndexException {
273+
SpatialIndex si = SpatialIndexIo.load(spatialIndexFile);
274+
SpatialIndexUtils.setSpatialIndex(dataset, si);
266275
}
267276

268277
/**
@@ -274,8 +283,12 @@ public static final void setupPrecomputedSpatialIndex(Dataset dataset, File spat
274283
* @param spatialIndexFile
275284
* @throws SpatialIndexException
276285
*/
277-
public static final void setupSpatialIndex(Dataset dataset, File spatialIndexFile) throws SpatialIndexException {
278-
SpatialIndex.buildSpatialIndex(dataset, spatialIndexFile);
286+
public static final void setupSpatialIndex(Dataset dataset, Path spatialIndexFile) throws SpatialIndexException {
287+
SpatialIndexIoKryo.buildSpatialIndex(dataset, spatialIndexFile);
288+
}
289+
290+
public static final void setupSpatialIndex(Dataset dataset, Path spatialIndexFile, boolean spatialIndexPerGraph) throws SpatialIndexException {
291+
SpatialIndexIoKryo.buildSpatialIndex(dataset, spatialIndexFile, spatialIndexPerGraph);
279292
}
280293

281294
/**
@@ -287,8 +300,8 @@ public static final void setupSpatialIndex(Dataset dataset, File spatialIndexFil
287300
* @param spatialIndexFile
288301
* @throws SpatialIndexException
289302
*/
290-
public static final void setupSpatialIndex(Dataset dataset, String srsURI, File spatialIndexFile) throws SpatialIndexException {
291-
SpatialIndex.buildSpatialIndex(dataset, srsURI, spatialIndexFile);
303+
public static final void setupSpatialIndex(Dataset dataset, String srsURI, Path spatialIndexFile) throws SpatialIndexException {
304+
SpatialIndexIoKryo.buildSpatialIndex(dataset, srsURI, spatialIndexFile);
292305
}
293306

294307
/**

0 commit comments

Comments
 (0)