Updated geoindexing, settings, geodoc and readme

SDI4Apps · Apr 19, 2016 · d55ccb8 · d55ccb8
1 parent 9854cd0
commit d55ccb8
Show file tree

Hide file tree

Showing 8 changed files with 175 additions and 60 deletions.
diff --git a/README.md b/README.md
@@ -4,16 +4,8 @@ The SDI4Apps OpenAPI is a set of web services exposed by the SDI4Apps Cloud plat
 
 The Open API consists partly of the protocols exposed by the components in the platform, i.e. Open Geospatial Consortium compliant web services like WMS, WFS and CS-W. Other parts of the API implements custom functionality that adds specific search/processing capabilities to the system. The code of these custom services are included in this repository.
 
-## System requirements
-* Java SDK
-* Maven
-* Apache Tomcat
-* GDAL/OGR Java SWIG bindings
-* Apache Lucene
-* PostgreSQL
-* PostGIS
-* pgRouting
+The code base of the project has been ported to this repository in the last week based on the content of two independent previous repositories where the actual development has taken place.
 
-## Installation
+# Web Service documentation
 
-Installation will be handled by the platform installation script...
+Documentation is currently being added to the site based on auto-generated JavaDocs from the source code. Please refer to the GitHub wiki for usage instructions.
diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/GeoDoc.java b/src/main/java/eu/sdi4apps/ftgeosearch/GeoDoc.java
@@ -5,6 +5,7 @@
  */
 package eu.sdi4apps.ftgeosearch;
 
+import eu.sdi4apps.openapi.config.Settings;
 import java.util.UUID;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
@@ -17,13 +18,12 @@
  * @author runarbe
  */
 public class GeoDoc {
-    
+
     /**
-     * The relevance score of the search result
-     * Only populated upon searches
+     * The relevance score of the search result Only populated upon searches
      */
     public float Score;
-    
+
     /**
      * Unique id of document
      */
@@ -33,7 +33,7 @@ public class GeoDoc {
      * The layer that the document belongs to
      */
     public String Layer;
-    
+
     /**
      * The type of object represented by the document
      */
@@ -88,6 +88,7 @@ public GeoDoc() {
 
     /**
      * Create a new GeoDoc
+     *
      * @param layer
      * @param fullGeom
      * @param pointGeom
@@ -129,17 +130,18 @@ public static GeoDoc create(
     }
 
     /**
-     * Add additional values to be indexed but  not displayed to the document
-     * 
-     * @param values 
+     * Add additional values to be indexed but not displayed to the document
+     *
+     * @param values
      */
     public void indexValues(String values) {
         this.IndexAdditional += values;
     }
 
     /**
      * Add custom JSON data object to the document
-     * @param jsonData 
+     *
+     * @param jsonData
      */
     public void setJsonData(String jsonData) {
         this.JsonData = jsonData;
@@ -156,17 +158,19 @@ public Document asLuceneDoc() {
             d.add(new Field("PointGeom", this.PointGeom, Store.YES, Index.NO));
             d.add(new Field("DisplayTitle", this.DisplayTitle, Store.YES, Index.NO));
             d.add(new Field("DisplayDescription", this.DisplayDescription, Store.YES, Index.NO));
-            
+
             Field indexTitle = new Field("IndexTitle", this.IndexTitle, Store.NO, Index.ANALYZED);
-            indexTitle.setBoost((float)1.2);
+            indexTitle.setBoost(Settings.TITLEBOOST);
             d.add(indexTitle);
-            
+
             Field indexDescription = new Field("IndexDescription", this.IndexDescription, Store.NO, Index.ANALYZED);
-            indexDescription.setBoost((float)1.1);
+            indexDescription.setBoost(Settings.DESCRIPTIONBOOST);
             d.add(indexDescription);
 
             d.add(new Field("IndexAdditional", this.IndexAdditional, Store.NO, Index.ANALYZED));
-            d.add(new Field("JsonData", Serializer.Serialize(this.JsonData), Store.YES, Index.NO));
+            if (this.JsonData != null) {
+                d.add(new Field("JsonData", Serializer.Serialize(this.JsonData), Store.YES, Index.NO));
+            }
         } catch (Exception e) {
             System.out.println("Error converting GeoDoc to LuceneDoc: " + e.toString());
         }

diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/Indexer.java b/src/main/java/eu/sdi4apps/ftgeosearch/Indexer.java
@@ -1,19 +1,24 @@
 package eu.sdi4apps.ftgeosearch;
 
+import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.shape.Shape;
 import eu.sdi4apps.openapi.utils.Logger;
-import eu.sdi4apps.ftgeosearch.drivers.ShapefileDriver;
 import eu.sdi4apps.openapi.config.Settings;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.sql.SQLException;
-import static java.util.Arrays.asList;
 import java.util.LinkedHashMap;
-import java.util.List;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.spatial.SpatialStrategy;
+import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy;
+import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree;
+import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import org.gdal.ogr.Feature;
@@ -35,6 +40,16 @@ public class Indexer {
 
     public static IndexWriterConfig indexWriterConfig = null;
 
+    public static SpatialContext spatialCtx = null;
+
+    public static SpatialStrategy spatialStrategy = null;
+
+    public static SpatialPrefixTree spatialPrefixTree = null;
+
+    public static int maxSpatialIndexLevels = 11;
+
+    public static int errorCount = 0;
+
     /**
      * Get the IndexWriter or null
      *
@@ -43,6 +58,12 @@ public class Indexer {
     public static IndexWriter getWriter() {
         try {
             if (indexWriter == null || indexWriter.isOpen() == false) {
+                spatialCtx = SpatialContext.GEO;
+
+                spatialPrefixTree = new GeohashPrefixTree(spatialCtx, maxSpatialIndexLevels);
+
+                spatialStrategy = new RecursivePrefixTreeStrategy(spatialPrefixTree, "GeoField");
+
                 analyzer = new StandardAnalyzer();
                 directory = FSDirectory.open(Paths.get(Settings.INDEXDIR));
                 indexWriterConfig = new IndexWriterConfig(analyzer);
@@ -79,15 +100,17 @@ public static void unlockIndex() {
 
     /**
      * Index a layer
-     * 
+     *
      * @param lyr
      * @param qi
-     * @param w 
+     * @param w
      */
     public static void indexLayer(Layer lyr, QueueItem qi, IndexWriter w) {
 
         try {
 
+            Indexer.errorCount = 0;
+
             qi.updateIndexingStatus(IndexingStatus.Indexing);
 
             Feature f = null;
@@ -107,6 +130,10 @@ public static void indexLayer(Layer lyr, QueueItem qi, IndexWriter w) {
 
                 indexFeature(w, qi, f, titleFieldMap, descriptionFieldMap, indexAdditionalFieldMap, jsonDataFieldMap);
 
+                if (Indexer.errorCount >= 50) {
+                    throw new Exception("More than 50 errors occurred, aborting indexing operation");
+                }
+
                 if (counter % batch == 0 || counter == totalFeatures) {
                     Logger.Log("Processed: " + counter + " items...");
                 }
@@ -146,12 +173,17 @@ public static void indexFeature(IndexWriter w,
 
             String compositeId = qi.layer + "-" + f.GetFID();
 
+            String pointWkt = g.PointOnSurface().ExportToWkt();
+
+            double[] shapeEnvelope = new double[4];
+            g.GetEnvelope(shapeEnvelope);
+
             GeoDoc gd = GeoDoc.create(
                     compositeId,
                     qi.layer,
                     qi.objtype,
                     g.ExportToWkt(),
-                    g.PointOnSurface().ExportToWkt(),
+                    pointWkt,
                     titleData[0],
                     descData[0],
                     titleData[1],
@@ -160,12 +192,35 @@ public static void indexFeature(IndexWriter w,
                     jsonData);
 
             if (gd != null) {
-                w.updateDocument(new Term("Id", compositeId), gd.asLuceneDoc());
+
+                /**
+                 * Retrieve Lucene document
+                 */
+                Document luceneGeoDoc = gd.asLuceneDoc();
+
+                /**
+                 * Add spatial indexing for bounding box of objects
+                 */
+                for (IndexableField geoField
+                        : spatialStrategy.createIndexableFields(
+                                spatialCtx.makeRectangle(shapeEnvelope[0],
+                                        shapeEnvelope[1],
+                                        shapeEnvelope[2],
+                                        shapeEnvelope[3]))) {
+                    luceneGeoDoc.add(geoField);
+                }
+
+                /**
+                 * Write the document to the index
+                 */
+                w.updateDocument(new Term("Id", compositeId), luceneGeoDoc);
             }
 
         } catch (Exception e) {
-            Logger.Log("An error occurred while writing feature to Lucene index: " + e.toString());
+            Indexer.errorCount++;
+            Logger.Log("An error occurred while writing feature to Lucene index: " + e.toString() + " (#" + Indexer.errorCount + ")");
         }
+
     }
 
     /**