From 9854cd022e7a606d48b726e30d973248ab8b6bfa Mon Sep 17 00:00:00 2001 From: Stein Runar Bergheim Date: Tue, 19 Apr 2016 09:29:59 +0200 Subject: [PATCH 1/2] Updates to search, indexing, config --- .gitignore | 3 +- pom.xml | 3 +- .../ftgeosearch/BackgroundJobManager.java | 1 - .../ftgeosearch/CheckIndexQueueJob.java | 63 +++++--- .../java/eu/sdi4apps/ftgeosearch/Console.java | 87 ----------- .../java/eu/sdi4apps/ftgeosearch/GdalExt.java | 5 +- .../java/eu/sdi4apps/ftgeosearch/GeoDoc.java | 19 ++- .../java/eu/sdi4apps/ftgeosearch/Indexer.java | 147 ++++++++++-------- .../eu/sdi4apps/ftgeosearch/IndexerQueue.java | 13 +- .../eu/sdi4apps/ftgeosearch/QueueItem.java | 78 ++++++---- .../eu/sdi4apps/ftgeosearch/SearchResult.java | 5 + .../eu/sdi4apps/ftgeosearch/Searcher.java | 137 +++++++++++++--- .../eu/sdi4apps/openapi/config/DbParams.java | 48 ++++-- .../eu/sdi4apps/openapi/config/Settings.java | 120 +++++++++++++- .../openapi/servlets/DescribeAPI.java | 97 ++++++++++++ .../eu/sdi4apps/openapi/servlets/Index.java | 53 ++++--- .../eu/sdi4apps/openapi/servlets/Search.java | 53 +++++-- .../java/eu/sdi4apps/openapi/types/BBox.java | 33 ++++ .../utils}/Logger.java | 23 ++- src/main/webapp/index.html | 3 +- 20 files changed, 698 insertions(+), 293 deletions(-) delete mode 100644 src/main/java/eu/sdi4apps/ftgeosearch/Console.java create mode 100644 src/main/java/eu/sdi4apps/openapi/servlets/DescribeAPI.java rename src/main/java/eu/sdi4apps/{ftgeosearch => openapi/utils}/Logger.java (54%) diff --git a/.gitignore b/.gitignore index 2216311..bffe608 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target/ nbactions.xml -nb-configuration.xml \ No newline at end of file +nb-configuration.xml +*-*.properties \ No newline at end of file diff --git a/pom.xml b/pom.xml index 1b8c850..b71dee5 100644 --- a/pom.xml +++ b/pom.xml @@ -23,12 +23,13 @@ org.postgresql postgresql - 9.4-1206-jdbc42 + 9.4.1208.jre7 org.gdal gdal 1.11.4 + provided joda-time diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/BackgroundJobManager.java b/src/main/java/eu/sdi4apps/ftgeosearch/BackgroundJobManager.java index 60e8ad6..fff2d20 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/BackgroundJobManager.java +++ b/src/main/java/eu/sdi4apps/ftgeosearch/BackgroundJobManager.java @@ -23,7 +23,6 @@ public class BackgroundJobManager implements ServletContextListener { @Override public void contextInitialized(ServletContextEvent event) { - scheduler = Executors.newSingleThreadScheduledExecutor(); scheduler.scheduleAtFixedRate(new CheckIndexQueueJob(), 0, 10, TimeUnit.SECONDS); System.out.println("Started background job manager"); diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/CheckIndexQueueJob.java b/src/main/java/eu/sdi4apps/ftgeosearch/CheckIndexQueueJob.java index 050973a..6d8e607 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/CheckIndexQueueJob.java +++ b/src/main/java/eu/sdi4apps/ftgeosearch/CheckIndexQueueJob.java @@ -5,16 +5,11 @@ */ package eu.sdi4apps.ftgeosearch; -import eu.sdi4apps.ftgeosearch.drivers.OGRDriver; +import eu.sdi4apps.openapi.utils.Logger; import eu.sdi4apps.ftgeosearch.drivers.ShapefileDriver; -import java.util.LinkedHashMap; -import java.util.List; +import java.io.File; import org.apache.lucene.index.IndexWriter; import org.gdal.ogr.DataSource; -import org.gdal.ogr.Feature; -import org.gdal.ogr.FeatureDefn; -import org.gdal.ogr.FieldDefn; -import org.gdal.ogr.Geometry; import org.gdal.ogr.Layer; import org.gdal.ogr.ogr; import org.joda.time.DateTime; @@ -30,9 +25,18 @@ public class CheckIndexQueueJob implements Runnable { private DateTime StartTime = DateTime.now(); + private Boolean IsWorking = false; + @Override public void run() { + if (IsWorking == true) { + Logger.Log("Previous indexing job still in progress"); + return; + } + + IsWorking = true; + try { DateTime currentTime = DateTime.now(); @@ -44,36 +48,53 @@ public void run() { for (QueueItem qi : IndexerQueue.top(1)) { - qi.updateIndexingStatus(IndexingStatus.Indexing); - + Logger.Log("Processing entry: " + qi.layer + " added " + qi.enqueued); + IndexWriter w = Indexer.getWriter(); switch (qi.datasettype) { case Shapefile: ShapefileDriver drv = (ShapefileDriver) qi.ogrdriver; - DataSource ds = ogr.Open(((ShapefileDriver) drv).filenameOfShpFile); - if (ds != null) { - Layer lyr = ds.GetLayer(0); - if (lyr == null) { - Logger.Log("No layer derived from source"); - return; - } - Indexer.indexLayer(lyr, qi, w); + if (drv == null) { + throw new Exception("Could not deserialize queue item into ogr driver: " + qi.ogrdriver); + } + File f = new File(drv.filenameOfShpFile); + + if (!f.exists()) { + throw new Exception("Enqueued source file '" + drv.filenameOfShpFile + "' does not exist"); + } + + if (!f.canRead()) { + throw new Exception("No file read access to '" + drv.filenameOfShpFile + "'"); + } + + DataSource ds = ogr.Open(drv.filenameOfShpFile, 0); + if (ds == null) { + throw new Exception("Could not open data source'" + drv.filenameOfShpFile + "' using gdal/ogr"); + } + + Layer lyr = ds.GetLayer(0); + if (lyr == null) { + throw new Exception("Could not get layer from source file '" + drv.filenameOfShpFile + "' using gdal/ogr"); } + + Indexer.indexLayer(lyr, qi, w); + break; default: + Logger.Log("Something went wrong"); drv = null; break; } - Indexer.closeWriter(); - qi.updateIndexingStatus(IndexingStatus.Indexed); - Logger.Log("Processing entry: " + qi.layer + " added " + qi.enqueued); } } catch (Exception e) { - Logger.Log(e.toString()); + Logger.Log("An indexing error occurred: " + e.toString()); + } finally { + Indexer.closeWriter(); + IsWorking = false; } } diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/Console.java b/src/main/java/eu/sdi4apps/ftgeosearch/Console.java deleted file mode 100644 index c42ef1d..0000000 --- a/src/main/java/eu/sdi4apps/ftgeosearch/Console.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package eu.sdi4apps.ftgeosearch; - -import eu.sdi4apps.ftgeosearch.drivers.ShapefileDriver; -import java.io.IOException; -import static java.util.Arrays.asList; -import java.util.List; -import org.apache.lucene.index.IndexWriter; -import org.gdal.ogr.DataSource; -import org.gdal.ogr.Layer; -import org.gdal.ogr.ogr; - -/** - * - * @author runarbe - */ -public class Console { - - /** - * @param args the command line arguments - */ - public static void main(String[] args) throws IOException { - - try { - - ogr.RegisterAll(); - - IndexerQueue.drop(); - IndexerQueue.create(); -// Indexer.dropIndex(); - - ShapefileDriver driver = new ShapefileDriver("C:/Users/runarbe/Documents/My Map Data/Natural Earth/10m_cultural/ne_10m_populated_places_simple.shp"); - List titleFields = asList("name"); - String titleFormat = "Karel Title: %s"; - List descriptionFields = asList("featurecla", "adm0name", "sov0name"); - String descriptionFormat = "Premek: %s, Tomas: %s, (Runar: %s)"; - List additionalFields = asList("featurecla", "sov0name", "adm0name"); - List jsonDataFields = asList("featurecla", "adm0name"); - - QueueItem entry = QueueItem.create("nepopulatedplaces", DatasetType.Shapefile, driver , titleFields, titleFormat, descriptionFields, descriptionFormat, null, null); - entry.jsonDataFields = jsonDataFields; - - IndexerQueue.enqueue(entry); - - ShapefileDriver drv2 = new ShapefileDriver("C:/Users/runarbe/Documents/My Map Data/Natural Earth/10m_cultural/ne_10m_admin_1_states_provinces_lakes_shp.shp"); - QueueItem e2 = QueueItem.create("states_provinces", DatasetType.Shapefile, drv2, "name", "admin"); - - IndexerQueue.enqueue(e2); - - ShapefileDriver drv; - - for (QueueItem qi : IndexerQueue.top(2)) { - - qi.updateIndexingStatus(IndexingStatus.Indexing); - - switch (qi.datasettype) { - case Shapefile: - drv = (ShapefileDriver) qi.ogrdriver; - break; - default: - drv = null; - break; - } - - DataSource ds = ogr.Open(drv.filenameOfShpFile); - - Layer lyr = ds.GetLayer(0); - - IndexWriter w = Indexer.getWriter(); - - Indexer.indexLayer(lyr, qi, w); - - Indexer.closeWriter(); - - qi.updateIndexingStatus(IndexingStatus.Indexed); - } - } catch (Exception e) { - Logger.Log(e.toString()); - } - - } - -} diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/GdalExt.java b/src/main/java/eu/sdi4apps/ftgeosearch/GdalExt.java index 3921a93..09c7cc2 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/GdalExt.java +++ b/src/main/java/eu/sdi4apps/ftgeosearch/GdalExt.java @@ -9,6 +9,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.commons.lang.StringUtils; import org.gdal.ogr.Feature; import org.gdal.ogr.FeatureDefn; import org.gdal.ogr.FieldDefn; @@ -69,7 +70,7 @@ public static String[] getFormattedAndIndexValuesAsString(Feature feat, LinkedHa i++; } s[0] = String.format(stringFormat, values); - s[1] = String.join(" ", indexValues); + s[1] = StringUtils.join(indexValues, " "); return s; } @@ -90,7 +91,7 @@ public static String getFieldValuesAsString(Feature feat, LinkedHashMap titleFieldMap = GdalExt.getFieldMap(lyr, qi.titlefields); LinkedHashMap descriptionFieldMap = GdalExt.getFieldMap(lyr, qi.descriptionfields); LinkedHashMap indexAdditionalFieldMap = GdalExt.getFieldMap(lyr, qi.additionalfields); - LinkedHashMap jsonDataFieldMap = GdalExt.getFieldMap(lyr, qi.jsonDataFields); - - Logger.Log(jsonDataFieldMap.toString()); + LinkedHashMap jsonDataFieldMap = GdalExt.getFieldMap(lyr, qi.jsondatafields); while (null != (f = lyr.GetNextFeature())) { - Geometry g = f.GetGeometryRef(); - - String[] titleData = GdalExt.getFormattedAndIndexValuesAsString(f, titleFieldMap, qi.titleformat); - String[] descData = GdalExt.getFormattedAndIndexValuesAsString(f, descriptionFieldMap, qi.descriptionformat); - String additionalData = GdalExt.getFieldValuesAsString(f, indexAdditionalFieldMap); - Object jsonData = GdalExt.getAsMapObject(f, jsonDataFieldMap); - - String compositeId = qi.layer + "-" + f.GetFID(); - - GeoDoc gd = GeoDoc.create( - compositeId, - qi.layer, - g.ExportToWkt(), - g.PointOnSurface().ExportToWkt(), - titleData[0], - descData[0], - titleData[1], - descData[1], - additionalData, - jsonData); - - if (gd != null) { - w.updateDocument(new Term("Id", compositeId), gd.asLuceneDoc()); - } + indexFeature(w, qi, f, titleFieldMap, descriptionFieldMap, indexAdditionalFieldMap, jsonDataFieldMap); if (counter % batch == 0 || counter == totalFeatures) { Logger.Log("Processed: " + counter + " items..."); } counter++; } + + qi.updateIndexingStatus(IndexingStatus.Indexed); + Logger.Log("Done"); + } catch (Exception ex) { - Logger.Log("An error occurred while indexing Layer", ex.toString()); + try { + Logger.Log("An error occurred while indexing Layer", ex.toString()); + qi.updateIndexingStatus(IndexingStatus.Error); + } catch (SQLException ex1) { + Logger.Log("Failed to set status of queue item to 'Error': " + ex.toString()); + } + } + } + + public static void indexFeature(IndexWriter w, + QueueItem qi, + Feature f, + LinkedHashMap titleFieldMap, + LinkedHashMap descriptionFieldMap, + LinkedHashMap indexAdditionalFieldMap, + LinkedHashMap jsonDataFieldMap) { + + try { + //Read the geometry + Geometry g = f.GetGeometryRef(); + + String[] titleData = GdalExt.getFormattedAndIndexValuesAsString(f, titleFieldMap, qi.titleformat); + String[] descData = GdalExt.getFormattedAndIndexValuesAsString(f, descriptionFieldMap, qi.descriptionformat); + String additionalData = GdalExt.getFieldValuesAsString(f, indexAdditionalFieldMap); + Object jsonData = GdalExt.getAsMapObject(f, jsonDataFieldMap); + + String compositeId = qi.layer + "-" + f.GetFID(); + + GeoDoc gd = GeoDoc.create( + compositeId, + qi.layer, + qi.objtype, + g.ExportToWkt(), + g.PointOnSurface().ExportToWkt(), + titleData[0], + descData[0], + titleData[1], + descData[1], + additionalData, + jsonData); + + if (gd != null) { + w.updateDocument(new Term("Id", compositeId), gd.asLuceneDoc()); + } + + } catch (Exception e) { + Logger.Log("An error occurred while writing feature to Lucene index: " + e.toString()); } } @@ -145,39 +194,7 @@ public static void closeWriter() { analyzer = null; } } catch (Exception e) { - System.out.println("Error during closing of writer: " + e.toString()); - } - } - - /** - * create a test index - * - * @param dropExistingQueueDatabase - * @throws IOException - */ - public static void create(boolean dropExistingQueueDatabase) throws IOException, SQLException { - - IndexerQueue.init(); - - if (dropExistingQueueDatabase) { - IndexerQueue.drop(); - IndexerQueue.create(); + Logger.Log("Error during closing of writer: " + e.toString()); } - - ShapefileDriver driver = new ShapefileDriver("C:/Users/runarbe/Documents/My Map Data/Natural Earth/10m_cultural/ne_10m_populated_places_simple.shp"); - List titleFields = asList("name"); - String titleFormat = "Title: %s"; - List descriptionFields = asList("featurecla"); - String descriptionFormat = "Description: %s..."; - List additionalFields = asList("featurecla", "sov0name", "adm0name"); - QueueItem entry = QueueItem.create("Natural Earth Simple", DatasetType.Shapefile, driver, titleFields, titleFormat, descriptionFields, descriptionFormat, null, null); - IndexerQueue.enqueue(entry); - - IndexWriter w = Indexer.getWriter(); - w.deleteAll(); - - w.commit(); - Indexer.closeWriter(); } - } diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/IndexerQueue.java b/src/main/java/eu/sdi4apps/ftgeosearch/IndexerQueue.java index 41e9ace..453559a 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/IndexerQueue.java +++ b/src/main/java/eu/sdi4apps/ftgeosearch/IndexerQueue.java @@ -5,6 +5,7 @@ */ package eu.sdi4apps.ftgeosearch; +import eu.sdi4apps.openapi.utils.Logger; import com.cedarsoftware.util.io.JsonReader; import eu.sdi4apps.ftgeosearch.drivers.ShapefileDriver; import eu.sdi4apps.openapi.config.Settings; @@ -15,10 +16,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import static java.sql.Types.NULL; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import org.joda.time.DateTime; @@ -34,7 +33,7 @@ public class IndexerQueue { private static Connection Conn; /** - * create Derby database + * create queue table if it doesn't already exist */ public static void create() { @@ -43,12 +42,14 @@ public static void create() { String sqlString = "CREATE TABLE IF NOT EXISTS queue (" + " id SERIAL NOT NULL PRIMARY KEY," + " layer VARCHAR(200) NOT NULL," + + " objtype VARCHAR(200) NOT NULL," + " titlefields VARCHAR(50)[] NOT NULL," + " titleformat VARCHAR(200) NOT NULL," + " descriptionfields VARCHAR(50)[] NOT NULL," + " descriptionformat VARCHAR(200) NOT NULL," + " additionalfields VARCHAR(50)[] NULL," + " jsondatafields VARCHAR(50)[] NULL," + + " srs INTEGER NOT NULL DEFAULT 4326," + " enqueued TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP," + " indexed TIMESTAMP," + " refresh INTEGER NOT NULL DEFAULT 1," @@ -133,7 +134,7 @@ public static void init() { try { if (IndexerQueue.Conn == null) { IndexerQueue.Conn = DriverManager.getConnection(Settings.INDEXDB.getJdbcUrl()); - System.out.println("Initialized queue database connection"); + Logger.Log("Initialized queue database connection"); } } catch (SQLException ex) { Logger.Log(ex.toString()); @@ -194,6 +195,8 @@ public static List top(int numberOfEntriesToReturn) { QueueItem qi = new QueueItem(); qi.id = r.getInt("id"); qi.layer = r.getString("layer"); + qi.srs = r.getInt("srs"); + qi.objtype = r.getString("objtype"); qi.datasettype = DatasetType.valueOf(r.getString("datasettype")); qi.status = IndexingStatus.valueOf(r.getString("indexingstatus")); qi.enqueued = new DateTime(r.getTimestamp("enqueued")); @@ -212,7 +215,7 @@ public static List top(int numberOfEntriesToReturn) { qi.additionalfields = Arrays.asList((String[]) r.getArray("additionalfields").getArray()); } if (r.getArray("jsondatafields") != null) { - qi.jsonDataFields = Arrays.asList((String[]) r.getArray("jsondatafields").getArray()); + qi.jsondatafields = Arrays.asList((String[]) r.getArray("jsondatafields").getArray()); } l.add(qi); } diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/QueueItem.java b/src/main/java/eu/sdi4apps/ftgeosearch/QueueItem.java index b45bf5c..4f0c6a1 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/QueueItem.java +++ b/src/main/java/eu/sdi4apps/ftgeosearch/QueueItem.java @@ -1,10 +1,6 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package eu.sdi4apps.ftgeosearch; +import eu.sdi4apps.openapi.utils.Logger; import eu.sdi4apps.ftgeosearch.drivers.OGRDriver; import com.cedarsoftware.util.io.JsonWriter; import java.sql.Array; @@ -14,12 +10,11 @@ import java.util.ArrayList; import static java.util.Arrays.asList; import java.util.List; +import org.apache.commons.lang.math.NumberUtils; import org.joda.time.DateTime; /** * Indexer queue item - * - * @author runarbe */ public class QueueItem { @@ -73,12 +68,23 @@ public class QueueItem { /** * Fields to be included as a JSON object */ - public List jsonDataFields = null; + public List jsondatafields = null; + /** * A list of fields to be excluded from indexing */ public List excludefields; + /** + * SRS id of the layer to be indexed + */ + public int srs = 4326; + + /** + * Object type contained within the layer to be indexed + */ + public String objtype = "general"; + /** * An interval at which the indexed data should be re-indexed specified in * minutes. @@ -115,6 +121,7 @@ public class QueueItem { * formats * * @param datasetname + * @param objtype * @param datasettype * @param ogrdriver * @param titlefields @@ -127,6 +134,7 @@ public class QueueItem { */ public static QueueItem create( String datasetname, + String objtype, DatasetType datasettype, OGRDriver ogrdriver, List titlefields, @@ -134,34 +142,39 @@ public static QueueItem create( List descriptionfields, String descriptionformat, List additionalfields, - List jsondatafields + List jsondatafields, + Integer srs ) { QueueItem item = new QueueItem(); item.layer = datasetname; item.datasettype = datasettype; + item.objtype = objtype; item.ogrdriver = ogrdriver; item.titlefields = titlefields; item.titleformat = titleformat; item.descriptionfields = descriptionfields; item.descriptionformat = descriptionformat; item.additionalfields = additionalfields; - item.jsonDataFields = jsondatafields; + item.jsondatafields = jsondatafields; + item.srs = srs; return item; } /** - * Index a layer with simple - * + * Index a layer with simple + * * @param datasetname + * @param objtype * @param datasettype * @param ogrdriver * @param titlefields * @param titleformat * @param descriptionfields * @param descriptionformat - * @return + * @return */ public static QueueItem create(String datasetname, + String objtype, DatasetType datasettype, OGRDriver ogrdriver, List titlefields, @@ -170,6 +183,7 @@ public static QueueItem create(String datasetname, String descriptionformat) { return create( datasetname, + objtype, datasettype, ogrdriver, titlefields, @@ -177,27 +191,31 @@ public static QueueItem create(String datasetname, descriptionfields, descriptionformat, null, - null); + null, + 4326); } /** * Index a layer with single field mapping to title and description - * + * * @param datasetname + * @param objtype * @param datasettype * @param ogrdriver * @param titleField * @param descriptionField - * @return + * @return */ public static QueueItem create( String datasetname, + String objtype, DatasetType datasettype, OGRDriver ogrdriver, String titleField, String descriptionField) { return create(datasetname, + objtype, datasettype, ogrdriver, asList(titleField), @@ -205,12 +223,13 @@ public static QueueItem create( asList(descriptionField), "%s", null, - null); + null, + 4326); } public void insert() throws SQLException { - PreparedStatement s = IndexerQueue.prepare("INSERT INTO queue (layer, datasettype, ogrdriver, titlefields, titleformat, descriptionfields, descriptionformat, additionalfields, jsondatafields)" - + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"); + PreparedStatement s = IndexerQueue.prepare("INSERT INTO queue (layer, datasettype, ogrdriver, titlefields, titleformat, descriptionfields, descriptionformat, additionalfields, jsondatafields, srs, objtype)" + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); s.setString(1, this.layer); s.setString(2, this.datasettype.toString()); @@ -219,22 +238,24 @@ public void insert() throws SQLException { s.setString(5, this.titleformat); s.setArray(6, IndexerQueue.createArrayOf(this.descriptionfields)); s.setString(7, this.descriptionformat); - if (this.additionalfields != null) { s.setArray(8, IndexerQueue.createArrayOf(this.additionalfields)); } else { s.setNull(8, NULL); } - if (this.jsonDataFields != null) { - s.setArray(9, IndexerQueue.createArrayOf(this.jsonDataFields)); + if (this.jsondatafields != null) { + s.setArray(9, IndexerQueue.createArrayOf(this.jsondatafields)); } else { s.setNull(9, NULL); } - s.execute(); + s.setInt(10, this.srs); + s.setString(11, this.objtype); - Logger.Log("Added layer: " + this.layer + " to indxing queue"); + s.execute(); + Logger.Log("Added layer '" + this.layer + "' to indxing queue"); + } public void updateIndexingStatus(IndexingStatus newIndexingStatus) throws SQLException { @@ -246,11 +267,10 @@ public void updateIndexingStatus(IndexingStatus newIndexingStatus) throws SQLExc } } - public void delete() { - String sql = String.format( - "DELETE FROM queue WHERE id = %s", - this.id); - IndexerQueue.executeUpdate(sql); + public void delete() throws SQLException { + PreparedStatement ps = IndexerQueue.prepare("DELETE FROM queue WHERE id = ?"); + ps.setInt(1, this.id); + ps.execute(); } } diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/SearchResult.java b/src/main/java/eu/sdi4apps/ftgeosearch/SearchResult.java index a0f265c..d801806 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/SearchResult.java +++ b/src/main/java/eu/sdi4apps/ftgeosearch/SearchResult.java @@ -32,6 +32,11 @@ public class SearchResult { * JSON options */ public static Map JsonOptions; + + /** + * Error message + */ + public String ErrorMessage = null; /** * Search result status, one of the constants diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/Searcher.java b/src/main/java/eu/sdi4apps/ftgeosearch/Searcher.java index b65ee04..bccdfc6 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/Searcher.java +++ b/src/main/java/eu/sdi4apps/ftgeosearch/Searcher.java @@ -6,14 +6,23 @@ package eu.sdi4apps.ftgeosearch; import eu.sdi4apps.openapi.config.Settings; +import eu.sdi4apps.openapi.types.BBox; +import eu.sdi4apps.openapi.utils.Logger; import java.io.IOException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import org.apache.commons.lang.StringUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.queries.TermFilter; import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.Filter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; @@ -26,37 +35,117 @@ */ public class Searcher { - public static SearchResult Search(String q) throws IOException, ParseException { + public static Analyzer analyzer; - SearchResult sr = new SearchResult(true); + public static Directory directory; - Analyzer analyzer = new StandardAnalyzer(); + public static DirectoryReader ireader = null; - Directory directory = FSDirectory.open(Settings.IndexDirectory); + public static IndexSearcher isearcher = null; - DirectoryReader ireader = DirectoryReader.open(directory); - IndexSearcher isearcher = new IndexSearcher(ireader); + static { + createSearcher(); + } + + public static IndexSearcher createSearcher() { + try { + analyzer = new StandardAnalyzer(); + directory = FSDirectory.open(Paths.get(Settings.INDEXDIR)); + ireader = DirectoryReader.open(directory); + isearcher = new IndexSearcher(ireader); + return isearcher; + } catch (Exception e) { + return null; + } + } + + public static void destroySearcher() { + try { + if (isearcher != null) { + isearcher = null; + } + + if (ireader != null) { + ireader.close(); + ireader = null; + } + + if (directory != null) { + directory.close(); + directory = null; + } + } catch (Exception e) { + Logger.Log("Failed to destroy searcher"); + } + } + + public static List Search(String q, + Integer maxresults, + String filter, + String extent) throws IOException, ParseException { + + List sr = new ArrayList<>(); + + try { + + /** + * Set default number of results to 100 + */ + if (maxresults == null) { + maxresults = 100; + } + + /** + * Split filter layers into an array + */ + String[] filterLayers; + if (filter != null) { + filterLayers = StringUtils.split(filter, ","); + } else { + filterLayers = null; + } + + /** + * Convert string extent to BBox object + */ + if (extent != null) { + BBox bbox = BBox.createFromString(extent); + if (bbox != null) { + Logger.Log(bbox.jsArray()); + } + } + + if (isearcher == null) { + createSearcher(); + } + + QueryParser parser = new MultiFieldQueryParser(new String[]{"IndexTitle", "IndexDescription", "IndexAdditional"}, analyzer); + Query query = parser.parse(q); + + ScoreDoc[] hits = isearcher.search(query, null, maxresults).scoreDocs; - QueryParser parser = new MultiFieldQueryParser(new String[]{"IndexTitle", "IndexDescription", "IndexAdditional"}, analyzer); - Query query = parser.parse(q); - ScoreDoc[] hits = isearcher.search(query, null, 1000).scoreDocs; + // Iterate through the results: + for (int i = 0; i < hits.length; i++) { + GeoDoc g = new GeoDoc(); + Document hitDoc = isearcher.doc(hits[i].doc); + g.Score = hits[i].score; + g.Id = hitDoc.get("Id"); + g.Layer = hitDoc.get("Layer"); + g.ObjType = hitDoc.get("ObjType"); + g.FullGeom = hitDoc.get("FullGeom"); + g.DisplayTitle = hitDoc.get("DisplayTitle"); + g.DisplayDescription = hitDoc.get("DisplayDescription"); + g.PointGeom = hitDoc.get("PointGeom"); + g.JsonData = Serializer.Deserialize(hitDoc.get("JsonData")); + sr.add(g); + } - // Iterate through the results: - for (int i = 0; i < hits.length; i++) { - GeoDoc g = new GeoDoc(); - Document hitDoc = isearcher.doc(hits[i].doc); - g.Score = hits[i].score; - g.Id = hitDoc.get("Id"); - g.Layer = hitDoc.get("Layer"); - g.FullGeom = hitDoc.get("FullGeom"); - g.DisplayTitle = hitDoc.get("DisplayTitle"); - g.DisplayDescription = hitDoc.get("DisplayDescription"); - g.PointGeom = hitDoc.get("PointGeom"); - g.JsonData = Serializer.Deserialize(hitDoc.get("JsonData")); - sr.addHit(g); + } catch (Exception e) { + Logger.Log("An exception occurred during search: " + e.toString()); + } finally { + destroySearcher(); + return sr; } - ireader.close(); - return sr; } diff --git a/src/main/java/eu/sdi4apps/openapi/config/DbParams.java b/src/main/java/eu/sdi4apps/openapi/config/DbParams.java index da84b4c..c236b11 100644 --- a/src/main/java/eu/sdi4apps/openapi/config/DbParams.java +++ b/src/main/java/eu/sdi4apps/openapi/config/DbParams.java @@ -17,24 +17,50 @@ public class DbParams { public int dbPort = 5432; public String dbHost = "localhost"; - public DbParams(String dbName, String dbUser, String dbPassword, int dbPort, String dbHost) { + public DbParams(String dbName, String dbUser, String dbPassword, String dbHost, Integer dbPort) throws Exception { - this.dbName = dbName; - this.dbUser = dbUser; - this.dbPassword = dbPassword; - this.dbPort = dbPort; - this.dbHost = dbHost; + try { + if (dbName == null) { + throw new Exception("Parameter dbName is null"); + } else { + this.dbName = dbName; + } + + if (dbUser == null) { + throw new Exception("Parameter dbUser is null"); + } else { + this.dbUser = dbUser; + } + + if (dbPassword == null) { + throw new Exception("Parameter dbPassword is null"); + } else { + this.dbPassword = dbPassword; + } + + this.dbPort = dbPort; + + this.dbHost = dbHost; + + } catch (Exception exception) { + throw new Exception("Failed to create database parameters: " + exception.toString()); + } } - public DbParams(String dbName, String dbUser, String dbPassword) { - this(dbName, dbUser, dbPassword, 5432, "localhost"); + public DbParams(String dbName, String dbUser, String dbPassword, String dbHost) throws Exception { + this(dbName, dbUser, dbPassword, dbHost, 5432); } - + + public DbParams(String dbName, String dbUser, String dbPassword) throws Exception { + this(dbName, dbUser, dbPassword, "localhost", 5432); + } + /** * Return the JDBC URL of the respective setting - * @return + * + * @return */ public String getJdbcUrl() { - return "jdbc:postgresql://" + dbHost + ":" + dbPort + "/" + dbName + "?user=" + dbUser + "&password=" + dbPassword; + return "jdbc:postgresql://" + dbHost + ":" + dbPort + "/" + dbName + "?user=" + dbUser + "&password=" + dbPassword; } } diff --git a/src/main/java/eu/sdi4apps/openapi/config/Settings.java b/src/main/java/eu/sdi4apps/openapi/config/Settings.java index 29b796f..d48af73 100644 --- a/src/main/java/eu/sdi4apps/openapi/config/Settings.java +++ b/src/main/java/eu/sdi4apps/openapi/config/Settings.java @@ -1,7 +1,13 @@ package eu.sdi4apps.openapi.config; +import eu.sdi4apps.ftgeosearch.IndexerQueue; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Properties; +import org.apache.commons.lang.math.NumberUtils; /** * Configuration parameters @@ -9,25 +15,125 @@ * @author runarbe */ public class Settings { - + + /** + * Instantiate static variables + */ + static { + + InputStream inputStream = null; + + try { + + Properties prop = new Properties(); + String propFileName = "openapi.properties"; + + inputStream = Settings.class.getClassLoader().getResourceAsStream(propFileName); + + if (inputStream != null) { + prop.load(inputStream); + } else { + throw new Exception("Configuration file: '" + propFileName + "' not found in the classpath"); + } + + // Load routingdb properties + String routingdb = prop.getProperty("routingdb"); + String routingdb_usr = prop.getProperty("routingdb_usr"); + String routingdb_pwd = prop.getProperty("routingdb_pwd"); + String routingdb_host = prop.getProperty("routingdb_host"); + Integer routingdb_port = NumberUtils.createInteger(prop.getProperty("routingdb_port")); + ROUTINGDB = new DbParams(routingdb, routingdb_usr, routingdb_pwd, routingdb_host, routingdb_port); + + // Load indexdb properties + String indexdb = prop.getProperty("indexdb"); + String indexdb_usr = prop.getProperty("indexdb_usr"); + String indexdb_pwd = prop.getProperty("indexdb_pwd"); + String indexdb_host = prop.getProperty("indexdb_host"); + Integer indexdb_port = NumberUtils.createInteger(prop.getProperty("indexdb_port")); + INDEXDB = new DbParams(indexdb, indexdb_usr, indexdb_pwd, indexdb_host, indexdb_port); + + // Load customdb properties + String customdb = prop.getProperty("customdb"); + String customdb_usr = prop.getProperty("customdb_usr"); + String customdb_pwd = prop.getProperty("customdb_pwd"); + String customdb_host = prop.getProperty("customdb_host"); + Integer customdb_port = NumberUtils.createInteger(prop.getProperty("customdb_port")); + CUSTOMDB = new DbParams(customdb, customdb_usr, customdb_pwd, customdb_host, customdb_port); + + // Load index directory configuration + String indexdir = prop.getProperty("indexdir"); + INDEXDIR = indexdir; + Files.createDirectories(Paths.get(indexdir)); + + // Load log directory configuration + String logdir = prop.getProperty("logdir"); + LOGDIR = logdir; + Files.createDirectories(Paths.get(logdir)); + + // Load shape directory configuration + String shapedir = prop.getProperty("shapedir"); + SHAPEDIR = shapedir; + Files.createDirectories(Paths.get(shapedir)); + + // Load GDAL/OGR + Class.forName("org.gdal.Loader"); + + // Initialize queue database + IndexerQueue.create(); + + // Set ready flag + isReady = true; + + } catch (Exception e) { + // If exception occurs, print error message to std out and set isReady flag to false + System.out.println("Configuration not loaded: " + e); + isReady = false; + } finally { + try { + if (inputStream != null) { + inputStream.close(); + } + } catch (IOException ex) { + // If exception occurs during close of inputStream, print error message to std out and set isReady flag to false + System.out.println("Unable to close configuration file: " + ex.toString()); + isReady = false; + } + } + } + + /** + * A flag to indicate whether the configuration file has been read properly + */ + public static Boolean isReady; + /** * Connection parameters for routing database */ - public static final DbParams ROUTINGDB = new DbParams("routing", "postgres", "postgres"); - + public static DbParams ROUTINGDB; //platform.sdi4apps.eu + /** * Connection parameters for custom data db */ - public static final DbParams CUSTOMDB = new DbParams("customdb", "postgres", "postgres"); - + public static DbParams CUSTOMDB; //platform.sdi4apps.eu + /** * Connection parameters for indexer db */ - public static final DbParams INDEXDB = new DbParams("indexdb", "postgres", "postgres"); + public static DbParams INDEXDB; //platform.sdi4apps.eu /** * The directory path to where the Lucene index should be stored */ - public static final Path IndexDirectory = Paths.get("C:/users/runarbe/documents/Lucene"); + public static String INDEXDIR; + + /** + * The directory where tomcat should write logs + */ + public static String LOGDIR; + + /** + * The directory where shape files to be indexed are located + */ + public static String SHAPEDIR; } diff --git a/src/main/java/eu/sdi4apps/openapi/servlets/DescribeAPI.java b/src/main/java/eu/sdi4apps/openapi/servlets/DescribeAPI.java new file mode 100644 index 0000000..30ab0b4 --- /dev/null +++ b/src/main/java/eu/sdi4apps/openapi/servlets/DescribeAPI.java @@ -0,0 +1,97 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package eu.sdi4apps.openapi.servlets; + +import eu.sdi4apps.openapi.config.Settings; +import eu.sdi4apps.openapi.types.DataResponse; +import eu.sdi4apps.openapi.types.Response; +import eu.sdi4apps.openapi.utils.Cors; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.util.Date; +import java.util.Properties; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * + * @author runarbe + */ +@WebServlet(name = "DescribeAPI", urlPatterns = {"/DescribeAPI"}) +public class DescribeAPI extends HttpServlet { + + /** + * Processes requests for both HTTP GET and POST + * methods. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + protected void processRequest(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + response = Cors.EnableCors(response); + + DataResponse ro = new DataResponse(); + ro.parameters = request.getQueryString(); + + try (PrintWriter out = response.getWriter()) { + if (Settings.isReady == true) { + ro.setSuccess(); + } else { + ro.setError("System not configured"); + } + out.println(ro.asJson()); + } + } + + // + /** + * Handles the HTTP GET method. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + processRequest(request, response); + } + + /** + * Handles the HTTP POST method. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + processRequest(request, response); + } + + /** + * Returns a short description of the servlet. + * + * @return a String containing servlet description + */ + @Override + public String getServletInfo() { + return "Short description"; + }// + +} diff --git a/src/main/java/eu/sdi4apps/openapi/servlets/Index.java b/src/main/java/eu/sdi4apps/openapi/servlets/Index.java index 5ebf465..9d96507 100644 --- a/src/main/java/eu/sdi4apps/openapi/servlets/Index.java +++ b/src/main/java/eu/sdi4apps/openapi/servlets/Index.java @@ -8,16 +8,16 @@ import eu.sdi4apps.ftgeosearch.DatasetType; import eu.sdi4apps.ftgeosearch.Indexer; import eu.sdi4apps.ftgeosearch.IndexerQueue; -import eu.sdi4apps.ftgeosearch.Logger; +import eu.sdi4apps.openapi.utils.Logger; import eu.sdi4apps.ftgeosearch.QueueItem; import eu.sdi4apps.ftgeosearch.drivers.ShapefileDriver; +import eu.sdi4apps.openapi.config.Settings; import eu.sdi4apps.openapi.types.DataResponse; -import eu.sdi4apps.openapi.types.Response; import eu.sdi4apps.openapi.utils.Cors; import eu.sdi4apps.openapi.utils.HttpParam; import java.io.IOException; import java.io.PrintWriter; -import java.sql.SQLException; +import java.nio.file.Paths; import static java.util.Arrays.asList; import java.util.HashMap; import java.util.List; @@ -51,32 +51,41 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re String action = HttpParam.GetString(request, "action", ""); switch (action) { - case "CreateIndex": - Indexer.create(false); - r.addMessage(action); - break; case "ViewQueue": int maxEntries = HttpParam.GetInt(request, "maxEntries", 10); r.setData(IndexerQueue.top(maxEntries), true); break; - case "EnqueueItem": - ShapefileDriver driver = new ShapefileDriver("C:/Users/runarbe/Documents/My Map Data/Natural Earth/10m_cultural/ne_10m_populated_places_simple.shp"); - List titleFields = asList("name"); - String titleFormat = "Title: %s"; - List descriptionFields = asList("featurecla", "adm0name", "sov0name"); - String descriptionFormat = "Class: %s, Adm0: %s, (Sov: %s)"; - List additionalFields = asList("featurecla", "sov0name", "adm0name"); - List jsonDataFields = asList("featurecla", "adm0name"); - QueueItem entry = QueueItem.create("Natural Earth Simple", DatasetType.Shapefile, driver, titleFields, titleFormat, descriptionFields, descriptionFormat, null, null); - entry.additionalfields = additionalFields; - entry.jsonDataFields = jsonDataFields; - IndexerQueue.enqueue(entry); - r.setData(entry, true); + case "UnlockIndex": + Indexer.unlockIndex(); break; - case "EnqueueShapefile": + case "DropIndex": + Indexer.dropIndex(); + break; + case "EnqueueItemTest": + IndexerQueue.create(); + ShapefileDriver driver2 = new ShapefileDriver(Settings.SHAPEDIR + "/ne_10m_populated_places_simple.shp"); + List titleFields2 = asList("name"); + String titleFormat2 = "Title: %s"; + List descriptionFields2 = asList("featurecla", "adm0name", "sov0name"); + String descriptionFormat2 = "Class: %s, Adm0: %s, (Sov: %s)"; + List additionalFields2 = asList("featurecla", "sov0name", "adm0name"); + List jsonDataFields2 = asList("featurecla", "adm0name"); + QueueItem entry2 = QueueItem.create("Natural Earth Simple", "places", DatasetType.Shapefile, driver2, titleFields2, titleFormat2, descriptionFields2, descriptionFormat2, null, null, 4326); + entry2.additionalfields = additionalFields2; + entry2.jsondatafields = jsonDataFields2; + IndexerQueue.enqueue(entry2); + ShapefileDriver drv3 = new ShapefileDriver(Settings.SHAPEDIR + "/gn1000.shp"); + List titleFields3 = asList("field_2"); + String titleFieldFormat3 = "%s"; + List descFields3 = asList("field_4", "field_8", "field_18"); + String descFieldFormat3 = "Alternative forms: %s. Type of name %s in %s"; + QueueItem entry3 = QueueItem.create("Geonames 1000", "names", DatasetType.Shapefile, drv3, titleFields3, titleFieldFormat3, descFields3, descFieldFormat3, null, null, 4326); + IndexerQueue.enqueue(entry3); + r.setData("Added two layers to index", true); + break; + case "EnqueueShapefile": Map reqFields = new HashMap(); - reqFields.put("shapefileName", true); reqFields.put("titleFields", true); reqFields.put("titleFormat", true); diff --git a/src/main/java/eu/sdi4apps/openapi/servlets/Search.java b/src/main/java/eu/sdi4apps/openapi/servlets/Search.java index 0f74aa1..5168ae0 100644 --- a/src/main/java/eu/sdi4apps/openapi/servlets/Search.java +++ b/src/main/java/eu/sdi4apps/openapi/servlets/Search.java @@ -2,8 +2,14 @@ import eu.sdi4apps.ftgeosearch.SearchResult; import eu.sdi4apps.ftgeosearch.Searcher; +import eu.sdi4apps.openapi.types.DataResponse; +import eu.sdi4apps.openapi.utils.Cors; +import eu.sdi4apps.openapi.utils.HttpParam; import java.io.IOException; import java.io.PrintWriter; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; import javax.servlet.ServletException; @@ -11,13 +17,14 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang.math.NumberUtils; import org.apache.lucene.queryparser.classic.ParseException; /** * * @author runarbe */ -@WebServlet(urlPatterns={"/search"}) +@WebServlet(urlPatterns = {"/search"}) public class Search extends HttpServlet { /** @@ -31,19 +38,41 @@ public class Search extends HttpServlet { */ protected void processRequest(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException, ParseException { - response.setContentType("application/json;charset=UTF-8"); - long startTime = System.nanoTime(); - - try (PrintWriter out = response.getWriter()) { - String q = request.getParameter("q"); - if (q != null) { - SearchResult sr = Searcher.Search(q); - sr.calculateTime(startTime); - out.println(sr.asJson()); - } else { - out.println("No query specified"); + + response = Cors.EnableCors(response); + + DataResponse r = new DataResponse(); + + r.parameters = request.getQueryString(); + + PrintWriter out = response.getWriter(); + + try { + + Map m = new LinkedHashMap<>(); + m.put("q", true); + m.put("filter", false); + m.put("maxresults", false); + m.put("extent", false); + + Map params = HttpParam.GetParameters(request, m, r); + if (r.status == "error") { + out.println(r.asJson()); + return; } + + List sres = Searcher.Search((String) params.get("q"), + NumberUtils.toInt((String) params.get("maxresults"), 100), + (String) params.get("filter"), + (String) params.get("extent")); + + r.setData(sres, true); + r.count = sres.size(); + + } catch (Exception e) { + r.addMessage("An error occurred: " + e.toString()); } + out.println(r.asJson()); } // diff --git a/src/main/java/eu/sdi4apps/openapi/types/BBox.java b/src/main/java/eu/sdi4apps/openapi/types/BBox.java index 88d4971..e813d95 100644 --- a/src/main/java/eu/sdi4apps/openapi/types/BBox.java +++ b/src/main/java/eu/sdi4apps/openapi/types/BBox.java @@ -2,7 +2,10 @@ import com.google.gson.Gson; import eu.sdi4apps.openapi.exceptions.GeneralException; +import eu.sdi4apps.openapi.utils.Logger; import java.io.IOException; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.math.NumberUtils; public class BBox { @@ -154,6 +157,36 @@ public static BBox createFromPoints(double lon1, double lat1, double lon2, doubl } + public static BBox createFromString(String extent) { + + if (extent == null) { + return null; + } + + String[] stringParts = StringUtils.split(extent, ","); + if (stringParts.length != 4) { + return null; + } + + Double[] doubleParts = new Double[4]; + + int i = 0; + for (String s : stringParts) { + doubleParts[i] = NumberUtils.toDouble(s); + i++; + } + + BBox bbox = new BBox(); + + bbox.minX = doubleParts[0]; + bbox.minY = doubleParts[1]; + bbox.maxX = doubleParts[2]; + bbox.maxY = doubleParts[3]; + + return bbox.normalize(); + + } + public String wktEnvelope() { return String.format("ST_MakeEnvelope(%s, %s, %s, %s, 4326)", this.minX, this.minY, this.maxX, this.maxY); } diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/Logger.java b/src/main/java/eu/sdi4apps/openapi/utils/Logger.java similarity index 54% rename from src/main/java/eu/sdi4apps/ftgeosearch/Logger.java rename to src/main/java/eu/sdi4apps/openapi/utils/Logger.java index 3732c5f..da3388e 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/Logger.java +++ b/src/main/java/eu/sdi4apps/openapi/utils/Logger.java @@ -3,7 +3,13 @@ * To change this template file, choose Tools | Templates * and open the template in the editor. */ -package eu.sdi4apps.ftgeosearch; +package eu.sdi4apps.openapi.utils; + +import eu.sdi4apps.openapi.config.Settings; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.io.PrintWriter; /** * Static logger class @@ -11,6 +17,17 @@ * @author runarbe */ public class Logger { + + public static void LogToFile(String msg) { + try(FileWriter fw = new FileWriter(Settings.LOGDIR + "/openapi.log", true); + BufferedWriter bw = new BufferedWriter(fw); + PrintWriter out = new PrintWriter(bw)) + { + out.println(msg); + } catch (IOException e) { + System.out.println("Logging to file failed: " + e.toString()); + } + } /** * Log a message @@ -19,6 +36,7 @@ public class Logger { */ public static void Log(String msg) { System.out.println(msg); + LogToFile(msg); } /** @@ -28,7 +46,8 @@ public static void Log(String msg) { * @param msg */ public static void Log(String title, String msg) { - Log(String.format("%s: %s", title, msg)); + String msg2 = String.format("%s: %s", title, msg); + Log(msg2); } diff --git a/src/main/webapp/index.html b/src/main/webapp/index.html index 3760b02..3279700 100644 --- a/src/main/webapp/index.html +++ b/src/main/webapp/index.html @@ -9,7 +9,8 @@

Feature search: Open API test suite

  • Search for 'os*'
  • Erase and restart the index
    • -
  • Enqueue test file for indexing
    • +
  • Unlock index
    • +
  • Enqueue test file for indexing
  • Enqueue dynamic Shapefile for indexing
  • View 5 last entries in queue
  • Find shortest path between a and b
    • From d55ccb8b5f064f2c68d7c2283cce2808226708d8 Mon Sep 17 00:00:00 2001 From: Stein Runar Bergheim Date: Tue, 19 Apr 2016 12:51:17 +0200 Subject: [PATCH 2/2] Updated geoindexing, settings, geodoc and readme --- README.md | 14 +--- .../java/eu/sdi4apps/ftgeosearch/GeoDoc.java | 32 +++++---- .../java/eu/sdi4apps/ftgeosearch/Indexer.java | 71 ++++++++++++++++--- .../eu/sdi4apps/ftgeosearch/Searcher.java | 71 +++++++++++++++---- .../eu/sdi4apps/openapi/config/Settings.java | 20 ++++++ .../eu/sdi4apps/openapi/servlets/Index.java | 8 +-- .../eu/sdi4apps/openapi/servlets/Search.java | 15 ++-- .../eu/sdi4apps/openapi/utils/HttpParam.java | 4 +- 8 files changed, 175 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 32607d7..2724ba0 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,8 @@ The SDI4Apps OpenAPI is a set of web services exposed by the SDI4Apps Cloud plat The Open API consists partly of the protocols exposed by the components in the platform, i.e. Open Geospatial Consortium compliant web services like WMS, WFS and CS-W. Other parts of the API implements custom functionality that adds specific search/processing capabilities to the system. The code of these custom services are included in this repository. -## System requirements -* Java SDK -* Maven -* Apache Tomcat -* GDAL/OGR Java SWIG bindings -* Apache Lucene -* PostgreSQL -* PostGIS -* pgRouting +The code base of the project has been ported to this repository in the last week based on the content of two independent previous repositories where the actual development has taken place. -## Installation +# Web Service documentation -Installation will be handled by the platform installation script... \ No newline at end of file +Documentation is currently being added to the site based on auto-generated JavaDocs from the source code. Please refer to the GitHub wiki for usage instructions. \ No newline at end of file diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/GeoDoc.java b/src/main/java/eu/sdi4apps/ftgeosearch/GeoDoc.java index dbe727e..76152e5 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/GeoDoc.java +++ b/src/main/java/eu/sdi4apps/ftgeosearch/GeoDoc.java @@ -5,6 +5,7 @@ */ package eu.sdi4apps.ftgeosearch; +import eu.sdi4apps.openapi.config.Settings; import java.util.UUID; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -17,13 +18,12 @@ * @author runarbe */ public class GeoDoc { - + /** - * The relevance score of the search result - * Only populated upon searches + * The relevance score of the search result Only populated upon searches */ public float Score; - + /** * Unique id of document */ @@ -33,7 +33,7 @@ public class GeoDoc { * The layer that the document belongs to */ public String Layer; - + /** * The type of object represented by the document */ @@ -88,6 +88,7 @@ public GeoDoc() { /** * Create a new GeoDoc + * * @param layer * @param fullGeom * @param pointGeom @@ -129,9 +130,9 @@ public static GeoDoc create( } /** - * Add additional values to be indexed but not displayed to the document - * - * @param values + * Add additional values to be indexed but not displayed to the document + * + * @param values */ public void indexValues(String values) { this.IndexAdditional += values; @@ -139,7 +140,8 @@ public void indexValues(String values) { /** * Add custom JSON data object to the document - * @param jsonData + * + * @param jsonData */ public void setJsonData(String jsonData) { this.JsonData = jsonData; @@ -156,17 +158,19 @@ public Document asLuceneDoc() { d.add(new Field("PointGeom", this.PointGeom, Store.YES, Index.NO)); d.add(new Field("DisplayTitle", this.DisplayTitle, Store.YES, Index.NO)); d.add(new Field("DisplayDescription", this.DisplayDescription, Store.YES, Index.NO)); - + Field indexTitle = new Field("IndexTitle", this.IndexTitle, Store.NO, Index.ANALYZED); - indexTitle.setBoost((float)1.2); + indexTitle.setBoost(Settings.TITLEBOOST); d.add(indexTitle); - + Field indexDescription = new Field("IndexDescription", this.IndexDescription, Store.NO, Index.ANALYZED); - indexDescription.setBoost((float)1.1); + indexDescription.setBoost(Settings.DESCRIPTIONBOOST); d.add(indexDescription); d.add(new Field("IndexAdditional", this.IndexAdditional, Store.NO, Index.ANALYZED)); - d.add(new Field("JsonData", Serializer.Serialize(this.JsonData), Store.YES, Index.NO)); + if (this.JsonData != null) { + d.add(new Field("JsonData", Serializer.Serialize(this.JsonData), Store.YES, Index.NO)); + } } catch (Exception e) { System.out.println("Error converting GeoDoc to LuceneDoc: " + e.toString()); } diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/Indexer.java b/src/main/java/eu/sdi4apps/ftgeosearch/Indexer.java index 22449d5..894e020 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/Indexer.java +++ b/src/main/java/eu/sdi4apps/ftgeosearch/Indexer.java @@ -1,19 +1,24 @@ package eu.sdi4apps.ftgeosearch; +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.shape.Shape; import eu.sdi4apps.openapi.utils.Logger; -import eu.sdi4apps.ftgeosearch.drivers.ShapefileDriver; import eu.sdi4apps.openapi.config.Settings; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.sql.SQLException; -import static java.util.Arrays.asList; import java.util.LinkedHashMap; -import java.util.List; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; +import org.apache.lucene.spatial.SpatialStrategy; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.gdal.ogr.Feature; @@ -35,6 +40,16 @@ public class Indexer { public static IndexWriterConfig indexWriterConfig = null; + public static SpatialContext spatialCtx = null; + + public static SpatialStrategy spatialStrategy = null; + + public static SpatialPrefixTree spatialPrefixTree = null; + + public static int maxSpatialIndexLevels = 11; + + public static int errorCount = 0; + /** * Get the IndexWriter or null * @@ -43,6 +58,12 @@ public class Indexer { public static IndexWriter getWriter() { try { if (indexWriter == null || indexWriter.isOpen() == false) { + spatialCtx = SpatialContext.GEO; + + spatialPrefixTree = new GeohashPrefixTree(spatialCtx, maxSpatialIndexLevels); + + spatialStrategy = new RecursivePrefixTreeStrategy(spatialPrefixTree, "GeoField"); + analyzer = new StandardAnalyzer(); directory = FSDirectory.open(Paths.get(Settings.INDEXDIR)); indexWriterConfig = new IndexWriterConfig(analyzer); @@ -79,15 +100,17 @@ public static void unlockIndex() { /** * Index a layer - * + * * @param lyr * @param qi - * @param w + * @param w */ public static void indexLayer(Layer lyr, QueueItem qi, IndexWriter w) { try { + Indexer.errorCount = 0; + qi.updateIndexingStatus(IndexingStatus.Indexing); Feature f = null; @@ -107,6 +130,10 @@ public static void indexLayer(Layer lyr, QueueItem qi, IndexWriter w) { indexFeature(w, qi, f, titleFieldMap, descriptionFieldMap, indexAdditionalFieldMap, jsonDataFieldMap); + if (Indexer.errorCount >= 50) { + throw new Exception("More than 50 errors occurred, aborting indexing operation"); + } + if (counter % batch == 0 || counter == totalFeatures) { Logger.Log("Processed: " + counter + " items..."); } @@ -146,12 +173,17 @@ public static void indexFeature(IndexWriter w, String compositeId = qi.layer + "-" + f.GetFID(); + String pointWkt = g.PointOnSurface().ExportToWkt(); + + double[] shapeEnvelope = new double[4]; + g.GetEnvelope(shapeEnvelope); + GeoDoc gd = GeoDoc.create( compositeId, qi.layer, qi.objtype, g.ExportToWkt(), - g.PointOnSurface().ExportToWkt(), + pointWkt, titleData[0], descData[0], titleData[1], @@ -160,12 +192,35 @@ public static void indexFeature(IndexWriter w, jsonData); if (gd != null) { - w.updateDocument(new Term("Id", compositeId), gd.asLuceneDoc()); + + /** + * Retrieve Lucene document + */ + Document luceneGeoDoc = gd.asLuceneDoc(); + + /** + * Add spatial indexing for bounding box of objects + */ + for (IndexableField geoField + : spatialStrategy.createIndexableFields( + spatialCtx.makeRectangle(shapeEnvelope[0], + shapeEnvelope[1], + shapeEnvelope[2], + shapeEnvelope[3]))) { + luceneGeoDoc.add(geoField); + } + + /** + * Write the document to the index + */ + w.updateDocument(new Term("Id", compositeId), luceneGeoDoc); } } catch (Exception e) { - Logger.Log("An error occurred while writing feature to Lucene index: " + e.toString()); + Indexer.errorCount++; + Logger.Log("An error occurred while writing feature to Lucene index: " + e.toString() + " (#" + Indexer.errorCount + ")"); } + } /** diff --git a/src/main/java/eu/sdi4apps/ftgeosearch/Searcher.java b/src/main/java/eu/sdi4apps/ftgeosearch/Searcher.java index bccdfc6..01b73cd 100644 --- a/src/main/java/eu/sdi4apps/ftgeosearch/Searcher.java +++ b/src/main/java/eu/sdi4apps/ftgeosearch/Searcher.java @@ -5,6 +5,12 @@ */ package eu.sdi4apps.ftgeosearch; +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.distance.DistanceUtils; +import static eu.sdi4apps.ftgeosearch.Indexer.maxSpatialIndexLevels; +import static eu.sdi4apps.ftgeosearch.Indexer.spatialCtx; +import static eu.sdi4apps.ftgeosearch.Indexer.spatialPrefixTree; +import static eu.sdi4apps.ftgeosearch.Indexer.spatialStrategy; import eu.sdi4apps.openapi.config.Settings; import eu.sdi4apps.openapi.types.BBox; import eu.sdi4apps.openapi.utils.Logger; @@ -12,20 +18,27 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.logging.Level; import org.apache.commons.lang.StringUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.queries.TermFilter; import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Filter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.spatial.SpatialStrategy; +import org.apache.lucene.spatial.prefix.RecursivePrefixTreeStrategy; +import org.apache.lucene.spatial.prefix.tree.GeohashPrefixTree; +import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree; +import org.apache.lucene.spatial.query.SpatialArgs; +import org.apache.lucene.spatial.query.SpatialOperation; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; @@ -43,12 +56,24 @@ public class Searcher { public static IndexSearcher isearcher = null; + public static SpatialContext spatialCtx = null; + + public static SpatialStrategy spatialStrategy = null; + + public static SpatialPrefixTree spatialPrefixTree = null; + + public static int maxSpatialIndexLevels = 11; + static { createSearcher(); } public static IndexSearcher createSearcher() { try { + spatialCtx = SpatialContext.GEO; + spatialPrefixTree = new GeohashPrefixTree(spatialCtx, maxSpatialIndexLevels); + spatialStrategy = new RecursivePrefixTreeStrategy(spatialPrefixTree, "GeoField"); + analyzer = new StandardAnalyzer(); directory = FSDirectory.open(Paths.get(Settings.INDEXDIR)); ireader = DirectoryReader.open(directory); @@ -84,15 +109,27 @@ public static List Search(String q, String filter, String extent) throws IOException, ParseException { - List sr = new ArrayList<>(); + List searchResults = new ArrayList<>(); try { + /** + * Create searcher if it does not exist + */ + if (isearcher == null) { + createSearcher(); + } + + /** + * Create a boolean query + */ + BooleanQuery combinedQuery = new BooleanQuery(); + /** * Set default number of results to 100 */ if (maxresults == null) { - maxresults = 100; + maxresults = Settings.NUMRESULTS; } /** @@ -105,24 +142,28 @@ public static List Search(String q, filterLayers = null; } + /** + * Create query clause for user specified term(s) + */ + QueryParser parser = new MultiFieldQueryParser(new String[]{"IndexTitle", "IndexDescription", "IndexAdditional"}, analyzer); + Query termQuery = parser.parse(q); + combinedQuery.add(termQuery, Occur.MUST); + /** * Convert string extent to BBox object */ if (extent != null) { BBox bbox = BBox.createFromString(extent); if (bbox != null) { - Logger.Log(bbox.jsArray()); + SpatialArgs args = new SpatialArgs(SpatialOperation.Intersects, + spatialCtx.makeRectangle(bbox.minX, bbox.maxX, bbox.minY, bbox.maxY)); + Query spatialQuery = spatialStrategy.makeQuery(args); + spatialQuery.setBoost(Settings.SPATIALBOOST); + combinedQuery.add(spatialQuery, Occur.SHOULD); } } - if (isearcher == null) { - createSearcher(); - } - - QueryParser parser = new MultiFieldQueryParser(new String[]{"IndexTitle", "IndexDescription", "IndexAdditional"}, analyzer); - Query query = parser.parse(q); - - ScoreDoc[] hits = isearcher.search(query, null, maxresults).scoreDocs; + ScoreDoc[] hits = isearcher.search(combinedQuery, maxresults).scoreDocs; // Iterate through the results: for (int i = 0; i < hits.length; i++) { @@ -137,14 +178,14 @@ public static List Search(String q, g.DisplayDescription = hitDoc.get("DisplayDescription"); g.PointGeom = hitDoc.get("PointGeom"); g.JsonData = Serializer.Deserialize(hitDoc.get("JsonData")); - sr.add(g); + searchResults.add(g); } } catch (Exception e) { Logger.Log("An exception occurred during search: " + e.toString()); } finally { destroySearcher(); - return sr; + return searchResults; } } diff --git a/src/main/java/eu/sdi4apps/openapi/config/Settings.java b/src/main/java/eu/sdi4apps/openapi/config/Settings.java index d48af73..32b9657 100644 --- a/src/main/java/eu/sdi4apps/openapi/config/Settings.java +++ b/src/main/java/eu/sdi4apps/openapi/config/Settings.java @@ -100,6 +100,26 @@ public class Settings { } } } + + /** + * Default number of results to return from a search + */ + public static int NUMRESULTS = 25; + + /** + * Default boost to apply to terms in title (at index time) + */ + public static final float TITLEBOOST = (float) 1.3; + + /** + * Default boost to apply to terms in description (at index time) + */ + public static final float DESCRIPTIONBOOST = (float) 1.2; + + /** + * Default boost to apply to matches within bounding box (at query time) + */ + public static final float SPATIALBOOST = (float) 3; /** * A flag to indicate whether the configuration file has been read properly diff --git a/src/main/java/eu/sdi4apps/openapi/servlets/Index.java b/src/main/java/eu/sdi4apps/openapi/servlets/Index.java index 9d96507..8583461 100644 --- a/src/main/java/eu/sdi4apps/openapi/servlets/Index.java +++ b/src/main/java/eu/sdi4apps/openapi/servlets/Index.java @@ -17,7 +17,6 @@ import eu.sdi4apps.openapi.utils.HttpParam; import java.io.IOException; import java.io.PrintWriter; -import java.nio.file.Paths; import static java.util.Arrays.asList; import java.util.HashMap; import java.util.List; @@ -81,7 +80,7 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re List descFields3 = asList("field_4", "field_8", "field_18"); String descFieldFormat3 = "Alternative forms: %s. Type of name %s in %s"; QueueItem entry3 = QueueItem.create("Geonames 1000", "names", DatasetType.Shapefile, drv3, titleFields3, titleFieldFormat3, descFields3, descFieldFormat3, null, null, 4326); - IndexerQueue.enqueue(entry3); + IndexerQueue.enqueue(entry3); r.setData("Added two layers to index", true); break; case "EnqueueShapefile": @@ -93,10 +92,7 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re reqFields.put("descriptionFormat", true); reqFields.put("additionalFields", true); reqFields.put("jsonDataFields", true); - - Map avFields = HttpParam.GetParameters(request, reqFields, r); - - Logger.Log(avFields.toString()); + Map avFields = HttpParam.GetParameters(request, reqFields, r); break; default: r.setError("Unsupported action: " + action); diff --git a/src/main/java/eu/sdi4apps/openapi/servlets/Search.java b/src/main/java/eu/sdi4apps/openapi/servlets/Search.java index 5168ae0..eefd987 100644 --- a/src/main/java/eu/sdi4apps/openapi/servlets/Search.java +++ b/src/main/java/eu/sdi4apps/openapi/servlets/Search.java @@ -55,16 +55,23 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re m.put("maxresults", false); m.put("extent", false); - Map params = HttpParam.GetParameters(request, m, r); + Map params = HttpParam.GetParameters(request, m, r); if (r.status == "error") { out.println(r.asJson()); return; } + Integer maxResults; + if (params.get("maxresults") != null) { + maxResults = NumberUtils.toInt(params.get("maxresults")); + } else { + maxResults = null; + } + List sres = Searcher.Search((String) params.get("q"), - NumberUtils.toInt((String) params.get("maxresults"), 100), - (String) params.get("filter"), - (String) params.get("extent")); + maxResults, + params.get("filter"), + params.get("extent")); r.setData(sres, true); r.count = sres.size(); diff --git a/src/main/java/eu/sdi4apps/openapi/utils/HttpParam.java b/src/main/java/eu/sdi4apps/openapi/utils/HttpParam.java index d421941..f25df89 100644 --- a/src/main/java/eu/sdi4apps/openapi/utils/HttpParam.java +++ b/src/main/java/eu/sdi4apps/openapi/utils/HttpParam.java @@ -17,9 +17,9 @@ */ public class HttpParam { - public static Map GetParameters(HttpServletRequest request, Map fields, Response r) { + public static Map GetParameters(HttpServletRequest request, Map fields, Response r) { - Map m = new LinkedHashMap(); + Map m = new LinkedHashMap(); for (Map.Entry e : fields.entrySet()) { String key = e.getKey();