From 1fbc13220aa568a392c067bfa33b78cd77b41a59 Mon Sep 17 00:00:00 2001 From: Michael Reichert Date: Mon, 6 Nov 2023 17:18:35 +0100 Subject: [PATCH] initial commit --- .gitignore | 5 + CMakeLists.txt | 90 +++ cmake/FindMapnik.cmake | 143 +++++ cmake/FindProtozero.cmake | 63 ++ src/CMakeLists.txt | 26 + src/mbtiles_vector_datasource.cpp | 282 +++++++++ src/mbtiles_vector_datasource.hpp | 80 +++ src/mbtiles_vector_featureset.cpp | 82 +++ src/mbtiles_vector_featureset.hpp | 63 ++ src/mvt_io.cpp | 276 +++++++++ src/mvt_io.hpp | 138 +++++ src/mvt_message.hpp | 54 ++ src/sqlite_connection.hpp | 181 ++++++ src/sqlite_resultset.hpp | 112 ++++ src/vector_tile_compression.cpp | 40 ++ src/vector_tile_compression.hpp | 41 ++ src/vector_tile_compression.ipp | 39 ++ src/vector_tile_geometry_decoder.cpp | 44 ++ src/vector_tile_geometry_decoder.hpp | 90 +++ src/vector_tile_geometry_decoder.ipp | 848 +++++++++++++++++++++++++++ src/vector_tile_projection.hpp | 31 + 21 files changed, 2728 insertions(+) create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 cmake/FindMapnik.cmake create mode 100644 cmake/FindProtozero.cmake create mode 100644 src/CMakeLists.txt create mode 100644 src/mbtiles_vector_datasource.cpp create mode 100644 src/mbtiles_vector_datasource.hpp create mode 100644 src/mbtiles_vector_featureset.cpp create mode 100644 src/mbtiles_vector_featureset.hpp create mode 100644 src/mvt_io.cpp create mode 100644 src/mvt_io.hpp create mode 100644 src/mvt_message.hpp create mode 100644 src/sqlite_connection.hpp create mode 100644 src/sqlite_resultset.hpp create mode 100644 src/vector_tile_compression.cpp create mode 100644 src/vector_tile_compression.hpp create mode 100644 src/vector_tile_compression.ipp create mode 100644 src/vector_tile_geometry_decoder.cpp create mode 100644 src/vector_tile_geometry_decoder.hpp create mode 100644 src/vector_tile_geometry_decoder.ipp create mode 100644 src/vector_tile_projection.hpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..78292d3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.sh +.*.swp +*~ +build/ +*.input diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..a76033d --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,90 @@ +cmake_minimum_required(VERSION 3.5.0) +project(mapnik-mbtiles-vector VERSION 0.0.1 LANGUAGES CXX C) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +set(AUTHOR "Michael Reichert ") + + +#----------------------------------------------------------------------------- +# +# Find external dependencies +# +#----------------------------------------------------------------------------- + +find_package(Mapnik REQUIRED) +find_package(Protozero REQUIRED) +find_package(SQLite3 REQUIRED) +find_package(ZLIB REQUIRED) +find_package(Boost 1.55.0 REQUIRED) + +#----------------------------------------------------------------------------- +# +# Decide which C++ version to use (Minimum/default: C++17). +# +#----------------------------------------------------------------------------- +if(NOT MSVC) + if(NOT USE_CPP_VERSION) + set(USE_CPP_VERSION c++17) + endif() + message(STATUS "Use C++ version: ${USE_CPP_VERSION}") + # following only available from cmake 2.8.12: + # add_compile_options(-std=${USE_CPP_VERSION}) + # so using this instead: + add_definitions(-std=${USE_CPP_VERSION}) +endif() + + +#----------------------------------------------------------------------------- +# +# Compiler and Linker flags +# +#----------------------------------------------------------------------------- +set(USUAL_COMPILE_OPTIONS "-O3 -g") + +set(CMAKE_CXX_FLAGS_DEV "${USUAL_COMPILE_OPTIONS}" + CACHE STRING "Flags used by the compiler during developer builds." + FORCE) + +set(CMAKE_EXE_LINKER_FLAGS_DEV "" + CACHE STRING "Flags used by the linker during developer builds." + FORCE) +mark_as_advanced( + CMAKE_CXX_FLAGS_DEV + CMAKE_EXE_LINKER_FLAGS_DEV +) + +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${USUAL_COMPILE_OPTIONS}" + CACHE STRING "Flags used by the compiler during RELWITHDEBINFO builds." + FORCE) + + +#----------------------------------------------------------------------------- +# +# Build Type +# +#----------------------------------------------------------------------------- +set(CMAKE_CONFIGURATION_TYPES "Debug Release RelWithDebInfo MinSizeRel Dev") + +# In 'Dev' mode: compile with very strict warnings and turn them into errors. +if(CMAKE_BUILD_TYPE STREQUAL "Dev") + if(NOT MSVC) + add_definitions(-Werror -fno-omit-frame-pointer) + endif() +endif() + +# Force RelWithDebInfo build type if none was given +if(CMAKE_BUILD_TYPE) + set(build_type ${CMAKE_BUILD_TYPE}) +else() + set(build_type "RelWithDebInfo") +endif() + +set(CMAKE_BUILD_TYPE ${build_type} + CACHE STRING + "Choose the type of build, options are: ${CMAKE_CONFIGURATION_TYPES}." + FORCE) + +add_subdirectory(src) + diff --git a/cmake/FindMapnik.cmake b/cmake/FindMapnik.cmake new file mode 100644 index 0000000..83aaa4e --- /dev/null +++ b/cmake/FindMapnik.cmake @@ -0,0 +1,143 @@ +# - Try to find Mapnik +# Once done this will define +# +# MAPNIK_FOUND - system has Mapnik +# MAPNIK_INCLUDE_DIRS - the Mapnik include directory +# MAPNIK_DEPS_INCLUDE_DIRS - the include directory where header-only +# dependencies of Mapnik can be found +# MAPNIK_LIBRARIES - Link these to use Mapnik +# MAPNIK_CONFIG - mapnik-config binary +# MAPNIK_CXXFLAGS - mapnik-config --cflags) +# MAPNIK_LDFLAGS - mapnik-config --libs) +# MAPNIK_PLUGINDIR - mapnik-config --input-plugins) +# +# Copyright (c) 2007 Andreas Schneider +# Copyright (c) 2014 Maxim Dementyev +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +if (MAPNIK_LIBRARIES AND MAPNIK_INCLUDE_DIRS AND MAPNIK_DEPS_INCLUDE_DIRS) + # in cache already + set(MAPNIK_FOUND TRUE) +else (MAPNIK_LIBRARIES AND MAPNIK_INCLUDE_DIRS AND MAPNIK_DEPS_INCLUDE_DIRS) + find_path(MAPNIK_INCLUDE_DIR + NAMES + mapnik/config.hpp + PATHS + /usr/include + /usr/local/include + /opt/local/include + /sw/include + ) + + find_path(MAPNIK_DEPS_INCLUDE_DIR + NAMES + mapbox/geometry/point.hpp + PATHS + /usr/include + /usr/local/include + /opt/local/include + /sw/include + PATH_SUFFIXES + mapnik/deps + ) + + find_library(MAPNIK_LIBRARY + NAMES + mapnik + mapnik2 + PATHS + /usr/lib + /usr/local/lib + /opt/local/lib + /sw/lib + ) + + if (MAPNIK_LIBRARY) + set(MAPNIK_FOUND TRUE) + endif (MAPNIK_LIBRARY) + + set(MAPNIK_INCLUDE_DIRS + ${MAPNIK_INCLUDE_DIR} + ) + + set(MAPNIK_DEPS_INCLUDE_DIRS + ${MAPNIK_DEPS_INCLUDE_DIR} + ) + + if (MAPNIK_FOUND) + set(MAPNIK_LIBRARIES + ${MAPNIK_LIBRARIES} + ${MAPNIK_LIBRARY} + ) + endif (MAPNIK_FOUND) + + if (MAPNIK_INCLUDE_DIRS AND MAPNIK_DEPS_INCLUDE_DIRS AND MAPNIK_LIBRARIES) + set(MAPNIK_FOUND TRUE) + endif (MAPNIK_INCLUDE_DIRS AND MAPNIK_DEPS_INCLUDE_DIRS AND MAPNIK_LIBRARIES) + + if (MAPNIK_FOUND) + if (NOT Mapnik_FIND_QUIETLY) + message(STATUS "Found Mapnik: ${MAPNIK_LIBRARIES}") + endif (NOT Mapnik_FIND_QUIETLY) + else (MAPNIK_FOUND) + if (Mapnik_FIND_REQUIRED) + message(FATAL_ERROR "Could not find Mapnik") + endif (Mapnik_FIND_REQUIRED) + endif (MAPNIK_FOUND) + + find_program(MAPNIK_CONFIG + NAMES + mapnik-config + PATHS + /usr/bin + /usr/local/bin + /opt/local/bin + ) + + if (MAPNIK_CONFIG) + message(STATUS "Found mapnik-config: ${MAPNIK_CONFIG}") + execute_process(COMMAND mapnik-config --cflags OUTPUT_VARIABLE MAPNIK_CXXFLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND mapnik-config --libs OUTPUT_VARIABLE MAPNIK_LDFLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND mapnik-config --input-plugins OUTPUT_VARIABLE MAPNIK_PLUGINDIR OUTPUT_STRIP_TRAILING_WHITESPACE) + message(STATUS "Mapnik default plugin path: ${MAPNIK_PLUGINDIR}") + execute_process(COMMAND mapnik-config --dep-includes OUTPUT_VARIABLE MAPNIK_INCLUDE_FLAGS OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE " " ";" MAPNIK_INCLUDE_FLAG_LIST ${MAPNIK_INCLUDE_FLAGS}) + foreach(INCLUDEFLAG ${MAPNIK_INCLUDE_FLAG_LIST}) + string(REPLACE "-I" "" INCLUDEPATH ${INCLUDEFLAG}) + set(MAPNIK_INCLUDE_DIRS ${MAPNIK_INCLUDE_DIRS} ${INCLUDEPATH}) + endforeach(INCLUDEFLAG) + endif (MAPNIK_CONFIG) + + # show the variables only in the advanced view + mark_as_advanced( + MAPNIK_INCLUDE_DIRS + MAPNIK_LIBRARIES + MAPNIK_CXXFLAGS + MAPNIK_LDFLAGS + MAPNIK_PLUGINDIR + ) + +endif (MAPNIK_LIBRARIES AND MAPNIK_INCLUDE_DIRS AND MAPNIK_DEPS_INCLUDE_DIRS) diff --git a/cmake/FindProtozero.cmake b/cmake/FindProtozero.cmake new file mode 100644 index 0000000..ad16cab --- /dev/null +++ b/cmake/FindProtozero.cmake @@ -0,0 +1,63 @@ +#---------------------------------------------------------------------- +# +# FindProtozero.cmake +# +# Find the protozero headers. +# +#---------------------------------------------------------------------- +# +# Usage: +# +# Copy this file somewhere into your project directory, where cmake can +# find it. Usually this will be a directory called "cmake" which you can +# add to the CMake module search path with the following line in your +# CMakeLists.txt: +# +# list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") +# +# Then add the following in your CMakeLists.txt: +# +# find_package(Protozero [version] [REQUIRED]) +# include_directories(SYSTEM ${PROTOZERO_INCLUDE_DIR}) +# +# The version number is optional. If it is not set, any version of +# protozero will do. +# +# if(NOT PROTOZERO_FOUND) +# message(WARNING "Protozero not found!\n") +# endif() +# +#---------------------------------------------------------------------- +# +# Variables: +# +# PROTOZERO_FOUND - True if Protozero was found. +# PROTOZERO_INCLUDE_DIR - Where to find include files. +# +#---------------------------------------------------------------------- + +# find include path +find_path(PROTOZERO_INCLUDE_DIR protozero/version.hpp + PATH_SUFFIXES include + PATHS ${CMAKE_SOURCE_DIR}/../protozero +) + +# Check version number +if(Protozero_FIND_VERSION) + file(STRINGS "${PROTOZERO_INCLUDE_DIR}/protozero/version.hpp" _version_define REGEX "#define PROTOZERO_VERSION_STRING") + if("${_version_define}" MATCHES "#define PROTOZERO_VERSION_STRING \"([0-9.]+)\"") + set(_version "${CMAKE_MATCH_1}") + else() + set(_version "unknown") + endif() +endif() + +#set(PROTOZERO_INCLUDE_DIRS "${PROTOZERO_INCLUDE_DIR}") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Protozero + REQUIRED_VARS PROTOZERO_INCLUDE_DIR + VERSION_VAR _version) + + +#---------------------------------------------------------------------- diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..53c2058 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,26 @@ +add_library(input-mbtiles_vector MODULE) +set_target_properties(input-mbtiles_vector PROPERTIES + OUTPUT_NAME mbtiles_vector + POSITION_INDEPENDENT_CODE ON + PREFIX "" + SUFFIX ".input" + #LIBRARY_OUTPUT_DIRECTORY "${MAPNIK_OUTPUT_DIR}/plugins/input" +) +#mapnik_install_plugin(input-mbtiles_vector) +target_sources(input-mbtiles_vector PRIVATE + mbtiles_vector_datasource.cpp + mbtiles_vector_featureset.cpp + mvt_io.cpp + vector_tile_compression.cpp + vector_tile_geometry_decoder.cpp +) +target_include_directories(input-mbtiles_vector PRIVATE + ${MAPNIK_INCLUDE_DIRS} + ${MAPNIK_DEPS_INCLUDE_DIRS} + ${PROTOZERO_INCLUDE_DIR} +) +target_link_libraries(input-mbtiles_vector PRIVATE + ${MAPNIK_LIBRARIES} + SQLite::SQLite3 + ZLIB::ZLIB +) diff --git a/src/mbtiles_vector_datasource.cpp b/src/mbtiles_vector_datasource.cpp new file mode 100644 index 0000000..692eb56 --- /dev/null +++ b/src/mbtiles_vector_datasource.cpp @@ -0,0 +1,282 @@ +/* + * mbtiles_vector_datasource.cpp + * + * Created on: 2023-09-13 + * Author: Michael Reichert + */ + +#include "mbtiles_vector_datasource.hpp" +#include "mbtiles_vector_featureset.hpp" +#include "vector_tile_projection.hpp" +#include +#include +#include +#include +#include +#include + + +DATASOURCE_PLUGIN_IMPL(mbtiles_vector_datasource_plugin, mbtiles_vector_datasource); +DATASOURCE_PLUGIN_EXPORT(mbtiles_vector_datasource_plugin); +DATASOURCE_PLUGIN_EMPTY_AFTER_LOAD(mbtiles_vector_datasource_plugin); +DATASOURCE_PLUGIN_EMPTY_BEFORE_UNLOAD(mbtiles_vector_datasource_plugin); + +mbtiles_vector_datasource::mbtiles_vector_datasource(mapnik::parameters const& params) + : datasource(params), + desc_(mbtiles_vector_datasource::name(), *params.get("encoding", "utf-8")) +{ + init(params); +} + +mbtiles_vector_datasource::~mbtiles_vector_datasource() {} + +mapnik::datasource::datasource_t mbtiles_vector_datasource::type() const +{ + return datasource::Vector; +} + +const char * mbtiles_vector_datasource::name() +{ + return "mbtiles_vector"; +} + +mapnik::layer_descriptor mbtiles_vector_datasource::get_descriptor() const +{ + return desc_; +} + +boost::optional mbtiles_vector_datasource::get_geometry_type() const +{ + return mapnik::datasource_geometry_t::Collection; +} + +mapnik::box2d mbtiles_vector_datasource::envelope() const +{ + return extent_; +} + +int mbtiles_vector_datasource::zoom_from_string(const std::string& z) +{ + return std::stoi(z); +} + +int mbtiles_vector_datasource::zoom_from_string(const char* z) +{ + char *end_ptr; + long int zoom = strtol(z, &end_ptr, 10); + if (*end_ptr == '\0') { + return static_cast(zoom); + } + throw mapnik::datasource_exception("MBTiles Plugin: " + database_path_ + ": invalid minzoom/maxzoom in table 'metadata'"); +} + +void mbtiles_vector_datasource::init(mapnik::parameters const& params) +{ + boost::optional file = params.get("file"); + if (!file) + { + throw mapnik::datasource_exception("mbtiles_vector Plugin: missing parameter"); + } + + boost::optional base = params.get("base"); + if (base) + { + database_path_ = *base + "/" + *file; + } + else + { + database_path_ = *file; + } + if (!mapnik::util::exists(database_path_)) + { + throw mapnik::datasource_exception("MBTiles Plugin: " + database_path_ + " does not exist"); + } + boost::optional layer = params.get("layer"); + if (!layer) + { + throw mapnik::datasource_exception("MBTiles Plugin: parameter 'layer' is missing."); + } + layer_ = layer.get(); + dataset_ = std::make_shared(database_path_); + // Ensure that the tileset contains vector tiles + std::shared_ptr result (dataset_->execute_query("SELECT value FROM metadata WHERE name = 'format';")); + if (result->is_valid() && result->step_next() && result->column_type(0) == SQLITE_TEXT && strcmp(result->column_text(0), "pbf")) + { + throw mapnik::datasource_exception("MBTiles Plugin: " + database_path_ + " has unsupported vector tile format, expected 'pbf'."); + } + // initialize envelope + boost::optional ext = params.get("extent"); + if (ext && !ext->empty()) + { + extent_.from_string(*ext); + } + else + { + result = dataset_->execute_query("SELECT value FROM metadata WHERE name = 'bounds';"); + if (result->is_valid() && result->step_next() && result->column_type(0) == SQLITE_TEXT) + { + extent_.from_string(result->column_text(0)); + } + } + if (!extent_.valid()) + { + throw mapnik::datasource_exception("MBTiles Plugin: " + database_path_ + " extent is invalid."); + } + // Bounds are specified in EPSG:4326, therefore transformation is required. + mapnik::lonlat2merc(extent_.minx_, extent_.miny_); + mapnik::lonlat2merc(extent_.maxx_, extent_.maxy_); + // initialize minzoom + result = dataset_->execute_query("SELECT value FROM metadata WHERE name = 'minzoom';"); + if (result->is_valid() && result->step_next() && result->column_type(0) == SQLITE_TEXT) + { + minzoom_ = zoom_from_string(result->column_text(0)); + } + // initialize maxzoom + result = dataset_->execute_query("SELECT value FROM metadata WHERE name = 'maxzoom';"); + if (result->is_valid() && result->step_next() && result->column_type(0) == SQLITE_TEXT) + { + maxzoom_ = zoom_from_string(result->column_text(0)); + } + //TODO make zoom level variable + boost::optional zoom = params.get("zoom"); + if (!zoom) + { + throw mapnik::datasource_exception("MBTiles Plugin: parameter 'zoom' missing"); + } + zoom_ = zoom_from_string(zoom.get()); + // Get 'json' field + result = dataset_->execute_query("SELECT value FROM metadata WHERE name = 'json';"); + if (result->is_valid() && result->step_next() && result->column_type(0) == SQLITE_TEXT) + { + json_ = result->column_text(0); + } + if (json_.empty()) + { + throw mapnik::datasource_exception("MBTiles Plugin: " + database_path_ + " has no 'json' entry in metadata table."); + } + parse_json(); +} + +void mbtiles_vector_datasource::raise_json_error(std::string message) +{ + throw mapnik::datasource_exception( + "MBTiles Plugin: " + database_path_ + + " has invalid 'json' entry in metadata table: " + message); +} + +void mbtiles_vector_datasource::raise_json_parse_error(size_t pos, rapidjson::ParseErrorCode code) +{ + throw mapnik::datasource_exception( + "MBTiles Plugin: " + database_path_ + + " has invalid 'json' entry in metadata table: JSON error at offset " + + std::to_string(pos) + + ": " + + rapidjson::GetParseError_En(code)); +} + +void mbtiles_vector_datasource::parse_json() +{ + rapidjson::Document doc; + doc.Parse(json_.c_str()); + if (doc.HasParseError()) + { + raise_json_parse_error(doc.GetErrorOffset(), doc.GetParseError()); + } + if (!doc.IsObject()) + { + raise_json_error("Root is not an object."); + } + if (!doc.HasMember("vector_layers")) + { + raise_json_error("vector_layers is missing."); + } + const auto vector_layers = doc.FindMember("vector_layers"); + if (vector_layers == doc.MemberEnd()) + { + raise_json_error("vector_layers is missing."); + } + if (!vector_layers->value.IsArray()) + { + raise_json_error("vector_layers is not an array."); + } + bool found = false; + for (const auto& l : vector_layers->value.GetArray()) + { + const auto id = l.FindMember("id"); + if (id == doc.MemberEnd() || !id->value.IsString()) + { + raise_json_error("vector_layers contains a layer without or with invalid 'id' property."); + } + if (id->value.GetString() == layer_) + { + found = true; + const auto minzoom = l.FindMember("minzoom"); + const auto maxzoom = l.FindMember("maxzoom"); + if (minzoom != l.MemberEnd() && minzoom->value.IsInt()) + { + minzoom_ = std::max(minzoom_, minzoom->value.GetInt()); + } + if (maxzoom != l.MemberEnd() && maxzoom->value.IsInt()) + { + maxzoom_ = std::min(maxzoom_, maxzoom->value.GetInt()); + } + const auto fields = l.FindMember("fields"); + if (fields == l.MemberEnd()) + { + raise_json_error("No member 'fields' for layer " + layer_); + } + for (const auto& field : fields->value.GetObject()) + { + std::string name = field.name.GetString(); + if (!field.value.IsString()) + { + raise_json_error("Layer '" + layer_ + "' has an invalid field definition: value of field '" + name + "' is not a of JSON type String."); + } + std::string value_type = field.value.GetString(); + if (value_type == "String") + { + desc_.add_descriptor(mapnik::attribute_descriptor(name, mapnik::String)); + } + else if (value_type == "Number") + { + desc_.add_descriptor(mapnik::attribute_descriptor(name, mapnik::Float)); + } + else if (value_type == "Boolean") + { + desc_.add_descriptor(mapnik::attribute_descriptor(name, mapnik::Boolean)); + } + else + { + raise_json_error("Layer '" + layer_ + "' field '" + name + "': invalid field type '" + value_type + "'"); + } + } + break; + } + } + if (!found) + { + raise_json_error("Requested layer '" + layer_ + "' not found."); + } + //TODO init bounds +} + +mapnik::featureset_ptr mbtiles_vector_datasource::features(mapnik::query const& q) const +{ +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, "mbtiles_vector_datasource::features"); +#endif + + mapnik::box2d const& box = q.get_bbox(); + return mapnik::featureset_ptr(new mbtiles_vector_featureset(dataset_, zoom_, box, layer_)); +} + +mapnik::featureset_ptr mbtiles_vector_datasource::features_at_point(mapnik::coord2d const& pt, double tol) const +{ +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, "mbtiles_vector_datasource::features"); +#endif + + mapnik::filter_at_point filter(pt, tol); + + return mapnik::featureset_ptr(new mbtiles_vector_featureset(dataset_, zoom_, filter.box_, layer_)); +} diff --git a/src/mbtiles_vector_datasource.hpp b/src/mbtiles_vector_datasource.hpp new file mode 100644 index 0000000..bbe5723 --- /dev/null +++ b/src/mbtiles_vector_datasource.hpp @@ -0,0 +1,80 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2015 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef MBTILES_VECTOR_DATASOURCE_HPP_ +#define MBTILES_VECTOR_DATASOURCE_HPP_ + +#include +//#include +#include "sqlite_connection.hpp" +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include + + +DATASOURCE_PLUGIN_DEF(mbtiles_vector_datasource_plugin, mbtiles_vector); + +class mbtiles_vector_datasource : public mapnik::datasource +{ +public: + mbtiles_vector_datasource(mapnik::parameters const& params); + virtual ~mbtiles_vector_datasource (); + mapnik::datasource::datasource_t type() const; + static const char * name(); + mapnik::featureset_ptr features(mapnik::query const& q) const; + mapnik::featureset_ptr features_at_point(mapnik::coord2d const& pt, double tol = 0) const; + mapnik::box2d envelope() const; + boost::optional get_geometry_type() const; + mapnik::layer_descriptor get_descriptor() const; + +private: + void init(mapnik::parameters const& params); + int zoom_from_string(const char* z); + int zoom_from_string(const std::string& z); + void parse_json(); + void raise_json_error(std::string message); + void raise_json_parse_error(size_t pos, rapidjson::ParseErrorCode code); + std::string database_path_; + std::shared_ptr dataset_; + mapnik::box2d extent_; + int minzoom_ = 0; + int maxzoom_ = 14; + int zoom_ = 14; + std::string json_; + std::string layer_; + mapnik::layer_descriptor desc_; +}; + + + +#endif /* MBTILES_VECTOR_DATASOURCE_HPP_ */ diff --git a/src/mbtiles_vector_featureset.cpp b/src/mbtiles_vector_featureset.cpp new file mode 100644 index 0000000..4b0a6e7 --- /dev/null +++ b/src/mbtiles_vector_featureset.cpp @@ -0,0 +1,82 @@ +/* + * mbtiles_vector_featureset.cpp + * + * Created on: 2023-09-15 + * Author: Michael Reichert + */ + +#include "mbtiles_vector_featureset.hpp" +#include "vector_tile_compression.hpp" +#include +#include + +mbtiles_vector_featureset::mbtiles_vector_featureset(std::shared_ptr database, + const int zoom, + mapnik::box2d const& extent, const std::string & layer) : + database_(database), + zoom_(zoom), + extent_(extent), + layer_(layer), + vector_tile_(nullptr) +{ + int tile_count = 1 << zoom; + constexpr double width = 2.0 * 6378137 * M_PI; + xmin_ = static_cast((extent_.minx() + width / 2) * (tile_count / width)); + xmax_ = static_cast((extent_.maxx() + width / 2) * (tile_count / width)); + ymin_ = static_cast(((width / 2) - extent_.miny()) * (tile_count / width)); + ymax_ = static_cast(((width / 2) - extent_.maxy()) * (tile_count / width)); + x_ = xmin_; + y_ = ymin_; +} + +mbtiles_vector_featureset::~mbtiles_vector_featureset() {} + +mapnik::feature_ptr mbtiles_vector_featureset::next() +{ + // If current tile is processed completely, go forward to the next tile. + // else step forward to the next feature + while (next_tile() && open_tile() && vector_tile_) + { + return vector_tile_->next(); + } + return mapnik::feature_ptr(); +} + +bool mbtiles_vector_featureset::next_tile() +{ + ++x_; + if (x_ <= xmax_) + { + return true; + } + x_ = xmin_; + ++y_; + return y_ <= ymax_; +} + +bool mbtiles_vector_featureset::open_tile() +{ + std::string sql = (boost::format("SELECT tile_data FROM tiles WHERE zoom_level = %1% AND tile_column = %2% AND tile_row = %3%") % zoom_ % x_ % y_).str(); + std::shared_ptr result (database_->execute_query(sql)); + int size = 0; + char const* blob = nullptr; + if (result->is_valid() && result->step_next() && result->column_type(0) == SQLITE_BLOB) + { + blob = result->column_blob(0, size); + } + else + { + return false; + } + std::cerr << "Opening vector tile " << zoom_ << '/' << x_ << '/' << y_ << '\n'; + if (mapnik::vector_tile_impl::is_gzip_compressed(blob, size) || + mapnik::vector_tile_impl::is_zlib_compressed(blob, size)) + { + std::cerr << "decompressing\n"; + std::string decompressed; + mapnik::vector_tile_impl::zlib_decompress(blob, size, decompressed); + vector_tile_.reset(new mvt_io(std::move(decompressed), x_, y_, zoom_, layer_)); + } + vector_tile_.reset(new mvt_io(std::string(blob, size), x_, y_, zoom_, layer_)); + return true; +} diff --git a/src/mbtiles_vector_featureset.hpp b/src/mbtiles_vector_featureset.hpp new file mode 100644 index 0000000..c380907 --- /dev/null +++ b/src/mbtiles_vector_featureset.hpp @@ -0,0 +1,63 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2015 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef MBTILES_VECTOR_FEATURESET_HPP_ +#define MBTILES_VECTOR_FEATURESET_HPP_ + +#include +// mapnik +#include +#include +// sqlite +#include "sqlite_connection.hpp" +#include "mvt_io.hpp" + +class mbtiles_vector_featureset : public mapnik::Featureset +{ +public: + mbtiles_vector_featureset(std::shared_ptr database, + const int zoom, + mapnik::box2d const& extent, + const std::string & layer); + + virtual ~mbtiles_vector_featureset(); + mapnik::feature_ptr next(); +private: + std::shared_ptr database_; + int zoom_; + mapnik::box2d const& extent_; + const std::string& layer_; + std::unique_ptr vector_tile_; + int xmin_; + int xmax_; + int ymin_; + int ymax_; + /// x index of the currently accessed tile + int x_ = 0; + /// y index of the currently accessed tile + int y_ = 0; + + bool next_tile(); + bool open_tile(); +}; + +#endif /* MBTILES_VECTOR_FEATURESET_HPP_ */ diff --git a/src/mvt_io.cpp b/src/mvt_io.cpp new file mode 100644 index 0000000..a0843a0 --- /dev/null +++ b/src/mvt_io.cpp @@ -0,0 +1,276 @@ +/* + * mvt_io.cpp + * + * Created on: 2023-09-18 + * Author: Michael Reichert + */ + +#include "mvt_io.hpp" +#include "vector_tile_geometry_decoder.hpp" +#include +#include + + +mvt_layer::mvt_layer(const uint32_t x, const uint32_t y, const uint32_t zoom) + : keys_(), + values_(), + tr_("utf-8"), + ctx_(std::make_shared()) +{ + resolution_ = mapnik::EARTH_CIRCUMFERENCE / (1 << zoom); + tile_x_ = -0.5 * mapnik::EARTH_CIRCUMFERENCE + x * resolution_; + tile_y_ = 0.5 * mapnik::EARTH_CIRCUMFERENCE - y * resolution_; +} + +void mvt_layer::add_feature(protozero::pbf_message&& feature) +{ + features_.push_back(feature); +} + +bool mvt_layer::has_features() const +{ + return !features_.empty(); +} + +void mvt_layer::add_key(std::string&& key) +{ + keys_.push_back(key); +} + +void mvt_layer::add_value(pbf_attr_value_type value) +{ + values_.push_back(value); +} + +void mvt_layer::set_name(std::string&& name) +{ + name_ = name; +} + +void mvt_layer::set_extent(uint32_t extent) +{ + extent_ = extent; +} + +void mvt_layer::finish_reading() +{ + num_keys_ = keys_.size(); + num_values_ = values_.size(); + scale_ = static_cast(extent_ / resolution_); +} + +mapnik::feature_ptr mvt_layer::next_feature() +{ + while (feature_index_ < features_.size()) + { + protozero::pbf_message f = features_.at(feature_index_); + mapnik::feature_ptr feature = mapnik::feature_factory::create(ctx_, feature_index_); + ++feature_index_; + mvt_message::geom_type geometry_type = mvt_message::geom_type::unknown; + bool has_geometry = false; + bool has_geometry_type = false; + mapnik::vector_tile_impl::GeometryPBF::pbf_itr geom_itr; + while (f.next()) + { + switch(f.tag()) + { + case mvt_message::feature::id: + feature->set_id(f.get_uint64()); + break; + case mvt_message::feature::tags: + { + auto tag_iterator = f.get_packed_uint32(); + for (auto _i = tag_iterator.begin(); _i != tag_iterator.end();) + { + std::size_t key_name = *(_i++); + if (_i == tag_iterator.end()) + { + throw std::runtime_error("Vector Tile has a feature with an odd number of tags, therefore the tile is invalid."); + } + std::size_t key_value = *(_i++); + if (key_name < num_keys_ && key_value < num_values_) + { + std::string const& name = keys_.at(key_name); + if (feature->has_key(name)) + { + pbf_attr_value_type val = values_.at(key_value); + value_visitor vv(tr_, feature, name); + mapnik::util::apply_visitor(vv, val); + } + } + } + } + break; + case mvt_message::feature::type: + has_geometry_type = true; + geometry_type = static_cast(f.get_enum()); + switch (geometry_type) + { + case mvt_message::geom_type::point: + case mvt_message::geom_type::linestring: + case mvt_message::geom_type::polygon: + break; + default: + throw std::runtime_error("Vector tile has an unknown geometry type " + + std::to_string(static_cast(geometry_type)) + " in feature"); + } + break; + case mvt_message::feature::geometry: + if (has_geometry) + { + throw std::runtime_error("Vector Tile has a feature with multiple geometry fields, it must have only one of them"); + } + has_geometry = true; + geom_itr = f.get_packed_uint32(); + break; + default: + throw std::runtime_error("Vector Tile contains unknown field type " + + std::to_string(static_cast(f.tag())) +" in feature"); + + } + } + if (has_geometry) + { + if (!has_geometry_type) + { + throw std::runtime_error("Vector Tile has a feature that does not define the required geometry type."); + } + mapnik::vector_tile_impl::GeometryPBF geoms(geom_itr); + mapnik::geometry::geometry geom = + mapnik::vector_tile_impl::decode_geometry(geoms, geometry_type, + 2, tile_x_, tile_y_, scale_, -1.0 * scale_); + if (geom.is()) + { + continue; + } +// #if defined(DEBUG) +// mapnik::box2d envelope = mapnik::geometry::envelope(geom); +// if (!filter_.pass(envelope)) +// { +// MAPNIK_LOG_ERROR(tile_featureset_pbf) << "tile_featureset_pbf: filter:pass should not get here"; +// continue; +// } +// #endif + feature->set_geometry(std::move(geom)); + return feature; + } + } + return mapnik::feature_ptr(); +} + +void mvt_io::read_layer(protozero::pbf_message& pbf_layer) +{ + layer_.reset(new mvt_layer(x_, y_, zoom_)); + while (pbf_layer.next()) + { + switch (pbf_layer.tag()) + { + case mvt_message::layer::name: + { + std::cerr << "Message layer\n"; + std::string name = pbf_layer.get_string(); + if (name != layer_name_) + { + return; + } + layer_->set_name(std::move(name)); + break; + } + case mvt_message::layer::extent: + layer_->set_extent(pbf_layer.get_uint32()); + break; + case mvt_message::layer::keys: + layer_->add_key(pbf_layer.get_string()); + break; + case mvt_message::layer::values: + { + protozero::pbf_message val_msg = pbf_layer.get_message(); + while (val_msg.next()) + { + switch(val_msg.tag()) { + case mvt_message::value::string_value: + layer_->add_value(val_msg.get_string()); + break; + case mvt_message::value::float_value: + layer_->add_value(val_msg.get_float()); + break; + case mvt_message::value::double_value: + layer_->add_value(val_msg.get_double()); + break; + case mvt_message::value::int_value: + layer_->add_value(val_msg.get_int64()); + break; + case mvt_message::value::uint_value: + layer_->add_value(val_msg.get_uint64()); + break; + case mvt_message::value::sint_value: + layer_->add_value(val_msg.get_sint64()); + break; + case mvt_message::value::bool_value: + layer_->add_value(val_msg.get_bool()); + break; + default: + throw std::runtime_error("unknown Value type " + std::to_string(static_cast(val_msg.tag())) + " in layer->values"); + } + } + break; + } + case mvt_message::layer::features: + { + protozero::pbf_message f_msg(pbf_layer.get_view()); + layer_->add_feature(std::move(f_msg)); + break; + } + default: + pbf_layer.skip(); + } + } + layer_->finish_reading(); +} + +void mvt_io::read_layers() +{ + protozero::pbf_message pbf_layer = message_.get_message(); +// protozero::pbf_reader pbf_layer = message_.get_message(); + + while (pbf_layer.next()) + { + switch (pbf_layer.tag()) + { + case mvt_message::tile::layer: + { + protozero::pbf_message msg_layer(pbf_layer.get_view()); + read_layer(msg_layer); + break; + } + default: + throw std::runtime_error{std::string("Unsupported message ") + std::to_string(static_cast(pbf_layer.tag())) + " in vector tile"}; + break; + } + } +} + +mapnik::feature_ptr mvt_io::next() +{ + if (!layer_ || !layer_->has_features()) + { + return mapnik::feature_ptr(); + } + return layer_->next_feature(); +} + +mvt_io::mvt_io(std::string&& data, const uint32_t x, const uint32_t y, const uint32_t zoom, std::string layer_name) + : message_(data), + x_(x), + y_(y), + zoom_(zoom), + layer_name_(layer_name) +{ + while (message_.next(mvt_message::tile::layer, protozero::pbf_wire_type::length_delimited)) + { + protozero::pbf_message msg_layer(message_.get_message()); + read_layer(msg_layer); +// read_layers(); + } +} + diff --git a/src/mvt_io.hpp b/src/mvt_io.hpp new file mode 100644 index 0000000..dafca1f --- /dev/null +++ b/src/mvt_io.hpp @@ -0,0 +1,138 @@ +/* + * mvt_io.hpp + * + * Created on: 2023-09-18 + * Author: Michael Reichert + */ + +#ifndef PLUGINS_INPUT_MBTILES_VECTOR_MVT_IO_HPP_ +#define PLUGINS_INPUT_MBTILES_VECTOR_MVT_IO_HPP_ + +#include +#include +#include +#include +#include +#include "mvt_message.hpp" + +//enum class mvt_value_type : char { +// unsupported = 0, +// value_integer = 1, +// value_double = 2, +// value_unicode = 3, +// value_bool = 4 +//}; + +using key_value_index = std::vector; + +//struct mvt_feature +//{ +// key_value_index key_indexes_; +// key_value_index value_indexes_; +// std::vector geom_encoded_; +// mvt_message::geom_type geom_type_ = mvt_message::geom_type::unknown; +//}; + +using pbf_attr_value_type = mapnik::util::variant; + +struct value_visitor +{ + mapnik::transcoder & tr_; + mapnik::feature_ptr & feature_; + std::string const& name_; + + value_visitor(mapnik::transcoder & tr, + mapnik::feature_ptr & feature, + std::string const& name) + : tr_(tr), + feature_(feature), + name_(name) {} + + void operator() (std::string const& val) + { + feature_->put(name_, tr_.transcode(val.data(), val.length())); + } + + void operator() (bool const& val) + { + feature_->put(name_, static_cast(val)); + } + + void operator() (int64_t const& val) + { + feature_->put(name_, static_cast(val)); + } + + void operator() (uint64_t const& val) + { + feature_->put(name_, static_cast(val)); + } + + void operator() (double const& val) + { + feature_->put(name_, static_cast(val)); + } + + void operator() (float const& val) + { + feature_->put(name_, static_cast(val)); + } +}; + +class mvt_layer +{ + std::vector> features_; + size_t feature_index_ = 0; + std::vector keys_; + std::vector values_; + std::size_t num_keys_ = 0; + std::size_t num_values_ = 0; + mapnik::transcoder tr_; + mapnik::context_ptr ctx_; + double tile_x_; + double tile_y_; + double resolution_; + double scale_ = 0; + std::string name_; + uint32_t extent_ = 4096; +public: + explicit mvt_layer(const uint32_t x, const uint32_t y, const uint32_t zoom); + void add_feature(protozero::pbf_message&& feature); + bool has_features() const; + void add_key(std::string&& key); + void add_value(pbf_attr_value_type value); + + void set_name(std::string&& name); + void set_extent(uint32_t extent); + mapnik::feature_ptr next_feature(); +// std::vector& keys() const; +// std::vector& values() const; + uint32_t extent() const; + void finish_reading(); +}; + +class mvt_io +{ + protozero::pbf_message message_; + const uint32_t x_; + const uint32_t y_; + const uint32_t zoom_; + std::string layer_name_; + const int tile_extent_ = -1; + std::unique_ptr layer_; + + void read_layer(protozero::pbf_message& l); + void read_layers(); + +// /// Transform pixel coordinates to Mercator coordinates, requires coordinate of top left corner of the tile. +// double pixel_x_to_mercator(const double x); +// double pixel_y_to_mercator(const double y); + +public: + explicit mvt_io(std::string&& data, const uint32_t x, const uint32_t y, const uint32_t zoom, std::string layer_name); + mapnik::feature_ptr next(); +}; + + + +#endif /* PLUGINS_INPUT_MBTILES_VECTOR_MVT_IO_HPP_ */ diff --git a/src/mvt_message.hpp b/src/mvt_message.hpp new file mode 100644 index 0000000..5c041d7 --- /dev/null +++ b/src/mvt_message.hpp @@ -0,0 +1,54 @@ +/* + * mvt_messages.hpp + * + * Created on: 2023-10-24 + * Author: Michael Reichert + */ + +#ifndef PLUGINS_INPUT_MBTILES_VECTOR_MVT_MESSAGE_HPP_ +#define PLUGINS_INPUT_MBTILES_VECTOR_MVT_MESSAGE_HPP_ + +namespace mvt_message +{ + enum class tile : protozero::pbf_tag_type + { + layer = 3 + }; + enum class layer : protozero::pbf_tag_type + { + version = 15, + name = 1, + features = 2, + keys = 3, + values = 4, + extent = 5 + }; + enum class value : protozero::pbf_tag_type + { + string_value = 1, + float_value = 2, + double_value = 3, + int_value = 4, + uint_value = 5, + sint_value = 6, + bool_value = 7 + }; + enum class feature : protozero::pbf_tag_type + { + id = 1, + tags = 2, + type = 3, + geometry = 4 + }; + enum class geom_type : int32_t + { + unknown = 0, + point = 1, + linestring = 2, + polygon = 3 + }; +}; + + + +#endif /* PLUGINS_INPUT_MBTILES_VECTOR_MVT_MESSAGE_HPP_ */ diff --git a/src/sqlite_connection.hpp b/src/sqlite_connection.hpp new file mode 100644 index 0000000..44767a7 --- /dev/null +++ b/src/sqlite_connection.hpp @@ -0,0 +1,181 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2021 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef MAPNIK_SQLITE_CONNECTION_HPP +#define MAPNIK_SQLITE_CONNECTION_HPP + +// stl +#include +#include + +// mapnik +#include +#include +#include + +#include +MAPNIK_DISABLE_WARNING_PUSH +#include +#include +MAPNIK_DISABLE_WARNING_POP + +// sqlite +extern "C" { +#include +} + +#include "sqlite_resultset.hpp" + +//============================================================================== + +class sqlite_connection +{ + public: + + sqlite_connection(std::string const& file) + : db_(0) + , file_(file) + { +#if SQLITE_VERSION_NUMBER >= 3005000 + int mode = SQLITE_OPEN_READWRITE; +#if SQLITE_VERSION_NUMBER >= 3006018 + // shared cache flag not available until >= 3.6.18 + // Don't use shared cache in SQLite prior to 3.7.15. + // https://github.com/mapnik/mapnik/issues/2483 + if (sqlite3_libversion_number() >= 3007015) + { + mode |= SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_SHAREDCACHE; + } +#endif + const int rc = sqlite3_open_v2(file_.c_str(), &db_, mode, 0); +#else + const int rc = sqlite3_open(file_.c_str(), &db_); +#endif + if (rc != SQLITE_OK) + { + std::ostringstream s; + s << "Sqlite Plugin: " << sqlite3_errmsg(db_); + + throw mapnik::datasource_exception(s.str()); + } + + sqlite3_busy_timeout(db_, 5000); + } + + sqlite_connection(std::string const& file, int flags) + : db_(0) + , file_(file) + { +#if SQLITE_VERSION_NUMBER >= 3005000 + const int rc = sqlite3_open_v2(file_.c_str(), &db_, flags, 0); +#else + const int rc = sqlite3_open(file_.c_str(), &db_); +#endif + if (rc != SQLITE_OK) + { + std::ostringstream s; + s << "Sqlite Plugin: " << sqlite3_errmsg(db_); + + throw mapnik::datasource_exception(s.str()); + } + } + + virtual ~sqlite_connection() + { + if (db_) + { + sqlite3_close(db_); + } + } + + void throw_sqlite_error(std::string const& sql) + { + std::ostringstream s; + s << "Sqlite Plugin: "; + if (db_) + s << "'" << sqlite3_errmsg(db_) << "'"; + else + s << "unknown error, lost connection"; + s << " (" << file_ << ")" + << "\nFull sql was: '" << sql << "'"; + + throw mapnik::datasource_exception(s.str()); + } + + std::shared_ptr execute_query(std::string const& sql) + { +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, std::string("sqlite_resultset::execute_query ") + sql); +#endif + sqlite3_stmt* stmt = 0; + + const int rc = sqlite3_prepare_v2(db_, sql.c_str(), -1, &stmt, 0); + if (rc != SQLITE_OK) + { + throw_sqlite_error(sql); + } + + return std::make_shared(stmt); + } + + void execute(std::string const& sql) + { +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, std::string("sqlite_resultset::execute ") + sql); +#endif + + const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0); + if (rc != SQLITE_OK) + { + throw_sqlite_error(sql); + } + } + + int execute_with_code(std::string const& sql) + { +#ifdef MAPNIK_STATS + mapnik::progress_timer __stats__(std::clog, std::string("sqlite_resultset::execute_with_code ") + sql); +#endif + + const int rc = sqlite3_exec(db_, sql.c_str(), 0, 0, 0); + return rc; + } + + sqlite3* operator*() + { + return db_; + } + + bool load_extension(std::string const& ext_path) + { + sqlite3_enable_load_extension(db_, 1); + int result = sqlite3_load_extension(db_, ext_path.c_str(), 0, 0); + return (result == SQLITE_OK) ? true : false; + } + + private: + + sqlite3* db_; + std::string file_; +}; + +#endif // MAPNIK_SQLITE_CONNECTION_HPP diff --git a/src/sqlite_resultset.hpp b/src/sqlite_resultset.hpp new file mode 100644 index 0000000..e3c318d --- /dev/null +++ b/src/sqlite_resultset.hpp @@ -0,0 +1,112 @@ +/***************************************************************************** + * + * This file is part of Mapnik (c++ mapping toolkit) + * + * Copyright (C) 2021 Artem Pavlenko + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + *****************************************************************************/ + +#ifndef MAPNIK_SQLITE_RESULTSET_HPP +#define MAPNIK_SQLITE_RESULTSET_HPP + +// mapnik +#include +#include +#include + +// stl +#include + +// sqlite +extern "C" { +#include +} + +//============================================================================== + +class sqlite_resultset +{ + public: + + sqlite_resultset(sqlite3_stmt* stmt) + : stmt_(stmt) + {} + + ~sqlite_resultset() + { + if (stmt_) + { + sqlite3_finalize(stmt_); + } + } + + bool is_valid() { return stmt_ != 0; } + + bool step_next() + { + const int status = sqlite3_step(stmt_); + if (status != SQLITE_ROW && status != SQLITE_DONE) + { + std::ostringstream s; + s << "SQLite Plugin: retrieving next row failed"; + std::string msg(sqlite3_errmsg(sqlite3_db_handle(stmt_))); + if (msg != "unknown error") + { + s << ": " << msg; + } + + throw mapnik::datasource_exception(s.str()); + } + return status == SQLITE_ROW; + } + + int column_count() { return sqlite3_column_count(stmt_); } + + int column_type(int col) { return sqlite3_column_type(stmt_, col); } + + const char* column_name(int col) { return sqlite3_column_name(stmt_, col); } + + bool column_isnull(int col) { return sqlite3_column_type(stmt_, col) == SQLITE_NULL; } + + int column_integer(int col) { return sqlite3_column_int(stmt_, col); } + + sqlite_int64 column_integer64(int col) { return sqlite3_column_int64(stmt_, col); } + + double column_double(int col) { return sqlite3_column_double(stmt_, col); } + + const char* column_text(int col, int& len) + { + len = sqlite3_column_bytes(stmt_, col); + return (const char*)sqlite3_column_text(stmt_, col); + } + + const char* column_text(int col) { return (const char*)sqlite3_column_text(stmt_, col); } + + const char* column_blob(int col, int& bytes) + { + bytes = sqlite3_column_bytes(stmt_, col); + return (const char*)sqlite3_column_blob(stmt_, col); + } + + sqlite3_stmt* get_statement() { return stmt_; } + + private: + + sqlite3_stmt* stmt_; +}; + +#endif // MAPNIK_SQLITE_RESULTSET_HPP diff --git a/src/vector_tile_compression.cpp b/src/vector_tile_compression.cpp new file mode 100644 index 0000000..337d3d2 --- /dev/null +++ b/src/vector_tile_compression.cpp @@ -0,0 +1,40 @@ +#include "vector_tile_compression.hpp" + +// zlib +#include + +// std +#include + +// decodes both zlib and gzip +// http://stackoverflow.com/a/1838702/2333354 +void mapnik::vector_tile_impl::zlib_decompress(const char * data, std::size_t size, std::string & output) +{ + z_stream inflate_s; + inflate_s.zalloc = Z_NULL; + inflate_s.zfree = Z_NULL; + inflate_s.opaque = Z_NULL; + inflate_s.avail_in = 0; + inflate_s.next_in = Z_NULL; + inflateInit2(&inflate_s, 32 + 15); + inflate_s.next_in = (Bytef *)data; + inflate_s.avail_in = size; + size_t length = 0; + do { + output.resize(length + 2 * size); + inflate_s.avail_out = 2 * size; + inflate_s.next_out = (Bytef *)(output.data() + length); + int ret = inflate(&inflate_s, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) + { + std::string error_msg = inflate_s.msg; + inflateEnd(&inflate_s); + throw std::runtime_error(error_msg); + } + + length += (2 * size - inflate_s.avail_out); + } while (inflate_s.avail_out == 0); + inflateEnd(&inflate_s); + output.resize(length); +} + diff --git a/src/vector_tile_compression.hpp b/src/vector_tile_compression.hpp new file mode 100644 index 0000000..1476d17 --- /dev/null +++ b/src/vector_tile_compression.hpp @@ -0,0 +1,41 @@ +#ifndef VECTOR_TILE_COMPRESSION_HPP_ +#define VECTOR_TILE_COMPRESSION_HPP_ + +#include +// zlib +#include + +namespace mapnik +{ + +namespace vector_tile_impl +{ + +inline bool is_zlib_compressed(const char * data, std::size_t size) +{ + return size > 2 && + static_cast(data[0]) == 0x78 && + ( + static_cast(data[1]) == 0x9C || + static_cast(data[1]) == 0x01 || + static_cast(data[1]) == 0xDA || + static_cast(data[1]) == 0x5E + ); +} + +inline bool is_gzip_compressed(const char * data, std::size_t size) +{ + return size > 2 && static_cast(data[0]) == 0x1F && static_cast(data[1]) == 0x8B; +} + +// decodes both zlib and gzip +// http://stackoverflow.com/a/1838702/2333354 +void zlib_decompress(const char * data, + std::size_t size, + std::string & output); + +} // end ns vector_tile_impl + +} // end ns mapnik + +#endif /* VECTOR_TILE_COMPRESSION_HPP_ */ diff --git a/src/vector_tile_compression.ipp b/src/vector_tile_compression.ipp new file mode 100644 index 0000000..10904d6 --- /dev/null +++ b/src/vector_tile_compression.ipp @@ -0,0 +1,39 @@ +// zlib +#include + +// std +#include + +#include "vector_tile_compression.hpp" + +// decodes both zlib and gzip +// http://stackoverflow.com/a/1838702/2333354 +void mapnik::vector_tile_impl::zlib_decompress(const char * data, std::size_t size, std::string & output) +{ + z_stream inflate_s; + inflate_s.zalloc = Z_NULL; + inflate_s.zfree = Z_NULL; + inflate_s.opaque = Z_NULL; + inflate_s.avail_in = 0; + inflate_s.next_in = Z_NULL; + inflateInit2(&inflate_s, 32 + 15); + inflate_s.next_in = (Bytef *)data; + inflate_s.avail_in = size; + size_t length = 0; + do { + output.resize(length + 2 * size); + inflate_s.avail_out = 2 * size; + inflate_s.next_out = (Bytef *)(output.data() + length); + int ret = inflate(&inflate_s, Z_FINISH); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) + { + std::string error_msg = inflate_s.msg; + inflateEnd(&inflate_s); + throw std::runtime_error(error_msg); + } + + length += (2 * size - inflate_s.avail_out); + } while (inflate_s.avail_out == 0); + inflateEnd(&inflate_s); + output.resize(length); +} diff --git a/src/vector_tile_geometry_decoder.cpp b/src/vector_tile_geometry_decoder.cpp new file mode 100644 index 0000000..e868068 --- /dev/null +++ b/src/vector_tile_geometry_decoder.cpp @@ -0,0 +1,44 @@ +#include "vector_tile_geometry_decoder.hpp" +#include "vector_tile_geometry_decoder.ipp" + +namespace mapnik +{ + +namespace vector_tile_impl +{ + +// decode geometry +template mapnik::geometry::geometry decode_geometry(GeometryPBF & paths, + mvt_message::geom_type geom_type, + unsigned version, + double tile_x, + double tile_y, + double scale_x, + double scale_y); +template mapnik::geometry::geometry decode_geometry(GeometryPBF & paths, + mvt_message::geom_type geom_type, + unsigned version, + std::int64_t tile_x, + std::int64_t tile_y, + double scale_x, + double scale_y); +template mapnik::geometry::geometry decode_geometry(GeometryPBF & paths, + mvt_message::geom_type geom_type, + unsigned version, + double tile_x, + double tile_y, + double scale_x, + double scale_y, + mapnik::box2d const& bbox); +template mapnik::geometry::geometry decode_geometry(GeometryPBF & paths, + mvt_message::geom_type geom_type, + unsigned version, + std::int64_t tile_x, + std::int64_t tile_y, + double scale_x, + double scale_y, + mapnik::box2d const& bbox); + +} // end ns vector_tile_impl + +} // end ns mapnik diff --git a/src/vector_tile_geometry_decoder.hpp b/src/vector_tile_geometry_decoder.hpp new file mode 100644 index 0000000..9268c8a --- /dev/null +++ b/src/vector_tile_geometry_decoder.hpp @@ -0,0 +1,90 @@ +#ifndef VECTOR_TILE_GEOMETRY_DECODER_HPP_ +#define VECTOR_TILE_GEOMETRY_DECODER_HPP_ + +//protozero +#include + +//mapnik +#include +#include +#if defined(DEBUG) +#include +#endif +#include "mvt_message.hpp" + +//std +#include +#include +#include + +namespace mapnik +{ + +namespace vector_tile_impl +{ + +// NOTE: this object is for one-time use. Once you've progressed to the end +// by calling next(), to re-iterate, you must construct a new object +class GeometryPBF +{ +public: + using value_type = std::int64_t; + using iterator_type = protozero::pbf_reader::const_uint32_iterator; + using pbf_itr = protozero::iterator_range; + + explicit GeometryPBF(pbf_itr const& geo_iterator); + + enum command : uint8_t + { + end = 0, + move_to = 1, + line_to = 2, + close = 7 + }; + + uint32_t get_length() const + { + return length; + } + + command point_next(value_type & rx, value_type & ry); + command line_next(value_type & rx, value_type & ry, bool skip_lineto_zero); + command ring_next(value_type & rx, value_type & ry, bool skip_lineto_zero); + +private: + iterator_type geo_itr_; + iterator_type geo_end_itr_; + value_type x, y; + value_type ox, oy; + uint32_t length; + uint8_t cmd; + #if defined(DEBUG) +public: + bool already_had_error; + #endif +}; + +template +mapnik::geometry::geometry decode_geometry(GeometryPBF & paths, + mvt_message::geom_type geom_type, + unsigned version, + value_type tile_x, + value_type tile_y, + double scale_x, + double scale_y, + mapnik::box2d const& bbox); + +template +mapnik::geometry::geometry decode_geometry(GeometryPBF & paths, + mvt_message::geom_type geom_type, + unsigned version, + value_type tile_x, + value_type tile_y, + double scale_x, + double scale_y); + +} // end ns vector_tile_impl + +} // end ns mapnik + +#endif /* VECTOR_TILE_GEOMETRY_DECODER_HPP_ */ diff --git a/src/vector_tile_geometry_decoder.ipp b/src/vector_tile_geometry_decoder.ipp new file mode 100644 index 0000000..16a6951 --- /dev/null +++ b/src/vector_tile_geometry_decoder.ipp @@ -0,0 +1,848 @@ +//protozero +#include + +//mapnik +#include +#include +#if defined(DEBUG) +#include +#endif + +//std +#include +#include +#include + +namespace mapnik +{ + +namespace vector_tile_impl +{ + +namespace detail +{ + +template +inline double calculate_segment_area(value_type const x0, value_type const y0, value_type const x1, value_type const y1) +{ + return (static_cast(x0) * static_cast(y1)) - (static_cast(y0) * static_cast(x1)); +} + +inline bool area_is_clockwise(double area) +{ + return (area < 0.0); +} + +template +inline bool scaling_reversed_orientation(value_type const scale_x_, value_type const scale_y_) +{ + return (scale_x_ * scale_y_) < 0; +} + +template +inline void move_cursor(value_type & x, value_type & y, std::int32_t dx, std::int32_t dy) +{ + x += static_cast(dx); + y += static_cast(dy); +} + + +template +inline value_type get_point_value(value_type const val, + double const scale_val, + double const tile_loc) +{ + return (tile_loc + static_cast(std::round(static_cast(val) / scale_val))); +} + +template <> +inline double get_point_value(double const val, + double const scale_val, + double const tile_loc) +{ + return tile_loc + (val / scale_val); +} + +constexpr std::size_t max_reserve() +{ + // Based on int64_t geometry being 16 bytes in size and + // maximum allocation size of 1 MB. + return (1024 * 1024) / 16; +} + +template +void decode_point(mapnik::geometry::geometry & geom, + GeometryPBF & paths, + geom_value_type const tile_x, + geom_value_type const tile_y, + double const scale_x, + double const scale_y, + mapnik::box2d const& bbox) +{ + typename GeometryPBF::command cmd; + using pbf_value_type = GeometryPBF::value_type; + pbf_value_type x1, y1; + mapnik::geometry::multi_point mp; + #if defined(DEBUG) + std::uint32_t previous_len = 0; + #endif + // Find first moveto inside bbox and then reserve points from size of geometry. + while (true) + { + cmd = paths.point_next(x1, y1); + geom_value_type x1_ = get_point_value(x1, scale_x, tile_x); + geom_value_type y1_ = get_point_value(y1, scale_y, tile_y); + if (cmd == GeometryPBF::end) + { + geom = mapnik::geometry::geometry_empty(); + return; + } + else if (bbox.intersects(x1_, y1_)) + { + #if defined(DEBUG) + if (previous_len <= paths.get_length() && !paths.already_had_error) + { + MAPNIK_LOG_WARN(decode_point) << "warning: encountered POINT geometry that might have MOVETO commands repeated that could be fewer commands"; + paths.already_had_error = true; + } + previous_len = paths.get_length(); + #endif + constexpr std::size_t max_size = max_reserve(); + if (paths.get_length() + 1 < max_size) + { + mp.reserve(paths.get_length() + 1); + } + else + { + mp.reserve(max_size); + } + mp.emplace_back(x1_, y1_); + break; + } + } + while ((cmd = paths.point_next(x1, y1)) != GeometryPBF::end) + { + #if defined(DEBUG) + if (previous_len <= paths.get_length() && !paths.already_had_error) + { + MAPNIK_LOG_WARN(decode_point) << "warning: encountered POINT geometry that might have MOVETO commands repeated that could be fewer commands"; + paths.already_had_error = true; + } + previous_len = paths.get_length(); + #endif + // TODO: consider profiling and trying to optimize this further + // when all points are within the bbox filter then the `mp.reserve` should be + // perfect, but when some points are thrown out we will allocate more than needed + // the "all points intersect" case I think is going to be more common/important + // however worth a future look to see if the "some or few points intersect" can be optimized + geom_value_type x1_ = get_point_value(x1, scale_x, tile_x); + geom_value_type y1_ = get_point_value(y1, scale_y, tile_y); + if (!bbox.intersects(x1_, y1_)) + { + continue; + } + mp.emplace_back(x1_, y1_); + } + std::size_t num_points = mp.size(); + if (num_points == 0) + { + geom = mapnik::geometry::geometry_empty(); + } + else if (num_points == 1) + { + geom = std::move(mp[0]); + } + else if (num_points > 1) + { + // return multipoint + geom = std::move(mp); + } +} + +template +void decode_linestring(mapnik::geometry::geometry & geom, + GeometryPBF & paths, + geom_value_type const tile_x, + geom_value_type const tile_y, + double scale_x, + double scale_y, + mapnik::box2d const& bbox, + unsigned version) +{ + using pbf_value_type = GeometryPBF::value_type; + typename GeometryPBF::command cmd; + pbf_value_type x0, y0; + pbf_value_type x1, y1; + geom_value_type x0_, y0_; + geom_value_type x1_, y1_; + mapnik::geometry::multi_line_string multi_line; + #if defined(DEBUG) + std::uint32_t previous_len = 0; + #endif + mapnik::box2d part_env; + cmd = paths.line_next(x0, y0, false); + if (cmd == GeometryPBF::end) + { + geom = mapnik::geometry::geometry_empty(); + return; + } + else if (cmd != GeometryPBF::move_to) + { + throw std::runtime_error("Vector Tile has LINESTRING type geometry where the first command is not MOVETO."); + } + + while (true) + { + cmd = paths.line_next(x1, y1, true); + if (cmd != GeometryPBF::line_to) + { + if (cmd == GeometryPBF::move_to) + { + if (version == 1) + { + // Version 1 of the spec wasn't clearly defined and therefore + // we shouldn't be strict on the reading of a tile that has two + // moveto commands that are repeated, lets ignore the previous moveto. + x0 = x1; + y0 = y1; + continue; + } + else + { + throw std::runtime_error("Vector Tile has LINESTRING type geometry with repeated MOVETO commands."); + } + } + else //cmd == GeometryPBF::end + { + if (version == 1) + { + // Version 1 of the spec wasn't clearly defined and therefore + // we shouldn't be strict on the reading of a tile that has only a moveto + // command. So lets just ignore this moveto command. + break; + } + else + { + throw std::runtime_error("Vector Tile has LINESTRING type geometry with a MOVETO command with no LINETO following."); + } + } + } + // add fresh line + multi_line.emplace_back(); + auto & line = multi_line.back(); + // reserve prior + constexpr std::size_t max_size = max_reserve(); + if (paths.get_length() + 2 < max_size) + { + line.reserve(paths.get_length() + 2); + } + else + { + line.reserve(max_size); + } + // add moveto command position + x0_ = get_point_value(x0, scale_x, tile_x); + y0_ = get_point_value(y0, scale_y, tile_y); + line.emplace_back(x0_, y0_); + part_env.init(x0_, y0_, x0_, y0_); + // add first lineto + x1_ = get_point_value(x1, scale_x, tile_x); + y1_ = get_point_value(y1, scale_y, tile_y); + line.emplace_back(x1_, y1_); + part_env.expand_to_include(x1_, y1_); + #if defined(DEBUG) + previous_len = paths.get_length(); + #endif + while ((cmd = paths.line_next(x1, y1, true)) == GeometryPBF::line_to) + { + x1_ = get_point_value(x1, scale_x, tile_x); + y1_ = get_point_value(y1, scale_y, tile_y); + line.emplace_back(x1_, y1_); + part_env.expand_to_include(x1_, y1_); + #if defined(DEBUG) + if (previous_len <= paths.get_length() && !paths.already_had_error) + { + MAPNIK_LOG_WARN(decode_linestring) << "warning: encountered LINESTRING geometry that might have LINETO commands repeated that could be fewer commands"; + paths.already_had_error = true; + } + previous_len = paths.get_length(); + #endif + } + if (!bbox.intersects(part_env)) + { + // remove last linestring + multi_line.pop_back(); + } + if (cmd == GeometryPBF::end) + { + break; + } + // else we are guaranteed it is a moveto + x0 = x1; + y0 = y1; + } + + std::size_t num_lines = multi_line.size(); + if (num_lines == 0) + { + geom = mapnik::geometry::geometry_empty(); + } + else if (num_lines == 1) + { + auto itr = std::make_move_iterator(multi_line.begin()); + if (itr->size() > 1) + { + geom = std::move(*itr); + } + else + { + geom = mapnik::geometry::geometry_empty(); + } + } + else if (num_lines > 1) + { + geom = std::move(multi_line); + } +} + +template +void decode_polygon(mapnik::geometry::geometry & geom, + GeometryPBF & paths, + geom_value_type const tile_x, + geom_value_type const tile_y, + double scale_x, + double scale_y, + mapnik::box2d const& bbox, + unsigned version) +{ + typename GeometryPBF::command cmd; + using pbf_value_type = GeometryPBF::value_type; + pbf_value_type x0, y0; + pbf_value_type x1, y1; + pbf_value_type x2, y2; + geom_value_type x0_, y0_; + geom_value_type x1_, y1_; + geom_value_type x2_, y2_; + #if defined(DEBUG) + std::uint32_t previous_len; + #endif + double ring_area = 0.0; + bool first_ring = true; + bool first_ring_is_clockwise = false; + bool last_exterior_not_included = false; + std::vector > rings; + std::vector rings_exterior; + mapnik::box2d part_env; + cmd = paths.ring_next(x0, y0, false); + if (cmd == GeometryPBF::end) + { + geom = mapnik::geometry::geometry_empty(); + return; + } + else if (cmd != GeometryPBF::move_to) + { + throw std::runtime_error("Vector Tile has POLYGON type geometry where the first command is not MOVETO."); + } + + while (true) + { + cmd = paths.ring_next(x1, y1, true); + if (cmd != GeometryPBF::line_to) + { + if (cmd == GeometryPBF::close && version == 1) + { + // Version 1 of the specification was not clear on the command requirements for polygons + // lets just to recover from this situation. + cmd = paths.ring_next(x0, y0, false); + if (cmd == GeometryPBF::end) + { + break; + } + else if (cmd == GeometryPBF::move_to) + { + continue; + } + else if (cmd == GeometryPBF::close) + { + throw std::runtime_error("Vector Tile has POLYGON type geometry where a CLOSE is followed by a CLOSE."); + } + else // cmd == GeometryPBF::line_to + { + throw std::runtime_error("Vector Tile has POLYGON type geometry where a CLOSE is followed by a LINETO."); + } + } + else // cmd == end || cmd == move_to + { + throw std::runtime_error("Vector Tile has POLYGON type geometry with a MOVETO command with out at least two LINETOs and CLOSE following."); + } + } + #if defined(DEBUG) + previous_len = paths.get_length(); + #endif + cmd = paths.ring_next(x2, y2, true); + if (cmd != GeometryPBF::line_to) + { + if (cmd == GeometryPBF::close && version == 1) + { + // Version 1 of the specification was not clear on the command requirements for polygons + // lets just to recover from this situation. + cmd = paths.ring_next(x0, y0, false); + if (cmd == GeometryPBF::end) + { + break; + } + else if (cmd == GeometryPBF::move_to) + { + continue; + } + else if (cmd == GeometryPBF::close) + { + throw std::runtime_error("Vector Tile has POLYGON type geometry where a CLOSE is followed by a CLOSE."); + } + else // cmd == GeometryPBF::line_to + { + throw std::runtime_error("Vector Tile has POLYGON type geometry where a CLOSE is followed by a LINETO."); + } + } + else // cmd == end || cmd == move_to + { + throw std::runtime_error("Vector Tile has POLYGON type geometry with a MOVETO command with out at least two LINETOs and CLOSE following."); + } + } + // add new ring to start adding to + rings.emplace_back(); + auto & ring = rings.back(); + // reserve prior + constexpr std::size_t max_size = max_reserve(); + if (paths.get_length() + 4 < max_size) + { + ring.reserve(paths.get_length() + 4); + } + else + { + ring.reserve(max_size); + } + // add moveto command position + x0_ = get_point_value(x0, scale_x, tile_x); + y0_ = get_point_value(y0, scale_y, tile_y); + ring.emplace_back(x0_, y0_); + part_env.init(x0_, y0_, x0_, y0_); + // add first lineto + x1_ = get_point_value(x1, scale_x, tile_x); + y1_ = get_point_value(y1, scale_y, tile_y); + ring.emplace_back(x1_, y1_); + part_env.expand_to_include(x1_, y1_); + ring_area += calculate_segment_area(x0, y0, x1, y1); + // add second lineto + x2_ = get_point_value(x2, scale_x, tile_x); + y2_ = get_point_value(y2, scale_y, tile_y); + ring.emplace_back(x2_, y2_); + part_env.expand_to_include(x2_, y2_); + ring_area += calculate_segment_area(x1, y1, x2, y2); + x1 = x2; + y1 = y2; + #if defined(DEBUG) + if (previous_len <= paths.get_length() && !paths.already_had_error) + { + MAPNIK_LOG_WARN(read_rings) << "warning: encountered POLYGON geometry that might have LINETO commands repeated that could be fewer commands"; + paths.already_had_error = true; + } + previous_len = paths.get_length(); + #endif + while ((cmd = paths.ring_next(x2, y2, true)) == GeometryPBF::line_to) + { + x2_ = get_point_value(x2, scale_x, tile_x); + y2_ = get_point_value(y2, scale_y, tile_y); + ring.emplace_back(x2_, y2_); + part_env.expand_to_include(x2_, y2_); + ring_area += calculate_segment_area(x1, y1, x2, y2); + x1 = x2; + y1 = y2; + #if defined(DEBUG) + if (previous_len <= paths.get_length() && !paths.already_had_error) + { + MAPNIK_LOG_WARN(read_rings) << "warning: encountered POLYGON geometry that might have LINETO commands repeated that could be fewer commands"; + paths.already_had_error = true; + } + previous_len = paths.get_length(); + #endif + } + // Make sure we are now on a close command + if (cmd != GeometryPBF::close) + { + throw std::runtime_error("Vector Tile has POLYGON type geometry with a ring not closed by a CLOSE command."); + } + if (ring.back().x != x0_ || ring.back().y != y0_) + { + // If the previous lineto didn't already close the polygon (WHICH IT SHOULD NOT) + // close out the polygon ring. + ring.emplace_back(x0_, y0_); + ring_area += calculate_segment_area(x1, y1, x0, y0); + } + if (ring.size() > 3) + { + if (first_ring) + { + first_ring_is_clockwise = area_is_clockwise(ring_area); + if (version != 1 && first_ring_is_clockwise) + { + throw std::runtime_error("Vector Tile has POLYGON with first ring clockwise. It is not valid according to v2 of VT spec."); + } + first_ring = false; + } + bool is_exterior = (first_ring_is_clockwise == area_is_clockwise(ring_area)); + if ((!is_exterior && last_exterior_not_included) || !bbox.intersects(part_env)) + { + // remove last linestring + if (is_exterior) + { + last_exterior_not_included = true; + } + rings.pop_back(); + } + else + { + if (is_exterior) + { + last_exterior_not_included = false; + } + rings_exterior.push_back(is_exterior); + } + } + else + { + rings.pop_back(); + } + ring_area = 0.0; + + cmd = paths.ring_next(x0, y0, false); + if (cmd == GeometryPBF::end) + { + break; + } + else if (cmd != GeometryPBF::move_to) + { + if (cmd == GeometryPBF::close) + { + throw std::runtime_error("Vector Tile has POLYGON type geometry where a CLOSE is followed by a CLOSE."); + } + else // cmd == GeometryPBF::line_to + { + throw std::runtime_error("Vector Tile has POLYGON type geometry where a CLOSE is followed by a LINETO."); + } + } + } + + if (rings.size() == 0) + { + geom = mapnik::geometry::geometry_empty(); + return; + } + + bool reverse_rings = (scaling_reversed_orientation(scale_x, scale_y) != first_ring_is_clockwise); + auto rings_itr = std::make_move_iterator(rings.begin()); + auto rings_end = std::make_move_iterator(rings.end()); + mapnik::geometry::multi_polygon multi_poly; + for (std::size_t i = 0; rings_itr != rings_end; ++rings_itr,++i) + { + if (rings_exterior[i]) multi_poly.emplace_back(); + auto & poly = multi_poly.back(); + if (reverse_rings) + { + std::reverse(rings_itr->begin(), rings_itr->end()); + } + poly.push_back(std::move(*rings_itr)); + } + auto num_poly = multi_poly.size(); + if (num_poly == 1) + { + auto itr = std::make_move_iterator(multi_poly.begin()); + geom = std::move(*itr); + } + else + { + geom = std::move(multi_poly); + } +} + +} // end ns detail + +GeometryPBF::GeometryPBF(pbf_itr const& geo_iterator) + : geo_itr_(geo_iterator.begin()), + geo_end_itr_(geo_iterator.end()), + x(0), + y(0), + ox(0), + oy(0), + length(0), + cmd(move_to) +{ + #if defined(DEBUG) + already_had_error = false; + #endif +} + +typename GeometryPBF::command GeometryPBF::point_next(value_type & rx, value_type & ry) +{ + if (length == 0) + { + if (geo_itr_ != geo_end_itr_) + { + uint32_t cmd_length = static_cast(*geo_itr_++); + cmd = cmd_length & 0x7; + length = cmd_length >> 3; + if (cmd == move_to) + { + if (length == 0) + { + throw std::runtime_error("Vector Tile has POINT geometry with a MOVETO command that has a command count of zero"); + } + } + else + { + if (cmd == line_to) + { + throw std::runtime_error("Vector Tile has POINT type geometry with a LINETO command."); + } + else if (cmd == close) + { + throw std::runtime_error("Vector Tile has POINT type geometry with a CLOSE command."); + } + else + { + throw std::runtime_error("Vector Tile has POINT type geometry with an unknown command."); + } + } + } + else + { + return end; + } + } + + --length; + // It is possible for the next to lines to throw because we can not check the length + // of the buffer to ensure that it is long enough. + // If an exception occurs it will likely be a end_of_buffer_exception with the text: + // "end of buffer exception" + // While this error message is not verbose a try catch here would slow down processing. + int32_t dx = protozero::decode_zigzag32(static_cast(*geo_itr_++)); + int32_t dy = protozero::decode_zigzag32(static_cast(*geo_itr_++)); + detail::move_cursor(x, y, dx, dy); + rx = x; + ry = y; + return move_to; +} + +typename GeometryPBF::command GeometryPBF::line_next(value_type & rx, + value_type & ry, + bool skip_lineto_zero) +{ + if (length == 0) + { + if (geo_itr_ != geo_end_itr_) + { + uint32_t cmd_length = static_cast(*geo_itr_++); + cmd = cmd_length & 0x7; + length = cmd_length >> 3; + if (cmd == move_to) + { + if (length != 1) + { + throw std::runtime_error("Vector Tile has LINESTRING with a MOVETO command that is given more then one pair of parameters or not enough parameters are provided"); + } + --length; + // It is possible for the next to lines to throw because we can not check the length + // of the buffer to ensure that it is long enough. + // If an exception occurs it will likely be a end_of_buffer_exception with the text: + // "end of buffer exception" + // While this error message is not verbose a try catch here would slow down processing. + int32_t dx = protozero::decode_zigzag32(static_cast(*geo_itr_++)); + int32_t dy = protozero::decode_zigzag32(static_cast(*geo_itr_++)); + detail::move_cursor(x, y, dx, dy); + rx = x; + ry = y; + return move_to; + } + else if (cmd == line_to) + { + if (length == 0) + { + throw std::runtime_error("Vector Tile has geometry with LINETO command that is not followed by a proper number of parameters"); + } + } + else + { + if (cmd == close) + { + throw std::runtime_error("Vector Tile has LINESTRING type geometry with a CLOSE command."); + } + else + { + throw std::runtime_error("Vector Tile has LINESTRING type geometry with an unknown command."); + } + } + } + else + { + return end; + } + } + + --length; + // It is possible for the next to lines to throw because we can not check the length + // of the buffer to ensure that it is long enough. + // If an exception occurs it will likely be a end_of_buffer_exception with the text: + // "end of buffer exception" + // While this error message is not verbose a try catch here would slow down processing. + int32_t dx = protozero::decode_zigzag32(static_cast(*geo_itr_++)); + int32_t dy = protozero::decode_zigzag32(static_cast(*geo_itr_++)); + if (skip_lineto_zero && dx == 0 && dy == 0) + { + // We are going to skip this vertex as the point doesn't move call line_next again + return line_next(rx, ry, true); + } + detail::move_cursor(x, y, dx, dy); + rx = x; + ry = y; + return line_to; +} + +typename GeometryPBF::command GeometryPBF::ring_next(value_type & rx, + value_type & ry, + bool skip_lineto_zero) +{ + if (length == 0) + { + if (geo_itr_ != geo_end_itr_) + { + uint32_t cmd_length = static_cast(*geo_itr_++); + cmd = cmd_length & 0x7; + length = cmd_length >> 3; + if (cmd == move_to) + { + if (length != 1) + { + throw std::runtime_error("Vector Tile has POLYGON with a MOVETO command that is given more then one pair of parameters or not enough parameters are provided"); + } + --length; + // It is possible for the next two lines to throw because we can not check the length + // of the buffer to ensure that it is long enough. + // If an exception occurs it will likely be a end_of_buffer_exception with the text: + // "end of buffer exception" + // While this error message is not verbose a try catch here would slow down processing. + int32_t dx = protozero::decode_zigzag32(static_cast(*geo_itr_++)); + int32_t dy = protozero::decode_zigzag32(static_cast(*geo_itr_++)); + detail::move_cursor(x, y, dx, dy); + rx = x; + ry = y; + ox = x; + oy = y; + return move_to; + } + else if (cmd == line_to) + { + if (length == 0) + { + throw std::runtime_error("Vector Tile has geometry with LINETO command that is not followed by a proper number of parameters"); + } + } + else if (cmd == close) + { + // Just set length in case a close command provides an invalid number here. + // While we could throw because V2 of the spec declares it incorrect, this is not + // difficult to fix and has no effect on the results. + length = 0; + rx = ox; + ry = oy; + return close; + } + else + { + throw std::runtime_error("Vector Tile has POLYGON type geometry with an unknown command."); + } + } + else + { + return end; + } + } + + --length; + // It is possible for the next to lines to throw because we can not check the length + // of the buffer to ensure that it is long enough. + // If an exception occurs it will likely be a end_of_buffer_exception with the text: + // "end of buffer exception" + // While this error message is not verbose a try catch here would slow down processing. + int32_t dx = protozero::decode_zigzag32(static_cast(*geo_itr_++)); + int32_t dy = protozero::decode_zigzag32(static_cast(*geo_itr_++)); + if (skip_lineto_zero && dx == 0 && dy == 0) + { + // We are going to skip this vertex as the point doesn't move call ring_next again + return ring_next(rx, ry, true); + } + detail::move_cursor(x, y, dx, dy); + rx = x; + ry = y; + return line_to; +} + +template +mapnik::geometry::geometry decode_geometry(GeometryPBF & paths, + mvt_message::geom_type geom_type, + unsigned version, + value_type tile_x, + value_type tile_y, + double scale_x, + double scale_y, + mapnik::box2d const& bbox) +{ + mapnik::geometry::geometry geom; // output geometry + switch (geom_type) + { + case mvt_message::geom_type::point: + { + detail::decode_point(geom, paths, tile_x, tile_y, scale_x, scale_y, bbox); + break; + } + case mvt_message::geom_type::linestring: + { + detail::decode_linestring(geom, paths, tile_x, tile_y, scale_x, scale_y, bbox, version); + break; + } + case mvt_message::geom_type::polygon: + { + detail::decode_polygon(geom, paths, tile_x, tile_y, scale_x, scale_y, bbox, version); + break; + } + case mvt_message::geom_type::unknown: + default: + { + // This was changed to not throw as unknown according to v2 of spec can simply be ignored and doesn't require + // it failing the processing + geom = mapnik::geometry::geometry_empty(); + break; + } + } + return geom; +} + +template +mapnik::geometry::geometry decode_geometry(GeometryPBF & paths, + mvt_message::geom_type geom_type, + unsigned version, + value_type tile_x, + value_type tile_y, + double scale_x, + double scale_y) +{ + mapnik::box2d bbox(std::numeric_limits::lowest(), + std::numeric_limits::lowest(), + std::numeric_limits::max(), + std::numeric_limits::max()); + return decode_geometry(paths, geom_type, version, tile_x, tile_y, scale_x, scale_y, bbox); +} + +} // end ns vector_tile_impl + +} // end ns mapnik diff --git a/src/vector_tile_projection.hpp b/src/vector_tile_projection.hpp new file mode 100644 index 0000000..f07271c --- /dev/null +++ b/src/vector_tile_projection.hpp @@ -0,0 +1,31 @@ +#ifndef __MAPNIK_VECTOR_TILE_PROJECTION_H__ +#define __MAPNIK_VECTOR_TILE_PROJECTION_H__ + +// mapnik +#include +#include + +namespace mapnik +{ + +namespace vector_tile_impl +{ + +inline mapnik::box2d tile_mercator_bbox(std::uint64_t x, + std::uint64_t y, + std::uint64_t z) +{ + const double half_of_equator = M_PI * EARTH_RADIUS; + const double tile_size = 2.0 * half_of_equator / (1ull << z); + double minx = -half_of_equator + x * tile_size; + double miny = half_of_equator - (y + 1.0) * tile_size; + double maxx = -half_of_equator + (x + 1.0) * tile_size; + double maxy = half_of_equator - y * tile_size; + return mapnik::box2d(minx,miny,maxx,maxy); +} + +} // end vector_tile_impl ns + +} // end mapnik ns + +#endif // __MAPNIK_VECTOR_TILE_PROJECTION_H__