From 0d5d0d39ccffba658062164f99711e3f2620ad2a Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Mon, 30 May 2022 21:47:30 +0200 Subject: [PATCH 01/10] Avoid memory overflow when excluding building key --- src/overpass_api/statements/query.cc | 143 +++++++++++++++++++++------ src/overpass_api/statements/query.h | 6 ++ 2 files changed, 119 insertions(+), 30 deletions(-) diff --git a/src/overpass_api/statements/query.cc b/src/overpass_api/statements/query.cc index 8d9db3b0f..6411a5345 100644 --- a/src/overpass_api/statements/query.cc +++ b/src/overpass_api/statements/query.cc @@ -178,7 +178,7 @@ void filter_id_list( { if (key_regex.matches(it.index().key) && it.index().value != void_tag_value() && val_regex.matches(it.index().value) && (!filtered || - binary_search(old_ids.begin(), old_ids.end(), std::make_pair(it.object().id, Uint31_Index(0u))))) + std::binary_search(old_ids.begin(), old_ids.end(), std::make_pair(it.object().id, Uint31_Index(0u))))) new_ids.push_back(std::make_pair(it.object().id, it.object().idx)); if (!filtered && new_ids.size() == 1024*1024) @@ -216,7 +216,7 @@ void filter_id_list( { if (key_regex.matches(it.index().key) && it.index().value != void_tag_value() && val_regex.matches(it.index().value) && - (!filtered || binary_search(old_ids.begin(), old_ids.end(), it.object()))) + (!filtered || std::binary_search(old_ids.begin(), old_ids.end(), it.object()))) new_ids.push_back(it.object()); } @@ -238,7 +238,7 @@ void filter_id_list( for (typename Container::const_iterator it = container.begin(); it != container.end(); ++it) { if (!filtered || - binary_search(old_ids.begin(), old_ids.end(), std::make_pair(it->first, Uint31_Index(0u)))) + std::binary_search(old_ids.begin(), old_ids.end(), std::make_pair(it->first, Uint31_Index(0u)))) new_ids.push_back(std::make_pair(it->first, it->second.second)); } @@ -431,6 +431,106 @@ std::vector< Id_Type > Query_Statement::collect_ids } +template< typename Id_Type > +class Id_Remover +{ +public: + Id_Remover(std::vector< std::pair< Id_Type, Uint31_Index > >& ids_) : ids(ids_), found(ids.size(), false) {} + ~Id_Remover() + { + auto t_it = ids.begin(); + decltype(found.size()) i = 0; + for (auto s_it = ids.begin(); s_it != ids.end(); ++s_it) + { + if (!found[i++]) + { + *t_it = *s_it; + ++t_it; + } + } + ids.erase(t_it, ids.end()); + } + void remove(Id_Type id) + { + auto it = std::lower_bound(ids.begin(), ids.end(), std::pair< Id_Type, Uint31_Index >{ id, Uint31_Index(0u) }); + if (it != ids.end() && it->first == id) + found[std::distance(ids.begin(), it)] = true; + } + +private: + std::vector< std::pair< Id_Type, Uint31_Index > >& ids; + std::vector< bool > found; +}; + + +template< class Id_Type > +void Query_Statement::filter_non_ids + (std::vector< std::pair< Id_Type, Uint31_Index > >& ids, + const File_Properties& file_prop, const File_Properties& attic_file_prop, + Resource_Manager& rman, uint64 timestamp) +{ + if (key_nvalues.empty() && key_nregexes.empty()) + return; + + Block_Backend< Tag_Index_Global, Tag_Object_Global< Id_Type > > tags_db + (rman.get_transaction()->data_index(&file_prop)); + Optional< Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > > > attic_tags_db + (timestamp == NOW ? 0 : + new Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > > + (rman.get_transaction()->data_index(&attic_file_prop))); + Id_Remover< Id_Type > idr(ids); + + // Handle Key-Non-Value pairs + for (std::vector< std::pair< std::string, std::string > >::const_iterator knvit = key_nvalues.begin(); + knvit != key_nvalues.end(); ++knvit) + { + if (timestamp == NOW) + { + std::set< Tag_Index_Global > tag_req = get_kv_req(knvit->first, knvit->second); + for (typename Block_Backend< Tag_Index_Global, Tag_Object_Global< Id_Type > >::Discrete_Iterator + it2(tags_db.discrete_begin(tag_req.begin(), tag_req.end())); + !(it2 == tags_db.discrete_end()); ++it2) + idr.remove(it2.object().id); + } + else + { + std::map< Id_Type, std::pair< uint64, Uint31_Index > > timestamp_per_id + = collect_attic_kv(knvit, timestamp, tags_db, *attic_tags_db.obj); + + for (typename std::map< Id_Type, std::pair< uint64, Uint31_Index > >::const_iterator + it = timestamp_per_id.begin(); it != timestamp_per_id.end(); ++it) + idr.remove(it->first); + } + rman.health_check(*this); + } + + // Handle Key-Regular-Expression-Non-Value pairs + for (std::vector< std::pair< std::string, Regular_Expression* > >::const_iterator knrit = key_nregexes.begin(); + knrit != key_nregexes.end(); ++knrit) + { + if (timestamp == NOW) + { + Ranges< Tag_Index_Global > ranges = get_k_req(knrit->first); + for (auto it2 = tags_db.range_begin(ranges); !it2.is_end(); ++it2) + { + if (knrit->second->matches(it2.index().value)) + idr.remove(it2.object().id); + } + } + else + { + std::map< Id_Type, std::pair< uint64, Uint31_Index > > timestamp_per_id + = collect_attic_kregv(knrit, timestamp, tags_db, *attic_tags_db.obj); + + for (typename std::map< Id_Type, std::pair< uint64, Uint31_Index > >::const_iterator + it = timestamp_per_id.begin(); it != timestamp_per_id.end(); ++it) + idr.remove(it->first); + } + rman.health_check(*this); + } +} + + template< class Id_Type > std::vector< std::pair< Id_Type, Uint31_Index > > Query_Statement::collect_non_ids (const File_Properties& file_prop, const File_Properties& attic_file_prop, @@ -804,7 +904,7 @@ void filter_by_ids( for (typename std::vector< TObject >::const_iterator eit = item_it->second.begin(); eit != item_it->second.end(); ++eit) { - if (binary_search(it->second.begin(), it->second.end(), eit->id)) + if (std::binary_search(it->second.begin(), it->second.end(), eit->id)) result[item_it->first.val()].push_back(*eit); } ++item_it; @@ -825,7 +925,7 @@ void filter_by_ids( for (typename std::vector< TObject >::const_iterator eit = item_it->second.begin(); eit != item_it->second.end(); ++eit) { - if (binary_search(it->second.begin(), it->second.end(), eit->id)) + if (std::binary_search(it->second.begin(), it->second.end(), eit->id)) result[item_it->first.val()].push_back(*eit); } ++item_it; @@ -837,7 +937,7 @@ void filter_by_ids( for (typename std::vector< Attic< TObject > >::const_iterator eit = attic_item_it->second.begin(); eit != attic_item_it->second.end(); ++eit) { - if (binary_search(it->second.begin(), it->second.end(), eit->id)) + if (std::binary_search(it->second.begin(), it->second.end(), eit->id)) attic_result[attic_item_it->first.val()].push_back(*eit); } ++attic_item_it; @@ -1041,7 +1141,7 @@ void Query_Statement::filter_by_tags for (typename std::vector< TObject >::const_iterator eit = item_it->second.begin(); eit != item_it->second.end(); ++eit) { - if (binary_search(ids_by_coarse_.begin(), ids_by_coarse_.end(), eit->id)) + if (std::binary_search(ids_by_coarse_.begin(), ids_by_coarse_.end(), eit->id)) result[item_it->first.val()].push_back(*eit); } ++item_it; @@ -1091,7 +1191,7 @@ void Query_Statement::filter_by_tags for (typename std::vector< TObject >::const_iterator eit = item_it->second.begin(); eit != item_it->second.end(); ++eit) { - if (binary_search(ids_by_coarse_.begin(), ids_by_coarse_.end(), eit->id)) + if (std::binary_search(ids_by_coarse_.begin(), ids_by_coarse_.end(), eit->id)) result[item_it->first.val()].push_back(*eit); } ++item_it; @@ -1233,29 +1333,12 @@ void Query_Statement::progress_1( collect_ids< Skeleton, Id_Type >(file_prop, attic_file_prop, rman, timestamp, check_keys_late, result_valid); if (!key_nvalues.empty() || (check_keys_late != prefer_ranges && !key_nregexes.empty())) + filter_non_ids< Id_Type >(id_idxs, file_prop, attic_file_prop, rman, timestamp); + + for (auto it = id_idxs.begin(); it != id_idxs.end(); ++it) { - std::vector< std::pair< Id_Type, Uint31_Index > > non_ids - = collect_non_ids< Id_Type >(file_prop, attic_file_prop, rman, timestamp); - std::vector< std::pair< Id_Type, Uint31_Index > > diff_ids(id_idxs.size()); - diff_ids.erase(set_difference(id_idxs.begin(), id_idxs.end(), non_ids.begin(), non_ids.end(), - diff_ids.begin()), diff_ids.end()); - ids.ids.clear(); - range_vec.clear(); - for (typename std::vector< std::pair< Id_Type, Uint31_Index > >::const_iterator it = diff_ids.begin(); - it != diff_ids.end(); ++it) - { - ids.ids.push_back(it->first); - range_vec.push_back(it->second); - } - } - else - { - for (typename std::vector< std::pair< Id_Type, Uint31_Index > >::const_iterator it = id_idxs.begin(); - it != id_idxs.end(); ++it) - { - ids.ids.push_back(it->first); - range_vec.push_back(it->second); - } + ids.ids.push_back(it->first); + range_vec.push_back(it->second); } if (ids.ids.empty() && result_valid) diff --git a/src/overpass_api/statements/query.h b/src/overpass_api/statements/query.h index 8f3e74ee0..52c477f67 100644 --- a/src/overpass_api/statements/query.h +++ b/src/overpass_api/statements/query.h @@ -190,6 +190,12 @@ class Query_Statement : public Output_Statement (const File_Properties& file_prop, Resource_Manager& rman, Query_Filter_Strategy check_keys_late); + template< class Id_Type > + void filter_non_ids + (std::vector< std::pair< Id_Type, Uint31_Index > >& ids, + const File_Properties& file_prop, const File_Properties& attic_file_prop, + Resource_Manager& rman, uint64 timestamp); + template< class Id_Type > std::vector< std::pair< Id_Type, Uint31_Index > > collect_non_ids (const File_Properties& file_prop, const File_Properties& attic_file_prop, From bb9ccb2dede0af6da07b9dcbb68f7e92041abfdf Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Tue, 31 May 2022 22:14:54 +0200 Subject: [PATCH 02/10] Avoid memory overflow when excluding building key also for attic data --- src/overpass_api/data/filter_by_tags.h | 22 +++++++++++++++++----- src/overpass_api/statements/query.cc | 4 ++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/overpass_api/data/filter_by_tags.h b/src/overpass_api/data/filter_by_tags.h index 7be532a60..be20aec3b 100644 --- a/src/overpass_api/data/filter_by_tags.h +++ b/src/overpass_api/data/filter_by_tags.h @@ -79,7 +79,8 @@ template< class Id_Type > std::map< Id_Type, std::pair< uint64, Uint31_Index > > collect_attic_kv( std::vector< std::pair< std::string, std::string > >::const_iterator kvit, uint64 timestamp, Block_Backend< Tag_Index_Global, Tag_Object_Global< Id_Type > >& tags_db, - Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > >& attic_tags_db) + Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > >& attic_tags_db, + std::vector< std::pair< Id_Type, Uint31_Index > >* relevant_ids = 0) { std::map< Id_Type, std::pair< uint64, Uint31_Index > > timestamp_per_id; std::set< Tag_Index_Global > tag_req = get_kv_req(kvit->first, kvit->second); @@ -87,13 +88,19 @@ std::map< Id_Type, std::pair< uint64, Uint31_Index > > collect_attic_kv( for (typename Block_Backend< Tag_Index_Global, Tag_Object_Global< Id_Type > >::Discrete_Iterator it2(tags_db.discrete_begin(tag_req.begin(), tag_req.end())); !(it2 == tags_db.discrete_end()); ++it2) - timestamp_per_id[it2.object().id] = std::make_pair(NOW, it2.object().idx); + { + if (!relevant_ids || std::binary_search( + relevant_ids->begin(), relevant_ids->end(), std::make_pair(it2.object().id, Uint31_Index(0u)))) + timestamp_per_id[it2.object().id] = std::make_pair(NOW, it2.object().idx); + } for (typename Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > >::Discrete_Iterator it2(attic_tags_db.discrete_begin(tag_req.begin(), tag_req.end())); !(it2 == attic_tags_db.discrete_end()); ++it2) { - if (it2.object().timestamp > timestamp) + if (it2.object().timestamp > timestamp && + (!relevant_ids || std::binary_search( + relevant_ids->begin(), relevant_ids->end(), std::make_pair(it2.object().id, Uint31_Index(0u))))) { std::pair< uint64, Uint31_Index >& ref = timestamp_per_id[it2.object().id]; if (ref.first == 0 || it2.object().timestamp < ref.first) @@ -165,20 +172,25 @@ template< class Id_Type > std::map< Id_Type, std::pair< uint64, Uint31_Index > > collect_attic_kregv( std::vector< std::pair< std::string, Regular_Expression* > >::const_iterator krit, uint64 timestamp, Block_Backend< Tag_Index_Global, Tag_Object_Global< Id_Type > >& tags_db, - Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > >& attic_tags_db) + Block_Backend< Tag_Index_Global, Attic< Tag_Object_Global< Id_Type > > >& attic_tags_db, + std::vector< std::pair< Id_Type, Uint31_Index > >* relevant_ids = 0) { std::map< Id_Type, std::pair< uint64, Uint31_Index > > timestamp_per_id; Ranges< Tag_Index_Global > ranges = get_k_req(krit->first); for (auto it2 = tags_db.range_begin(ranges); !(it2 == tags_db.range_end()); ++it2) { - if (krit->second->matches(it2.index().value)) + if ((!relevant_ids || std::binary_search( + relevant_ids->begin(), relevant_ids->end(), std::make_pair(it2.object().id, Uint31_Index(0u)))) + && krit->second->matches(it2.index().value)) timestamp_per_id[it2.object().id] = std::make_pair(NOW, it2.object().idx); } for (auto it2 = attic_tags_db.range_begin(ranges); !(it2 == attic_tags_db.range_end()); ++it2) { if (it2.object().timestamp > timestamp && it2.index().value != void_tag_value() + && (!relevant_ids || std::binary_search( + relevant_ids->begin(), relevant_ids->end(), std::make_pair(it2.object().id, Uint31_Index(0u)))) && krit->second->matches(it2.index().value)) { std::pair< uint64, Uint31_Index >& ref = timestamp_per_id[it2.object().id]; diff --git a/src/overpass_api/statements/query.cc b/src/overpass_api/statements/query.cc index 6411a5345..91b798ee3 100644 --- a/src/overpass_api/statements/query.cc +++ b/src/overpass_api/statements/query.cc @@ -495,7 +495,7 @@ void Query_Statement::filter_non_ids else { std::map< Id_Type, std::pair< uint64, Uint31_Index > > timestamp_per_id - = collect_attic_kv(knvit, timestamp, tags_db, *attic_tags_db.obj); + = collect_attic_kv(knvit, timestamp, tags_db, *attic_tags_db.obj, &ids); for (typename std::map< Id_Type, std::pair< uint64, Uint31_Index > >::const_iterator it = timestamp_per_id.begin(); it != timestamp_per_id.end(); ++it) @@ -520,7 +520,7 @@ void Query_Statement::filter_non_ids else { std::map< Id_Type, std::pair< uint64, Uint31_Index > > timestamp_per_id - = collect_attic_kregv(knrit, timestamp, tags_db, *attic_tags_db.obj); + = collect_attic_kregv(knrit, timestamp, tags_db, *attic_tags_db.obj, &ids); for (typename std::map< Id_Type, std::pair< uint64, Uint31_Index > >::const_iterator it = timestamp_per_id.begin(); it != timestamp_per_id.end(); ++it) From 2cf59bd1020afbcbc558017cecac9d6da34e1f5b Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Tue, 31 May 2022 22:20:07 +0200 Subject: [PATCH 03/10] Automated commit for release 0.7.58.1 --- src/Makefile.am | 6 +++--- src/configure.ac | 6 +++--- src/overpass_api/core/settings.cc | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 9c2b3e858..ddcf8e58c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -210,7 +210,7 @@ bin_bbox_brim_query_LDADD = libexpatwrapper.la bin_translate_xapi_SOURCES = expat/translate_xapi.cc expat/escape_xml.cc bin_translate_xapi_LDADD = -distdir = osm-3s_v0.7.58 +distdir = osm-3s_v0.7.58.1 nobase_dist_HEADERS = \ expat/escape_json.h\ @@ -353,8 +353,8 @@ EXTRA_DIST = \ rules/areas_delta.osm3s # Put test-bin here and test-bin/Makefile in configure.ac to activate test-bin -SUBDIRS = test-bin -#SUBDIRS = +#SUBDIRS = test-bin +SUBDIRS = AM_CXXFLAGS = -std=c++11 diff --git a/src/configure.ac b/src/configure.ac index 9c0ad00f4..fd6ce5bc8 100644 --- a/src/configure.ac +++ b/src/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.61]) -AC_INIT([OverpassAPI], [0.7.58], [roland@olbricht.nrw]) +AC_INIT([OverpassAPI], [0.7.58.1], [roland@olbricht.nrw]) AC_CONFIG_SRCDIR([template_db/file_blocks.h]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) @@ -73,7 +73,7 @@ fi AC_SUBST(COMPRESS_LIBS, ["$COMPRESS_LIBS"]) -AC_CONFIG_FILES([Makefile test-bin/Makefile]) -#AC_CONFIG_FILES([Makefile]) +#AC_CONFIG_FILES([Makefile test-bin/Makefile]) +AC_CONFIG_FILES([Makefile]) AC_OUTPUT AC_SYS_LARGEFILE diff --git a/src/overpass_api/core/settings.cc b/src/overpass_api/core/settings.cc index 80b309308..3926b9c38 100644 --- a/src/overpass_api/core/settings.cc +++ b/src/overpass_api/core/settings.cc @@ -96,8 +96,8 @@ Basic_Settings::Basic_Settings() base_directory("./"), logfile_name("transactions.log"), shared_name_base("/osm3s_v0.7.58"), - version("0.7.58"), - source_hash("af71d8c8d17360a3b2956cf12edf463d6167fc0d"), + version("0.7.58.1"), + source_hash("bb9ccb2dede0af6da07b9dcbb68f7e92041abfdf"), #ifdef HAVE_LZ4 compression_method(File_Blocks_Index< Uint31_Index >::LZ4_COMPRESSION), #else From 2b5354b14bd520d6cdefea30d2f70c9be4b9cf23 Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Wed, 1 Jun 2022 23:02:28 +0200 Subject: [PATCH 04/10] Ensure that invert flag is set to true for key-values with huge count --- src/Makefile.am | 4 ++-- src/configure.ac | 4 ++-- src/overpass_api/statements/query.cc | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index ddcf8e58c..534960adc 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -353,8 +353,8 @@ EXTRA_DIST = \ rules/areas_delta.osm3s # Put test-bin here and test-bin/Makefile in configure.ac to activate test-bin -#SUBDIRS = test-bin -SUBDIRS = +SUBDIRS = test-bin +#SUBDIRS = AM_CXXFLAGS = -std=c++11 diff --git a/src/configure.ac b/src/configure.ac index fd6ce5bc8..ba1637d21 100644 --- a/src/configure.ac +++ b/src/configure.ac @@ -73,7 +73,7 @@ fi AC_SUBST(COMPRESS_LIBS, ["$COMPRESS_LIBS"]) -#AC_CONFIG_FILES([Makefile test-bin/Makefile]) -AC_CONFIG_FILES([Makefile]) +AC_CONFIG_FILES([Makefile test-bin/Makefile]) +#AC_CONFIG_FILES([Makefile]) AC_OUTPUT AC_SYS_LARGEFILE diff --git a/src/overpass_api/statements/query.cc b/src/overpass_api/statements/query.cc index 91b798ee3..b6d9ee5e5 100644 --- a/src/overpass_api/statements/query.cc +++ b/src/overpass_api/statements/query.cc @@ -1328,9 +1328,9 @@ void Query_Statement::progress_1( && (!keys.empty() || !key_regexes.empty() || !regkey_regexes.empty()))) { bool result_valid = true; - ids.invert = false; std::vector< std::pair< Id_Type, Uint31_Index > > id_idxs = collect_ids< Skeleton, Id_Type >(file_prop, attic_file_prop, rman, timestamp, check_keys_late, result_valid); + ids.invert = !result_valid; if (!key_nvalues.empty() || (check_keys_late != prefer_ranges && !key_nregexes.empty())) filter_non_ids< Id_Type >(id_idxs, file_prop, attic_file_prop, rman, timestamp); From 40bf20b9c97ed1b14cea7e2910726014a122690d Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Wed, 1 Jun 2022 23:02:33 +0200 Subject: [PATCH 05/10] Automated commit for release 0.7.58.2 --- src/Makefile.am | 2 +- src/configure.ac | 2 +- src/overpass_api/core/settings.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 534960adc..9ed39fcd2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -210,7 +210,7 @@ bin_bbox_brim_query_LDADD = libexpatwrapper.la bin_translate_xapi_SOURCES = expat/translate_xapi.cc expat/escape_xml.cc bin_translate_xapi_LDADD = -distdir = osm-3s_v0.7.58.1 +distdir = osm-3s_v0.7.58.2 nobase_dist_HEADERS = \ expat/escape_json.h\ diff --git a/src/configure.ac b/src/configure.ac index ba1637d21..722e43449 100644 --- a/src/configure.ac +++ b/src/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.61]) -AC_INIT([OverpassAPI], [0.7.58.1], [roland@olbricht.nrw]) +AC_INIT([OverpassAPI], [0.7.58.2], [roland@olbricht.nrw]) AC_CONFIG_SRCDIR([template_db/file_blocks.h]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/src/overpass_api/core/settings.cc b/src/overpass_api/core/settings.cc index 3926b9c38..8d178aa55 100644 --- a/src/overpass_api/core/settings.cc +++ b/src/overpass_api/core/settings.cc @@ -96,8 +96,8 @@ Basic_Settings::Basic_Settings() base_directory("./"), logfile_name("transactions.log"), shared_name_base("/osm3s_v0.7.58"), - version("0.7.58.1"), - source_hash("bb9ccb2dede0af6da07b9dcbb68f7e92041abfdf"), + version("0.7.58.2"), + source_hash("2b5354b14bd520d6cdefea30d2f70c9be4b9cf23"), #ifdef HAVE_LZ4 compression_method(File_Blocks_Index< Uint31_Index >::LZ4_COMPRESSION), #else From b33c3320ea54db7aba1576d37dc4fee181237d58 Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Fri, 17 Jun 2022 08:10:41 +0200 Subject: [PATCH 06/10] Fix an infinite loop --- src/overpass_api/statements/coord_query.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/overpass_api/statements/coord_query.cc b/src/overpass_api/statements/coord_query.cc index c80fbe47e..f7f93f85a 100644 --- a/src/overpass_api/statements/coord_query.cc +++ b/src/overpass_api/statements/coord_query.cc @@ -387,7 +387,7 @@ void Coord_Query_Statement::execute(Resource_Manager& rman) while (!tai.is_end() && tai.get_idx().val() < idx.val()) tai.next(); if (tai.is_end()) - continue; + break; if (cur_it->first == tai.get_idx()) { From b0e83fa7aa8cc3e55f474ccf01d75cccf814477d Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Fri, 17 Jun 2022 08:10:47 +0200 Subject: [PATCH 07/10] Automated commit for release 0.7.58.3 --- src/Makefile.am | 2 +- src/configure.ac | 2 +- src/overpass_api/core/settings.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 9ed39fcd2..5433e298f 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -210,7 +210,7 @@ bin_bbox_brim_query_LDADD = libexpatwrapper.la bin_translate_xapi_SOURCES = expat/translate_xapi.cc expat/escape_xml.cc bin_translate_xapi_LDADD = -distdir = osm-3s_v0.7.58.2 +distdir = osm-3s_v0.7.58.3 nobase_dist_HEADERS = \ expat/escape_json.h\ diff --git a/src/configure.ac b/src/configure.ac index 722e43449..adb01401b 100644 --- a/src/configure.ac +++ b/src/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.61]) -AC_INIT([OverpassAPI], [0.7.58.2], [roland@olbricht.nrw]) +AC_INIT([OverpassAPI], [0.7.58.3], [roland@olbricht.nrw]) AC_CONFIG_SRCDIR([template_db/file_blocks.h]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/src/overpass_api/core/settings.cc b/src/overpass_api/core/settings.cc index 8d178aa55..3d5341393 100644 --- a/src/overpass_api/core/settings.cc +++ b/src/overpass_api/core/settings.cc @@ -96,8 +96,8 @@ Basic_Settings::Basic_Settings() base_directory("./"), logfile_name("transactions.log"), shared_name_base("/osm3s_v0.7.58"), - version("0.7.58.2"), - source_hash("2b5354b14bd520d6cdefea30d2f70c9be4b9cf23"), + version("0.7.58.3"), + source_hash("b33c3320ea54db7aba1576d37dc4fee181237d58"), #ifdef HAVE_LZ4 compression_method(File_Blocks_Index< Uint31_Index >::LZ4_COMPRESSION), #else From 01f3aeb1ebcb22983e2c1ea77d5d571d4e1fb861 Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Tue, 19 Jul 2022 23:18:42 +0200 Subject: [PATCH 08/10] Automated commit for release 0.7.58.4 --- src/Makefile.am | 2 +- src/configure.ac | 2 +- src/overpass_api/core/settings.cc | 4 ++-- src/overpass_api/osm-backend/area_updater.cc | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 5433e298f..42d2e4ed8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -210,7 +210,7 @@ bin_bbox_brim_query_LDADD = libexpatwrapper.la bin_translate_xapi_SOURCES = expat/translate_xapi.cc expat/escape_xml.cc bin_translate_xapi_LDADD = -distdir = osm-3s_v0.7.58.3 +distdir = osm-3s_v0.7.58.4 nobase_dist_HEADERS = \ expat/escape_json.h\ diff --git a/src/configure.ac b/src/configure.ac index adb01401b..9726effb7 100644 --- a/src/configure.ac +++ b/src/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.61]) -AC_INIT([OverpassAPI], [0.7.58.3], [roland@olbricht.nrw]) +AC_INIT([OverpassAPI], [0.7.58.4], [roland@olbricht.nrw]) AC_CONFIG_SRCDIR([template_db/file_blocks.h]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/src/overpass_api/core/settings.cc b/src/overpass_api/core/settings.cc index 3d5341393..94a758408 100644 --- a/src/overpass_api/core/settings.cc +++ b/src/overpass_api/core/settings.cc @@ -96,8 +96,8 @@ Basic_Settings::Basic_Settings() base_directory("./"), logfile_name("transactions.log"), shared_name_base("/osm3s_v0.7.58"), - version("0.7.58.3"), - source_hash("b33c3320ea54db7aba1576d37dc4fee181237d58"), + version("0.7.58.4"), + source_hash("b0e83fa7aa8cc3e55f474ccf01d75cccf814477d"), #ifdef HAVE_LZ4 compression_method(File_Blocks_Index< Uint31_Index >::LZ4_COMPRESSION), #else diff --git a/src/overpass_api/osm-backend/area_updater.cc b/src/overpass_api/osm-backend/area_updater.cc index 2039c851e..c2f8aad9d 100644 --- a/src/overpass_api/osm-backend/area_updater.cc +++ b/src/overpass_api/osm-backend/area_updater.cc @@ -224,6 +224,7 @@ void Area_Updater::prepare_tags Ranges< Tag_Index_Local > ranges; for (auto it = to_delete_coarse.begin(); it != to_delete_coarse.end(); ++it) ranges.push_back({ it->first, "", "" }, { it->first + 1, "", "" }); + ranges.sort(); // iterate over the result Block_Backend< Tag_Index_Local, Uint32_Index > areas_db From b0c4acbb1aca720244f95d62ad14fc85fb12467e Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Wed, 20 Jul 2022 12:50:51 +0200 Subject: [PATCH 09/10] Fixed duplicate tags due to unsorted range condition --- src/overpass_api/osm-backend/area_updater.cc | 63 +++----------------- src/overpass_api/osm-backend/area_updater.h | 3 - 2 files changed, 7 insertions(+), 59 deletions(-) diff --git a/src/overpass_api/osm-backend/area_updater.cc b/src/overpass_api/osm-backend/area_updater.cc index c2f8aad9d..0f70bcb7c 100644 --- a/src/overpass_api/osm-backend/area_updater.cc +++ b/src/overpass_api/osm-backend/area_updater.cc @@ -158,13 +158,15 @@ void Area_Updater::prepare_delete_tags Ranges< Tag_Index_Local > ranges; for (auto it = to_delete_coarse.begin(); it != to_delete_coarse.end(); ++it) ranges.push_back({ it->first, "", "" }, { it->first + 1, "", "" }); - + ranges.sort(); + // iterate over the result Block_Backend< Tag_Index_Local, Uint32_Index > areas_db (transaction->data_index(area_settings().AREA_TAGS_LOCAL)); Tag_Index_Local current_index; Tag_Entry< uint32 > tag_entry; current_index.index = 0xffffffff; + std::set< Area::Id_Type >* handle = 0; for (auto it = areas_db.range_begin(ranges); !(it == areas_db.range_end()); ++it) { if (!(current_index == it.index())) @@ -176,16 +178,18 @@ void Area_Updater::prepare_delete_tags tag_entry.key = it.index().key; tag_entry.value = it.index().value; tag_entry.ids.clear(); + + handle = &to_delete_coarse[it.index().index]; } - std::set< Area::Id_Type >& handle(to_delete_coarse[it.index().index]); - if (handle.find(it.object().val()) != handle.end()) + if (handle->find(it.object().val()) != handle->end()) tag_entry.ids.push_back(it.object().val()); } if ((current_index.index != 0xffffffff) && (!tag_entry.ids.empty())) tags_to_delete.push_back(tag_entry); } + Area_Location* binary_search_for_id (std::vector< std::pair< Area_Location, Uint31_Index > >& vect, uint32 id) { @@ -205,59 +209,6 @@ Area_Location* binary_search_for_id return 0; } -void Area_Updater::prepare_tags - (std::vector< Tag_Entry< uint32 > >& tags_to_delete, - const std::map< uint32, std::vector< uint32 > >& to_delete) -{ - // make indices appropriately coarse - std::map< uint32, std::set< uint32 > > to_delete_coarse; - for (std::map< uint32, std::vector< uint32 > >::const_iterator - it(to_delete.begin()); it != to_delete.end(); ++it) - { - std::set< uint32 >& handle(to_delete_coarse[it->first & 0xffffff00]); - for (std::vector< uint32 >::const_iterator it2(it->second.begin()); - it2 != it->second.end(); ++it2) - handle.insert(*it2); - } - - // formulate range query - Ranges< Tag_Index_Local > ranges; - for (auto it = to_delete_coarse.begin(); it != to_delete_coarse.end(); ++it) - ranges.push_back({ it->first, "", "" }, { it->first + 1, "", "" }); - ranges.sort(); - - // iterate over the result - Block_Backend< Tag_Index_Local, Uint32_Index > areas_db - (transaction->data_index(area_settings().AREA_TAGS_LOCAL)); - Tag_Index_Local current_index; - Tag_Entry< uint32 > tag_entry; - current_index.index = 0xffffffff; - for (auto it = areas_db.range_begin(ranges); !(it == areas_db.range_end()); ++it) - { - if (!(current_index == it.index())) - { - if ((current_index.index != 0xffffffff) && (!tag_entry.ids.empty())) - tags_to_delete.push_back(tag_entry); - current_index = it.index(); - tag_entry.index = it.index().index; - tag_entry.key = it.index().key; - tag_entry.value = it.index().value; - tag_entry.ids.clear(); - } - - std::set< uint32 >& handle(to_delete_coarse[it.index().index]); - if (handle.find(it.object().val()) != handle.end()) - { - Area_Location* area(binary_search_for_id(areas_to_insert, it.object().val())); - if (area != 0) - area->tags.push_back(std::make_pair(it.index().key, it.index().value)); - tag_entry.ids.push_back(it.object().val()); - } - } - if ((current_index.index != 0xffffffff) && (!tag_entry.ids.empty())) - tags_to_delete.push_back(tag_entry); -} - void Area_Updater::update_area_tags_local (const std::vector< Tag_Entry< uint32 > >& tags_to_delete) { diff --git a/src/overpass_api/osm-backend/area_updater.h b/src/overpass_api/osm-backend/area_updater.h index ec8961007..e878081e0 100644 --- a/src/overpass_api/osm-backend/area_updater.h +++ b/src/overpass_api/osm-backend/area_updater.h @@ -86,9 +86,6 @@ struct Area_Updater : public Area_Usage_Listener void prepare_delete_tags (std::vector< Tag_Entry< uint32 > >& tags_to_delete, const std::map< Uint31_Index, std::set< Area_Skeleton > >& to_delete); - void prepare_tags - (std::vector< Tag_Entry< uint32 > >& tags_to_delete, - const std::map< uint32, std::vector< uint32 > >& to_delete); void update_area_tags_local (const std::vector< Tag_Entry< uint32 > >& tags_to_delete); void update_area_tags_global From 44246a431313ad2af05e4dd30d9db0049c183382 Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Wed, 20 Jul 2022 12:50:57 +0200 Subject: [PATCH 10/10] Automated commit for release 0.7.58.5 --- src/Makefile.am | 2 +- src/configure.ac | 2 +- src/overpass_api/core/settings.cc | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 42d2e4ed8..a8a3abcbf 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -210,7 +210,7 @@ bin_bbox_brim_query_LDADD = libexpatwrapper.la bin_translate_xapi_SOURCES = expat/translate_xapi.cc expat/escape_xml.cc bin_translate_xapi_LDADD = -distdir = osm-3s_v0.7.58.4 +distdir = osm-3s_v0.7.58.5 nobase_dist_HEADERS = \ expat/escape_json.h\ diff --git a/src/configure.ac b/src/configure.ac index 9726effb7..850b222b6 100644 --- a/src/configure.ac +++ b/src/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.61]) -AC_INIT([OverpassAPI], [0.7.58.4], [roland@olbricht.nrw]) +AC_INIT([OverpassAPI], [0.7.58.5], [roland@olbricht.nrw]) AC_CONFIG_SRCDIR([template_db/file_blocks.h]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/src/overpass_api/core/settings.cc b/src/overpass_api/core/settings.cc index 94a758408..40491f451 100644 --- a/src/overpass_api/core/settings.cc +++ b/src/overpass_api/core/settings.cc @@ -96,8 +96,8 @@ Basic_Settings::Basic_Settings() base_directory("./"), logfile_name("transactions.log"), shared_name_base("/osm3s_v0.7.58"), - version("0.7.58.4"), - source_hash("b0e83fa7aa8cc3e55f474ccf01d75cccf814477d"), + version("0.7.58.5"), + source_hash("b0c4acbb1aca720244f95d62ad14fc85fb12467e"), #ifdef HAVE_LZ4 compression_method(File_Blocks_Index< Uint31_Index >::LZ4_COMPRESSION), #else