From 84d91b30263e07c87bca8d19f8f26fe81a882d73 Mon Sep 17 00:00:00 2001 From: Brian Weed Date: Thu, 15 Aug 2024 13:06:04 -0400 Subject: [PATCH 01/67] Added missing const empty() is a simple read-only getter, and therefore should be const. --- src/boost/locale/shared/message.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/boost/locale/shared/message.cpp b/src/boost/locale/shared/message.cpp index da8b4b93..c3252316 100644 --- a/src/boost/locale/shared/message.cpp +++ b/src/boost/locale/shared/message.cpp @@ -205,7 +205,7 @@ namespace boost { namespace locale { namespace gnu_gettext { size_t size() const { return size_; } - bool empty() { return size_ == 0; } + bool empty() const { return size_ == 0; } private: uint32_t get(unsigned offset) const From 9277c65aa7ebc7e2f76fa3f1e937273ec9fa7e85 Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Mon, 11 Mar 2024 08:38:17 -0500 Subject: [PATCH 02/67] Make the library modular usable. --- build.jam | 26 ++++++++++++++++++++ build/Jamfile.v2 | 62 ++++++++++++++++++++++++------------------------ test/Jamfile.v2 | 2 +- 3 files changed, 58 insertions(+), 32 deletions(-) create mode 100644 build.jam diff --git a/build.jam b/build.jam new file mode 100644 index 00000000..75d4e07d --- /dev/null +++ b/build.jam @@ -0,0 +1,26 @@ +# Copyright René Ferdinand Rivera Morell 2023-2024 +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +import project ; + +project /boost/locale + : common-requirements + /boost/assert//boost_assert + /boost/config//boost_config + /boost/core//boost_core + /boost/iterator//boost_iterator + /boost/predef//boost_predef + /boost/thread//boost_thread + include + ; + +explicit + [ alias boost_locale : build//boost_locale ] + [ alias all : boost_locale test ] + ; + +call-if : boost-library locale + : install boost_locale + ; diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index b7268c58..d9403048 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -15,7 +15,7 @@ import toolset ; path-constant TOP : .. ; -project /boost/locale +project : source-location $(TOP)/src/boost/locale ; @@ -46,7 +46,7 @@ explicit has_iconv ; ICONV_PATH = [ modules.peek : ICONV_PATH ] ; # There may also be an external iconv library -lib iconv : +searched-lib iconv : : $(ICONV_PATH)/lib shared shared : : $(ICONV_PATH)/include ; @@ -127,54 +127,54 @@ if [ modules.peek : ICU_ICUIN_NAME ] if $(ICU_ICUUC_NAME) { - lib icuuc : : $(ICU_ICUUC_NAME) @path_options ; + searched-lib icuuc : : $(ICU_ICUUC_NAME) @path_options ; debug-message Using "$(ICU_ICUUC_NAME)" for library "icuuc" ; } else { - lib icuuc : : shared @path_options ; - lib icuuc : : msvc debug icuucd shared @path_options ; - lib icuuc : : intel windows debug icuucd shared @path_options ; - lib icuuc : : sicuuc static @path_options ; - lib icuuc : : msvc debug sicuucd static @path_options ; - lib icuuc : : intel windows debug sicuucd static @path_options ; - lib icuuc : : this_is_an_invalid_library_name ; + searched-lib icuuc : : shared @path_options ; + searched-lib icuuc : : msvc debug icuucd shared @path_options ; + searched-lib icuuc : : intel windows debug icuucd shared @path_options ; + searched-lib icuuc : : sicuuc static @path_options ; + searched-lib icuuc : : msvc debug sicuucd static @path_options ; + searched-lib icuuc : : intel windows debug sicuucd static @path_options ; + searched-lib icuuc : : this_is_an_invalid_library_name ; } if $(ICU_ICUDT_NAME) { - lib icudt : : $(ICU_ICUDT_NAME) @path_options ; + searched-lib icudt : : $(ICU_ICUDT_NAME) @path_options ; debug-message Using "$(ICU_ICUDT_NAME)" for library "icudt" ; } else { - lib icudt : : icudata shared @path_options ; - lib icudt : : icudt msvc shared @path_options ; - lib icudt : : icudt intel windows shared @path_options ; - lib icudt : : sicudata static @path_options ; - lib icudt : : sicudt msvc static @path_options ; - lib icudt : : sicudt intel windows static @path_options ; - lib icudt : : this_is_an_invalid_library_name ; + searched-lib icudt : : icudata shared @path_options ; + searched-lib icudt : : icudt msvc shared @path_options ; + searched-lib icudt : : icudt intel windows shared @path_options ; + searched-lib icudt : : sicudata static @path_options ; + searched-lib icudt : : sicudt msvc static @path_options ; + searched-lib icudt : : sicudt intel windows static @path_options ; + searched-lib icudt : : this_is_an_invalid_library_name ; } if $(ICU_ICUIN_NAME) { - lib icuin : : $(ICU_ICUIN_NAME) @path_options ; + searched-lib icuin : : $(ICU_ICUIN_NAME) @path_options ; debug-message Using "$(ICU_ICUIN_NAME)" for library "icuin" ; } else { - lib icuin : : icui18n shared @path_options ; - lib icuin : : msvc debug icuind shared @path_options ; - lib icuin : : msvc icuin shared @path_options ; - lib icuin : : intel windows debug icuind shared @path_options ; - lib icuin : : intel windows icuin shared @path_options ; - lib icuin : : sicui18n static @path_options ; - lib icuin : : msvc debug sicuind static @path_options ; - lib icuin : : msvc sicuin static @path_options ; - lib icuin : : intel windows debug sicuind static @path_options ; - lib icuin : : intel windows sicuin static @path_options ; - lib icuin : : this_is_an_invalid_library_name ; + searched-lib icuin : : icui18n shared @path_options ; + searched-lib icuin : : msvc debug icuind shared @path_options ; + searched-lib icuin : : msvc icuin shared @path_options ; + searched-lib icuin : : intel windows debug icuind shared @path_options ; + searched-lib icuin : : intel windows icuin shared @path_options ; + searched-lib icuin : : sicui18n static @path_options ; + searched-lib icuin : : msvc debug sicuind static @path_options ; + searched-lib icuin : : msvc sicuin static @path_options ; + searched-lib icuin : : intel windows debug sicuind static @path_options ; + searched-lib icuin : : intel windows sicuin static @path_options ; + searched-lib icuin : : this_is_an_invalid_library_name ; } ICU_OPTS = @@ -421,7 +421,7 @@ local cxx_requirements = [ requires cxx11_variadic_templates ] ; -boost-lib locale +lib boost_locale : encoding/codepage.cpp shared/date_time.cpp diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index 40f4b6f1..e3b85592 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -6,7 +6,7 @@ # https://www.boost.org/LICENSE_1_0.txt import config : requires ; -import ../../predef/tools/check/predef : check : predef-check ; +import predef : check : predef-check ; project : requirements /boost/locale//boost_locale From 14f84eef49c887df0be76beaea91fae4a3caeda1 Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Fri, 29 Mar 2024 21:15:59 -0500 Subject: [PATCH 03/67] Switch to library requirements instead of source. As source puts extra source in install targets. --- build.jam | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build.jam b/build.jam index 75d4e07d..0c0b695e 100644 --- a/build.jam +++ b/build.jam @@ -7,12 +7,12 @@ import project ; project /boost/locale : common-requirements - /boost/assert//boost_assert - /boost/config//boost_config - /boost/core//boost_core - /boost/iterator//boost_iterator - /boost/predef//boost_predef - /boost/thread//boost_thread + /boost/assert//boost_assert + /boost/config//boost_config + /boost/core//boost_core + /boost/iterator//boost_iterator + /boost/predef//boost_predef + /boost/thread//boost_thread include ; From cf272ebf153066714748d7566d1c6cb59894758d Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Sun, 28 Apr 2024 20:15:58 -0500 Subject: [PATCH 04/67] Add missing NO_LIB usage requirements. --- build/Jamfile.v2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index d9403048..0a9cbc9a 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -441,8 +441,6 @@ lib boost_locale : $(cxx_requirements) BOOST_LOCALE_SOURCE - # Don't link explicitly, not required - BOOST_THREAD_NO_LIB=1 $(TOP)/src multi windows:_CRT_SECURE_NO_WARNINGS @@ -450,4 +448,5 @@ lib boost_locale # Meanwhile remove this @configure : : $(cxx_requirements) + BOOST_LOCALE_NO_LIB=1 ; From 36dff671ba82dd76fefec749ef7f783d802a7749 Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Sat, 4 May 2024 23:30:15 -0500 Subject: [PATCH 05/67] Add missing import-search for cconfig/predef checks. --- build/Jamfile.v2 | 1 + 1 file changed, 1 insertion(+) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 0a9cbc9a..8c88e26a 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -5,6 +5,7 @@ # Distributed under the Boost Software License, Version 1.0. # https://www.boost.org/LICENSE_1_0.txt. +import-search /boost/config/checks ; import config : requires ; import configure ; import errors ; From ed2a38bfe7c1c3fdf7c09b5d2288fbb06c3988c3 Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Sun, 5 May 2024 09:00:01 -0500 Subject: [PATCH 06/67] Add requires-b2 check to top-level build file. --- build.jam | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.jam b/build.jam index 0c0b695e..56491ef7 100644 --- a/build.jam +++ b/build.jam @@ -3,7 +3,7 @@ # (See accompanying file LICENSE_1_0.txt or copy at # http://www.boost.org/LICENSE_1_0.txt) -import project ; +require-b2 5.2 ; project /boost/locale : common-requirements From 8fe7802f0e0a0ddfe8418a25d0ea19b6474151fe Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Tue, 23 Jul 2024 22:34:23 -0500 Subject: [PATCH 07/67] Move inter-lib dependencies to a project variable and into the build targets. --- build.jam | 15 +++++++++------ build/Jamfile.v2 | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/build.jam b/build.jam index 56491ef7..ecaa9a08 100644 --- a/build.jam +++ b/build.jam @@ -5,14 +5,16 @@ require-b2 5.2 ; +constant boost_dependencies : + /boost/assert//boost_assert + /boost/config//boost_config + /boost/core//boost_core + /boost/iterator//boost_iterator + /boost/predef//boost_predef + /boost/thread//boost_thread ; + project /boost/locale : common-requirements - /boost/assert//boost_assert - /boost/config//boost_config - /boost/core//boost_core - /boost/iterator//boost_iterator - /boost/predef//boost_predef - /boost/thread//boost_thread include ; @@ -24,3 +26,4 @@ explicit call-if : boost-library locale : install boost_locale ; + diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 8c88e26a..d72b5bf5 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -18,6 +18,7 @@ path-constant TOP : .. ; project : source-location $(TOP)/src/boost/locale + : common-requirements $(boost_dependencies) ; # Features From 4bf61b7a29f43d4ca96a5c3bed05a90918bdc001 Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Wed, 24 Jul 2024 21:44:35 -0500 Subject: [PATCH 08/67] Fix dynamic build to signal to export symbols. --- build/Jamfile.v2 | 1 + 1 file changed, 1 insertion(+) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index d72b5bf5..9c4936d0 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -442,6 +442,7 @@ lib boost_locale util/locale_data.cpp : $(cxx_requirements) + shared:BOOST_LOCALE_DYN_LINK=1 BOOST_LOCALE_SOURCE $(TOP)/src multi From f66f62cb2b8e9d52252cdbe1eebde42f5609413b Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Fri, 26 Jul 2024 17:19:44 -0500 Subject: [PATCH 09/67] Split b2 dependencies into public and private. --- build.jam | 3 +-- build/Jamfile.v2 | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/build.jam b/build.jam index ecaa9a08..ce78c15d 100644 --- a/build.jam +++ b/build.jam @@ -10,8 +10,7 @@ constant boost_dependencies : /boost/config//boost_config /boost/core//boost_core /boost/iterator//boost_iterator - /boost/predef//boost_predef - /boost/thread//boost_thread ; + /boost/utility//boost_utility ; project /boost/locale : common-requirements diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 9c4936d0..b3633693 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -16,9 +16,15 @@ import toolset ; path-constant TOP : .. ; +constant boost_dependencies_private : + /boost/predef//boost_predef + /boost/thread//boost_thread + ; + project : source-location $(TOP)/src/boost/locale : common-requirements $(boost_dependencies) + : requirements $(boost_dependencies_private) ; # Features From 03657559f8d255bbbb4f397c9e7e2aaeab8ec247 Mon Sep 17 00:00:00 2001 From: Rene Rivera Date: Thu, 15 Aug 2024 20:39:23 -0500 Subject: [PATCH 10/67] The locale features need to be at the root as they need to be defined for all subprojects on load. --- build.jam | 10 ++++++++++ build/Jamfile.v2 | 9 --------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/build.jam b/build.jam index ce78c15d..573ca20a 100644 --- a/build.jam +++ b/build.jam @@ -5,6 +5,16 @@ require-b2 5.2 ; +import feature ; + +# Features + +feature.feature boost.locale.iconv : on off : optional propagated ; +feature.feature boost.locale.icu : on off : optional propagated ; +feature.feature boost.locale.posix : on off : optional propagated ; +feature.feature boost.locale.std : on off : optional propagated ; +feature.feature boost.locale.winapi : on off : optional propagated ; + constant boost_dependencies : /boost/assert//boost_assert /boost/config//boost_config diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index b3633693..0c89a4fb 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -9,7 +9,6 @@ import-search /boost/config/checks ; import config : requires ; import configure ; import errors ; -import feature ; import os ; import path ; import toolset ; @@ -27,14 +26,6 @@ project : requirements $(boost_dependencies_private) ; -# Features - -feature.feature boost.locale.iconv : on off : optional propagated ; -feature.feature boost.locale.icu : on off : optional propagated ; -feature.feature boost.locale.posix : on off : optional propagated ; -feature.feature boost.locale.std : on off : optional propagated ; -feature.feature boost.locale.winapi : on off : optional propagated ; - local rule debug-message ( message * ) { if --debug-configuration in [ modules.peek : ARGV ] From 3ff8e5a3f029b125f5d286957f979fc71f264cde Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 26 Aug 2024 17:16:29 +0200 Subject: [PATCH 11/67] Fix dynamic build imports --- build/Jamfile.v2 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 0c89a4fb..7889db6b 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -437,7 +437,7 @@ lib boost_locale util/encoding.cpp util/info.cpp util/locale_data.cpp - : + : # requirements $(cxx_requirements) shared:BOOST_LOCALE_DYN_LINK=1 BOOST_LOCALE_SOURCE @@ -447,6 +447,8 @@ lib boost_locale windows:_SCL_SECURE_NO_WARNINGS # Meanwhile remove this @configure - : : $(cxx_requirements) - BOOST_LOCALE_NO_LIB=1 + : : # usage-requirements + $(cxx_requirements) + shared:BOOST_LOCALE_DYN_LINK=1 + BOOST_LOCALE_NO_LIB=1 ; From ff917062e50b7d789a58279bf2416037486d5901 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 13 Sep 2024 09:06:22 +0200 Subject: [PATCH 12/67] Fix typo in preprocessor guard Also add copyright notice for previous changes. Fixes #239 --- include/boost/locale/gnu_gettext.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/boost/locale/gnu_gettext.hpp b/include/boost/locale/gnu_gettext.hpp index aff76b1c..6612e84f 100644 --- a/include/boost/locale/gnu_gettext.hpp +++ b/include/boost/locale/gnu_gettext.hpp @@ -1,11 +1,12 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2022-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#ifndef BOOST_LOCLAE_GNU_GETTEXT_HPP -#define BOOST_LOCLAE_GNU_GETTEXT_HPP +#ifndef BOOST_LOCALE_GNU_GETTEXT_HPP +#define BOOST_LOCALE_GNU_GETTEXT_HPP #include #include From 9a346dd5f8f61555839018dd50cfd88daf9c0b7e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 10 Nov 2024 19:09:22 +0100 Subject: [PATCH 13/67] Fix parsing of numbers in floating point format to integers When parsing a string like "123.456" to an integer the ICU backend would first parse it greedily to a floating point value and then cast/truncate it to an integer. Set the flag to only parse integers when parsing to an integral number. Care must be taken not to set that when parsing e.g. a currency or date to an integer where the truncation is intended. --- doc/changelog.txt | 2 + src/boost/locale/icu/formatter.cpp | 10 +++- test/formatting_common.hpp | 82 ++++++++++++++++++++++++++++++ test/test_formatting.cpp | 3 ++ test/test_posix_formatting.cpp | 2 + test/test_std_formatting.cpp | 5 ++ test/test_winapi_formatting.cpp | 2 + 7 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 test/formatting_common.hpp diff --git a/doc/changelog.txt b/doc/changelog.txt index da943877..3e4fe789 100644 --- a/doc/changelog.txt +++ b/doc/changelog.txt @@ -8,6 +8,8 @@ /*! \page changelog Changelog +- 1.88.0 + - Fix parsing of numbers in floating point format to integers - 1.86.0 - Make ICU implementation of `to_title` threadsafe - Add allocator support to `utf_to_utf` diff --git a/src/boost/locale/icu/formatter.cpp b/src/boost/locale/icu/formatter.cpp index 12cb7363..955b3d18 100644 --- a/src/boost/locale/icu/formatter.cpp +++ b/src/boost/locale/icu/formatter.cpp @@ -54,7 +54,9 @@ namespace boost { namespace locale { namespace impl_icu { public: typedef std::basic_string string_type; - number_format(icu::NumberFormat& fmt, std::string codepage) : cvt_(codepage), icu_fmt_(fmt) {} + number_format(icu::NumberFormat& fmt, const std::string& codepage, bool isNumberOnly = false) : + cvt_(codepage), icu_fmt_(fmt), isNumberOnly_(isNumberOnly) + {} string_type format(double value, size_t& code_points) const override { return do_format(value, code_points); } string_type format(int64_t value, size_t& code_points) const override { return do_format(value, code_points); } @@ -107,6 +109,9 @@ namespace boost { namespace locale { namespace impl_icu { icu::ParsePosition pp; icu::UnicodeString tmp = cvt_.icu(str.data(), str.data() + str.size()); + // For the plain number parsing (no currency etc) parse "123.456" as 2 ints + // not a float later converted to int + icu_fmt_.setParseIntegerOnly(std::is_integral::value && isNumberOnly_); icu_fmt_.parse(tmp, val, pp); ValueType tmp_v; @@ -122,6 +127,7 @@ namespace boost { namespace locale { namespace impl_icu { icu_std_converter cvt_; icu::NumberFormat& icu_fmt_; + const bool isNumberOnly_; }; template @@ -355,7 +361,7 @@ namespace boost { namespace locale { namespace impl_icu { icu::NumberFormat& nf = cache.number_format((how == std::ios_base::scientific) ? num_fmt_type::sci : num_fmt_type::number); set_fraction_digits(nf, how, ios.precision()); - return ptr_type(new number_format(nf, encoding)); + return ptr_type(new number_format(nf, encoding, true)); } case currency: { icu::NumberFormat& nf = cache.number_format( diff --git a/test/formatting_common.hpp b/test/formatting_common.hpp new file mode 100644 index 00000000..f5662598 --- /dev/null +++ b/test/formatting_common.hpp @@ -0,0 +1,82 @@ +// +// Copyright (c) 2024 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#include +#include +#include +#include +#include + +#include "../src/boost/locale/util/foreach_char.hpp" +#include "boostLocale/test/tools.hpp" +#include "boostLocale/test/unit_test.hpp" + +template +void test_parse_multi_number_by_char(const std::locale& locale) +{ + std::basic_istringstream stream; + stream.imbue(locale); + stream.str(ascii_to("42.12,345")); + stream >> boost::locale::as::number; + + IntType value; + TEST_REQUIRE(stream >> value); + TEST_EQ(value, IntType(42)); + TEST_EQ(static_cast(stream.get()), '.'); + TEST_REQUIRE(stream >> value); + TEST_EQ(value, IntType(12345)); + TEST_REQUIRE(!(stream >> value)); + TEST(stream.eof()); + + stream.str(ascii_to("42.25,678")); + stream.clear(); + float fValue; + TEST_REQUIRE(stream >> fValue); + TEST_EQ(fValue, 42.25); + TEST_EQ(static_cast(stream.get()), ','); + TEST_REQUIRE(stream >> value); + TEST_EQ(value, IntType(678)); + TEST_REQUIRE(!(stream >> value)); + TEST(stream.eof()); + + // Parsing a floating point currency to integer truncates the floating point value but fully parses it + stream.str(ascii_to("USD1,234.55,67.89")); + stream.clear(); + TEST_REQUIRE(!(stream >> value)); + stream.clear(); + stream >> boost::locale::as::currency >> boost::locale::as::currency_iso; + if(stream >> value) { // Parsing currencies not fully supported by WinAPI backend + TEST_EQ(value, IntType(1234)); + TEST_EQ(static_cast(stream.get()), ','); + TEST_REQUIRE(stream >> boost::locale::as::number >> value); + TEST_EQ(value, IntType(67)); + TEST(!stream.eof()); + } +} + +/// Test that parsing multiple numbers without any spaces works as expected +void test_parse_multi_number() +{ + const auto locale = boost::locale::generator{}("en_US.UTF-8"); + +#define BOOST_LOCALE_CALL_I(T, I) \ + std::cout << "\t" #I << std::endl; \ + test_parse_multi_number_by_char(locale); + +#define BOOST_LOCALE_CALL(T) \ + std::cout << "test_parse_multi_number " #T << std::endl; \ + BOOST_LOCALE_CALL_I(T, int16_t); \ + BOOST_LOCALE_CALL_I(T, uint16_t); \ + BOOST_LOCALE_CALL_I(T, int32_t); \ + BOOST_LOCALE_CALL_I(T, uint32_t); \ + BOOST_LOCALE_CALL_I(T, int64_t); \ + BOOST_LOCALE_CALL_I(T, uint64_t); + + BOOST_LOCALE_CALL(char); + BOOST_LOCALE_CALL(wchar_t); +#undef BOOST_LOCALE_CALL +#undef BOOST_LOCALE_CALL_I +} \ No newline at end of file diff --git a/test/test_formatting.cpp b/test/test_formatting.cpp index a88f06bc..d418c583 100644 --- a/test/test_formatting.cpp +++ b/test/test_formatting.cpp @@ -21,6 +21,7 @@ #include "boostLocale/test/tools.hpp" #include "boostLocale/test/unit_test.hpp" +#include "formatting_common.hpp" const std::string test_locale_name = "en_US"; std::string message_path = "./"; @@ -928,6 +929,8 @@ void test_main(int argc, char** argv) test_manip(); test_format_class(); #endif + + test_parse_multi_number(); } // boostinspect:noascii diff --git a/test/test_posix_formatting.cpp b/test/test_posix_formatting.cpp index c6c46a0f..15c4d6d2 100644 --- a/test/test_posix_formatting.cpp +++ b/test/test_posix_formatting.cpp @@ -19,6 +19,7 @@ #endif #include "boostLocale/test/tools.hpp" #include "boostLocale/test/unit_test.hpp" +#include "formatting_common.hpp" #ifdef BOOST_LOCALE_NO_POSIX_BACKEND // Dummy just to make it compile @@ -185,6 +186,7 @@ void test_main(int /*argc*/, char** /*argv*/) TEST(v == "12345,45" || v == "12 345,45" || v == "12.345,45"); } } + test_parse_multi_number(); } // boostinspect:noascii diff --git a/test/test_std_formatting.cpp b/test/test_std_formatting.cpp index 35085c1b..79156ad0 100644 --- a/test/test_std_formatting.cpp +++ b/test/test_std_formatting.cpp @@ -14,6 +14,7 @@ #include "boostLocale/test/tools.hpp" #include "boostLocale/test/unit_test.hpp" +#include "formatting_common.hpp" template void test_by_char(const std::locale& l, const std::locale& lreal) @@ -230,6 +231,10 @@ void test_main(int /*argc*/, char** /*argv*/) } } } + // Std backend silently falls back to the C locale when the locale is not supported + // which breaks the test assumptions + if(has_std_locale("en_US.UTF-8")) + test_parse_multi_number(); } // boostinspect:noascii diff --git a/test/test_winapi_formatting.cpp b/test/test_winapi_formatting.cpp index 4ccb10b2..499e92a8 100644 --- a/test/test_winapi_formatting.cpp +++ b/test/test_winapi_formatting.cpp @@ -23,6 +23,7 @@ #include "../src/boost/locale/win32/lcid.hpp" #include "boostLocale/test/tools.hpp" #include "boostLocale/test/unit_test.hpp" +#include "formatting_common.hpp" template void test_by_char(const std::locale& l, std::string name, int lcid) @@ -176,6 +177,7 @@ void test_main(int /*argc*/, char** /*argv*/) test_by_char(l, name, name_lcid.second); } } + test_parse_multi_number(); std::cout << "- Testing strftime" << std::endl; test_date_time(gen("en_US.UTF-8")); } From e3b9e2eea6b29e68d9cc578e04a93241999e4465 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 12 Nov 2024 16:16:07 +0100 Subject: [PATCH 14/67] Workaround missing thousand sep in some POSIX backends The Drone CI fails because the en_US.UTF-8 `locale_t` does not have a grouping and thousand separator so "12,345" fails to parse. Format instead to have the expected format as input. --- test/formatting_common.hpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/formatting_common.hpp b/test/formatting_common.hpp index f5662598..97ef7919 100644 --- a/test/formatting_common.hpp +++ b/test/formatting_common.hpp @@ -17,9 +17,17 @@ template void test_parse_multi_number_by_char(const std::locale& locale) { + // thousandsNum will mostly be 12,345 but some systems + // don't have the thousand separator for the POSIX locale. + // So use the formatted output. + const IntType expectedInt = 12345; + std::basic_ostringstream thousandsNum; + thousandsNum.imbue(locale); + thousandsNum << boost::locale::as::number << expectedInt; + std::basic_istringstream stream; stream.imbue(locale); - stream.str(ascii_to("42.12,345")); + stream.str(ascii_to("42.") + thousandsNum.str()); stream >> boost::locale::as::number; IntType value; @@ -27,7 +35,7 @@ void test_parse_multi_number_by_char(const std::locale& locale) TEST_EQ(value, IntType(42)); TEST_EQ(static_cast(stream.get()), '.'); TEST_REQUIRE(stream >> value); - TEST_EQ(value, IntType(12345)); + TEST_EQ(value, expectedInt); TEST_REQUIRE(!(stream >> value)); TEST(stream.eof()); From 1e9e85546ed0811b87a374a6e363b6b20c633a42 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 12 Nov 2024 16:19:31 +0100 Subject: [PATCH 15/67] Appveyor: Split Cygwin job which is running into 1h timeout --- .appveyor.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 1767c353..c18806d8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -111,11 +111,18 @@ environment: # (Currently) the images up to 2017 use an older Cygwin # This tests that the library works with more recent versions - - FLAVOR: cygwin (64-bit, latest) + - FLAVOR: cygwin (64-bit, latest) C++11 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 ADDPATH: C:\cygwin64\bin; B2_ADDRESS_MODEL: 64 - B2_CXXSTD: 11,1z + B2_CXXSTD: 11 + B2_TOOLSET: gcc + # Split to avoid 1h timeout for multi-config runs + - FLAVOR: cygwin (64-bit, latest) C++17 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 + ADDPATH: C:\cygwin64\bin; + B2_ADDRESS_MODEL: 64 + B2_CXXSTD: 1z B2_TOOLSET: gcc - FLAVOR: mingw64 (32-bit) From 3743c9e0da2219931e2506b6711df1d34abc7412 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 27 Nov 2024 19:25:04 +0100 Subject: [PATCH 16/67] GHA: Handle removal of Node 16 Download an unofficial Node 20 build against glibc 2.17 --- .github/workflows/ci.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43f4b935..9e0a855e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,8 +134,12 @@ jobs: timeout-minutes: 120 runs-on: ${{matrix.os}} - container: ${{matrix.container}} - env: {B2_USE_CCACHE: 1, ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true} + container: + image: ${{matrix.container}} + volumes: + - /node20217:/node20217:rw,rshared + - ${{ startsWith(matrix.container, 'ubuntu:1') && '/node20217:/__e/node20:ro,rshared' || ' ' }} + env: {B2_USE_CCACHE: 1} steps: - name: Setup environment @@ -158,14 +162,18 @@ jobs: echo "B2_USE_CCACHE=0" >> $GITHUB_ENV fi git config --global pack.threads 0 + if [[ "${{matrix.container}}" == "ubuntu:1"* ]]; then + # Node 20 doesn't work with Ubuntu 16/18 glibc: https://github.com/actions/checkout/issues/1590 + curl -sL https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz | tar -xJ --strip-components 1 -C /node20217 + fi - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # For coverage builds fetch the whole history, else only 1 commit using a 'fake ternary' fetch-depth: ${{ matrix.coverage && '0' || '1' }} - name: Cache ccache - uses: actions/cache@v3 + uses: actions/cache@v4 if: env.B2_USE_CCACHE with: path: ~/.ccache @@ -173,7 +181,7 @@ jobs: restore-keys: ${{matrix.os}}-${{matrix.container}}-${{matrix.compiler}}- - name: Fetch Boost.CI - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: boostorg/boost-ci ref: master From 675b6f28676d118d6e3ddb7b6a7a490612637ad5 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 28 Nov 2024 08:57:30 +0100 Subject: [PATCH 17/67] GHA: Update OS versions, remove containers where possible and add new compilers --- .github/workflows/ci.yml | 63 +++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e0a855e..2532d589 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,39 +86,48 @@ jobs: include: # Linux, gcc # GCC 5 is the first to have enough C++11 support, so don't test anything older except for one to verify the checks - - { compiler: gcc-4.9, cxxstd: '03,11', os: ubuntu-20.04, container: 'ubuntu:16.04' } - - { compiler: gcc-5, cxxstd: '03,11,14,1z', os: ubuntu-22.04, container: 'ubuntu:18.04' } - - { compiler: gcc-6, cxxstd: '11,14,17', os: ubuntu-22.04, container: 'ubuntu:18.04' } - - { compiler: gcc-7, cxxstd: '11,14,17', os: ubuntu-22.04, container: 'ubuntu:18.04' } - - { compiler: gcc-8, cxxstd: '11,14,17,2a', os: ubuntu-22.04, container: 'ubuntu:18.04' } - - { compiler: gcc-9, cxxstd: '11,14,17,2a', os: ubuntu-22.04, container: 'ubuntu:18.04' } - - { compiler: gcc-10, cxxstd: '11,14,17,20', os: ubuntu-20.04 } - - { compiler: gcc-11, cxxstd: '11,14,17,20', os: ubuntu-20.04 } + - { compiler: gcc-4.9, cxxstd: '03,11', os: ubuntu-latest, container: 'ubuntu:16.04' } + - { compiler: gcc-5, cxxstd: '03,11,14,1z', os: ubuntu-latest, container: 'ubuntu:18.04' } + - { compiler: gcc-6, cxxstd: '11,14,17', os: ubuntu-latest, container: 'ubuntu:18.04' } + # GCC 7 & 8 on Ubuntu 20 show failures in basic_filebuf::_M_convert_to_external conversion + - { compiler: gcc-7, cxxstd: '11,14,17', os: ubuntu-latest, container: 'ubuntu:18.04' } + - { compiler: gcc-8, cxxstd: '11,14,17,2a', os: ubuntu-latest, container: 'ubuntu:18.04' } + - { compiler: gcc-9, cxxstd: '11,14,17,2a', os: ubuntu-20.04 } + - { compiler: gcc-10, cxxstd: '11,14,17,20', os: ubuntu-22.04 } + - { compiler: gcc-11, cxxstd: '11,14,17,20', os: ubuntu-22.04 } + - { compiler: gcc-12, cxxstd: '11,14,17,20', os: ubuntu-22.04 } + - { compiler: gcc-13, cxxstd: '11,14,17,20,2b', os: ubuntu-24.04 } + - { compiler: gcc-14, cxxstd: '11,14,17,20,2b', os: ubuntu-24.04 } + - { name: GCC w/ sanitizers, sanitize: yes, - compiler: gcc-12, cxxstd: '11,14,17,20', os: ubuntu-22.04 } + compiler: gcc-13, cxxstd: '11,14,17,20', os: ubuntu-24.04 } - { name: Collect coverage, coverage: yes, - compiler: gcc-8, cxxstd: '11,2a', os: ubuntu-20.04, install: 'g++-8-multilib', address-model: '32,64' } + compiler: gcc-8, cxxstd: '11,2a', os: ubuntu-20.04, install: 'g++-8-multilib', address-model: '32,64' } # Linux, clang - - { compiler: clang-3.5, cxxstd: '03,11', os: ubuntu-20.04, container: 'ubuntu:16.04' } - - { compiler: clang-3.6, cxxstd: '11,14', os: ubuntu-20.04, container: 'ubuntu:16.04' } - - { compiler: clang-3.7, cxxstd: '11,14', os: ubuntu-20.04, container: 'ubuntu:16.04' } - - { compiler: clang-3.8, cxxstd: '11,14', os: ubuntu-20.04, container: 'ubuntu:16.04' } - - { compiler: clang-3.9, cxxstd: '11,14', os: ubuntu-22.04, container: 'ubuntu:18.04' } - - { compiler: clang-4.0, cxxstd: '11,14', os: ubuntu-22.04, container: 'ubuntu:18.04' } - - { compiler: clang-5.0, cxxstd: '11,14,1z', os: ubuntu-22.04, container: 'ubuntu:18.04' } - - { compiler: clang-6.0, cxxstd: '11,14,17', os: ubuntu-22.04, container: 'ubuntu:18.04' } - - { compiler: clang-7, cxxstd: '11,14,17', os: ubuntu-22.04, container: 'ubuntu:18.04' } + - { compiler: clang-3.5, cxxstd: '11', os: ubuntu-latest, container: 'ubuntu:16.04' } + - { compiler: clang-3.6, cxxstd: '11,14', os: ubuntu-latest, container: 'ubuntu:16.04' } + - { compiler: clang-3.7, cxxstd: '11,14', os: ubuntu-latest, container: 'ubuntu:16.04' } + - { compiler: clang-3.8, cxxstd: '11,14', os: ubuntu-latest, container: 'ubuntu:16.04' } + - { compiler: clang-3.9, cxxstd: '11,14', os: ubuntu-latest, container: 'ubuntu:18.04' } + - { compiler: clang-4.0, cxxstd: '11,14', os: ubuntu-latest, container: 'ubuntu:18.04' } + - { compiler: clang-5.0, cxxstd: '11,14,1z', os: ubuntu-latest, container: 'ubuntu:18.04' } + - { compiler: clang-6.0, cxxstd: '11,14,17', os: ubuntu-20.04 } + - { compiler: clang-7, cxxstd: '11,14,17', os: ubuntu-20.04 } # Note: clang-8 does not fully support C++20, so it is not compatible with some libstdc++ versions in this mode - - { compiler: clang-8, cxxstd: '11,14,17,2a', os: ubuntu-22.04, container: 'ubuntu:18.04', install: 'clang-8 g++-7', gcc_toolchain: 7 } - - { compiler: clang-9, cxxstd: '11,14,17,2a', os: ubuntu-20.04 } - - { compiler: clang-10, cxxstd: '11,14,17,20', os: ubuntu-20.04 } - - { compiler: clang-11, cxxstd: '11,14,17,20', os: ubuntu-20.04 } - - { compiler: clang-12, cxxstd: '11,14,17,20', os: ubuntu-20.04 } + - { compiler: clang-8, cxxstd: '11,14,17,2a', os: ubuntu-20.04 , install: 'clang-8 g++-7', gcc_toolchain: 7 } + - { compiler: clang-9, cxxstd: '11,14,17,2a', os: ubuntu-20.04 } + - { compiler: clang-10, cxxstd: '11,14,17,20', os: ubuntu-20.04 } + - { compiler: clang-11, cxxstd: '11,14,17,20', os: ubuntu-20.04 } + - { compiler: clang-12, cxxstd: '11,14,17,20', os: ubuntu-20.04 } # Clang isn't compatible with libstdc++-13, so use the slightly older one - - { compiler: clang-13, cxxstd: '11,14,17,20', os: ubuntu-22.04, install: 'clang-13 g++-12', gcc_toolchain: 12 } - - { compiler: clang-14, cxxstd: '11,14,17,20', os: ubuntu-22.04, install: 'clang-14 g++-12', gcc_toolchain: 12 } - - { compiler: clang-15, cxxstd: '11,14,17,20', os: ubuntu-22.04, install: 'clang-15 g++-12', gcc_toolchain: 12 } + - { compiler: clang-13, cxxstd: '11,14,17,20', os: ubuntu-22.04, install: 'clang-13 g++-12', gcc_toolchain: 12 } + - { compiler: clang-14, cxxstd: '11,14,17,20', os: ubuntu-22.04, install: 'clang-14 g++-12', gcc_toolchain: 12 } + - { compiler: clang-15, cxxstd: '11,14,17,20', os: ubuntu-22.04, install: 'clang-15 g++-12', gcc_toolchain: 12 } + - { compiler: clang-16, cxxstd: '11,14,17,20,2b', os: ubuntu-24.04 } + # https://github.com/llvm/llvm-project/issues/59827: disabled 2b/23 for clang-17 with libstdc++13 in 24.04 + - { compiler: clang-17, cxxstd: '11,14,17,20', os: ubuntu-24.04 } + - { compiler: clang-18, cxxstd: '11,14,17,20,23,2c', os: ubuntu-24.04 } # libc++ - { compiler: clang-6.0, cxxstd: '11,14', os: ubuntu-22.04, container: 'ubuntu:18.04', stdlib: libc++, install: 'clang-6.0 libc++-dev libc++abi-dev' } From 5159cb6218b1247b5362703ae2a09926d8342ca1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 17 Nov 2024 16:28:10 +0100 Subject: [PATCH 18/67] Minor refactoring using const and type alias --- src/boost/locale/icu/numeric.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/boost/locale/icu/numeric.cpp b/src/boost/locale/icu/numeric.cpp index d81f55f7..9002a233 100644 --- a/src/boost/locale/icu/numeric.cpp +++ b/src/boost/locale/icu/numeric.cpp @@ -108,19 +108,20 @@ namespace boost { namespace locale { namespace impl_icu { if(detail::use_parent(ios, val)) return std::num_put::do_put(out, ios, fill, val); - const auto formatter = formatter_type::create(ios, loc_, enc_); + const std::unique_ptr formatter = formatter_type::create(ios, loc_, enc_); if(!formatter) return std::num_put::do_put(out, ios, fill, val); + using icu_type = typename detail::icu_format_type::type; size_t code_points; - typedef typename detail::icu_format_type::type icu_type; - const string_type& str = formatter->format(static_cast(val), code_points); + const string_type str = formatter->format(static_cast(val), code_points); + std::streamsize on_left = 0, on_right = 0, points = code_points; if(points < ios.width()) { - std::streamsize n = ios.width() - points; + const std::streamsize n = ios.width() - points; - std::ios_base::fmtflags flags = ios.flags() & std::ios_base::adjustfield; + const std::ios_base::fmtflags flags = ios.flags() & std::ios_base::adjustfield; // We do not really know internal point, so we assume that it does not // exist. So according to the standard field should be right aligned @@ -242,7 +243,7 @@ namespace boost { namespace locale { namespace impl_icu { if(!stream_ptr || detail::use_parent(ios, ValueType(0))) return std::num_get::do_get(in, end, ios, err, val); - const auto formatter = formatter_type::create(ios, loc_, enc_); + const std::unique_ptr formatter = formatter_type::create(ios, loc_, enc_); if(!formatter) return std::num_get::do_get(in, end, ios, err, val); @@ -256,7 +257,7 @@ namespace boost { namespace locale { namespace impl_icu { while(tmp.size() < 4096 && in != end && *in != '\n') tmp += *in++; - typedef typename detail::icu_format_type::type icu_type; + using icu_type = typename detail::icu_format_type::type; icu_type value; size_t parsed_chars; From 397f013506cb781175a967565ef1ebcc054d6747 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 16 Aug 2024 12:58:25 +0200 Subject: [PATCH 19/67] Remove copy operators from `generator` `hold_ptr` already ensures the proper semantic. --- include/boost/locale/generator.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/boost/locale/generator.hpp b/include/boost/locale/generator.hpp index 675aa604..119e6565 100644 --- a/include/boost/locale/generator.hpp +++ b/include/boost/locale/generator.hpp @@ -190,9 +190,6 @@ namespace locale { private: void set_all_options(localization_backend& backend, const std::string& id) const; - generator(const generator&); - void operator=(const generator&); - struct data; hold_ptr d; }; From 69e0e30d062177ed479a2e97df7a69bdfb0453ed Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 16 Aug 2024 13:01:38 +0200 Subject: [PATCH 20/67] Improve const correctness in `generator` --- src/boost/locale/shared/generator.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/boost/locale/shared/generator.cpp b/src/boost/locale/shared/generator.cpp index baf424be..ddff3ba6 100644 --- a/src/boost/locale/shared/generator.cpp +++ b/src/boost/locale/shared/generator.cpp @@ -1,5 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -93,9 +94,7 @@ namespace boost { namespace locale { std::locale generator::generate(const std::string& id) const { - std::locale base = std::locale::classic(); - - return generate(base, id); + return generate(std::locale::classic(), id); } std::locale generator::generate(const std::locale& base, const std::string& id) const @@ -110,8 +109,8 @@ namespace boost { namespace locale { set_all_options(*backend, id); std::locale result = base; - category_t facets = d->cats; - char_facet_t chars = d->chars; + const category_t facets = d->cats; + const char_facet_t chars = d->chars; for(category_t facet = per_character_facet_first; facet <= per_character_facet_last; ++facet) { if(!(facets & facet)) From 9f4424d7db3c61e7e67ecd73bfad7743504ddfb7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 24 Nov 2024 19:38:03 +0100 Subject: [PATCH 21/67] test: Include file in error message Also move the error reporting to a single function --- test/boostLocale/test/unit_test.hpp | 160 ++++++++++++++-------------- test/test_catalog.cpp | 2 +- test/test_date_time.cpp | 2 +- test/test_formatting.cpp | 19 ++-- 4 files changed, 93 insertions(+), 90 deletions(-) diff --git a/test/boostLocale/test/unit_test.hpp b/test/boostLocale/test/unit_test.hpp index 81db1829..953f414f 100644 --- a/test/boostLocale/test/unit_test.hpp +++ b/test/boostLocale/test/unit_test.hpp @@ -23,6 +23,12 @@ # include #endif +#ifndef BOOST_LOCALE_ERROR_LIMIT +# define BOOST_LOCALE_ERROR_LIMIT 20 +#endif + +#define BOOST_LOCALE_STRINGIZE(x) #x + namespace boost { namespace locale { namespace test { /// Name/path of current executable std::string exe_name; @@ -48,50 +54,47 @@ namespace boost { namespace locale { namespace test { static test_result instance; return instance; } -}}} // namespace boost::locale::test -#ifndef BOOST_LOCALE_ERROR_LIMIT -# define BOOST_LOCALE_ERROR_LIMIT 20 -#endif + inline void report_error(const char* expr, const char* file, int line) + { + std::cerr << "Error at " << file << '#' << line << ": " << expr << std::endl; + if(++boost::locale::test::results().error_counter > BOOST_LOCALE_ERROR_LIMIT) + throw std::runtime_error("Error limits reached, stopping unit test"); + } +}}} // namespace boost::locale::test -#define BOOST_LOCALE_STRINGIZE(x) #x +#define BOOST_LOCALE_TEST_REPORT_ERROR(expr) boost::locale::test::report_error(expr, __FILE__, __LINE__) -#define THROW_IF_TOO_BIG(X) \ - if((X) > BOOST_LOCALE_ERROR_LIMIT) \ - throw std::runtime_error("Error limits reached, stopping unit test") - -#define TEST(X) \ - do { \ - boost::locale::test::results().test_counter++; \ - if(X) \ - break; \ - std::cerr << "Error in line:" << __LINE__ << " " #X << std::endl; \ - THROW_IF_TOO_BIG(boost::locale::test::results().error_counter++); \ - BOOST_LOCALE_START_CONST_CONDITION \ +#define TEST(X) \ + do { \ + boost::locale::test::results().test_counter++; \ + if(X) \ + break; \ + BOOST_LOCALE_TEST_REPORT_ERROR(#X); \ + BOOST_LOCALE_START_CONST_CONDITION \ } while(0) BOOST_LOCALE_END_CONST_CONDITION -#define TEST_REQUIRE(X) \ - do { \ - boost::locale::test::results().test_counter++; \ - if(X) \ - break; \ - std::cerr << "Error in line " << __LINE__ << ": " #X << std::endl; \ - throw std::runtime_error("Critical test " #X " failed"); \ - BOOST_LOCALE_START_CONST_CONDITION \ +#define TEST_REQUIRE(X) \ + do { \ + boost::locale::test::results().test_counter++; \ + if(X) \ + break; \ + BOOST_LOCALE_TEST_REPORT_ERROR(#X); \ + throw std::runtime_error("Critical test " #X " failed"); \ + BOOST_LOCALE_START_CONST_CONDITION \ } while(0) BOOST_LOCALE_END_CONST_CONDITION -#define TEST_THROWS(X, E) \ - do { \ - boost::locale::test::results().test_counter++; \ - try { \ - X; \ - } catch(E const& /*e*/) { \ - break; \ - } catch(...) { \ - } \ - std::cerr << "Error in line " << __LINE__ << ": " #X << std::endl; \ - THROW_IF_TOO_BIG(boost::locale::test::results().error_counter++); \ - BOOST_LOCALE_START_CONST_CONDITION \ +#define TEST_THROWS(X, E) \ + do { \ + boost::locale::test::results().test_counter++; \ + try { \ + X; \ + } catch(E const& /*e*/) { \ + break; \ + } catch(...) { \ + } \ + BOOST_LOCALE_TEST_REPORT_ERROR(#X); \ + BOOST_LOCALE_START_CONST_CONDITION \ } while(0) BOOST_LOCALE_END_CONST_CONDITION void test_main(int argc, char** argv); @@ -138,6 +141,11 @@ const std::string& to_string(const std::string& s) return s; } +std::string to_string(std::nullptr_t) +{ + return ""; +} + template std::string to_string(const std::vector& v) { @@ -223,58 +231,52 @@ std::string to_string(const char32_t c) #endif template -void test_impl(bool success, T const& l, U const& r, const char* expr, const char* fail_expr, int line) +void test_impl(bool success, + T const& l, + U const& r, + const char* expr, + const char* fail_expr, + const char* file, + int line) { boost::locale::test::results().test_counter++; if(!success) { - std::cerr << "Error in line " << line << ": " << expr << std::endl; - std::cerr << "---- [" << to_string(l) << "] " << fail_expr << " [" << to_string(r) << "]" << std::endl; - THROW_IF_TOO_BIG(boost::locale::test::results().error_counter++); + if(fail_expr) { + std::ostringstream s; + s << expr << '\n' << "---- [" << to_string(l) << "] " << fail_expr << " [" << to_string(r) << "]"; + boost::locale::test::report_error(s.str().c_str(), file, line); + } else + boost::locale::test::report_error(expr, file, line); } } -template -void test_eq_impl(T const& l, U const& r, const char* expr, int line) -{ - test_impl(l == r, l, r, expr, "!=", line); -} - -template -void test_ne_impl(T const& l, U const& r, const char* expr, int line) -{ - test_impl(l != r, l, r, expr, "==", line); -} - -template -void test_le_impl(T const& l, U const& r, const char* expr, int line) +void test_impl(bool success, const char* reason, const char* file, int line) { - test_impl(l <= r, l, r, expr, ">", line); + test_impl(success, nullptr, nullptr, reason, nullptr, file, line); } -template -void test_lt_impl(T const& l, U const& r, const char* expr, int line) -{ - test_impl(l < r, l, r, expr, ">=", line); -} - -template -void test_ge_impl(T const& l, U const& r, const char* expr, int line) -{ - test_impl(l >= r, l, r, expr, "<", line); -} - -template -void test_gt_impl(T const& l, U const& r, const char* expr, int line) -{ - test_impl(l > r, l, r, expr, "<=", line); -} +#define BOOST_LOCALE_TEST_OP_IMPL(name, test_op, fail_op) \ + template \ + void test_##name##_impl(T const& l, U const& r, const char* expr, const char* file, int line) \ + { \ + test_impl(l test_op r, l, r, expr, #fail_op, file, line); \ + } -#define TEST_EQ(x, y) test_eq_impl(x, y, BOOST_LOCALE_STRINGIZE(x == y), __LINE__) -#define TEST_NE(x, y) test_ne_impl(x, y, BOOST_LOCALE_STRINGIZE(x != y), __LINE__) -#define TEST_LE(x, y) test_le_impl(x, y, BOOST_LOCALE_STRINGIZE(x <= y), __LINE__) -#define TEST_LT(x, y) test_lt_impl(x, y, BOOST_LOCALE_STRINGIZE(x < y), __LINE__) -#define TEST_GE(x, y) test_ge_impl(x, y, BOOST_LOCALE_STRINGIZE(x >= y), __LINE__) -#define TEST_GT(x, y) test_gt_impl(x, y, BOOST_LOCALE_STRINGIZE(x > y), __LINE__) +BOOST_LOCALE_TEST_OP_IMPL(eq, ==, !=) +BOOST_LOCALE_TEST_OP_IMPL(ne, !=, ==) +BOOST_LOCALE_TEST_OP_IMPL(le, <=, >) +BOOST_LOCALE_TEST_OP_IMPL(lt, <, >=) +BOOST_LOCALE_TEST_OP_IMPL(ge, >=, <) +BOOST_LOCALE_TEST_OP_IMPL(gt, >, <=) + +#undef BOOST_LOCALE_TEST_OP_IMPL + +#define TEST_EQ(x, y) test_eq_impl(x, y, BOOST_LOCALE_STRINGIZE(x == y), __FILE__, __LINE__) +#define TEST_NE(x, y) test_ne_impl(x, y, BOOST_LOCALE_STRINGIZE(x != y), __FILE__, __LINE__) +#define TEST_LE(x, y) test_le_impl(x, y, BOOST_LOCALE_STRINGIZE(x <= y), __FILE__, __LINE__) +#define TEST_LT(x, y) test_lt_impl(x, y, BOOST_LOCALE_STRINGIZE(x < y), __FILE__, __LINE__) +#define TEST_GE(x, y) test_ge_impl(x, y, BOOST_LOCALE_STRINGIZE(x >= y), __FILE__, __LINE__) +#define TEST_GT(x, y) test_gt_impl(x, y, BOOST_LOCALE_STRINGIZE(x > y), __FILE__, __LINE__) #if BOOST_LOCALE_SPACESHIP_NULLPTR_WARNING # pragma clang diagnostic pop diff --git a/test/test_catalog.cpp b/test/test_catalog.cpp index 29a34c69..6b884b99 100644 --- a/test/test_catalog.cpp +++ b/test/test_catalog.cpp @@ -47,7 +47,7 @@ void test_plural_expr() TEST(ptr); \ return ptr; \ }() -#define TEST_EQ_EXPR(expr, rhs) test_eq_impl(COMPILE_PLURAL_EXPR(expr)(0), rhs, expr, __LINE__) +#define TEST_EQ_EXPR(expr, rhs) test_eq_impl(COMPILE_PLURAL_EXPR(expr)(0), rhs, expr, __FILE__, __LINE__) // Number only TEST_EQ_EXPR("0", 0); TEST_EQ_EXPR("42", 42); diff --git a/test/test_date_time.cpp b/test/test_date_time.cpp index b9e6ca0e..38280caf 100644 --- a/test/test_date_time.cpp +++ b/test/test_date_time.cpp @@ -30,7 +30,7 @@ #define TEST_EQ_FMT(t, X) \ empty_stream(ss) << (t); \ - test_eq_impl(ss.str(), X, #t "==" #X, __LINE__) + test_eq_impl(ss.str(), X, #t "==" #X, __FILE__, __LINE__) // Very simple container for a part of the tests. Counts its instances struct mock_calendar : public boost::locale::abstract_calendar { diff --git a/test/test_formatting.cpp b/test/test_formatting.cpp index d418c583..e2df2adc 100644 --- a/test/test_formatting.cpp +++ b/test/test_formatting.cpp @@ -159,17 +159,17 @@ void test_fmt_impl(std::basic_ostringstream& ss, const std::basic_string& expected, int line) { - ss << value; - test_eq_impl(ss.str(), expected, "", line); + test_impl(!!(ss << value), "Formatting failed", __FILE__, line); + test_eq_impl(ss.str(), expected, "", __FILE__, line); } template void test_parse_impl(std::basic_istringstream& ss, const T& expected, int line) { T v; - ss >> v >> std::ws; - test_eq_impl(v, expected, "v == expected", line); - test_eq_impl(ss.eof(), true, "ss.eof()", line); + test_impl(!!(ss >> v), "Parsing failed", __FILE__, line); + test_eq_impl(v, expected, "v == expected", __FILE__, line); + test_eq_impl((ss >> std::ws).eof(), true, "ss.eof()", __FILE__, line); } template @@ -178,8 +178,9 @@ void test_parse_at_impl(std::basic_istringstream& ss, const T& expecte T v; CharType c_at; ss >> v >> std::skipws >> c_at; - test_eq_impl(v, expected, "v == expected", line); - test_eq_impl(c_at, '@', "c_at == @", line); + test_impl(!!ss, "Parsing failed", __FILE__, line); + test_eq_impl(v, expected, "v == expected", __FILE__, line); + test_eq_impl(c_at, '@', "c_at == @", __FILE__, line); } template @@ -187,7 +188,7 @@ void test_parse_fail_impl(std::basic_istringstream& ss, int line) { T v; ss >> v; - test_eq_impl(ss.fail(), true, "ss.fail()", line); + test_eq_impl(ss.fail(), true, "ss.fail()", __FILE__, line); } #define TEST_FMT(manip, value, expected) \ @@ -641,7 +642,7 @@ void test_format_class_impl(const std::string& fmt_string, format_type fmt(std::basic_string(fmt_string.begin(), fmt_string.end())); fmt % value; std::basic_string expected_str_loc(to_correct_string(expected_str, loc)); - test_eq_impl(fmt.str(loc), expected_str_loc, ("Format: " + fmt_string).c_str(), line); + test_eq_impl(fmt.str(loc), expected_str_loc, ("Format: " + fmt_string).c_str(), __FILE__, line); } template From f0d95b079b4be457327350cddd35e76ef3ff4553 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 24 Nov 2024 20:01:01 +0100 Subject: [PATCH 22/67] Add TEST_CONTEXT Avoid to require manual output in anticipation of failures. --- test/boostLocale/test/unit_test.hpp | 37 ++++++++++++++++++++++++++--- test/test_catalog.cpp | 6 ++--- test/test_codecvt.cpp | 4 +--- test/test_encoding.cpp | 14 +++++------ test/test_formatting.cpp | 2 ++ 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/test/boostLocale/test/unit_test.hpp b/test/boostLocale/test/unit_test.hpp index 953f414f..cbfc8cd4 100644 --- a/test/boostLocale/test/unit_test.hpp +++ b/test/boostLocale/test/unit_test.hpp @@ -10,6 +10,7 @@ #define BOOST_LOCALE_UNIT_TEST_HPP #include +#include #include #include #include @@ -33,8 +34,10 @@ namespace boost { namespace locale { namespace test { /// Name/path of current executable std::string exe_name; + class test_context; + struct test_result { - test_result() : error_counter(0), test_counter(0) + test_result() { #if defined(_MSC_VER) && (_MSC_VER > 1310) // disable message boxes on assert(), abort() @@ -46,8 +49,9 @@ namespace boost { namespace locale { namespace test { _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR); #endif } - int error_counter; - int test_counter; + int error_counter = 0; + int test_counter = 0; + const test_context* context = nullptr; }; inline test_result& results() { @@ -55,9 +59,32 @@ namespace boost { namespace locale { namespace test { return instance; } + class test_context { + const test_context* oldCtx_; + const std::string msg_; + + public: + test_context(std::string ctx) : oldCtx_(results().context), msg_(std::move(ctx)) { results().context = this; } + ~test_context() { results().context = oldCtx_; } + friend std::ostream& operator<<(std::ostream& os, const test_context& c) + { + const test_context* current = &c; + os << "CONTEXT: "; + std::string indent = "\n\t"; + do { + os << indent << current->msg_; + indent += '\t'; + } while((current = current->oldCtx_) != nullptr); + return os; + } + }; + inline void report_error(const char* expr, const char* file, int line) { std::cerr << "Error at " << file << '#' << line << ": " << expr << std::endl; + const auto* context = results().context; + if(context) + std::cerr << ' ' << *context << std::endl; if(++boost::locale::test::results().error_counter > BOOST_LOCALE_ERROR_LIMIT) throw std::runtime_error("Error limits reached, stopping unit test"); } @@ -97,6 +124,10 @@ namespace boost { namespace locale { namespace test { BOOST_LOCALE_START_CONST_CONDITION \ } while(0) BOOST_LOCALE_END_CONST_CONDITION +#define TEST_CONTEXT(expr) \ + boost::locale::test::test_context BOOST_JOIN(test_context_, __COUNTER__)( \ + static_cast(std::stringstream{} << expr).str()) + void test_main(int argc, char** argv); int main(int argc, char** argv) diff --git a/test/test_catalog.cpp b/test/test_catalog.cpp index 6b884b99..bc06805f 100644 --- a/test/test_catalog.cpp +++ b/test/test_catalog.cpp @@ -31,10 +31,8 @@ void test_plural_expr_rand(const T& ref, const char* expr) const auto n = getRandValue(minVal, maxVal); const auto result = ptr(n); const auto refResult = ref(n); - if(result != refResult) { - std::cerr << "Expression: " << expr << "; n=" << n << '\n'; // LCOV_EXCL_LINE - TEST_EQ(result, refResult); // LCOV_EXCL_LINE - } + TEST_CONTEXT("Expression: " << expr << "; n=" << n); + TEST_EQ(result, refResult); } } diff --git a/test/test_codecvt.cpp b/test/test_codecvt.cpp index 7df704d3..504da439 100644 --- a/test/test_codecvt.cpp +++ b/test/test_codecvt.cpp @@ -53,10 +53,8 @@ void test_codecvt_in_n_m(const cvt_type& cvt, int n, int m) std::mbstate_t mb2 = mb; std::codecvt_base::result r = cvt.in(mb, from, end, from_next, to, to_end, to_next); - int count = cvt.length(mb2, from, end, to_end - to); + const int count = cvt.length(mb2, from, end, to_end - to); TEST_EQ(memcmp(&mb, &mb2, sizeof(mb)), 0); - if(count != from_next - from) - std::cout << count << " " << from_next - from << std::endl; // LCOV_EXCL_LINE TEST_EQ(count, from_next - from); if(r == cvt_type::partial) { diff --git a/test/test_encoding.cpp b/test/test_encoding.cpp index 4de7e610..7fb07c2f 100644 --- a/test/test_encoding.cpp +++ b/test/test_encoding.cpp @@ -832,10 +832,9 @@ void test_simple_encodings() const auto encodings = get_simple_encodings(); for(auto it = encodings.begin(), end = encodings.end(); it != end; ++it) { TEST_EQ(normalize_encoding(*it), *it); // Must be normalized - const auto it2 = std::find(it + 1, end, *it); - TEST(it2 == end); - if(it2 != end) - std::cerr << "Duplicate entry: " << *it << '\n'; // LCOV_EXCL_LINE + TEST_CONTEXT("Entry: " << *it); + // Must be unique + TEST(std::find(it + 1, end, *it) == end); } const auto it = std::is_sorted_until(encodings.begin(), encodings.end()); TEST(it == encodings.end()); @@ -852,10 +851,9 @@ void test_win_codepages() auto is_same_win_codepage = [&it](const windows_encoding& rhs) -> bool { return it->codepage == rhs.codepage && std::strcmp(it->name, rhs.name) == 0; }; - const auto* it2 = std::find_if(it + 1, end, is_same_win_codepage); - TEST(it2 == end); - if(it2 != end) - std::cerr << "Duplicate entry: " << it->name << ':' << it->codepage << '\n'; // LCOV_EXCL_LINE + TEST_CONTEXT("Entry: " << it->name << ':' << it->codepage); + // Must be unique + TEST(std::find_if(it + 1, end, is_same_win_codepage) == end); } const auto cmp = [](const windows_encoding& rhs, const windows_encoding& lhs) -> bool { return rhs < lhs.name; }; const auto* it = std::is_sorted_until(all_windows_encodings, std::end(all_windows_encodings), cmp); diff --git a/test/test_formatting.cpp b/test/test_formatting.cpp index e2df2adc..41c30a0f 100644 --- a/test/test_formatting.cpp +++ b/test/test_formatting.cpp @@ -300,6 +300,7 @@ void test_parse_fail_impl(std::basic_istringstream& ss, int line) #define TEST_MIN_MAX_POSIX(type) \ do { \ + TEST_CONTEXT(#type); \ const std::string minval = as_posix_string(std::numeric_limits::min()); \ const std::string maxval = as_posix_string(std::numeric_limits::max()); \ TEST_MIN_MAX_FMT(as::posix, type, minval, maxval); \ @@ -340,6 +341,7 @@ void test_as_posix(const std::string& e_charset = "UTF-8") localization_backend_manager::global(backend); for(const std::string name : {"en_US", "ru_RU", "de_DE"}) { const std::locale loc = boost::locale::generator{}(name + "." + e_charset); + TEST_CONTEXT("Locale " << (name + "." + e_charset)); TEST_MIN_MAX_POSIX(int16_t); TEST_MIN_MAX_POSIX(uint16_t); From 396ec24b5869e0481d6335804eeede0f5a5e1b53 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 1 Dec 2024 18:23:54 +0100 Subject: [PATCH 23/67] Add support for ISO-15924 abbreviation script code to `locale_data` The value is (currently) ignored but this allows to parse locale names returned by ICU. --- include/boost/locale/util/locale_data.hpp | 11 +++- src/boost/locale/util/locale_data.cpp | 75 ++++++++++++++++++----- test/test_util.cpp | 63 +++++++++++++++++-- 3 files changed, 126 insertions(+), 23 deletions(-) diff --git a/include/boost/locale/util/locale_data.hpp b/include/boost/locale/util/locale_data.hpp index 397c404d..cbd7f696 100644 --- a/include/boost/locale/util/locale_data.hpp +++ b/include/boost/locale/util/locale_data.hpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2023 Alexander Grund +// Copyright (c) 2023-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -21,6 +21,7 @@ namespace boost { namespace locale { namespace util { /// Holder and parser for locale names/identifiers class BOOST_LOCALE_DECL locale_data { std::string language_; + std::string script_; std::string country_; std::string encoding_; std::string variant_; @@ -36,6 +37,8 @@ namespace boost { namespace locale { namespace util { /// Return language (usually 2 lowercase letters, i.e. ISO-639 or 'C') const std::string& language() const { return language_; } + /// Return the ISO-15924 abbreviation script code if present + const std::string& script() const { return script_; } /// Return country (usually 2 uppercase letters, i.e. ISO-3166) const std::string& country() const { return country_; } /// Return encoding/codeset, e.g. ISO8859-1 or UTF-8 @@ -48,12 +51,13 @@ namespace boost { namespace locale { namespace util { /// Return iff the encoding is UTF-8 bool is_utf8() const { return utf8_; } - /// Parse a locale identifier of the form `[language[_territory][.codeset][@modifier]]` + /// Parse a locale identifier of the form `[language[_script][_territory][.codeset][@modifier]]` /// /// Allows a dash as the delimiter: `[language-territory]` /// Return true if the identifier is valid: /// - `language` is given and consists of ASCII letters - /// - `territory`, if given, consists of ASCII letters + /// - `script` is only considered if it consists of exactly 4 ASCII letters + /// - `territory`, if given, consists of ASCII letters (usually ISO-3166) /// - Any field started by a delimiter (`_`, `-`, `.`, `@`) is not empty /// Otherwise parsing is aborted. Valid values already parsed stay set, other are defaulted. bool parse(const std::string& locale_name); @@ -65,6 +69,7 @@ namespace boost { namespace locale { namespace util { private: void reset(); bool parse_from_lang(const std::string& input); + bool parse_from_script(const std::string& input); bool parse_from_country(const std::string& input); bool parse_from_encoding(const std::string& input); bool parse_from_variant(const std::string& input); diff --git a/src/boost/locale/util/locale_data.cpp b/src/boost/locale/util/locale_data.cpp index f95d28a5..41c0bc88 100644 --- a/src/boost/locale/util/locale_data.cpp +++ b/src/boost/locale/util/locale_data.cpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2022-2023 Alexander Grund +// Copyright (c) 2022-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -14,6 +14,26 @@ #include namespace boost { namespace locale { namespace util { + /// Convert uppercase ASCII to lower case, return true if converted + static bool make_lower(char& c) + { + if(is_upper_ascii(c)) { + c += 'a' - 'A'; + return true; + } else + return false; + } + + /// Convert lowercase ASCII to upper case, return true if converted + static bool make_upper(char& c) + { + if(is_lower_ascii(c)) { + c += 'A' - 'a'; + return true; + } else + return false; + } + locale_data::locale_data() { reset(); @@ -28,6 +48,7 @@ namespace boost { namespace locale { namespace util { void locale_data::reset() { language_ = "C"; + script_.clear(); country_.clear(); encoding_ = "US-ASCII"; variant_.clear(); @@ -37,6 +58,8 @@ namespace boost { namespace locale { namespace util { std::string locale_data::to_string() const { std::string result = language_; + if(!script_.empty()) + (result += '_') += script_; if(!country_.empty()) (result += '_') += country_; if(!encoding_.empty() && !util::are_encodings_equal(encoding_, "US-ASCII")) @@ -60,14 +83,39 @@ namespace boost { namespace locale { namespace util { return false; // lowercase ASCII for(char& c : tmp) { - if(is_upper_ascii(c)) - c += 'a' - 'A'; - else if(!is_lower_ascii(c)) + if(!is_lower_ascii(c) && !make_lower(c)) return false; } if(tmp != "c" && tmp != "posix") // Keep default language_ = tmp; + if(end >= input.size()) + return true; + else if(input[end] == '-' || input[end] == '_') + return parse_from_script(input.substr(end + 1)); + else if(input[end] == '.') + return parse_from_encoding(input.substr(end + 1)); + else { + BOOST_ASSERT_MSG(input[end] == '@', "Unexpected delimiter"); + return parse_from_variant(input.substr(end + 1)); + } + } + + bool locale_data::parse_from_script(const std::string& input) + { + const auto end = input.find_first_of("-_@."); + std::string tmp = input.substr(0, end); + // Script is exactly 4 ASCII characters, otherwise it is not present + if(tmp.length() != 4) + return parse_from_country(input); + + for(char& c : tmp) { + if(!is_lower_ascii(c) && !make_lower(c)) + return parse_from_country(input); + } + make_upper(tmp[0]); // Capitalize first letter only + script_ = tmp; + if(end >= input.size()) return true; else if(input[end] == '-' || input[end] == '_') @@ -91,10 +139,9 @@ namespace boost { namespace locale { namespace util { return false; // Make uppercase - for(char& c : tmp) { - if(util::is_lower_ascii(c)) - c += 'A' - 'a'; - } + for(char& c : tmp) + make_upper(c); + // If it's ALL uppercase ASCII, assume ISO 3166 country id if(std::find_if_not(tmp.begin(), tmp.end(), util::is_upper_ascii) != tmp.end()) { // else handle special cases: @@ -142,20 +189,16 @@ namespace boost { namespace locale { namespace util { return false; variant_ = input; // No assumptions, just make it lowercase - for(char& c : variant_) { - if(util::is_upper_ascii(c)) - c += 'a' - 'A'; - } + for(char& c : variant_) + make_lower(c); return true; } locale_data& locale_data::encoding(std::string new_encoding, const bool uppercase) { if(uppercase) { - for(char& c : new_encoding) { - if(util::is_lower_ascii(c)) - c += 'A' - 'a'; - } + for(char& c : new_encoding) + make_upper(c); } encoding_ = std::move(new_encoding); utf8_ = util::normalize_encoding(encoding_) == "utf8"; diff --git a/test/test_util.cpp b/test/test_util.cpp index 5b4b90d8..cf773ba1 100644 --- a/test/test_util.cpp +++ b/test/test_util.cpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2022-2023 Alexander Grund +// Copyright (c) 2022-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -112,6 +112,34 @@ void test_get_system_locale() TEST_EQ(get_system_locale(true), "barlang.bar"); } +#ifndef BOOST_LOCALE_WITH_ICU +void verify_against_icu(){}; +#else +# include +void verify_against_icu() +{ + int32_t count; + auto* cur_locale = icu::Locale::getAvailableLocales(count); + boost::locale::util::locale_data data; + for(int i = 0; i < count; i++, cur_locale++) { + const std::string loc_name = cur_locale->getName(); + if(loc_name == "en_US_POSIX") + continue; // Parsed as "C", tested elsewhere + for(const bool add_utf8 : {false, true}) { + // Also test with added encoding to verify input is fully parsed + const std::string curName = add_utf8 ? loc_name + ".UTF-8" : loc_name; + TEST_CONTEXT(curName); + TEST(data.parse(curName)); + TEST_EQ(data.language(), cur_locale->getLanguage()); + TEST_EQ(data.country(), cur_locale->getCountry()); + TEST_EQ(data.encoding(), add_utf8 ? "UTF-8" : "US-ASCII"); + TEST_EQ(data.variant(), cur_locale->getVariant()); + TEST_EQ(data.to_string(), curName); + } + } +} +#endif + void test_locale_data() { boost::locale::util::locale_data data; @@ -131,6 +159,7 @@ void test_locale_data() TEST(data.parse("C")); TEST_EQ(data.language(), "C"); + TEST_EQ(data.script(), ""); TEST_EQ(data.country(), ""); TEST_EQ(data.encoding(), "US-ASCII"); TEST(!data.is_utf8()); @@ -138,6 +167,7 @@ void test_locale_data() TEST(data.parse("ku_TR.UTF-8@sorani")); TEST_EQ(data.language(), "ku"); + TEST_EQ(data.script(), ""); TEST_EQ(data.country(), "TR"); TEST_EQ(data.encoding(), "UTF-8"); TEST(data.is_utf8()); @@ -200,6 +230,17 @@ void test_locale_data() TEST(data.is_utf8()); TEST_EQ(data.variant(), ""); + // Script used, optionally with dashes instead of underscores + for(const std::string name : {"pa_Arab_PK.UTF-8", "pa-Arab_PK.UTF-8", "pa_Arab-PK.UTF-8"}) { + TEST(data.parse("pa_Arab_PK.UTF-8")); + TEST_EQ(data.language(), "pa"); + TEST_EQ(data.script(), "Arab"); + TEST_EQ(data.country(), "PK"); + TEST_EQ(data.encoding(), "UTF-8"); + TEST(data.is_utf8()); + TEST_EQ(data.variant(), ""); + } + // to_string yields the input (if format is correct already) for(const std::string name : {"C", "en_US.UTF-8", @@ -211,8 +252,18 @@ void test_locale_data() "th_TH.TIS620", "zh_TW.UTF-8@radical", "en_001", - "en_150.UTF-8"}) + "en_150.UTF-8", + // Different variation with parts missing + "pa_Arab_PK.UTF-8", + "pa_Arab_PK@euro", + "pa_Arab.UTF-8", + "pa_Arab@euro", + "pa.UTF-8", + "pa@euro", + "pa_PK.UTF-8", + "pa_PK@euro"}) { + TEST_CONTEXT(name); TEST(data.parse(name)); TEST_EQ(data.to_string(), name); } @@ -224,16 +275,18 @@ void test_locale_data() // Unify casing: // - language: lowercase + // - script: Capitalized // - region: uppercase // - encoding: uppercase // - variant: lowercase - TEST(data.parse("EN_us.utf-8@EUro")); + TEST(data.parse("EN_sCrI_us.utf-8@EUro")); TEST_EQ(data.language(), "en"); + TEST_EQ(data.script(), "Scri"); TEST_EQ(data.country(), "US"); TEST_EQ(data.encoding(), "UTF-8"); TEST(data.is_utf8()); TEST_EQ(data.variant(), "euro"); - TEST_EQ(data.to_string(), "en_US.UTF-8@euro"); + TEST_EQ(data.to_string(), "en_Scri_US.UTF-8@euro"); TEST(data.parse("lAnGUagE_cOunTRy.eNCo-d123inG@Va-r1_Ant")); TEST_EQ(data.to_string(), "language_COUNTRY.ENCO-D123ING@va-r1_ant"); @@ -313,6 +366,8 @@ void test_locale_data() // Construct from string TEST_EQ(boost::locale::util::locale_data("en_US.UTF-8").to_string(), "en_US.UTF-8"); TEST_THROWS(boost::locale::util::locale_data invalid("en_UÖ.UTF-8"), std::invalid_argument); + + verify_against_icu(); } #include "../src/boost/locale/util/numeric.hpp" From bce1f9ea808583e0452717b6ea3371f554d9b4f8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 8 Dec 2024 19:48:05 +0100 Subject: [PATCH 24/67] Use node from boost.io --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2532d589..72561a0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: git config --global pack.threads 0 if [[ "${{matrix.container}}" == "ubuntu:1"* ]]; then # Node 20 doesn't work with Ubuntu 16/18 glibc: https://github.com/actions/checkout/issues/1590 - curl -sL https://unofficial-builds.nodejs.org/download/release/v20.9.0/node-v20.9.0-linux-x64-glibc-217.tar.xz | tar -xJ --strip-components 1 -C /node20217 + curl -sL https://archives.boost.io/misc/node/node-v20.9.0-linux-x64-glibc-217.tar.xz | tar -xJ --strip-components 1 -C /node20217 fi - uses: actions/checkout@v4 From 6b79d8bbeba538e99f4f2fc645f9f3d8f3a90ed3 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 10 Dec 2024 19:38:30 +0100 Subject: [PATCH 25/67] Refactor ICU search for more clarity --- build/Jamfile.v2 | 52 ++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 7889db6b..f38e3e26 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -95,7 +95,7 @@ if $(.icu-path) debug-message ICU path set to "$(.icu-path)" ; } -rule path_options ( properties * ) +rule icu_search_paths ( properties * ) { local result ; if 64 in $(properties) && msvc in $(properties) @@ -126,53 +126,53 @@ if [ modules.peek : ICU_ICUIN_NAME ] if $(ICU_ICUUC_NAME) { - searched-lib icuuc : : $(ICU_ICUUC_NAME) @path_options ; + searched-lib icuuc : : $(ICU_ICUUC_NAME) @icu_search_paths ; debug-message Using "$(ICU_ICUUC_NAME)" for library "icuuc" ; } else { - searched-lib icuuc : : shared @path_options ; - searched-lib icuuc : : msvc debug icuucd shared @path_options ; - searched-lib icuuc : : intel windows debug icuucd shared @path_options ; - searched-lib icuuc : : sicuuc static @path_options ; - searched-lib icuuc : : msvc debug sicuucd static @path_options ; - searched-lib icuuc : : intel windows debug sicuucd static @path_options ; + searched-lib icuuc : : icuuc shared @icu_search_paths ; + searched-lib icuuc : : msvc debug icuucd shared @icu_search_paths ; + searched-lib icuuc : : intel windows debug icuucd shared @icu_search_paths ; + searched-lib icuuc : : sicuuc static @icu_search_paths ; + searched-lib icuuc : : msvc debug sicuucd static @icu_search_paths ; + searched-lib icuuc : : intel windows debug sicuucd static @icu_search_paths ; searched-lib icuuc : : this_is_an_invalid_library_name ; } if $(ICU_ICUDT_NAME) { - searched-lib icudt : : $(ICU_ICUDT_NAME) @path_options ; + searched-lib icudt : : $(ICU_ICUDT_NAME) @icu_search_paths ; debug-message Using "$(ICU_ICUDT_NAME)" for library "icudt" ; } else { - searched-lib icudt : : icudata shared @path_options ; - searched-lib icudt : : icudt msvc shared @path_options ; - searched-lib icudt : : icudt intel windows shared @path_options ; - searched-lib icudt : : sicudata static @path_options ; - searched-lib icudt : : sicudt msvc static @path_options ; - searched-lib icudt : : sicudt intel windows static @path_options ; + searched-lib icudt : : icudata shared @icu_search_paths ; + searched-lib icudt : : msvc icudt shared @icu_search_paths ; + searched-lib icudt : : intel windows icudt shared @icu_search_paths ; + searched-lib icudt : : sicudata static @icu_search_paths ; + searched-lib icudt : : msvc sicudt static @icu_search_paths ; + searched-lib icudt : : intel windows sicudt static @icu_search_paths ; searched-lib icudt : : this_is_an_invalid_library_name ; } if $(ICU_ICUIN_NAME) { - searched-lib icuin : : $(ICU_ICUIN_NAME) @path_options ; + searched-lib icuin : : $(ICU_ICUIN_NAME) @icu_search_paths ; debug-message Using "$(ICU_ICUIN_NAME)" for library "icuin" ; } else { - searched-lib icuin : : icui18n shared @path_options ; - searched-lib icuin : : msvc debug icuind shared @path_options ; - searched-lib icuin : : msvc icuin shared @path_options ; - searched-lib icuin : : intel windows debug icuind shared @path_options ; - searched-lib icuin : : intel windows icuin shared @path_options ; - searched-lib icuin : : sicui18n static @path_options ; - searched-lib icuin : : msvc debug sicuind static @path_options ; - searched-lib icuin : : msvc sicuin static @path_options ; - searched-lib icuin : : intel windows debug sicuind static @path_options ; - searched-lib icuin : : intel windows sicuin static @path_options ; + searched-lib icuin : : icui18n shared @icu_search_paths ; + searched-lib icuin : : msvc debug icuind shared @icu_search_paths ; + searched-lib icuin : : msvc icuin shared @icu_search_paths ; + searched-lib icuin : : intel windows debug icuind shared @icu_search_paths ; + searched-lib icuin : : intel windows icuin shared @icu_search_paths ; + searched-lib icuin : : sicui18n static @icu_search_paths ; + searched-lib icuin : : msvc debug sicuind static @icu_search_paths ; + searched-lib icuin : : msvc sicuin static @icu_search_paths ; + searched-lib icuin : : intel windows debug sicuind static @icu_search_paths ; + searched-lib icuin : : intel windows sicuin static @icu_search_paths ; searched-lib icuin : : this_is_an_invalid_library_name ; } From 087dfb61d619d0f5b3faddd9eb1e06dbec6ad319 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 19 Dec 2024 15:58:03 +0100 Subject: [PATCH 26/67] GHA: Remove MacOS 12 job Runner is removed --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72561a0b..68b01c92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,6 @@ jobs: compiler: clang-12, cxxstd: '11,14,17,20', os: ubuntu-20.04, stdlib: libc++, install: 'clang-12 libc++-12-dev libc++abi-12-dev' } # OSX, clang - - { compiler: clang, cxxstd: '11,14,17,20', os: macos-12 } - { name: MacOS w/ clang and sanitizers, compiler: clang, cxxstd: '11,14,17,20,2b', os: macos-13, ubsan: yes } # TODO: Iconv issue From d52908bb3c2f7bd43e3726f842fa79a6ef691fb4 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sat, 21 Dec 2024 12:16:52 +0100 Subject: [PATCH 27/67] Refactor test_formatting - Make spelling of "ICU" consistently uppercase (except at start of names) - Move definitions related to existance of ICU together and sort by name --- test/formatting_common.hpp | 1 - test/test_formatting.cpp | 117 +++++++++++++++++++------------------ 2 files changed, 60 insertions(+), 58 deletions(-) diff --git a/test/formatting_common.hpp b/test/formatting_common.hpp index 97ef7919..260bc20d 100644 --- a/test/formatting_common.hpp +++ b/test/formatting_common.hpp @@ -10,7 +10,6 @@ #include #include -#include "../src/boost/locale/util/foreach_char.hpp" #include "boostLocale/test/tools.hpp" #include "boostLocale/test/unit_test.hpp" diff --git a/test/test_formatting.cpp b/test/test_formatting.cpp index 41c30a0f..8f111230 100644 --- a/test/test_formatting.cpp +++ b/test/test_formatting.cpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2021-2022 Alexander Grund +// Copyright (c) 2021-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -26,103 +26,111 @@ const std::string test_locale_name = "en_US"; std::string message_path = "./"; -#ifdef BOOST_LOCALE_WITH_ICU +#ifndef BOOST_LOCALE_WITH_ICU +# define BOOST_LOCALE_ICU_VERSION 0 +# define BOOST_LOCALE_ICU_VERSION_EXACT 0 +#else # include # include # include # include # define BOOST_LOCALE_ICU_VERSION (U_ICU_VERSION_MAJOR_NUM * 100 + U_ICU_VERSION_MINOR_NUM) # define BOOST_LOCALE_ICU_VERSION_EXACT (BOOST_LOCALE_ICU_VERSION * 100 + U_ICU_VERSION_PATCHLEVEL_NUM) +#endif -const icu::Locale& get_icu_test_locale() +using format_style_t = std::ios_base&(std::ios_base&); + +namespace { +#ifndef BOOST_LOCALE_WITH_ICU +const std::string icu_full_gmt_name; +// clang-format off +#if BOOST_LOCALE_ICU_VERSION >= 402 +std::string get_ICU_currency_iso(...){ return ""; } // LCOV_EXCL_LINE +#endif +std::string get_ICU_date(...){ return ""; } // LCOV_EXCL_LINE +std::string get_ICU_datetime(...){ return ""; } // LCOV_EXCL_LINE +std::string get_ICU_time(...){ return ""; } // LCOV_EXCL_LINE +// clang-format on +#else +const icu::Locale& get_ICU_test_locale() { static icu::Locale locale = icu::Locale::createCanonical(test_locale_name.c_str()); return locale; } -std::string from_icu_string(const icu::UnicodeString& str) +std::string from_ICU_string(const icu::UnicodeString& str) { return boost::locale::conv::utf_to_utf(str.getBuffer(), str.getBuffer() + str.length()); } -#else -# define BOOST_LOCALE_ICU_VERSION 0 -# define BOOST_LOCALE_ICU_VERSION_EXACT 0 -#endif // Currency style changes between ICU versions, so get "real" value from ICU -#if BOOST_LOCALE_ICU_VERSION >= 402 +# if BOOST_LOCALE_ICU_VERSION >= 402 -std::string get_icu_currency_iso(const double value) +std::string get_ICU_currency_iso(const double value) { -# if BOOST_LOCALE_ICU_VERSION >= 408 +# if BOOST_LOCALE_ICU_VERSION >= 408 auto styleIso = UNUM_CURRENCY_ISO; -# else +# else auto styleIso = icu::NumberFormat::kIsoCurrencyStyle; -# endif +# endif UErrorCode err = U_ZERO_ERROR; - std::unique_ptr fmt(icu::NumberFormat::createInstance(get_icu_test_locale(), styleIso, err)); + std::unique_ptr fmt(icu::NumberFormat::createInstance(get_ICU_test_locale(), styleIso, err)); TEST_REQUIRE(U_SUCCESS(err) && fmt.get()); icu::UnicodeString tmp; - return from_icu_string(fmt->format(value, tmp)); + return from_ICU_string(fmt->format(value, tmp)); } -#endif - -using format_style_t = std::ios_base&(std::ios_base&); - -#ifdef BOOST_LOCALE_WITH_ICU -std::string get_icu_gmt_name(icu::TimeZone::EDisplayType style) +# endif +std::string get_ICU_gmt_name(icu::TimeZone::EDisplayType style) { icu::UnicodeString tmp; - return from_icu_string(icu::TimeZone::getGMT()->getDisplayName(false, style, get_icu_test_locale(), tmp)); + return from_ICU_string(icu::TimeZone::getGMT()->getDisplayName(false, style, get_ICU_test_locale(), tmp)); } // This changes between ICU versions, e.g. "GMT" or "Greenwich Mean Time" -const std::string icu_full_gmt_name = get_icu_gmt_name(icu::TimeZone::EDisplayType::LONG); +const std::string icu_full_gmt_name = get_ICU_gmt_name(icu::TimeZone::EDisplayType::LONG); -std::string get_ICU_time(format_style_t style, const time_t ts, const char* tz = nullptr) +std::string get_ICU_date(format_style_t style, const time_t ts) { using icu::DateFormat; DateFormat::EStyle icu_style = DateFormat::kDefault; namespace as = boost::locale::as; - if(style == as::time_short) + if(style == as::date_short) icu_style = DateFormat::kShort; - else if(style == as::time_medium) + else if(style == as::date_medium) icu_style = DateFormat::kMedium; - else if(style == as::time_long) + else if(style == as::date_long) icu_style = DateFormat::kLong; - else if(style == as::time_full) + else if(style == as::date_full) icu_style = DateFormat::kFull; - std::unique_ptr fmt(icu::DateFormat::createTimeInstance(icu_style, get_icu_test_locale())); - if(!tz) - fmt->setTimeZone(*icu::TimeZone::getGMT()); - else - fmt->adoptTimeZone(icu::TimeZone::createTimeZone(icu::UnicodeString::fromUTF8(tz))); + std::unique_ptr fmt(icu::DateFormat::createDateInstance(icu_style, get_ICU_test_locale())); + fmt->setTimeZone(*icu::TimeZone::getGMT()); icu::UnicodeString s; - return from_icu_string(fmt->format(ts * 1000., s)); + return from_ICU_string(fmt->format(ts * 1000., s)); } -std::string get_ICU_date(format_style_t style, const time_t ts) +std::string get_ICU_datetime(format_style_t style, const time_t ts) { using icu::DateFormat; DateFormat::EStyle icu_style = DateFormat::kDefault; namespace as = boost::locale::as; - if(style == as::date_short) + if(style == as::time_short) icu_style = DateFormat::kShort; - else if(style == as::date_medium) + else if(style == as::time_medium) icu_style = DateFormat::kMedium; - else if(style == as::date_long) + else if(style == as::time_long) icu_style = DateFormat::kLong; - else if(style == as::date_full) + else if(style == as::time_full) icu_style = DateFormat::kFull; - std::unique_ptr fmt(icu::DateFormat::createDateInstance(icu_style, get_icu_test_locale())); + std::unique_ptr fmt( + icu::DateFormat::createDateTimeInstance(icu_style, icu_style, get_ICU_test_locale())); fmt->setTimeZone(*icu::TimeZone::getGMT()); icu::UnicodeString s; - return from_icu_string(fmt->format(ts * 1000., s)); + return from_ICU_string(fmt->format(ts * 1000., s)); } -std::string get_ICU_datetime(format_style_t style, const time_t ts) +std::string get_ICU_time(format_style_t style, const time_t ts, const char* tz = nullptr) { using icu::DateFormat; DateFormat::EStyle icu_style = DateFormat::kDefault; @@ -135,21 +143,16 @@ std::string get_ICU_datetime(format_style_t style, const time_t ts) icu_style = DateFormat::kLong; else if(style == as::time_full) icu_style = DateFormat::kFull; - std::unique_ptr fmt( - icu::DateFormat::createDateTimeInstance(icu_style, icu_style, get_icu_test_locale())); - fmt->setTimeZone(*icu::TimeZone::getGMT()); + std::unique_ptr fmt(icu::DateFormat::createTimeInstance(icu_style, get_ICU_test_locale())); + if(!tz) + fmt->setTimeZone(*icu::TimeZone::getGMT()); + else + fmt->adoptTimeZone(icu::TimeZone::createTimeZone(icu::UnicodeString::fromUTF8(tz))); icu::UnicodeString s; - return from_icu_string(fmt->format(ts * 1000., s)); + return from_ICU_string(fmt->format(ts * 1000., s)); } - -#else -const std::string icu_full_gmt_name; -// clang-format off -std::string get_ICU_time(...){ return ""; } // LCOV_EXCL_LINE -std::string get_ICU_datetime(...){ return ""; } // LCOV_EXCL_LINE -std::string get_ICU_date(...){ return ""; } // LCOV_EXCL_LINE -// clang-format on #endif +} // namespace using namespace boost::locale; @@ -427,8 +430,8 @@ void test_manip(std::string e_charset = "UTF-8") #if BOOST_LOCALE_ICU_VERSION >= 402 TEST_FMT_PARSE_2(as::currency, as::currency_national, 1345, "$1,345.00"); TEST_FMT_PARSE_2(as::currency, as::currency_national, 1345.34, "$1,345.34"); - TEST_FMT_PARSE_2(as::currency, as::currency_iso, 1345, get_icu_currency_iso(1345)); - TEST_FMT_PARSE_2(as::currency, as::currency_iso, 1345.34, get_icu_currency_iso(1345.34)); + TEST_FMT_PARSE_2(as::currency, as::currency_iso, 1345, get_ICU_currency_iso(1345)); + TEST_FMT_PARSE_2(as::currency, as::currency_iso, 1345.34, get_ICU_currency_iso(1345.34)); #endif TEST_FMT_PARSE_1(as::spellout, 10, "ten"); #if 402 <= BOOST_LOCALE_ICU_VERSION && BOOST_LOCALE_ICU_VERSION < 408 @@ -800,7 +803,7 @@ void test_format_class(std::string charset = "UTF-8") #if BOOST_LOCALE_ICU_VERSION >= 402 TEST_FORMAT_CLS("{1,cur=nat}", 1234, "$1,234.00"); TEST_FORMAT_CLS("{1,cur=national}", 1234, "$1,234.00"); - TEST_FORMAT_CLS("{1,cur=iso}", 1234, get_icu_currency_iso(1234)); + TEST_FORMAT_CLS("{1,cur=iso}", 1234, get_ICU_currency_iso(1234)); #endif TEST_FORMAT_CLS("{1,spell}", 10, "ten"); TEST_FORMAT_CLS("{1,spellout}", 10, "ten"); From ea76606d36955467c030692e6ef47ea445012d35 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Dec 2024 17:24:14 +0100 Subject: [PATCH 28/67] Reduce number of C++11 compiler checks Speed up the configuration step: - defaulted moves implies defaulted functions and rvalues -> Clang 3.0, MSVC 19.0/2015 -> decltype, noexcept, auto & variadic templates also available in all compilers that support those - override requires GCC 4.7+ - hdr type_traits implies hdr functional for all known stdlibs --- build/Jamfile.v2 | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index f38e3e26..b6979d3c 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -403,21 +403,9 @@ rule configure-flags ( properties * ) alias build_flags : : : : @configure-flags ; local cxx_requirements = [ requires - cxx11_auto_declarations - cxx11_decltype - cxx11_defaulted_functions cxx11_defaulted_moves - cxx11_hdr_functional cxx11_hdr_type_traits - cxx11_noexcept - cxx11_nullptr cxx11_override - cxx11_range_based_for - cxx11_rvalue_references - cxx11_scoped_enums - cxx11_smart_ptr - cxx11_static_assert - cxx11_variadic_templates ] ; lib boost_locale From 791d7a14928c97b6622d7de3d1e5f77635c115ca Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 26 Dec 2024 16:55:58 +0100 Subject: [PATCH 29/67] Build: Make SOURCES variables `local` --- build/Jamfile.v2 | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index b6979d3c..ab23d59e 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -229,7 +229,7 @@ rule configure-full ( properties * : flags-only ) found-icu = true ; flags-result += BOOST_LOCALE_WITH_ICU=1 $(ICU_OPTS) ; - ICU_SOURCES = + local ICU_SOURCES = boundary codecvt collator @@ -292,7 +292,7 @@ rule configure-full ( properties * : flags-only ) flags-result += BOOST_LOCALE_NO_STD_BACKEND=1 ; } else { - STD_SOURCES = + local STD_SOURCES = codecvt collate converter @@ -329,7 +329,7 @@ rule configure-full ( properties * : flags-only ) flags-result += BOOST_LOCALE_NO_WINAPI_BACKEND=1 ; } else { - WINAPI_SOURCES = + local WINAPI_SOURCES = collate converter numeric @@ -362,7 +362,7 @@ rule configure-full ( properties * : flags-only ) flags-result += BOOST_LOCALE_NO_POSIX_BACKEND=1 ; } else { - POSIX_SOURCES = + local POSIX_SOURCES = collate converter numeric @@ -395,8 +395,7 @@ rule configure ( properties * ) rule configure-flags ( properties * ) { - local result ; - result = [ configure-full $(properties) : "flags" ] ; + local result = [ configure-full $(properties) : "flags" ] ; return $(result) ; } From bbfb763a982bc727464f34899b4c912efce91373 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 25 Dec 2024 17:05:45 +0100 Subject: [PATCH 30/67] Exclude ICU 50.1 This version has a regression with Integer parsing. See https://unicode-org.atlassian.net/browse/ICU-9780 --- CMakeLists.txt | 6 +++++- README.md | 3 ++- build/has_icu_test.cpp | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f47ad394..45cd6946 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2020, 2021 Peter Dimov +# Copyright 2022-2024 Alexander Grund # Distributed under the Boost Software License, Version 1.0. # https://www.boost.org/LICENSE_1_0.txt @@ -123,7 +124,10 @@ endif() if(BOOST_LOCALE_ENABLE_ICU) find_package(ICU COMPONENTS data i18n uc REQUIRED) - + if(ICU_VERSION VERSION_GREATER_EQUAL "50.1" AND ICU_VERSION VERSION_LESS "50.1.1") + # https://unicode-org.atlassian.net/browse/ICU-9780 + message(FATAL_ERROR "ICU 50.1.0 has a bug with integer parsing and cannot be used reliably") + endif() target_compile_definitions(boost_locale PRIVATE BOOST_LOCALE_WITH_ICU=1) target_link_libraries(boost_locale PRIVATE ICU::data ICU::i18n ICU::uc) diff --git a/README.md b/README.md index ea18e02d..b52ea7ba 100644 --- a/README.md +++ b/README.md @@ -35,9 +35,10 @@ Sacrificing some less important features, Boost.Locale becomes less powerful but Distributed under the [Boost Software License, Version 1.0](https://www.boost.org/LICENSE_1_0.txt). -### Properties +### Properties / Requirements * C++11 +* ICU for full feature support. ICU 50.1 is not supported, please use a newer version. * Formatted with clang-format, see [`tools/format_sources.sh`](https://github.com/boostorg/locale/blob/develop/tools/format_sources.sh) ### Build Status diff --git a/build/has_icu_test.cpp b/build/has_icu_test.cpp index 3f43f883..5e0e29fa 100644 --- a/build/has_icu_test.cpp +++ b/build/has_icu_test.cpp @@ -1,4 +1,5 @@ // Copyright (c) 2010 John Maddock +// Copyright (c) 2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -9,6 +10,22 @@ #include #include +// Sanity check to avoid false negatives for version checks +#ifndef U_ICU_VERSION_MAJOR_NUM +# error "U_ICU_VERSION_MAJOR_NUM is missing" +#endif +#ifndef U_ICU_VERSION_MINOR_NUM +# error "U_ICU_VERSION_MINOR_NUM is missing" +#endif +#ifndef U_ICU_VERSION_PATCHLEVEL_NUM +# error "U_ICU_VERSION_PATCHLEVEL_NUM is missing" +#endif + +#if(U_ICU_VERSION_MAJOR_NUM == 50) && (U_ICU_VERSION_MINOR_NUM == 1) && (U_ICU_VERSION_PATCHLEVEL_NUM < 1) +// https://unicode-org.atlassian.net/browse/ICU-9780 +# error "ICU 50.1.0 has a bug with integer parsing and cannot be used reliably" +#endif + int main() { icu::Locale loc; From 34611eadaedd7acc5918f17d2cfda6acf00a3640 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 5 Dec 2024 20:51:43 +0100 Subject: [PATCH 31/67] Raise minimum ICU version to 4.2 StringPiece is only available in 4.2+ and is required for proper parsing and formatting. As this version is now "old enough", just assume 4.2+ removing many conditionals. --- build/has_icu_test.cpp | 6 +++-- doc/changelog.txt | 1 + src/boost/locale/icu/collator.cpp | 13 +++-------- src/boost/locale/icu/conversion.cpp | 19 +++------------- src/boost/locale/icu/date_time.cpp | 21 ++++-------------- src/boost/locale/icu/formatters_cache.cpp | 7 ++---- src/boost/locale/icu/time_zone.cpp | 6 ++--- test/test_formatting.cpp | 27 +++++------------------ 8 files changed, 25 insertions(+), 75 deletions(-) diff --git a/build/has_icu_test.cpp b/build/has_icu_test.cpp index 5e0e29fa..bc36c345 100644 --- a/build/has_icu_test.cpp +++ b/build/has_icu_test.cpp @@ -1,11 +1,12 @@ // Copyright (c) 2010 John Maddock -// Copyright (c) 2024 Alexander Grund +// Copyright (c) 2022-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt #include #include +#include #include #include #include @@ -31,5 +32,6 @@ int main() icu::Locale loc; UErrorCode err = U_ZERO_ERROR; UChar32 c = ::u_charFromName(U_UNICODE_CHAR_NAME, "GREEK SMALL LETTER ALPHA", &err); - return err; + icu::StringPiece sp; + return U_SUCCESS(err) && sp.empty() && (c != 0u); } diff --git a/doc/changelog.txt b/doc/changelog.txt index 3e4fe789..1809777c 100644 --- a/doc/changelog.txt +++ b/doc/changelog.txt @@ -10,6 +10,7 @@ - 1.88.0 - Fix parsing of numbers in floating point format to integers + - Require ICU 4.2 or later - 1.86.0 - Make ICU implementation of `to_title` threadsafe - Add allocator support to `utf_to_utf` diff --git a/src/boost/locale/icu/collator.cpp b/src/boost/locale/icu/collator.cpp index 72ade74f..a57d3694 100644 --- a/src/boost/locale/icu/collator.cpp +++ b/src/boost/locale/icu/collator.cpp @@ -1,5 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2022-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -16,13 +17,8 @@ #include #include #include +#include #include -#if BOOST_LOCALE_ICU_VERSION >= 402 -# define BOOST_LOCALE_WITH_STRINGPIECE 1 -# include -#else -# define BOOST_LOCALE_WITH_STRINGPIECE 0 -#endif #ifdef BOOST_MSVC # pragma warning(disable : 4244) // 'argument' : conversion from 'int' @@ -43,7 +39,6 @@ namespace boost { namespace locale { namespace impl_icu { return res; } -#if BOOST_LOCALE_WITH_STRINGPIECE int do_utf8_compare(collate_level level, const char* b1, const char* e1, @@ -55,7 +50,6 @@ namespace boost { namespace locale { namespace impl_icu { icu::StringPiece right(b2, e2 - b2); return get_collator(level).compareUTF8(left, right, status); } -#endif int do_ustring_compare(collate_level level, const CharType* b1, @@ -159,7 +153,6 @@ namespace boost { namespace locale { namespace impl_icu { bool is_utf8_; }; -#if BOOST_LOCALE_WITH_STRINGPIECE template<> int collate_impl::do_real_compare(collate_level level, const char* b1, @@ -173,7 +166,7 @@ namespace boost { namespace locale { namespace impl_icu { else return do_ustring_compare(level, b1, e1, b2, e2, status); } -#endif + std::locale create_collate(const std::locale& in, const cdata& cd, char_facet_t type) { switch(type) { diff --git a/src/boost/locale/icu/conversion.cpp b/src/boost/locale/icu/conversion.cpp index c619dd94..829ceabd 100644 --- a/src/boost/locale/icu/conversion.cpp +++ b/src/boost/locale/icu/conversion.cpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2022-2023 Alexander Grund +// Copyright (c) 2022-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -13,11 +13,8 @@ #include #include #include +#include #include -#if BOOST_LOCALE_ICU_VERSION >= 308 -# include -# define BOOST_LOCALE_WITH_CASEMAP -#endif #include namespace boost { namespace locale { namespace impl_icu { @@ -72,7 +69,6 @@ namespace boost { namespace locale { namespace impl_icu { std::string encoding_; }; // converter_impl -#ifdef BOOST_LOCALE_WITH_CASEMAP template struct get_casemap_size_type; @@ -193,26 +189,17 @@ namespace boost { namespace locale { namespace impl_icu { raii_casemap map_; }; // converter_impl -#endif // BOOST_LOCALE_WITH_CASEMAP - std::locale create_convert(const std::locale& in, const cdata& cd, char_facet_t type) { switch(type) { case char_facet_t::nochar: break; case char_facet_t::char_f: -#ifdef BOOST_LOCALE_WITH_CASEMAP if(cd.is_utf8()) return std::locale(in, new utf8_converter_impl(cd)); -#endif return std::locale(in, new converter_impl(cd)); case char_facet_t::wchar_f: return std::locale(in, new converter_impl(cd)); #ifndef BOOST_LOCALE_NO_CXX20_STRING8 - case char_facet_t::char8_f: -# if defined(BOOST_LOCALE_WITH_CASEMAP) - return std::locale(in, new utf8_converter_impl(cd)); -# else - return std::locale(in, new converter_impl(cd)); -# endif + case char_facet_t::char8_f: return std::locale(in, new utf8_converter_impl(cd)); #elif defined(__cpp_char8_t) case char_facet_t::char8_f: break; #endif diff --git a/src/boost/locale/icu/date_time.cpp b/src/boost/locale/icu/date_time.cpp index 14e21d9f..2dd7824c 100644 --- a/src/boost/locale/icu/date_time.cpp +++ b/src/boost/locale/icu/date_time.cpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2021-2022 Alexander Grund +// Copyright (c) 2021-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -67,10 +67,6 @@ namespace boost { namespace locale { namespace impl_icu { const double rounded_time = std::floor(calendar_->getTime(err) / U_MILLIS_PER_SECOND) * U_MILLIS_PER_SECOND; calendar_->setTime(rounded_time, err); check_and_throw_dt(err); -#if BOOST_LOCALE_ICU_VERSION < 402 - // workaround old/invalid data, it should be 4 in general - calendar_->setMinimalDaysInFirstWeek(4); -#endif encoding_ = dat.encoding(); } calendar_impl(const calendar_impl& other) @@ -79,15 +75,9 @@ namespace boost { namespace locale { namespace impl_icu { encoding_ = other.encoding_; } - calendar_impl* clone() const override - { - return new calendar_impl(*this); - } + calendar_impl* clone() const override { return new calendar_impl(*this); } - void set_value(period::marks::period_mark p, int value) override - { - calendar_->set(to_icu(p), int32_t(value)); - } + void set_value(period::marks::period_mark p, int value) override { calendar_->set(to_icu(p), int32_t(value)); } int get_value(period::marks::period_mark p, value_type type) const override { @@ -202,10 +192,7 @@ namespace boost { namespace locale { namespace impl_icu { check_and_throw_dt(err); return diff; } - void set_timezone(const std::string& tz) override - { - calendar_->adoptTimeZone(get_time_zone(tz)); - } + void set_timezone(const std::string& tz) override { calendar_->adoptTimeZone(get_time_zone(tz)); } std::string get_timezone() const override { icu::UnicodeString tz; diff --git a/src/boost/locale/icu/formatters_cache.cpp b/src/boost/locale/icu/formatters_cache.cpp index c9a8c463..2da6af6a 100644 --- a/src/boost/locale/icu/formatters_cache.cpp +++ b/src/boost/locale/icu/formatters_cache.cpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2021-2022 Alexander Grund +// Copyright (c) 2021-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -92,16 +92,13 @@ namespace boost { namespace locale { namespace impl_icu { case num_fmt_type::curr_iso: return icu::NumberFormat::createInstance(locale_, UNUM_CURRENCY_ISO, err); break; -#elif BOOST_LOCALE_ICU_VERSION >= 402 +#else case num_fmt_type::curr_nat: return icu::NumberFormat::createInstance(locale_, icu::NumberFormat::kCurrencyStyle, err); break; case num_fmt_type::curr_iso: return icu::NumberFormat::createInstance(locale_, icu::NumberFormat::kIsoCurrencyStyle, err); break; -#else - case num_fmt_type::curr_nat: - case num_fmt_type::curr_iso: return icu::NumberFormat::createCurrencyInstance(locale_, err); break; #endif case num_fmt_type::percent: return icu::NumberFormat::createPercentInstance(locale_, err); break; case num_fmt_type::spell: return new icu::RuleBasedNumberFormat(icu::URBNF_SPELLOUT, locale_, err); break; diff --git a/src/boost/locale/icu/time_zone.cpp b/src/boost/locale/icu/time_zone.cpp index 2f11862a..e1f37023 100644 --- a/src/boost/locale/icu/time_zone.cpp +++ b/src/boost/locale/icu/time_zone.cpp @@ -1,5 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2022-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -13,14 +14,13 @@ // Bug - when ICU tries to find a file that is equivalent to /etc/localtime it finds /usr/share/zoneinfo/localtime // that is just a symbolic link to /etc/localtime. // -// It started in 4.0 and was fixed in version 4.6, also the fix was backported to the 4.4 branch so it should be +// It was fixed in version 4.6, also the fix was backported to the 4.4 branch so it should be // available from 4.4.3... So we test if the workaround is required // // It is also relevant only for Linux, BSD and Apple (as I see in ICU code) // -#if BOOST_LOCALE_ICU_VERSION >= 400 && BOOST_LOCALE_ICU_VERSION <= 406 \ - && (BOOST_LOCALE_ICU_VERSION != 404 || U_ICU_VERSION_PATCHLEVEL_NUM >= 3) +#if BOOST_LOCALE_ICU_VERSION <= 406 && (BOOST_LOCALE_ICU_VERSION != 404 || U_ICU_VERSION_PATCHLEVEL_NUM >= 3) # if BOOST_OS_LINUX || BOOST_OS_BSD_FREE || defined(__APPLE__) # define BOOST_LOCALE_WORKAROUND_ICU_BUG # endif diff --git a/test/test_formatting.cpp b/test/test_formatting.cpp index 8f111230..52c2095f 100644 --- a/test/test_formatting.cpp +++ b/test/test_formatting.cpp @@ -63,16 +63,13 @@ std::string from_ICU_string(const icu::UnicodeString& str) return boost::locale::conv::utf_to_utf(str.getBuffer(), str.getBuffer() + str.length()); } -// Currency style changes between ICU versions, so get "real" value from ICU -# if BOOST_LOCALE_ICU_VERSION >= 402 - std::string get_ICU_currency_iso(const double value) { -# if BOOST_LOCALE_ICU_VERSION >= 408 +# if BOOST_LOCALE_ICU_VERSION >= 408 auto styleIso = UNUM_CURRENCY_ISO; -# else +# else auto styleIso = icu::NumberFormat::kIsoCurrencyStyle; -# endif +# endif UErrorCode err = U_ZERO_ERROR; std::unique_ptr fmt(icu::NumberFormat::createInstance(get_ICU_test_locale(), styleIso, err)); TEST_REQUIRE(U_SUCCESS(err) && fmt.get()); @@ -81,7 +78,6 @@ std::string get_ICU_currency_iso(const double value) return from_ICU_string(fmt->format(value, tmp)); } -# endif std::string get_ICU_gmt_name(icu::TimeZone::EDisplayType style) { icu::UnicodeString tmp; @@ -427,12 +423,11 @@ void test_manip(std::string e_charset = "UTF-8") TEST_PARSE_FAILS(as::currency, "$", double); -#if BOOST_LOCALE_ICU_VERSION >= 402 TEST_FMT_PARSE_2(as::currency, as::currency_national, 1345, "$1,345.00"); TEST_FMT_PARSE_2(as::currency, as::currency_national, 1345.34, "$1,345.34"); TEST_FMT_PARSE_2(as::currency, as::currency_iso, 1345, get_ICU_currency_iso(1345)); TEST_FMT_PARSE_2(as::currency, as::currency_iso, 1345.34, get_ICU_currency_iso(1345.34)); -#endif + TEST_FMT_PARSE_1(as::spellout, 10, "ten"); #if 402 <= BOOST_LOCALE_ICU_VERSION && BOOST_LOCALE_ICU_VERSION < 408 if(e_charset == "UTF-8") @@ -505,7 +500,6 @@ void test_manip(std::string e_charset = "UTF-8") a_datetime, icu_time_long, a_time + a_timesec); -#if !(BOOST_LOCALE_ICU_VERSION == 308 && defined(__CYGWIN__)) // Known failure due to ICU issue TEST_PARSE(as::time >> as::time_full >> as::time_zone("GMT+01:00"), "4:33:13 PM GMT+01:00", a_time + a_timesec); TEST_FMT_PARSE_3_2(as::time, as::time_full, @@ -513,7 +507,6 @@ void test_manip(std::string e_charset = "UTF-8") a_datetime, icu_time_full, a_time + a_timesec); -#endif const std::string icu_def = get_ICU_datetime(as::time, a_datetime); const std::string icu_short = get_ICU_datetime(as::time_short, a_datetime); @@ -762,11 +755,7 @@ void test_format_class(std::string charset = "UTF-8") // format with locale & encoding { -#if BOOST_LOCALE_ICU_VERSION >= 400 const auto expected = boost::locale::conv::utf_to_utf("10,00\xC2\xA0€"); -#else - const auto expected = boost::locale::conv::utf_to_utf("10,00 €"); // LCOV_EXCL_LINE -#endif TEST_EQ(do_format(loc, "{1,cur,locale=de_DE.UTF-8}", 10), expected); } @@ -794,20 +783,14 @@ void test_format_class(std::string charset = "UTF-8") TEST_FORMAT_CLS("{1,cur}", 1234, "$1,234.00"); TEST_FORMAT_CLS("{1,currency}", 1234, "$1,234.00"); if(charset == "UTF-8") { -#if BOOST_LOCALE_ICU_VERSION >= 400 TEST_FORMAT_CLS("{1,cur,locale=de_DE}", 10, "10,00\xC2\xA0€"); -#else - TEST_FORMAT_CLS("{1,cur,locale=de_DE}", 10, "10,00 €"); // LCOV_EXCL_LINE -#endif } -#if BOOST_LOCALE_ICU_VERSION >= 402 TEST_FORMAT_CLS("{1,cur=nat}", 1234, "$1,234.00"); TEST_FORMAT_CLS("{1,cur=national}", 1234, "$1,234.00"); TEST_FORMAT_CLS("{1,cur=iso}", 1234, get_ICU_currency_iso(1234)); -#endif TEST_FORMAT_CLS("{1,spell}", 10, "ten"); TEST_FORMAT_CLS("{1,spellout}", 10, "ten"); -#if 402 <= BOOST_LOCALE_ICU_VERSION && BOOST_LOCALE_ICU_VERSION < 408 +#if BOOST_LOCALE_ICU_VERSION < 408 if(charset == "UTF-8") { TEST_FORMAT_CLS("{1,ord}", 1, "1\xcb\xa2\xe1\xb5\x97"); TEST_FORMAT_CLS("{1,ordinal}", 1, "1\xcb\xa2\xe1\xb5\x97"); From ec1676280e4c91610da8748324c7697c0d411712 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 26 Dec 2024 13:35:59 +0100 Subject: [PATCH 32/67] Raise minimum ICU version to 4.8.1 A lot of checks and a major workaround are for ICU 4.8 or earlier which can be removed. An annoying bug (Parsing of timezones like "GMT" in "full" format followed by unrelated text) fixed in 4.8.1 is worth avoiding by requiring this version over 4.8.0. --- CMakeLists.txt | 5 +- README.md | 3 +- build/Jamfile.v2 | 1 - build/has_icu_test.cpp | 12 +- src/boost/locale/icu/formatters_cache.cpp | 9 - src/boost/locale/icu/time_zone.cpp | 212 ---------------------- src/boost/locale/icu/time_zone.hpp | 14 +- test/test_date_time.cpp | 2 +- test/test_formatting.cpp | 40 +--- test/test_generator.cpp | 3 +- 10 files changed, 34 insertions(+), 267 deletions(-) delete mode 100644 src/boost/locale/icu/time_zone.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 45cd6946..977c50aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,7 +73,7 @@ endif() # Build Options -find_package(ICU COMPONENTS data i18n uc QUIET) +find_package(ICU 4.8.1 COMPONENTS data i18n uc QUIET) option(BOOST_LOCALE_ENABLE_ICU "Boost.Locale: enable ICU backend" ${ICU_FOUND}) # Workaround for an issue e.g. on FreeBSD where including ICU may change the used libiconv @@ -123,7 +123,7 @@ endif() if(BOOST_LOCALE_ENABLE_ICU) - find_package(ICU COMPONENTS data i18n uc REQUIRED) + find_package(ICU 4.8.1 COMPONENTS data i18n uc REQUIRED) if(ICU_VERSION VERSION_GREATER_EQUAL "50.1" AND ICU_VERSION VERSION_LESS "50.1.1") # https://unicode-org.atlassian.net/browse/ICU-9780 message(FATAL_ERROR "ICU 50.1.0 has a bug with integer parsing and cannot be used reliably") @@ -148,7 +148,6 @@ if(BOOST_LOCALE_ENABLE_ICU) src/boost/locale/icu/icu_backend.hpp src/boost/locale/icu/icu_util.hpp src/boost/locale/icu/numeric.cpp - src/boost/locale/icu/time_zone.cpp src/boost/locale/icu/time_zone.hpp src/boost/locale/icu/uconv.hpp ) diff --git a/README.md b/README.md index b52ea7ba..127384b0 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ Distributed under the [Boost Software License, Version 1.0](https://www.boost.or ### Properties / Requirements * C++11 -* ICU for full feature support. ICU 50.1 is not supported, please use a newer version. +* ICU 4.8.1 or newer for full feature support. + ICU 50.1 is not supported, please use a newer version. * Formatted with clang-format, see [`tools/format_sources.sh`](https://github.com/boostorg/locale/blob/develop/tools/format_sources.sh) ### Build Status diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index ab23d59e..f494f779 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -239,7 +239,6 @@ rule configure-full ( properties * : flags-only ) formatters_cache icu_backend numeric - time_zone ; result += icu/$(ICU_SOURCES).cpp diff --git a/build/has_icu_test.cpp b/build/has_icu_test.cpp index bc36c345..5abe66a5 100644 --- a/build/has_icu_test.cpp +++ b/build/has_icu_test.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -22,7 +23,15 @@ # error "U_ICU_VERSION_PATCHLEVEL_NUM is missing" #endif -#if(U_ICU_VERSION_MAJOR_NUM == 50) && (U_ICU_VERSION_MINOR_NUM == 1) && (U_ICU_VERSION_PATCHLEVEL_NUM < 1) +#define BOOST_LOCALE_MAKE_VERSION(maj, min, patch) (((maj)*100 + (min)) * 100 + patch) +#define BOOST_LOCALE_ICU_VERSION \ + BOOST_LOCALE_MAKE_VERSION(U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MAJOR_NUM, U_ICU_VERSION_MINOR_NUM) + +#if BOOST_LOCALE_ICU_VERSION < BOOST_LOCALE_MAKE_VERSION(4, 8, 1) +// 4.8.0 fails parsing "GMT" as a full time zone if followed by unrelated text +# error "ICU 4.8.1 or higher is required" +#endif +#if BOOST_LOCALE_ICU_VERSION == BOOST_LOCALE_MAKE_VERSION(50, 1, 0) // https://unicode-org.atlassian.net/browse/ICU-9780 # error "ICU 50.1.0 has a bug with integer parsing and cannot be used reliably" #endif @@ -32,6 +41,7 @@ int main() icu::Locale loc; UErrorCode err = U_ZERO_ERROR; UChar32 c = ::u_charFromName(U_UNICODE_CHAR_NAME, "GREEK SMALL LETTER ALPHA", &err); + delete icu::NumberFormat::createInstance(loc, UNUM_CURRENCY_ISO, err); icu::StringPiece sp; return U_SUCCESS(err) && sp.empty() && (c != 0u); } diff --git a/src/boost/locale/icu/formatters_cache.cpp b/src/boost/locale/icu/formatters_cache.cpp index 2da6af6a..81591b9d 100644 --- a/src/boost/locale/icu/formatters_cache.cpp +++ b/src/boost/locale/icu/formatters_cache.cpp @@ -87,19 +87,10 @@ namespace boost { namespace locale { namespace impl_icu { switch(type) { case num_fmt_type::number: return icu::NumberFormat::createInstance(locale_, err); break; case num_fmt_type::sci: return icu::NumberFormat::createScientificInstance(locale_, err); break; -#if BOOST_LOCALE_ICU_VERSION >= 408 case num_fmt_type::curr_nat: return icu::NumberFormat::createInstance(locale_, UNUM_CURRENCY, err); break; case num_fmt_type::curr_iso: return icu::NumberFormat::createInstance(locale_, UNUM_CURRENCY_ISO, err); break; -#else - case num_fmt_type::curr_nat: - return icu::NumberFormat::createInstance(locale_, icu::NumberFormat::kCurrencyStyle, err); - break; - case num_fmt_type::curr_iso: - return icu::NumberFormat::createInstance(locale_, icu::NumberFormat::kIsoCurrencyStyle, err); - break; -#endif case num_fmt_type::percent: return icu::NumberFormat::createPercentInstance(locale_, err); break; case num_fmt_type::spell: return new icu::RuleBasedNumberFormat(icu::URBNF_SPELLOUT, locale_, err); break; case num_fmt_type::ordinal: return new icu::RuleBasedNumberFormat(icu::URBNF_ORDINAL, locale_, err); break; diff --git a/src/boost/locale/icu/time_zone.cpp b/src/boost/locale/icu/time_zone.cpp deleted file mode 100644 index e1f37023..00000000 --- a/src/boost/locale/icu/time_zone.cpp +++ /dev/null @@ -1,212 +0,0 @@ -// -// Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2022-2024 Alexander Grund -// -// Distributed under the Boost Software License, Version 1.0. -// https://www.boost.org/LICENSE_1_0.txt - -#include "boost/locale/icu/time_zone.hpp" -#include -#include "boost/locale/icu/icu_util.hpp" -#include - -// -// Bug - when ICU tries to find a file that is equivalent to /etc/localtime it finds /usr/share/zoneinfo/localtime -// that is just a symbolic link to /etc/localtime. -// -// It was fixed in version 4.6, also the fix was backported to the 4.4 branch so it should be -// available from 4.4.3... So we test if the workaround is required -// -// It is also relevant only for Linux, BSD and Apple (as I see in ICU code) -// - -#if BOOST_LOCALE_ICU_VERSION <= 406 && (BOOST_LOCALE_ICU_VERSION != 404 || U_ICU_VERSION_PATCHLEVEL_NUM >= 3) -# if BOOST_OS_LINUX || BOOST_OS_BSD_FREE || defined(__APPLE__) -# define BOOST_LOCALE_WORKAROUND_ICU_BUG -# endif -#endif - -#ifdef BOOST_LOCALE_WORKAROUND_ICU_BUG -# include -# include -# include -# include -# include -# include -# include -# include -#endif - -namespace boost { namespace locale { namespace impl_icu { - -#ifndef BOOST_LOCALE_WORKAROUND_ICU_BUG - - // This is normal behavior - - icu::TimeZone* get_time_zone(const std::string& time_zone) - { - if(time_zone.empty()) - return icu::TimeZone::createDefault(); - else - return icu::TimeZone::createTimeZone(time_zone.c_str()); - } - -#else - - // This is a workaround for an ICU timezone detection bug. - // It is \b very ICU specific and should not be used - // in general. It is also designed to work only on - // specific patforms: Linux, BSD and Apple, where this bug may actually - // occur - namespace { - - // Under BSD, Linux and Mac OS X dirent has normal size - // so no issues with readdir_r - - class directory { - public: - directory(const char* name) : d(0), read_result(0) - { - d = opendir(name); - if(!d) - return; - } - ~directory() - { - if(d) - closedir(d); - } - bool is_open() { return d; } - const char* next() - { - if(d && readdir_r(d, &de, &read_result) == 0 && read_result != 0) - return de.d_name; - return nullptr; - } - - private: - DIR* d; - struct dirent de; - struct dirent* read_result; - }; - - bool files_equal(const std::string& left, const std::string& right) - { - char l[256], r[256]; - std::ifstream ls(left); - if(!ls) - return false; - std::ifstream rs(right); - if(!rs) - return false; - do { - ls.read(l, sizeof(l)); - rs.read(r, sizeof(r)); - size_t n; - if((n = ls.gcount()) != size_t(rs.gcount())) - return false; - if(memcmp(l, r, n) != 0) - return false; - } while(!ls.eof() || !rs.eof()); - if(bool(ls.eof()) != bool(rs.eof())) - return false; - return true; - } - - std::string find_file_in(const std::string& ref, size_t size, const std::string& dir) - { - directory d(dir.c_str()); - if(!d.is_open()) - return std::string(); - - const char* name = nullptr; - while((name = d.next()) != 0) { - std::string file_name = name; - if(file_name == "." || file_name == ".." || file_name == "posixrules" || file_name == "localtime") - continue; - struct stat st; - std::string path = dir + "/" + file_name; - if(stat(path.c_str(), &st) == 0) { - if(S_ISDIR(st.st_mode)) { - std::string res = find_file_in(ref, size, path); - if(!res.empty()) - return file_name + "/" + res; - } else { - if(size_t(st.st_size) == size && files_equal(path, ref)) - return file_name; - } - } - } - return std::string(); - } - - // This actually emulates ICU's search - // algorithm... just it ignores localtime - std::string detect_correct_time_zone() - { - const char* tz_dir = "/usr/share/zoneinfo"; - const char* tz_file = "/etc/localtime"; - - struct stat st; - if(::stat(tz_file, &st) != 0) - return std::string(); - size_t size = st.st_size; - std::string r = find_file_in(tz_file, size, tz_dir); - if(r.empty()) - return r; - if(r.compare(0, 6, "posix/") == 0 || r.compare(0, 6, "right/", 6) == 0) - return r.substr(6); - return r; - } - - // Using pthread as: - // - This bug is relevant for only Linux, BSD, Mac OS X and - // pthreads are native threading API - // - The dependency on boost.thread may be removed when using - // more recent ICU versions (so TLS would not be needed) - // - // This the dependency on Boost.Thread is eliminated - - pthread_once_t init_tz = PTHREAD_ONCE_INIT; - std::string default_time_zone_name; - - extern "C" { - static void init_tz_proc() - { - try { - default_time_zone_name = detect_correct_time_zone(); - } catch(...) { - } - } - } - - std::string get_time_zone_name() - { - pthread_once(&init_tz, init_tz_proc); - return default_time_zone_name; - } - - } // namespace - - icu::TimeZone* get_time_zone(const std::string& time_zone) - { - if(!time_zone.empty()) - return icu::TimeZone::createTimeZone(time_zone.c_str()); - hold_ptr tz(icu::TimeZone::createDefault()); - icu::UnicodeString id; - tz->getID(id); - // Check if there is a bug? - if(id != icu::UnicodeString("localtime")) - return tz.release(); - // Now let's deal with the bug and run the fixed - // search loop as that of ICU - std::string real_id = get_time_zone_name(); - if(real_id.empty()) { - // if we failed fallback to ICU's time zone - return tz.release(); - } - return icu::TimeZone::createTimeZone(real_id.c_str()); - } -#endif // bug workaround - -}}} // namespace boost::locale::impl_icu diff --git a/src/boost/locale/icu/time_zone.hpp b/src/boost/locale/icu/time_zone.hpp index 87ab0c44..d265b1d6 100644 --- a/src/boost/locale/icu/time_zone.hpp +++ b/src/boost/locale/icu/time_zone.hpp @@ -1,5 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2022-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -14,9 +15,14 @@ namespace boost { namespace locale { namespace impl_icu { - // Provides a workaround for an ICU default timezone bug and also - // handles time_zone string correctly - if empty returns default - // otherwise returns the instance created with time_zone - icu::TimeZone* get_time_zone(const std::string& time_zone); + // Return an ICU time zone instance. + // If the argument is empty returns the default timezone. + inline icu::TimeZone* get_time_zone(const std::string& time_zone) + { + if(time_zone.empty()) + return icu::TimeZone::createDefault(); + else + return icu::TimeZone::createTimeZone(time_zone.c_str()); + } }}} // namespace boost::locale::impl_icu #endif diff --git a/test/test_date_time.cpp b/test/test_date_time.cpp index 38280caf..a989bf62 100644 --- a/test/test_date_time.cpp +++ b/test/test_date_time.cpp @@ -599,7 +599,7 @@ void test_main(int /*argc*/, char** /*argv*/) TEST_EQ(time_point.get(week_of_year()), 6); // cldr changes -#if BOOST_LOCALE_ICU_VERSION >= 408 && BOOST_LOCALE_ICU_VERSION <= 6000 +#if BOOST_LOCALE_ICU_VERSION <= 6000 const bool ICU_cldr_issue = backend_name == "icu"; #else const bool ICU_cldr_issue = false; diff --git a/test/test_formatting.cpp b/test/test_formatting.cpp index 52c2095f..4920e8c0 100644 --- a/test/test_formatting.cpp +++ b/test/test_formatting.cpp @@ -26,16 +26,11 @@ const std::string test_locale_name = "en_US"; std::string message_path = "./"; -#ifndef BOOST_LOCALE_WITH_ICU -# define BOOST_LOCALE_ICU_VERSION 0 -# define BOOST_LOCALE_ICU_VERSION_EXACT 0 -#else +#ifdef BOOST_LOCALE_WITH_ICU # include # include # include # include -# define BOOST_LOCALE_ICU_VERSION (U_ICU_VERSION_MAJOR_NUM * 100 + U_ICU_VERSION_MINOR_NUM) -# define BOOST_LOCALE_ICU_VERSION_EXACT (BOOST_LOCALE_ICU_VERSION * 100 + U_ICU_VERSION_PATCHLEVEL_NUM) #endif using format_style_t = std::ios_base&(std::ios_base&); @@ -44,9 +39,7 @@ namespace { #ifndef BOOST_LOCALE_WITH_ICU const std::string icu_full_gmt_name; // clang-format off -#if BOOST_LOCALE_ICU_VERSION >= 402 std::string get_ICU_currency_iso(...){ return ""; } // LCOV_EXCL_LINE -#endif std::string get_ICU_date(...){ return ""; } // LCOV_EXCL_LINE std::string get_ICU_datetime(...){ return ""; } // LCOV_EXCL_LINE std::string get_ICU_time(...){ return ""; } // LCOV_EXCL_LINE @@ -65,13 +58,9 @@ std::string from_ICU_string(const icu::UnicodeString& str) std::string get_ICU_currency_iso(const double value) { -# if BOOST_LOCALE_ICU_VERSION >= 408 - auto styleIso = UNUM_CURRENCY_ISO; -# else - auto styleIso = icu::NumberFormat::kIsoCurrencyStyle; -# endif UErrorCode err = U_ZERO_ERROR; - std::unique_ptr fmt(icu::NumberFormat::createInstance(get_ICU_test_locale(), styleIso, err)); + std::unique_ptr fmt( + icu::NumberFormat::createInstance(get_ICU_test_locale(), UNUM_CURRENCY_ISO, err)); TEST_REQUIRE(U_SUCCESS(err) && fmt.get()); icu::UnicodeString tmp; @@ -429,12 +418,7 @@ void test_manip(std::string e_charset = "UTF-8") TEST_FMT_PARSE_2(as::currency, as::currency_iso, 1345.34, get_ICU_currency_iso(1345.34)); TEST_FMT_PARSE_1(as::spellout, 10, "ten"); -#if 402 <= BOOST_LOCALE_ICU_VERSION && BOOST_LOCALE_ICU_VERSION < 408 - if(e_charset == "UTF-8") - TEST_FMT(as::ordinal, 1, "1\xcb\xa2\xe1\xb5\x97"); // 1st with st as ligatures -#else TEST_FMT(as::ordinal, 1, "1st"); -#endif time_t a_date = 3600 * 24 * (31 + 4); // Feb 5th time_t a_time = 3600 * 15 + 60 * 33; // 15:33:05 @@ -466,11 +450,9 @@ void test_manip(std::string e_charset = "UTF-8") TEST_PARSE(as::time >> as::time_long >> as::gmt, "3:33:13 PM GMT", a_time + a_timesec); TEST_FMT_PARSE_3_2(as::time, as::time_long, as::gmt, a_datetime, icu_time_long, a_time + a_timesec); - // ICU 4.8.0 has a bug which makes parsing the full time fail when anything follows the time zone -#if BOOST_LOCALE_ICU_VERSION_EXACT != 40800 + TEST_PARSE(as::time >> as::time_full >> as::gmt, "3:33:13 PM GMT+00:00", a_time + a_timesec); TEST_FMT_PARSE_3_2(as::time, as::time_full, as::gmt, a_datetime, icu_time_full, a_time + a_timesec); -#endif TEST_PARSE_FAILS(as::time, "AM", double); icu_time_def = get_ICU_time(as::time, a_datetime, "GMT+01:00"); @@ -527,13 +509,11 @@ void test_manip(std::string e_charset = "UTF-8") "February 5, 1970 3:33:13 PM GMT", a_datetime); TEST_FMT_PARSE_4(as::datetime, as::date_long, as::time_long, as::gmt, a_datetime, icu_long); -#if BOOST_LOCALE_ICU_VERSION_EXACT != 40800 - // ICU 4.8.0 has a bug which makes parsing the full time fail when anything follows the time zone + TEST_PARSE(as::datetime >> as::date_full >> as::time_full >> as::gmt, "Thursday, February 5, 1970 3:33:13 PM Greenwich Mean Time", a_datetime); TEST_FMT_PARSE_4(as::datetime, as::date_full, as::time_full, as::gmt, a_datetime, icu_full); -#endif const std::pair mark_test_cases[] = { std::make_pair('a', "Thu"), @@ -782,23 +762,15 @@ void test_format_class(std::string charset = "UTF-8") TEST_FORMAT_CLS("{percent,1}", 0.1, "10%"); TEST_FORMAT_CLS("{1,cur}", 1234, "$1,234.00"); TEST_FORMAT_CLS("{1,currency}", 1234, "$1,234.00"); - if(charset == "UTF-8") { + if(charset == "UTF-8") TEST_FORMAT_CLS("{1,cur,locale=de_DE}", 10, "10,00\xC2\xA0€"); - } TEST_FORMAT_CLS("{1,cur=nat}", 1234, "$1,234.00"); TEST_FORMAT_CLS("{1,cur=national}", 1234, "$1,234.00"); TEST_FORMAT_CLS("{1,cur=iso}", 1234, get_ICU_currency_iso(1234)); TEST_FORMAT_CLS("{1,spell}", 10, "ten"); TEST_FORMAT_CLS("{1,spellout}", 10, "ten"); -#if BOOST_LOCALE_ICU_VERSION < 408 - if(charset == "UTF-8") { - TEST_FORMAT_CLS("{1,ord}", 1, "1\xcb\xa2\xe1\xb5\x97"); - TEST_FORMAT_CLS("{1,ordinal}", 1, "1\xcb\xa2\xe1\xb5\x97"); - } -#else TEST_FORMAT_CLS("{1,ord}", 1, "1st"); TEST_FORMAT_CLS("{1,ordinal}", 1, "1st"); -#endif // formatted time { diff --git a/test/test_generator.cpp b/test/test_generator.cpp index 084101bf..b2928ce8 100644 --- a/test/test_generator.cpp +++ b/test/test_generator.cpp @@ -82,7 +82,8 @@ bool hasLocaleForBackend(const std::string& locale_name, const std::string& back return has_posix_locale(locale_name); else { BOOST_ASSERT(backendName == "icu"); - return BOOST_LOCALE_ICU_VERSION >= 5901; // First version to use (correct) CLDR data + return (locale_name.substr(0, 3) != "en_") + || (BOOST_LOCALE_ICU_VERSION >= 5901); // First version to use (correct) CLDR data for en_* locales } } From 9e6e9545c6d06eb2c960d95b172d007267f59d27 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sat, 28 Dec 2024 10:42:51 +0100 Subject: [PATCH 33/67] GHA: Uprev codecov action to v5 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68b01c92..861e8ff3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -330,7 +330,7 @@ jobs: - name: Upload coverage if: matrix.coverage - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: disable_search: true file: coverage.info @@ -450,7 +450,7 @@ jobs: - name: Upload coverage if: matrix.coverage - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: disable_search: true file: __out/cobertura.xml From 843fdc91577a34cb13e0181756874171d1d33f7f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 2 Jan 2025 19:09:35 +0100 Subject: [PATCH 34/67] Replace (semi-)absolute includes to implementation headers by relative ones Makes it easier to see the distinction, especially for tools. --- CMakeLists.txt | 2 +- build/Jamfile.v2 | 1 - src/boost/locale/encoding/codepage.cpp | 10 +++++----- src/boost/locale/encoding/iconv_converter.hpp | 4 ++-- src/boost/locale/encoding/uconv_converter.hpp | 4 ++-- src/boost/locale/encoding/wconv_converter.hpp | 2 +- src/boost/locale/icu/boundary.cpp | 12 ++++++------ src/boost/locale/icu/codecvt.cpp | 14 +++++++------- src/boost/locale/icu/collator.cpp | 12 ++++++------ src/boost/locale/icu/conversion.cpp | 8 ++++---- src/boost/locale/icu/date_time.cpp | 10 +++++----- src/boost/locale/icu/formatter.cpp | 12 ++++++------ src/boost/locale/icu/formatters_cache.cpp | 2 +- src/boost/locale/icu/formatters_cache.hpp | 2 +- src/boost/locale/icu/icu_backend.cpp | 10 +++++----- src/boost/locale/icu/numeric.cpp | 8 ++++---- src/boost/locale/icu/uconv.hpp | 2 +- src/boost/locale/posix/codecvt.cpp | 6 +++--- src/boost/locale/posix/collate.cpp | 5 ++--- src/boost/locale/posix/converter.cpp | 4 ++-- src/boost/locale/posix/numeric.cpp | 4 ++-- src/boost/locale/posix/posix_backend.cpp | 10 +++++----- src/boost/locale/shared/format.cpp | 2 +- src/boost/locale/shared/formatting.cpp | 2 +- src/boost/locale/shared/iconv_codecvt.cpp | 6 +++--- src/boost/locale/shared/ids.cpp | 2 +- src/boost/locale/shared/localization_backend.cpp | 8 ++++---- src/boost/locale/shared/message.cpp | 10 +++++----- src/boost/locale/shared/mo_lambda.cpp | 2 +- src/boost/locale/std/codecvt.cpp | 2 +- src/boost/locale/std/collate.cpp | 2 +- src/boost/locale/std/converter.cpp | 2 +- src/boost/locale/std/numeric.cpp | 4 ++-- src/boost/locale/std/std_backend.cpp | 16 ++++++++-------- src/boost/locale/util/codecvt_converter.cpp | 4 ++-- src/boost/locale/util/encoding.cpp | 6 +++--- src/boost/locale/util/gregorian.cpp | 4 ++-- src/boost/locale/util/locale_data.cpp | 4 ++-- src/boost/locale/win32/api.hpp | 2 +- src/boost/locale/win32/collate.cpp | 6 +++--- src/boost/locale/win32/converter.cpp | 4 ++-- src/boost/locale/win32/lcid.cpp | 2 +- src/boost/locale/win32/numeric.cpp | 6 +++--- src/boost/locale/win32/win_backend.cpp | 12 ++++++------ test/CMakeLists.txt | 7 +------ test/boostLocale/test/tools.hpp | 4 ++-- test/test_helpers.cpp | 2 +- 47 files changed, 129 insertions(+), 136 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 977c50aa..69fd067c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,7 +41,7 @@ add_library(boost_locale add_library(Boost::locale ALIAS boost_locale) -target_include_directories(boost_locale PUBLIC include PRIVATE src) +target_include_directories(boost_locale PUBLIC include) target_compile_features(boost_locale PUBLIC cxx_std_11) target_link_libraries(boost_locale diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index f494f779..35f0f4fc 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -427,7 +427,6 @@ lib boost_locale $(cxx_requirements) shared:BOOST_LOCALE_DYN_LINK=1 BOOST_LOCALE_SOURCE - $(TOP)/src multi windows:_CRT_SECURE_NO_WARNINGS windows:_SCL_SECURE_NO_WARNINGS diff --git a/src/boost/locale/encoding/codepage.cpp b/src/boost/locale/encoding/codepage.cpp index 0ac9659a..0f052b82 100644 --- a/src/boost/locale/encoding/codepage.cpp +++ b/src/boost/locale/encoding/codepage.cpp @@ -1,24 +1,24 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2022-2023 Alexander Grund +// Copyright (c) 2022-2025 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt #include -#include "boost/locale/util/make_std_unique.hpp" +#include "../util/make_std_unique.hpp" #if BOOST_LOCALE_USE_WIN32_API # define BOOST_LOCALE_WITH_WCONV #endif #ifdef BOOST_LOCALE_WITH_ICONV -# include "boost/locale/encoding/iconv_converter.hpp" +# include "iconv_converter.hpp" #endif #ifdef BOOST_LOCALE_WITH_ICU -# include "boost/locale/encoding/uconv_converter.hpp" +# include "uconv_converter.hpp" #endif #ifdef BOOST_LOCALE_WITH_WCONV -# include "boost/locale/encoding/wconv_converter.hpp" +# include "wconv_converter.hpp" #endif namespace boost { namespace locale { namespace conv { diff --git a/src/boost/locale/encoding/iconv_converter.hpp b/src/boost/locale/encoding/iconv_converter.hpp index 2877dbca..8d26287b 100644 --- a/src/boost/locale/encoding/iconv_converter.hpp +++ b/src/boost/locale/encoding/iconv_converter.hpp @@ -8,8 +8,8 @@ #define BOOST_LOCALE_IMPL_ICONV_CODEPAGE_HPP #include -#include "boost/locale/util/encoding.hpp" -#include "boost/locale/util/iconv.hpp" +#include "../util/encoding.hpp" +#include "../util/iconv.hpp" #include #include #include diff --git a/src/boost/locale/encoding/uconv_converter.hpp b/src/boost/locale/encoding/uconv_converter.hpp index 53a14055..ae891376 100644 --- a/src/boost/locale/encoding/uconv_converter.hpp +++ b/src/boost/locale/encoding/uconv_converter.hpp @@ -8,8 +8,8 @@ #define BOOST_LOCALE_IMPL_UCONV_CODEPAGE_HPP #include #include -#include "boost/locale/icu/icu_util.hpp" -#include "boost/locale/icu/uconv.hpp" +#include "../icu/icu_util.hpp" +#include "../icu/uconv.hpp" #include #include diff --git a/src/boost/locale/encoding/wconv_converter.hpp b/src/boost/locale/encoding/wconv_converter.hpp index 32cd78e2..e29b562e 100644 --- a/src/boost/locale/encoding/wconv_converter.hpp +++ b/src/boost/locale/encoding/wconv_converter.hpp @@ -11,7 +11,7 @@ # define NOMINMAX #endif #include -#include "boost/locale/util/encoding.hpp" +#include "../util/encoding.hpp" #include #include #include diff --git a/src/boost/locale/icu/boundary.cpp b/src/boost/locale/icu/boundary.cpp index 4adada4f..339cab50 100644 --- a/src/boost/locale/icu/boundary.cpp +++ b/src/boost/locale/icu/boundary.cpp @@ -1,17 +1,17 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2021-2022 Alexander Grund +// Copyright (c) 2021-2025 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt #include #include -#include "boost/locale/icu/all_generator.hpp" -#include "boost/locale/icu/cdata.hpp" -#include "boost/locale/icu/icu_util.hpp" -#include "boost/locale/icu/uconv.hpp" -#include "boost/locale/util/encoding.hpp" +#include "../util/encoding.hpp" +#include "all_generator.hpp" +#include "cdata.hpp" +#include "icu_util.hpp" +#include "uconv.hpp" #if BOOST_LOCALE_ICU_VERSION >= 5502 # include #endif diff --git a/src/boost/locale/icu/codecvt.cpp b/src/boost/locale/icu/codecvt.cpp index cdf958ab..3e6e24f7 100644 --- a/src/boost/locale/icu/codecvt.cpp +++ b/src/boost/locale/icu/codecvt.cpp @@ -1,20 +1,20 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2022-2023 Alexander Grund +// Copyright (c) 2022-2025 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/icu/codecvt.hpp" +#include "codecvt.hpp" #include #include #include #include -#include "boost/locale/icu/all_generator.hpp" -#include "boost/locale/icu/icu_util.hpp" -#include "boost/locale/icu/uconv.hpp" -#include "boost/locale/util/encoding.hpp" -#include "boost/locale/util/make_std_unique.hpp" +#include "../util/encoding.hpp" +#include "../util/make_std_unique.hpp" +#include "all_generator.hpp" +#include "icu_util.hpp" +#include "uconv.hpp" #include #include diff --git a/src/boost/locale/icu/collator.cpp b/src/boost/locale/icu/collator.cpp index a57d3694..43d921d5 100644 --- a/src/boost/locale/icu/collator.cpp +++ b/src/boost/locale/icu/collator.cpp @@ -7,12 +7,12 @@ #include #include -#include "boost/locale/icu/all_generator.hpp" -#include "boost/locale/icu/cdata.hpp" -#include "boost/locale/icu/icu_util.hpp" -#include "boost/locale/icu/uconv.hpp" -#include "boost/locale/shared/mo_hash.hpp" -#include "boost/locale/shared/std_collate_adapter.hpp" +#include "../shared/mo_hash.hpp" +#include "../shared/std_collate_adapter.hpp" +#include "all_generator.hpp" +#include "cdata.hpp" +#include "icu_util.hpp" +#include "uconv.hpp" #include #include #include diff --git a/src/boost/locale/icu/conversion.cpp b/src/boost/locale/icu/conversion.cpp index 829ceabd..130a0d4e 100644 --- a/src/boost/locale/icu/conversion.cpp +++ b/src/boost/locale/icu/conversion.cpp @@ -6,10 +6,10 @@ // https://www.boost.org/LICENSE_1_0.txt #include -#include "boost/locale/icu/all_generator.hpp" -#include "boost/locale/icu/cdata.hpp" -#include "boost/locale/icu/icu_util.hpp" -#include "boost/locale/icu/uconv.hpp" +#include "all_generator.hpp" +#include "cdata.hpp" +#include "icu_util.hpp" +#include "uconv.hpp" #include #include #include diff --git a/src/boost/locale/icu/date_time.cpp b/src/boost/locale/icu/date_time.cpp index 2dd7824c..76785ec6 100644 --- a/src/boost/locale/icu/date_time.cpp +++ b/src/boost/locale/icu/date_time.cpp @@ -9,11 +9,11 @@ #include #include #include -#include "boost/locale/icu/all_generator.hpp" -#include "boost/locale/icu/cdata.hpp" -#include "boost/locale/icu/icu_util.hpp" -#include "boost/locale/icu/time_zone.hpp" -#include "boost/locale/icu/uconv.hpp" +#include "all_generator.hpp" +#include "cdata.hpp" +#include "icu_util.hpp" +#include "time_zone.hpp" +#include "uconv.hpp" #include #include #include diff --git a/src/boost/locale/icu/formatter.cpp b/src/boost/locale/icu/formatter.cpp index 955b3d18..ccdab80c 100644 --- a/src/boost/locale/icu/formatter.cpp +++ b/src/boost/locale/icu/formatter.cpp @@ -5,14 +5,14 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/icu/formatter.hpp" +#include "formatter.hpp" #include #include -#include "boost/locale/icu/formatters_cache.hpp" -#include "boost/locale/icu/icu_util.hpp" -#include "boost/locale/icu/time_zone.hpp" -#include "boost/locale/icu/uconv.hpp" -#include "boost/locale/util/foreach_char.hpp" +#include "../util/foreach_char.hpp" +#include "formatters_cache.hpp" +#include "icu_util.hpp" +#include "time_zone.hpp" +#include "uconv.hpp" #include #include #ifdef BOOST_MSVC diff --git a/src/boost/locale/icu/formatters_cache.cpp b/src/boost/locale/icu/formatters_cache.cpp index 81591b9d..101ffcc6 100644 --- a/src/boost/locale/icu/formatters_cache.cpp +++ b/src/boost/locale/icu/formatters_cache.cpp @@ -5,7 +5,7 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/icu/formatters_cache.hpp" +#include "formatters_cache.hpp" #include #include #include diff --git a/src/boost/locale/icu/formatters_cache.hpp b/src/boost/locale/icu/formatters_cache.hpp index d0e4978d..2d4c950c 100644 --- a/src/boost/locale/icu/formatters_cache.hpp +++ b/src/boost/locale/icu/formatters_cache.hpp @@ -9,7 +9,7 @@ #define BOOST_LOCALE_PREDEFINED_FORMATTERS_HPP_INCLUDED #include -#include "boost/locale/icu/icu_util.hpp" +#include "icu_util.hpp" #include #include diff --git a/src/boost/locale/icu/icu_backend.cpp b/src/boost/locale/icu/icu_backend.cpp index ecb78ae4..4b70e975 100644 --- a/src/boost/locale/icu/icu_backend.cpp +++ b/src/boost/locale/icu/icu_backend.cpp @@ -5,14 +5,14 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/icu/icu_backend.hpp" +#include "icu_backend.hpp" #include #include #include -#include "boost/locale/icu/all_generator.hpp" -#include "boost/locale/icu/cdata.hpp" -#include "boost/locale/shared/message.hpp" -#include "boost/locale/util/make_std_unique.hpp" +#include "../shared/message.hpp" +#include "../util/make_std_unique.hpp" +#include "all_generator.hpp" +#include "cdata.hpp" #include diff --git a/src/boost/locale/icu/numeric.cpp b/src/boost/locale/icu/numeric.cpp index 9002a233..1fcf044a 100644 --- a/src/boost/locale/icu/numeric.cpp +++ b/src/boost/locale/icu/numeric.cpp @@ -5,10 +5,10 @@ // https://www.boost.org/LICENSE_1_0.txt #include -#include "boost/locale/icu/all_generator.hpp" -#include "boost/locale/icu/cdata.hpp" -#include "boost/locale/icu/formatter.hpp" -#include "boost/locale/icu/formatters_cache.hpp" +#include "all_generator.hpp" +#include "cdata.hpp" +#include "formatter.hpp" +#include "formatters_cache.hpp" #include #include #include diff --git a/src/boost/locale/icu/uconv.hpp b/src/boost/locale/icu/uconv.hpp index 6563afb6..f801f10e 100644 --- a/src/boost/locale/icu/uconv.hpp +++ b/src/boost/locale/icu/uconv.hpp @@ -9,7 +9,7 @@ #define BOOST_SRC_LOCALE_ICU_UCONV_HPP #include -#include "boost/locale/icu/icu_util.hpp" +#include "icu_util.hpp" #include #include diff --git a/src/boost/locale/posix/codecvt.cpp b/src/boost/locale/posix/codecvt.cpp index d2c5acf0..3f0a0ed4 100644 --- a/src/boost/locale/posix/codecvt.cpp +++ b/src/boost/locale/posix/codecvt.cpp @@ -6,9 +6,9 @@ // https://www.boost.org/LICENSE_1_0.txt #include -#include "boost/locale/posix/all_generator.hpp" -#include "boost/locale/shared/iconv_codecvt.hpp" -#include "boost/locale/util/encoding.hpp" +#include "../shared/iconv_codecvt.hpp" +#include "../util/encoding.hpp" +#include "all_generator.hpp" #include namespace boost { namespace locale { namespace impl_posix { diff --git a/src/boost/locale/posix/collate.cpp b/src/boost/locale/posix/collate.cpp index c9fb0251..bd8ab360 100644 --- a/src/boost/locale/posix/collate.cpp +++ b/src/boost/locale/posix/collate.cpp @@ -8,6 +8,8 @@ #if defined(__FreeBSD__) # include #endif +#include "../shared/mo_hash.hpp" +#include "all_generator.hpp" #include #include #include @@ -17,9 +19,6 @@ #include #include -#include "boost/locale/posix/all_generator.hpp" -#include "boost/locale/shared/mo_hash.hpp" - namespace boost { namespace locale { namespace impl_posix { template diff --git a/src/boost/locale/posix/converter.cpp b/src/boost/locale/posix/converter.cpp index b787a562..38a4bb69 100644 --- a/src/boost/locale/posix/converter.cpp +++ b/src/boost/locale/posix/converter.cpp @@ -7,7 +7,7 @@ #include #include #include -#include "boost/locale/util/encoding.hpp" +#include "../util/encoding.hpp" #include #include #include @@ -18,7 +18,7 @@ # include #endif -#include "boost/locale/posix/all_generator.hpp" +#include "all_generator.hpp" namespace boost { namespace locale { namespace impl_posix { diff --git a/src/boost/locale/posix/numeric.cpp b/src/boost/locale/posix/numeric.cpp index a3ee2c40..2810b821 100644 --- a/src/boost/locale/posix/numeric.cpp +++ b/src/boost/locale/posix/numeric.cpp @@ -27,8 +27,8 @@ #include #include -#include "boost/locale/posix/all_generator.hpp" -#include "boost/locale/util/numeric.hpp" +#include "../util/numeric.hpp" +#include "all_generator.hpp" namespace boost { namespace locale { namespace impl_posix { diff --git a/src/boost/locale/posix/posix_backend.cpp b/src/boost/locale/posix/posix_backend.cpp index 020d0c77..98aadf21 100644 --- a/src/boost/locale/posix/posix_backend.cpp +++ b/src/boost/locale/posix/posix_backend.cpp @@ -5,7 +5,7 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/posix/posix_backend.hpp" +#include "posix_backend.hpp" #include #include #include @@ -19,10 +19,10 @@ # include #endif -#include "boost/locale/posix/all_generator.hpp" -#include "boost/locale/shared/message.hpp" -#include "boost/locale/util/gregorian.hpp" -#include "boost/locale/util/make_std_unique.hpp" +#include "../shared/message.hpp" +#include "../util/gregorian.hpp" +#include "../util/make_std_unique.hpp" +#include "all_generator.hpp" namespace boost { namespace locale { namespace impl_posix { diff --git a/src/boost/locale/shared/format.cpp b/src/boost/locale/shared/format.cpp index 7ee809fe..ed2fb2ea 100644 --- a/src/boost/locale/shared/format.cpp +++ b/src/boost/locale/shared/format.cpp @@ -7,7 +7,7 @@ #include #include #include -#include "boost/locale/util/numeric.hpp" +#include "../util/numeric.hpp" #include #include #include diff --git a/src/boost/locale/shared/formatting.cpp b/src/boost/locale/shared/formatting.cpp index 457ba782..1dc9b3a9 100644 --- a/src/boost/locale/shared/formatting.cpp +++ b/src/boost/locale/shared/formatting.cpp @@ -6,7 +6,7 @@ #include #include -#include "boost/locale/shared/ios_prop.hpp" +#include "ios_prop.hpp" namespace boost { namespace locale { ios_info::ios_info() : flags_(0), domain_id_(0), time_zone_(time_zone::global()) {} diff --git a/src/boost/locale/shared/iconv_codecvt.cpp b/src/boost/locale/shared/iconv_codecvt.cpp index 2ed618a0..48197307 100644 --- a/src/boost/locale/shared/iconv_codecvt.cpp +++ b/src/boost/locale/shared/iconv_codecvt.cpp @@ -12,9 +12,9 @@ #include #include #ifdef BOOST_LOCALE_WITH_ICONV -# include "boost/locale/util/encoding.hpp" -# include "boost/locale/util/iconv.hpp" -# include "boost/locale/util/make_std_unique.hpp" +# include "../util/encoding.hpp" +# include "../util/iconv.hpp" +# include "../util/make_std_unique.hpp" #endif namespace boost { namespace locale { diff --git a/src/boost/locale/shared/ids.cpp b/src/boost/locale/shared/ids.cpp index 819de7f9..2d5ef255 100644 --- a/src/boost/locale/shared/ids.cpp +++ b/src/boost/locale/shared/ids.cpp @@ -11,7 +11,7 @@ #include #include #include -#include "boost/locale/util/foreach_char.hpp" +#include "../util/foreach_char.hpp" #include namespace boost { namespace locale { diff --git a/src/boost/locale/shared/localization_backend.cpp b/src/boost/locale/shared/localization_backend.cpp index 951cb3be..beace3e3 100644 --- a/src/boost/locale/shared/localization_backend.cpp +++ b/src/boost/locale/shared/localization_backend.cpp @@ -12,19 +12,19 @@ #include #ifdef BOOST_LOCALE_WITH_ICU -# include "boost/locale/icu/icu_backend.hpp" +# include "../icu/icu_backend.hpp" #endif #ifndef BOOST_LOCALE_NO_POSIX_BACKEND -# include "boost/locale/posix/posix_backend.hpp" +# include "../posix/posix_backend.hpp" #endif #ifndef BOOST_LOCALE_NO_STD_BACKEND -# include "boost/locale/std/std_backend.hpp" +# include "../std/std_backend.hpp" #endif #ifndef BOOST_LOCALE_NO_WINAPI_BACKEND -# include "boost/locale/win32/win_backend.hpp" +# include "../win32/win_backend.hpp" #endif namespace boost { namespace locale { diff --git a/src/boost/locale/shared/message.cpp b/src/boost/locale/shared/message.cpp index c3252316..27aaa671 100644 --- a/src/boost/locale/shared/message.cpp +++ b/src/boost/locale/shared/message.cpp @@ -20,11 +20,11 @@ #include #include -#include "boost/locale/shared/message.hpp" -#include "boost/locale/shared/mo_hash.hpp" -#include "boost/locale/shared/mo_lambda.hpp" -#include "boost/locale/util/encoding.hpp" -#include "boost/locale/util/foreach_char.hpp" +#include "../util/encoding.hpp" +#include "../util/foreach_char.hpp" +#include "message.hpp" +#include "mo_hash.hpp" +#include "mo_lambda.hpp" #include #include #include diff --git a/src/boost/locale/shared/mo_lambda.cpp b/src/boost/locale/shared/mo_lambda.cpp index 7b37ca91..e2a16c8c 100644 --- a/src/boost/locale/shared/mo_lambda.cpp +++ b/src/boost/locale/shared/mo_lambda.cpp @@ -5,7 +5,7 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/shared/mo_lambda.hpp" +#include "mo_lambda.hpp" #include #include #include diff --git a/src/boost/locale/std/codecvt.cpp b/src/boost/locale/std/codecvt.cpp index 22af5d76..58d4ff8d 100644 --- a/src/boost/locale/std/codecvt.cpp +++ b/src/boost/locale/std/codecvt.cpp @@ -5,7 +5,7 @@ // https://www.boost.org/LICENSE_1_0.txt #include -#include "boost/locale/std/all_generator.hpp" +#include "all_generator.hpp" #include namespace boost { namespace locale { namespace impl_std { diff --git a/src/boost/locale/std/collate.cpp b/src/boost/locale/std/collate.cpp index 48d80bcc..d7e5e7af 100644 --- a/src/boost/locale/std/collate.cpp +++ b/src/boost/locale/std/collate.cpp @@ -5,7 +5,7 @@ // https://www.boost.org/LICENSE_1_0.txt #include -#include "boost/locale/std/all_generator.hpp" +#include "all_generator.hpp" #include #include #include diff --git a/src/boost/locale/std/converter.cpp b/src/boost/locale/std/converter.cpp index ce08606b..fcfb7df7 100644 --- a/src/boost/locale/std/converter.cpp +++ b/src/boost/locale/std/converter.cpp @@ -11,7 +11,7 @@ #include #include -#include "boost/locale/std/all_generator.hpp" +#include "all_generator.hpp" namespace boost { namespace locale { namespace impl_std { diff --git a/src/boost/locale/std/numeric.cpp b/src/boost/locale/std/numeric.cpp index fe3dc095..b88cbd56 100644 --- a/src/boost/locale/std/numeric.cpp +++ b/src/boost/locale/std/numeric.cpp @@ -14,8 +14,8 @@ #include #include -#include "boost/locale/std/all_generator.hpp" -#include "boost/locale/util/numeric.hpp" +#include "../util/numeric.hpp" +#include "all_generator.hpp" namespace boost { namespace locale { namespace impl_std { /// Forwarding time_put facet diff --git a/src/boost/locale/std/std_backend.cpp b/src/boost/locale/std/std_backend.cpp index 4a134c25..0bcd2800 100644 --- a/src/boost/locale/std/std_backend.cpp +++ b/src/boost/locale/std/std_backend.cpp @@ -5,7 +5,7 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/std/std_backend.hpp" +#include "std_backend.hpp" #include #include #include @@ -20,15 +20,15 @@ # ifndef NOMINMAX # define NOMINMAX # endif -# include "boost/locale/win32/lcid.hpp" +# include "../win32/lcid.hpp" # include #endif -#include "boost/locale/shared/message.hpp" -#include "boost/locale/std/all_generator.hpp" -#include "boost/locale/util/encoding.hpp" -#include "boost/locale/util/gregorian.hpp" -#include "boost/locale/util/make_std_unique.hpp" -#include "boost/locale/util/numeric.hpp" +#include "../shared/message.hpp" +#include "../util/encoding.hpp" +#include "../util/gregorian.hpp" +#include "../util/make_std_unique.hpp" +#include "../util/numeric.hpp" +#include "all_generator.hpp" namespace { struct windows_name { diff --git a/src/boost/locale/util/codecvt_converter.cpp b/src/boost/locale/util/codecvt_converter.cpp index b1cf9df9..26af5005 100644 --- a/src/boost/locale/util/codecvt_converter.cpp +++ b/src/boost/locale/util/codecvt_converter.cpp @@ -15,8 +15,8 @@ #include #include -#include "boost/locale/util/encoding.hpp" -#include "boost/locale/util/make_std_unique.hpp" +#include "encoding.hpp" +#include "make_std_unique.hpp" #ifdef BOOST_MSVC # pragma warning(disable : 4244) // loose data diff --git a/src/boost/locale/util/encoding.cpp b/src/boost/locale/util/encoding.cpp index 8a6bf183..18b658fa 100644 --- a/src/boost/locale/util/encoding.cpp +++ b/src/boost/locale/util/encoding.cpp @@ -5,10 +5,10 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/util/encoding.hpp" -#include "boost/locale/util/string.hpp" +#include "encoding.hpp" +#include #if BOOST_LOCALE_USE_WIN32_API -# include "boost/locale/util/win_codepages.hpp" +# include "win_codepages.hpp" # ifndef NOMINMAX # define NOMINMAX # endif diff --git a/src/boost/locale/util/gregorian.cpp b/src/boost/locale/util/gregorian.cpp index b0bc6e8a..3183d4ae 100644 --- a/src/boost/locale/util/gregorian.cpp +++ b/src/boost/locale/util/gregorian.cpp @@ -4,11 +4,11 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/util/gregorian.hpp" +#include "gregorian.hpp" #include #include #include -#include "boost/locale/util/timezone.hpp" +#include "timezone.hpp" #include #include #include diff --git a/src/boost/locale/util/locale_data.cpp b/src/boost/locale/util/locale_data.cpp index 41c0bc88..1fded47a 100644 --- a/src/boost/locale/util/locale_data.cpp +++ b/src/boost/locale/util/locale_data.cpp @@ -6,8 +6,8 @@ // https://www.boost.org/LICENSE_1_0.txt #include -#include "boost/locale/util/encoding.hpp" -#include "boost/locale/util/string.hpp" +#include +#include "encoding.hpp" #include #include #include diff --git a/src/boost/locale/win32/api.hpp b/src/boost/locale/win32/api.hpp index 33791ef9..e9b95b7f 100644 --- a/src/boost/locale/win32/api.hpp +++ b/src/boost/locale/win32/api.hpp @@ -15,7 +15,7 @@ #include #include -#include "boost/locale/win32/lcid.hpp" +#include "lcid.hpp" #ifndef NOMINMAX # define NOMINMAX diff --git a/src/boost/locale/win32/collate.cpp b/src/boost/locale/win32/collate.cpp index 16fe38d1..dcdb2c99 100644 --- a/src/boost/locale/win32/collate.cpp +++ b/src/boost/locale/win32/collate.cpp @@ -6,9 +6,9 @@ #include #include -#include "boost/locale/shared/mo_hash.hpp" -#include "boost/locale/shared/std_collate_adapter.hpp" -#include "boost/locale/win32/api.hpp" +#include "../shared/mo_hash.hpp" +#include "../shared/std_collate_adapter.hpp" +#include "api.hpp" #include #include #include diff --git a/src/boost/locale/win32/converter.cpp b/src/boost/locale/win32/converter.cpp index ed29aedc..455fef6c 100644 --- a/src/boost/locale/win32/converter.cpp +++ b/src/boost/locale/win32/converter.cpp @@ -7,8 +7,8 @@ #include #include #include -#include "boost/locale/win32/all_generator.hpp" -#include "boost/locale/win32/api.hpp" +#include "all_generator.hpp" +#include "api.hpp" #include #include #include diff --git a/src/boost/locale/win32/lcid.cpp b/src/boost/locale/win32/lcid.cpp index 14354557..e1b998d6 100644 --- a/src/boost/locale/win32/lcid.cpp +++ b/src/boost/locale/win32/lcid.cpp @@ -8,7 +8,7 @@ # define NOMINMAX #endif -#include "boost/locale/win32/lcid.hpp" +#include "lcid.hpp" #include #include #include diff --git a/src/boost/locale/win32/numeric.cpp b/src/boost/locale/win32/numeric.cpp index 556de79a..3286424e 100644 --- a/src/boost/locale/win32/numeric.cpp +++ b/src/boost/locale/win32/numeric.cpp @@ -4,12 +4,12 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/util/numeric.hpp" +#include "../util/numeric.hpp" #include #include #include -#include "boost/locale/win32/all_generator.hpp" -#include "boost/locale/win32/api.hpp" +#include "all_generator.hpp" +#include "api.hpp" #include #include #include diff --git a/src/boost/locale/win32/win_backend.cpp b/src/boost/locale/win32/win_backend.cpp index 48155d6c..2ce4d460 100644 --- a/src/boost/locale/win32/win_backend.cpp +++ b/src/boost/locale/win32/win_backend.cpp @@ -5,17 +5,17 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "boost/locale/win32/win_backend.hpp" +#include "win_backend.hpp" #include #include #include #include #include -#include "boost/locale/shared/message.hpp" -#include "boost/locale/util/gregorian.hpp" -#include "boost/locale/util/make_std_unique.hpp" -#include "boost/locale/win32/all_generator.hpp" -#include "boost/locale/win32/api.hpp" +#include "../shared/message.hpp" +#include "../util/gregorian.hpp" +#include "../util/make_std_unique.hpp" +#include "all_generator.hpp" +#include "api.hpp" #include #include #include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 412fbb21..10284197 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,13 +4,8 @@ include(BoostTestJamfile) -add_library(boost_locale_test INTERFACE) -# Add test folder to include directories, especially for systems -# where the current folder is not automatically added to the search path -target_include_directories(boost_locale_test INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}") - set(BOOST_TEST_COMPILE_DEFINITIONS "") -set(BOOST_TEST_LINK_LIBRARIES Boost::locale boost_locale_test) +set(BOOST_TEST_LINK_LIBRARIES Boost::locale) set(BOOST_TEST_COMPILE_OPTIONS ${BOOST_LOCALE_WARNING_OPTIONS}) if(MSVC OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11) diff --git a/test/boostLocale/test/tools.hpp b/test/boostLocale/test/tools.hpp index f386c047..92fb7e24 100644 --- a/test/boostLocale/test/tools.hpp +++ b/test/boostLocale/test/tools.hpp @@ -8,8 +8,8 @@ #define BOOST_LOCALE_TEST_TOOLS_HPP #include -#include "boostLocale/test/posix_tools.hpp" -#include "boostLocale/test/unit_test.hpp" +#include "posix_tools.hpp" +#include "unit_test.hpp" #include #include #include diff --git a/test/test_helpers.cpp b/test/test_helpers.cpp index f7e5a327..97ddb55f 100644 --- a/test/test_helpers.cpp +++ b/test/test_helpers.cpp @@ -17,7 +17,7 @@ # endif #endif -#include +#include "boostLocale/test/test_helpers.hpp" #include #ifdef BOOST_WINDOWS # include From b1cf7585d4f4d7c1e95abaede1c3133242eee3ff Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 7 Jan 2025 20:05:46 +0100 Subject: [PATCH 35/67] GHA: Fix parameter for codecov action --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 861e8ff3..ba58b786 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -333,7 +333,7 @@ jobs: uses: codecov/codecov-action@v5 with: disable_search: true - file: coverage.info + files: coverage.info name: Github Actions token: ${{secrets.CODECOV_TOKEN}} verbose: true @@ -453,7 +453,7 @@ jobs: uses: codecov/codecov-action@v5 with: disable_search: true - file: __out/cobertura.xml + files: __out/cobertura.xml name: Github Actions token: ${{secrets.CODECOV_TOKEN}} verbose: true From d20388d31b4142837ebe934c90975f95534c4ac2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 8 Jan 2025 14:31:41 +0100 Subject: [PATCH 36/67] Update coverage step - Use new "collect" argument - Fail CI on error --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba58b786..8ab417f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -324,14 +324,13 @@ jobs: - name: Collect coverage if: matrix.coverage - run: ci/codecov.sh "upload" - env: - BOOST_CI_CODECOV_IO_UPLOAD: skip + run: ci/codecov.sh "collect" - name: Upload coverage if: matrix.coverage uses: codecov/codecov-action@v5 with: + fail_ci_if_error: true disable_search: true files: coverage.info name: Github Actions From f00d128633c7f0ce2b0d2df17ae2b7e35be8026f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 7 Jan 2025 18:46:39 +0100 Subject: [PATCH 37/67] GHA: Remove ubuntu-toolchain ppa --- .github/workflows/ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ab417f1..ac596b82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -206,13 +206,6 @@ jobs: run: | SOURCE_KEYS=(${{join(matrix.source_keys, ' ')}}) SOURCES=(${{join(matrix.sources, ' ')}}) - # Add these by default - SOURCE_KEYS+=( - 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x1E9377A2BA9EF27F' - 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x40976EAF437D05B5' - 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x3B4FE6ACC0B21F32' - ) - SOURCES+=(ppa:ubuntu-toolchain-r/test) ci/add-apt-keys.sh "${SOURCE_KEYS[@]}" # Initial update before adding sources required to get e.g. keys From 1bb178a507713c46e6cf830bf1a67304412ecfc8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sat, 4 Jan 2025 19:14:51 +0100 Subject: [PATCH 38/67] GHA: Use libstdc++-11 for Clang 13/14 They are not yet compatible with libstdc++-12 and up. --- .github/workflows/ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac596b82..362bf75e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ # Copyright 2020-2021 Peter Dimov # Copyright 2021 Andrey Semashev -# Copyright 2021-2024 Alexander Grund +# Copyright 2021-2025 Alexander Grund # Copyright 2022 James E. King III # # Distributed under the Boost Software License, Version 1.0. @@ -115,15 +115,15 @@ jobs: - { compiler: clang-6.0, cxxstd: '11,14,17', os: ubuntu-20.04 } - { compiler: clang-7, cxxstd: '11,14,17', os: ubuntu-20.04 } # Note: clang-8 does not fully support C++20, so it is not compatible with some libstdc++ versions in this mode - - { compiler: clang-8, cxxstd: '11,14,17,2a', os: ubuntu-20.04 , install: 'clang-8 g++-7', gcc_toolchain: 7 } + - { compiler: clang-8, cxxstd: '11,14,17,2a', os: ubuntu-20.04 , gcc_toolchain: 7 } - { compiler: clang-9, cxxstd: '11,14,17,2a', os: ubuntu-20.04 } - { compiler: clang-10, cxxstd: '11,14,17,20', os: ubuntu-20.04 } - { compiler: clang-11, cxxstd: '11,14,17,20', os: ubuntu-20.04 } - { compiler: clang-12, cxxstd: '11,14,17,20', os: ubuntu-20.04 } # Clang isn't compatible with libstdc++-13, so use the slightly older one - - { compiler: clang-13, cxxstd: '11,14,17,20', os: ubuntu-22.04, install: 'clang-13 g++-12', gcc_toolchain: 12 } - - { compiler: clang-14, cxxstd: '11,14,17,20', os: ubuntu-22.04, install: 'clang-14 g++-12', gcc_toolchain: 12 } - - { compiler: clang-15, cxxstd: '11,14,17,20', os: ubuntu-22.04, install: 'clang-15 g++-12', gcc_toolchain: 12 } + - { compiler: clang-13, cxxstd: '11,14,17,20', os: ubuntu-22.04, gcc_toolchain: 11 } + - { compiler: clang-14, cxxstd: '11,14,17,20', os: ubuntu-22.04, gcc_toolchain: 11 } + - { compiler: clang-15, cxxstd: '11,14,17,20', os: ubuntu-22.04, gcc_toolchain: 12 } - { compiler: clang-16, cxxstd: '11,14,17,20,2b', os: ubuntu-24.04 } # https://github.com/llvm/llvm-project/issues/59827: disabled 2b/23 for clang-17 with libstdc++13 in 24.04 - { compiler: clang-17, cxxstd: '11,14,17,20', os: ubuntu-24.04 } @@ -216,6 +216,7 @@ jobs: if [[ -z "${{matrix.install}}" ]]; then pkgs="${{matrix.compiler}}" pkgs="${pkgs/gcc-/g++-}" + [[ -z "${{matrix.gcc_toolchain}}" ]] || pkgs+=" g++-${{matrix.gcc_toolchain}}" else pkgs="${{matrix.install}}" fi From 676ba05b82fec61e5996b9de68b152c14e3af2de Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 6 Jan 2025 09:32:20 +0100 Subject: [PATCH 39/67] Show output of config step and tests in GHA CI --- .github/workflows/ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 362bf75e..d68fd936 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -307,6 +307,26 @@ jobs: B2_TARGETS="libs/$SELF/test//show_config --verbose-test" ci/build.sh ci/build.sh + - name: Show config.log + if: '!matrix.coverity && always()' + run: cat "$BOOST_ROOT/bin.v2/config.log" + + - name: Test output + if: '!matrix.coverity && always()' + run: | + for f in $(find "$BOOST_ROOT/bin.v2/libs/$SELF/test" -type f -name 'test_*.run'); do + name=$(basename "$f") + name=${name%.run} + config=$(dirname "$f") + config=${config#*.test/*/} + config=${config/\/visibility-hidden/} + echo "::group::$name" + echo "$config" + echo "" + cat "$f" + echo "::endgroup::" + done + - name: Run tests with iconv only if: '!matrix.coverity' run: ci/build.sh From 219dff158444b50fe4b24c7104d09b3b97c99d1f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 7 Jan 2025 18:50:16 +0100 Subject: [PATCH 40/67] GHA: Install 32bit ICU if compiling for 32bit --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d68fd936..8a454415 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,6 +207,8 @@ jobs: SOURCE_KEYS=(${{join(matrix.source_keys, ' ')}}) SOURCES=(${{join(matrix.sources, ' ')}}) + [[ "${{matrix.address-model}}" != *32* ]] || sudo dpkg --add-architecture i386 + ci/add-apt-keys.sh "${SOURCE_KEYS[@]}" # Initial update before adding sources required to get e.g. keys sudo apt-get -o Acquire::Retries=$NET_RETRY_COUNT update @@ -220,6 +222,7 @@ jobs: else pkgs="${{matrix.install}}" fi + [[ "${{matrix.address-model}}" != *32* ]] || pkgs+=" libicu-dev:i386" sudo apt-get -o Acquire::Retries=$NET_RETRY_COUNT install -y $pkgs libicu-dev - name: Setup GCC Toolchain From 806fc4351a592a035d3fb713ee60b5b67f6b880e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 9 Jan 2025 09:47:11 +0100 Subject: [PATCH 41/67] Avoid missing coverage Recompiling with different options/defines might remove covered lines that no longer exist. This is e.g. the case for the IConv code paths when compiling with ICU and without IConv in the last test run before collecting the coverage data. To avoid this collect coverage data after each test run into separate files and upload all of them. --- .github/workflows/ci.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a454415..da2f0443 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -330,18 +330,25 @@ jobs: echo "::endgroup::" done + - name: Collect coverage + if: matrix.coverage + run: ci/codecov.sh "collect" && mv coverage.info coverage.all.info + - name: Run tests with iconv only if: '!matrix.coverity' run: ci/build.sh env: {B2_FLAGS: -a boost.locale.icu=off boost.locale.iconv=on} + - name: Collect coverage + if: matrix.coverage + run: ci/codecov.sh "collect" && mv coverage.info coverage.iconv.info + - name: Run tests with ICU only if: '!matrix.coverity' run: ci/build.sh env: {B2_FLAGS: -a boost.locale.icu=on boost.locale.iconv=off} - - name: Collect coverage if: matrix.coverage - run: ci/codecov.sh "collect" + run: ci/codecov.sh "collect" && mv coverage.info coverage.icu.info - name: Upload coverage if: matrix.coverage @@ -349,7 +356,7 @@ jobs: with: fail_ci_if_error: true disable_search: true - files: coverage.info + files: coverage.all.info,coverage.iconv.info,coverage.icu.info name: Github Actions token: ${{secrets.CODECOV_TOKEN}} verbose: true From 5a58e65bdacd48578206de7ef5b90d6cf61f0449 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 9 Jan 2025 13:46:39 +0100 Subject: [PATCH 42/67] Remove superflous subfolders of `src` The shorter paths make the sources easier to navigate. Basically revert of #99 / f44b3bb771f01fb94614260d57e1d09cdfe1cdff after the underlying issue is resolved. --- CMakeLists.txt | 140 +++++++++--------- build/Jamfile.v2 | 2 +- src/{boost/locale => }/encoding/codepage.cpp | 0 .../locale => }/encoding/iconv_converter.hpp | 0 .../locale => }/encoding/uconv_converter.hpp | 0 .../locale => }/encoding/wconv_converter.hpp | 0 src/{boost/locale => }/icu/all_generator.hpp | 0 src/{boost/locale => }/icu/boundary.cpp | 0 src/{boost/locale => }/icu/cdata.hpp | 0 src/{boost/locale => }/icu/codecvt.cpp | 0 src/{boost/locale => }/icu/codecvt.hpp | 0 src/{boost/locale => }/icu/collator.cpp | 0 src/{boost/locale => }/icu/conversion.cpp | 0 src/{boost/locale => }/icu/date_time.cpp | 0 src/{boost/locale => }/icu/formatter.cpp | 0 src/{boost/locale => }/icu/formatter.hpp | 0 .../locale => }/icu/formatters_cache.cpp | 0 .../locale => }/icu/formatters_cache.hpp | 0 src/{boost/locale => }/icu/icu_backend.cpp | 0 src/{boost/locale => }/icu/icu_backend.hpp | 0 src/{boost/locale => }/icu/icu_util.hpp | 0 src/{boost/locale => }/icu/numeric.cpp | 0 src/{boost/locale => }/icu/time_zone.hpp | 0 src/{boost/locale => }/icu/uconv.hpp | 0 .../locale => }/posix/all_generator.hpp | 0 src/{boost/locale => }/posix/codecvt.cpp | 0 src/{boost/locale => }/posix/collate.cpp | 0 src/{boost/locale => }/posix/converter.cpp | 0 src/{boost/locale => }/posix/numeric.cpp | 0 .../locale => }/posix/posix_backend.cpp | 0 .../locale => }/posix/posix_backend.hpp | 0 src/{boost/locale => }/shared/date_time.cpp | 0 src/{boost/locale => }/shared/format.cpp | 0 src/{boost/locale => }/shared/formatting.cpp | 0 src/{boost/locale => }/shared/generator.cpp | 0 .../locale => }/shared/iconv_codecvt.cpp | 0 .../locale => }/shared/iconv_codecvt.hpp | 0 src/{boost/locale => }/shared/ids.cpp | 0 src/{boost/locale => }/shared/ios_prop.hpp | 0 .../shared/localization_backend.cpp | 0 src/{boost/locale => }/shared/message.cpp | 0 src/{boost/locale => }/shared/message.hpp | 0 src/{boost/locale => }/shared/mo_hash.hpp | 0 src/{boost/locale => }/shared/mo_lambda.cpp | 0 src/{boost/locale => }/shared/mo_lambda.hpp | 0 .../shared/std_collate_adapter.hpp | 0 src/{boost/locale => }/std/all_generator.hpp | 0 src/{boost/locale => }/std/codecvt.cpp | 0 src/{boost/locale => }/std/collate.cpp | 0 src/{boost/locale => }/std/converter.cpp | 0 src/{boost/locale => }/std/numeric.cpp | 0 src/{boost/locale => }/std/std_backend.cpp | 0 src/{boost/locale => }/std/std_backend.hpp | 0 .../locale => }/util/codecvt_converter.cpp | 0 .../locale => }/util/default_locale.cpp | 0 src/{boost/locale => }/util/encoding.cpp | 0 src/{boost/locale => }/util/encoding.hpp | 0 src/{boost/locale => }/util/foreach_char.hpp | 0 src/{boost/locale => }/util/gregorian.cpp | 0 src/{boost/locale => }/util/gregorian.hpp | 0 src/{boost/locale => }/util/iconv.hpp | 0 src/{boost/locale => }/util/info.cpp | 0 src/{boost/locale => }/util/locale_data.cpp | 0 .../locale => }/util/make_std_unique.hpp | 0 src/{boost/locale => }/util/numeric.hpp | 0 src/{boost/locale => }/util/timezone.hpp | 0 src/{boost/locale => }/util/win_codepages.hpp | 0 .../locale => }/win32/all_generator.hpp | 0 src/{boost/locale => }/win32/api.hpp | 0 src/{boost/locale => }/win32/collate.cpp | 0 src/{boost/locale => }/win32/converter.cpp | 0 src/{boost/locale => }/win32/lcid.cpp | 0 src/{boost/locale => }/win32/lcid.hpp | 0 src/{boost/locale => }/win32/numeric.cpp | 0 src/{boost/locale => }/win32/win_backend.cpp | 0 src/{boost/locale => }/win32/win_backend.hpp | 0 test/boostLocale/test/tools.hpp | 2 +- test/test_catalog.cpp | 2 +- test/test_codepage_converter.cpp | 4 +- test/test_encoding.cpp | 4 +- test/test_generator.cpp | 2 +- test/test_ios_prop.cpp | 2 +- test/test_util.cpp | 2 +- test/test_winapi_formatting.cpp | 2 +- 84 files changed, 81 insertions(+), 81 deletions(-) rename src/{boost/locale => }/encoding/codepage.cpp (100%) rename src/{boost/locale => }/encoding/iconv_converter.hpp (100%) rename src/{boost/locale => }/encoding/uconv_converter.hpp (100%) rename src/{boost/locale => }/encoding/wconv_converter.hpp (100%) rename src/{boost/locale => }/icu/all_generator.hpp (100%) rename src/{boost/locale => }/icu/boundary.cpp (100%) rename src/{boost/locale => }/icu/cdata.hpp (100%) rename src/{boost/locale => }/icu/codecvt.cpp (100%) rename src/{boost/locale => }/icu/codecvt.hpp (100%) rename src/{boost/locale => }/icu/collator.cpp (100%) rename src/{boost/locale => }/icu/conversion.cpp (100%) rename src/{boost/locale => }/icu/date_time.cpp (100%) rename src/{boost/locale => }/icu/formatter.cpp (100%) rename src/{boost/locale => }/icu/formatter.hpp (100%) rename src/{boost/locale => }/icu/formatters_cache.cpp (100%) rename src/{boost/locale => }/icu/formatters_cache.hpp (100%) rename src/{boost/locale => }/icu/icu_backend.cpp (100%) rename src/{boost/locale => }/icu/icu_backend.hpp (100%) rename src/{boost/locale => }/icu/icu_util.hpp (100%) rename src/{boost/locale => }/icu/numeric.cpp (100%) rename src/{boost/locale => }/icu/time_zone.hpp (100%) rename src/{boost/locale => }/icu/uconv.hpp (100%) rename src/{boost/locale => }/posix/all_generator.hpp (100%) rename src/{boost/locale => }/posix/codecvt.cpp (100%) rename src/{boost/locale => }/posix/collate.cpp (100%) rename src/{boost/locale => }/posix/converter.cpp (100%) rename src/{boost/locale => }/posix/numeric.cpp (100%) rename src/{boost/locale => }/posix/posix_backend.cpp (100%) rename src/{boost/locale => }/posix/posix_backend.hpp (100%) rename src/{boost/locale => }/shared/date_time.cpp (100%) rename src/{boost/locale => }/shared/format.cpp (100%) rename src/{boost/locale => }/shared/formatting.cpp (100%) rename src/{boost/locale => }/shared/generator.cpp (100%) rename src/{boost/locale => }/shared/iconv_codecvt.cpp (100%) rename src/{boost/locale => }/shared/iconv_codecvt.hpp (100%) rename src/{boost/locale => }/shared/ids.cpp (100%) rename src/{boost/locale => }/shared/ios_prop.hpp (100%) rename src/{boost/locale => }/shared/localization_backend.cpp (100%) rename src/{boost/locale => }/shared/message.cpp (100%) rename src/{boost/locale => }/shared/message.hpp (100%) rename src/{boost/locale => }/shared/mo_hash.hpp (100%) rename src/{boost/locale => }/shared/mo_lambda.cpp (100%) rename src/{boost/locale => }/shared/mo_lambda.hpp (100%) rename src/{boost/locale => }/shared/std_collate_adapter.hpp (100%) rename src/{boost/locale => }/std/all_generator.hpp (100%) rename src/{boost/locale => }/std/codecvt.cpp (100%) rename src/{boost/locale => }/std/collate.cpp (100%) rename src/{boost/locale => }/std/converter.cpp (100%) rename src/{boost/locale => }/std/numeric.cpp (100%) rename src/{boost/locale => }/std/std_backend.cpp (100%) rename src/{boost/locale => }/std/std_backend.hpp (100%) rename src/{boost/locale => }/util/codecvt_converter.cpp (100%) rename src/{boost/locale => }/util/default_locale.cpp (100%) rename src/{boost/locale => }/util/encoding.cpp (100%) rename src/{boost/locale => }/util/encoding.hpp (100%) rename src/{boost/locale => }/util/foreach_char.hpp (100%) rename src/{boost/locale => }/util/gregorian.cpp (100%) rename src/{boost/locale => }/util/gregorian.hpp (100%) rename src/{boost/locale => }/util/iconv.hpp (100%) rename src/{boost/locale => }/util/info.cpp (100%) rename src/{boost/locale => }/util/locale_data.cpp (100%) rename src/{boost/locale => }/util/make_std_unique.hpp (100%) rename src/{boost/locale => }/util/numeric.hpp (100%) rename src/{boost/locale => }/util/timezone.hpp (100%) rename src/{boost/locale => }/util/win_codepages.hpp (100%) rename src/{boost/locale => }/win32/all_generator.hpp (100%) rename src/{boost/locale => }/win32/api.hpp (100%) rename src/{boost/locale => }/win32/collate.cpp (100%) rename src/{boost/locale => }/win32/converter.cpp (100%) rename src/{boost/locale => }/win32/lcid.cpp (100%) rename src/{boost/locale => }/win32/lcid.hpp (100%) rename src/{boost/locale => }/win32/numeric.cpp (100%) rename src/{boost/locale => }/win32/win_backend.cpp (100%) rename src/{boost/locale => }/win32/win_backend.hpp (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 69fd067c..32f10c9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,31 +11,31 @@ project(boost_locale VERSION "${BOOST_SUPERPROJECT_VERSION}" LANGUAGES CXX) file(GLOB_RECURSE headers include/*.hpp) add_library(boost_locale - src/boost/locale/encoding/codepage.cpp - src/boost/locale/encoding/iconv_converter.hpp - src/boost/locale/encoding/uconv_converter.hpp - src/boost/locale/encoding/wconv_converter.hpp - src/boost/locale/shared/date_time.cpp - src/boost/locale/shared/format.cpp - src/boost/locale/shared/formatting.cpp - src/boost/locale/shared/generator.cpp - src/boost/locale/shared/iconv_codecvt.cpp - src/boost/locale/shared/iconv_codecvt.hpp - src/boost/locale/shared/ids.cpp - src/boost/locale/shared/localization_backend.cpp - src/boost/locale/shared/message.cpp - src/boost/locale/shared/mo_lambda.cpp - src/boost/locale/shared/std_collate_adapter.hpp - src/boost/locale/util/codecvt_converter.cpp - src/boost/locale/util/default_locale.cpp - src/boost/locale/util/encoding.cpp - src/boost/locale/util/encoding.hpp - src/boost/locale/util/foreach_char.hpp - src/boost/locale/util/info.cpp - src/boost/locale/util/locale_data.cpp - src/boost/locale/util/make_std_unique.hpp - src/boost/locale/util/numeric.hpp - src/boost/locale/util/timezone.hpp + src/encoding/codepage.cpp + src/encoding/iconv_converter.hpp + src/encoding/uconv_converter.hpp + src/encoding/wconv_converter.hpp + src/shared/date_time.cpp + src/shared/format.cpp + src/shared/formatting.cpp + src/shared/generator.cpp + src/shared/iconv_codecvt.cpp + src/shared/iconv_codecvt.hpp + src/shared/ids.cpp + src/shared/localization_backend.cpp + src/shared/message.cpp + src/shared/mo_lambda.cpp + src/shared/std_collate_adapter.hpp + src/util/codecvt_converter.cpp + src/util/default_locale.cpp + src/util/encoding.cpp + src/util/encoding.hpp + src/util/foreach_char.hpp + src/util/info.cpp + src/util/locale_data.cpp + src/util/make_std_unique.hpp + src/util/numeric.hpp + src/util/timezone.hpp ${headers} ) @@ -117,7 +117,7 @@ if(BOOST_LOCALE_ENABLE_ICONV) target_link_libraries(boost_locale PRIVATE Iconv::Iconv) target_sources(boost_locale PRIVATE - src/boost/locale/util/iconv.hpp + src/util/iconv.hpp ) endif() @@ -132,24 +132,24 @@ if(BOOST_LOCALE_ENABLE_ICU) target_link_libraries(boost_locale PRIVATE ICU::data ICU::i18n ICU::uc) target_sources(boost_locale PRIVATE - src/boost/locale/icu/all_generator.hpp - src/boost/locale/icu/boundary.cpp - src/boost/locale/icu/cdata.hpp - src/boost/locale/icu/codecvt.cpp - src/boost/locale/icu/codecvt.hpp - src/boost/locale/icu/collator.cpp - src/boost/locale/icu/conversion.cpp - src/boost/locale/icu/date_time.cpp - src/boost/locale/icu/formatter.cpp - src/boost/locale/icu/formatter.hpp - src/boost/locale/icu/formatters_cache.cpp - src/boost/locale/icu/formatters_cache.hpp - src/boost/locale/icu/icu_backend.cpp - src/boost/locale/icu/icu_backend.hpp - src/boost/locale/icu/icu_util.hpp - src/boost/locale/icu/numeric.cpp - src/boost/locale/icu/time_zone.hpp - src/boost/locale/icu/uconv.hpp + src/icu/all_generator.hpp + src/icu/boundary.cpp + src/icu/cdata.hpp + src/icu/codecvt.cpp + src/icu/codecvt.hpp + src/icu/collator.cpp + src/icu/conversion.cpp + src/icu/date_time.cpp + src/icu/formatter.cpp + src/icu/formatter.hpp + src/icu/formatters_cache.cpp + src/icu/formatters_cache.hpp + src/icu/icu_backend.cpp + src/icu/icu_backend.hpp + src/icu/icu_util.hpp + src/icu/numeric.cpp + src/icu/time_zone.hpp + src/icu/uconv.hpp ) endif() @@ -157,13 +157,13 @@ endif() if(BOOST_LOCALE_ENABLE_STD) target_sources(boost_locale PRIVATE - src/boost/locale/std/all_generator.hpp - src/boost/locale/std/codecvt.cpp - src/boost/locale/std/collate.cpp - src/boost/locale/std/converter.cpp - src/boost/locale/std/numeric.cpp - src/boost/locale/std/std_backend.cpp - src/boost/locale/std/std_backend.hpp + src/std/all_generator.hpp + src/std/codecvt.cpp + src/std/collate.cpp + src/std/converter.cpp + src/std/numeric.cpp + src/std/std_backend.cpp + src/std/std_backend.hpp ) else() @@ -175,13 +175,13 @@ endif() if(BOOST_LOCALE_ENABLE_WINAPI) target_sources(boost_locale PRIVATE - src/boost/locale/win32/all_generator.hpp - src/boost/locale/win32/api.hpp - src/boost/locale/win32/collate.cpp - src/boost/locale/win32/converter.cpp - src/boost/locale/win32/numeric.cpp - src/boost/locale/win32/win_backend.cpp - src/boost/locale/win32/win_backend.hpp + src/win32/all_generator.hpp + src/win32/api.hpp + src/win32/collate.cpp + src/win32/converter.cpp + src/win32/numeric.cpp + src/win32/win_backend.cpp + src/win32/win_backend.hpp ) else() @@ -193,8 +193,8 @@ endif() if(BOOST_LOCALE_ENABLE_WINAPI OR (BOOST_LOCALE_ENABLE_STD AND WIN32)) target_sources(boost_locale PRIVATE - src/boost/locale/win32/lcid.cpp - src/boost/locale/win32/lcid.hpp + src/win32/lcid.cpp + src/win32/lcid.hpp ) endif() @@ -202,13 +202,13 @@ endif() if(BOOST_LOCALE_ENABLE_POSIX) target_sources(boost_locale PRIVATE - src/boost/locale/posix/all_generator.hpp - src/boost/locale/posix/codecvt.cpp - src/boost/locale/posix/collate.cpp - src/boost/locale/posix/converter.cpp - src/boost/locale/posix/numeric.cpp - src/boost/locale/posix/posix_backend.cpp - src/boost/locale/posix/posix_backend.hpp + src/posix/all_generator.hpp + src/posix/codecvt.cpp + src/posix/collate.cpp + src/posix/converter.cpp + src/posix/numeric.cpp + src/posix/posix_backend.cpp + src/posix/posix_backend.hpp ) else() @@ -220,8 +220,8 @@ endif() if(BOOST_LOCALE_ENABLE_WINAPI OR BOOST_LOCALE_ENABLE_STD OR BOOST_LOCALE_ENABLE_POSIX) target_sources(boost_locale PRIVATE - src/boost/locale/util/gregorian.cpp - src/boost/locale/util/gregorian.hpp + src/util/gregorian.cpp + src/util/gregorian.hpp ) endif() @@ -231,7 +231,7 @@ set(sources_src ${sources}) list(FILTER sources_src INCLUDE REGEX "^src/") set(sources_inc ${sources}) list(FILTER sources_inc EXCLUDE REGEX "^src/") -source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}/src/boost/locale" PREFIX "sources" FILES ${sources_src}) +source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}/src" PREFIX "sources" FILES ${sources_src}) source_group(TREE "${CMAKE_CURRENT_SOURCE_DIR}/include/boost/locale" PREFIX "headers" FILES ${sources_inc}) source_group("headers" FILES "include/boost/locale.hpp") diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 35f0f4fc..7a07e787 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -21,7 +21,7 @@ constant boost_dependencies_private : ; project - : source-location $(TOP)/src/boost/locale + : source-location $(TOP)/src : common-requirements $(boost_dependencies) : requirements $(boost_dependencies_private) ; diff --git a/src/boost/locale/encoding/codepage.cpp b/src/encoding/codepage.cpp similarity index 100% rename from src/boost/locale/encoding/codepage.cpp rename to src/encoding/codepage.cpp diff --git a/src/boost/locale/encoding/iconv_converter.hpp b/src/encoding/iconv_converter.hpp similarity index 100% rename from src/boost/locale/encoding/iconv_converter.hpp rename to src/encoding/iconv_converter.hpp diff --git a/src/boost/locale/encoding/uconv_converter.hpp b/src/encoding/uconv_converter.hpp similarity index 100% rename from src/boost/locale/encoding/uconv_converter.hpp rename to src/encoding/uconv_converter.hpp diff --git a/src/boost/locale/encoding/wconv_converter.hpp b/src/encoding/wconv_converter.hpp similarity index 100% rename from src/boost/locale/encoding/wconv_converter.hpp rename to src/encoding/wconv_converter.hpp diff --git a/src/boost/locale/icu/all_generator.hpp b/src/icu/all_generator.hpp similarity index 100% rename from src/boost/locale/icu/all_generator.hpp rename to src/icu/all_generator.hpp diff --git a/src/boost/locale/icu/boundary.cpp b/src/icu/boundary.cpp similarity index 100% rename from src/boost/locale/icu/boundary.cpp rename to src/icu/boundary.cpp diff --git a/src/boost/locale/icu/cdata.hpp b/src/icu/cdata.hpp similarity index 100% rename from src/boost/locale/icu/cdata.hpp rename to src/icu/cdata.hpp diff --git a/src/boost/locale/icu/codecvt.cpp b/src/icu/codecvt.cpp similarity index 100% rename from src/boost/locale/icu/codecvt.cpp rename to src/icu/codecvt.cpp diff --git a/src/boost/locale/icu/codecvt.hpp b/src/icu/codecvt.hpp similarity index 100% rename from src/boost/locale/icu/codecvt.hpp rename to src/icu/codecvt.hpp diff --git a/src/boost/locale/icu/collator.cpp b/src/icu/collator.cpp similarity index 100% rename from src/boost/locale/icu/collator.cpp rename to src/icu/collator.cpp diff --git a/src/boost/locale/icu/conversion.cpp b/src/icu/conversion.cpp similarity index 100% rename from src/boost/locale/icu/conversion.cpp rename to src/icu/conversion.cpp diff --git a/src/boost/locale/icu/date_time.cpp b/src/icu/date_time.cpp similarity index 100% rename from src/boost/locale/icu/date_time.cpp rename to src/icu/date_time.cpp diff --git a/src/boost/locale/icu/formatter.cpp b/src/icu/formatter.cpp similarity index 100% rename from src/boost/locale/icu/formatter.cpp rename to src/icu/formatter.cpp diff --git a/src/boost/locale/icu/formatter.hpp b/src/icu/formatter.hpp similarity index 100% rename from src/boost/locale/icu/formatter.hpp rename to src/icu/formatter.hpp diff --git a/src/boost/locale/icu/formatters_cache.cpp b/src/icu/formatters_cache.cpp similarity index 100% rename from src/boost/locale/icu/formatters_cache.cpp rename to src/icu/formatters_cache.cpp diff --git a/src/boost/locale/icu/formatters_cache.hpp b/src/icu/formatters_cache.hpp similarity index 100% rename from src/boost/locale/icu/formatters_cache.hpp rename to src/icu/formatters_cache.hpp diff --git a/src/boost/locale/icu/icu_backend.cpp b/src/icu/icu_backend.cpp similarity index 100% rename from src/boost/locale/icu/icu_backend.cpp rename to src/icu/icu_backend.cpp diff --git a/src/boost/locale/icu/icu_backend.hpp b/src/icu/icu_backend.hpp similarity index 100% rename from src/boost/locale/icu/icu_backend.hpp rename to src/icu/icu_backend.hpp diff --git a/src/boost/locale/icu/icu_util.hpp b/src/icu/icu_util.hpp similarity index 100% rename from src/boost/locale/icu/icu_util.hpp rename to src/icu/icu_util.hpp diff --git a/src/boost/locale/icu/numeric.cpp b/src/icu/numeric.cpp similarity index 100% rename from src/boost/locale/icu/numeric.cpp rename to src/icu/numeric.cpp diff --git a/src/boost/locale/icu/time_zone.hpp b/src/icu/time_zone.hpp similarity index 100% rename from src/boost/locale/icu/time_zone.hpp rename to src/icu/time_zone.hpp diff --git a/src/boost/locale/icu/uconv.hpp b/src/icu/uconv.hpp similarity index 100% rename from src/boost/locale/icu/uconv.hpp rename to src/icu/uconv.hpp diff --git a/src/boost/locale/posix/all_generator.hpp b/src/posix/all_generator.hpp similarity index 100% rename from src/boost/locale/posix/all_generator.hpp rename to src/posix/all_generator.hpp diff --git a/src/boost/locale/posix/codecvt.cpp b/src/posix/codecvt.cpp similarity index 100% rename from src/boost/locale/posix/codecvt.cpp rename to src/posix/codecvt.cpp diff --git a/src/boost/locale/posix/collate.cpp b/src/posix/collate.cpp similarity index 100% rename from src/boost/locale/posix/collate.cpp rename to src/posix/collate.cpp diff --git a/src/boost/locale/posix/converter.cpp b/src/posix/converter.cpp similarity index 100% rename from src/boost/locale/posix/converter.cpp rename to src/posix/converter.cpp diff --git a/src/boost/locale/posix/numeric.cpp b/src/posix/numeric.cpp similarity index 100% rename from src/boost/locale/posix/numeric.cpp rename to src/posix/numeric.cpp diff --git a/src/boost/locale/posix/posix_backend.cpp b/src/posix/posix_backend.cpp similarity index 100% rename from src/boost/locale/posix/posix_backend.cpp rename to src/posix/posix_backend.cpp diff --git a/src/boost/locale/posix/posix_backend.hpp b/src/posix/posix_backend.hpp similarity index 100% rename from src/boost/locale/posix/posix_backend.hpp rename to src/posix/posix_backend.hpp diff --git a/src/boost/locale/shared/date_time.cpp b/src/shared/date_time.cpp similarity index 100% rename from src/boost/locale/shared/date_time.cpp rename to src/shared/date_time.cpp diff --git a/src/boost/locale/shared/format.cpp b/src/shared/format.cpp similarity index 100% rename from src/boost/locale/shared/format.cpp rename to src/shared/format.cpp diff --git a/src/boost/locale/shared/formatting.cpp b/src/shared/formatting.cpp similarity index 100% rename from src/boost/locale/shared/formatting.cpp rename to src/shared/formatting.cpp diff --git a/src/boost/locale/shared/generator.cpp b/src/shared/generator.cpp similarity index 100% rename from src/boost/locale/shared/generator.cpp rename to src/shared/generator.cpp diff --git a/src/boost/locale/shared/iconv_codecvt.cpp b/src/shared/iconv_codecvt.cpp similarity index 100% rename from src/boost/locale/shared/iconv_codecvt.cpp rename to src/shared/iconv_codecvt.cpp diff --git a/src/boost/locale/shared/iconv_codecvt.hpp b/src/shared/iconv_codecvt.hpp similarity index 100% rename from src/boost/locale/shared/iconv_codecvt.hpp rename to src/shared/iconv_codecvt.hpp diff --git a/src/boost/locale/shared/ids.cpp b/src/shared/ids.cpp similarity index 100% rename from src/boost/locale/shared/ids.cpp rename to src/shared/ids.cpp diff --git a/src/boost/locale/shared/ios_prop.hpp b/src/shared/ios_prop.hpp similarity index 100% rename from src/boost/locale/shared/ios_prop.hpp rename to src/shared/ios_prop.hpp diff --git a/src/boost/locale/shared/localization_backend.cpp b/src/shared/localization_backend.cpp similarity index 100% rename from src/boost/locale/shared/localization_backend.cpp rename to src/shared/localization_backend.cpp diff --git a/src/boost/locale/shared/message.cpp b/src/shared/message.cpp similarity index 100% rename from src/boost/locale/shared/message.cpp rename to src/shared/message.cpp diff --git a/src/boost/locale/shared/message.hpp b/src/shared/message.hpp similarity index 100% rename from src/boost/locale/shared/message.hpp rename to src/shared/message.hpp diff --git a/src/boost/locale/shared/mo_hash.hpp b/src/shared/mo_hash.hpp similarity index 100% rename from src/boost/locale/shared/mo_hash.hpp rename to src/shared/mo_hash.hpp diff --git a/src/boost/locale/shared/mo_lambda.cpp b/src/shared/mo_lambda.cpp similarity index 100% rename from src/boost/locale/shared/mo_lambda.cpp rename to src/shared/mo_lambda.cpp diff --git a/src/boost/locale/shared/mo_lambda.hpp b/src/shared/mo_lambda.hpp similarity index 100% rename from src/boost/locale/shared/mo_lambda.hpp rename to src/shared/mo_lambda.hpp diff --git a/src/boost/locale/shared/std_collate_adapter.hpp b/src/shared/std_collate_adapter.hpp similarity index 100% rename from src/boost/locale/shared/std_collate_adapter.hpp rename to src/shared/std_collate_adapter.hpp diff --git a/src/boost/locale/std/all_generator.hpp b/src/std/all_generator.hpp similarity index 100% rename from src/boost/locale/std/all_generator.hpp rename to src/std/all_generator.hpp diff --git a/src/boost/locale/std/codecvt.cpp b/src/std/codecvt.cpp similarity index 100% rename from src/boost/locale/std/codecvt.cpp rename to src/std/codecvt.cpp diff --git a/src/boost/locale/std/collate.cpp b/src/std/collate.cpp similarity index 100% rename from src/boost/locale/std/collate.cpp rename to src/std/collate.cpp diff --git a/src/boost/locale/std/converter.cpp b/src/std/converter.cpp similarity index 100% rename from src/boost/locale/std/converter.cpp rename to src/std/converter.cpp diff --git a/src/boost/locale/std/numeric.cpp b/src/std/numeric.cpp similarity index 100% rename from src/boost/locale/std/numeric.cpp rename to src/std/numeric.cpp diff --git a/src/boost/locale/std/std_backend.cpp b/src/std/std_backend.cpp similarity index 100% rename from src/boost/locale/std/std_backend.cpp rename to src/std/std_backend.cpp diff --git a/src/boost/locale/std/std_backend.hpp b/src/std/std_backend.hpp similarity index 100% rename from src/boost/locale/std/std_backend.hpp rename to src/std/std_backend.hpp diff --git a/src/boost/locale/util/codecvt_converter.cpp b/src/util/codecvt_converter.cpp similarity index 100% rename from src/boost/locale/util/codecvt_converter.cpp rename to src/util/codecvt_converter.cpp diff --git a/src/boost/locale/util/default_locale.cpp b/src/util/default_locale.cpp similarity index 100% rename from src/boost/locale/util/default_locale.cpp rename to src/util/default_locale.cpp diff --git a/src/boost/locale/util/encoding.cpp b/src/util/encoding.cpp similarity index 100% rename from src/boost/locale/util/encoding.cpp rename to src/util/encoding.cpp diff --git a/src/boost/locale/util/encoding.hpp b/src/util/encoding.hpp similarity index 100% rename from src/boost/locale/util/encoding.hpp rename to src/util/encoding.hpp diff --git a/src/boost/locale/util/foreach_char.hpp b/src/util/foreach_char.hpp similarity index 100% rename from src/boost/locale/util/foreach_char.hpp rename to src/util/foreach_char.hpp diff --git a/src/boost/locale/util/gregorian.cpp b/src/util/gregorian.cpp similarity index 100% rename from src/boost/locale/util/gregorian.cpp rename to src/util/gregorian.cpp diff --git a/src/boost/locale/util/gregorian.hpp b/src/util/gregorian.hpp similarity index 100% rename from src/boost/locale/util/gregorian.hpp rename to src/util/gregorian.hpp diff --git a/src/boost/locale/util/iconv.hpp b/src/util/iconv.hpp similarity index 100% rename from src/boost/locale/util/iconv.hpp rename to src/util/iconv.hpp diff --git a/src/boost/locale/util/info.cpp b/src/util/info.cpp similarity index 100% rename from src/boost/locale/util/info.cpp rename to src/util/info.cpp diff --git a/src/boost/locale/util/locale_data.cpp b/src/util/locale_data.cpp similarity index 100% rename from src/boost/locale/util/locale_data.cpp rename to src/util/locale_data.cpp diff --git a/src/boost/locale/util/make_std_unique.hpp b/src/util/make_std_unique.hpp similarity index 100% rename from src/boost/locale/util/make_std_unique.hpp rename to src/util/make_std_unique.hpp diff --git a/src/boost/locale/util/numeric.hpp b/src/util/numeric.hpp similarity index 100% rename from src/boost/locale/util/numeric.hpp rename to src/util/numeric.hpp diff --git a/src/boost/locale/util/timezone.hpp b/src/util/timezone.hpp similarity index 100% rename from src/boost/locale/util/timezone.hpp rename to src/util/timezone.hpp diff --git a/src/boost/locale/util/win_codepages.hpp b/src/util/win_codepages.hpp similarity index 100% rename from src/boost/locale/util/win_codepages.hpp rename to src/util/win_codepages.hpp diff --git a/src/boost/locale/win32/all_generator.hpp b/src/win32/all_generator.hpp similarity index 100% rename from src/boost/locale/win32/all_generator.hpp rename to src/win32/all_generator.hpp diff --git a/src/boost/locale/win32/api.hpp b/src/win32/api.hpp similarity index 100% rename from src/boost/locale/win32/api.hpp rename to src/win32/api.hpp diff --git a/src/boost/locale/win32/collate.cpp b/src/win32/collate.cpp similarity index 100% rename from src/boost/locale/win32/collate.cpp rename to src/win32/collate.cpp diff --git a/src/boost/locale/win32/converter.cpp b/src/win32/converter.cpp similarity index 100% rename from src/boost/locale/win32/converter.cpp rename to src/win32/converter.cpp diff --git a/src/boost/locale/win32/lcid.cpp b/src/win32/lcid.cpp similarity index 100% rename from src/boost/locale/win32/lcid.cpp rename to src/win32/lcid.cpp diff --git a/src/boost/locale/win32/lcid.hpp b/src/win32/lcid.hpp similarity index 100% rename from src/boost/locale/win32/lcid.hpp rename to src/win32/lcid.hpp diff --git a/src/boost/locale/win32/numeric.cpp b/src/win32/numeric.cpp similarity index 100% rename from src/boost/locale/win32/numeric.cpp rename to src/win32/numeric.cpp diff --git a/src/boost/locale/win32/win_backend.cpp b/src/win32/win_backend.cpp similarity index 100% rename from src/boost/locale/win32/win_backend.cpp rename to src/win32/win_backend.cpp diff --git a/src/boost/locale/win32/win_backend.hpp b/src/win32/win_backend.hpp similarity index 100% rename from src/boost/locale/win32/win_backend.hpp rename to src/win32/win_backend.hpp diff --git a/test/boostLocale/test/tools.hpp b/test/boostLocale/test/tools.hpp index 92fb7e24..7e9a0453 100644 --- a/test/boostLocale/test/tools.hpp +++ b/test/boostLocale/test/tools.hpp @@ -16,7 +16,7 @@ #include #include #ifndef BOOST_LOCALE_NO_WINAPI_BACKEND -# include "../src/boost/locale/win32/lcid.hpp" +# include "../src/win32/lcid.hpp" #else # include #endif diff --git a/test/test_catalog.cpp b/test/test_catalog.cpp index bc06805f..603c5219 100644 --- a/test/test_catalog.cpp +++ b/test/test_catalog.cpp @@ -5,7 +5,7 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "../src/boost/locale/shared/mo_lambda.hpp" +#include "../src/shared/mo_lambda.hpp" #include "boostLocale/test/unit_test.hpp" #include #include diff --git a/test/test_codepage_converter.cpp b/test/test_codepage_converter.cpp index 3f03cae0..458ab217 100644 --- a/test/test_codepage_converter.cpp +++ b/test/test_codepage_converter.cpp @@ -6,9 +6,9 @@ #include #ifdef BOOST_LOCALE_WITH_ICU -# include "../src/boost/locale/icu/codecvt.hpp" +# include "../src/icu/codecvt.hpp" #endif -#include "../src/boost/locale/shared/iconv_codecvt.hpp" +#include "../src/shared/iconv_codecvt.hpp" #include #include diff --git a/test/test_encoding.cpp b/test/test_encoding.cpp index 7fb07c2f..0a01d43d 100644 --- a/test/test_encoding.cpp +++ b/test/test_encoding.cpp @@ -813,8 +813,8 @@ bool isLittleEndian() return reinterpret_cast(&endianMark)[0] == 1; } -#include "../src/boost/locale/util/encoding.hpp" -#include "../src/boost/locale/util/win_codepages.hpp" +#include "../src/util/encoding.hpp" +#include "../src/util/win_codepages.hpp" void test_utf_name() { diff --git a/test/test_generator.cpp b/test/test_generator.cpp index b2928ce8..ed3ac3cb 100644 --- a/test/test_generator.cpp +++ b/test/test_generator.cpp @@ -7,7 +7,7 @@ #include #include -#include "../src/boost/locale/win32/lcid.hpp" +#include "../src/win32/lcid.hpp" #include "boostLocale/test/tools.hpp" #include "boostLocale/test/unit_test.hpp" #include diff --git a/test/test_ios_prop.cpp b/test/test_ios_prop.cpp index 6628a15f..1a41a0b4 100644 --- a/test/test_ios_prop.cpp +++ b/test/test_ios_prop.cpp @@ -5,7 +5,7 @@ // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt -#include "../src/boost/locale/shared/ios_prop.hpp" +#include "../src/shared/ios_prop.hpp" #include "boostLocale/test/unit_test.hpp" #include #include diff --git a/test/test_util.cpp b/test/test_util.cpp index cf773ba1..caa5983e 100644 --- a/test/test_util.cpp +++ b/test/test_util.cpp @@ -370,7 +370,7 @@ void test_locale_data() verify_against_icu(); } -#include "../src/boost/locale/util/numeric.hpp" +#include "../src/util/numeric.hpp" #include #include #include diff --git a/test/test_winapi_formatting.cpp b/test/test_winapi_formatting.cpp index 499e92a8..00a7216c 100644 --- a/test/test_winapi_formatting.cpp +++ b/test/test_winapi_formatting.cpp @@ -20,7 +20,7 @@ # endif # include #endif -#include "../src/boost/locale/win32/lcid.hpp" +#include "../src/win32/lcid.hpp" #include "boostLocale/test/tools.hpp" #include "boostLocale/test/unit_test.hpp" #include "formatting_common.hpp" From 6826cca15913f385103cc0d783cee9563d445c0a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Dec 2024 11:30:44 +0100 Subject: [PATCH 43/67] Consistently pass `string_view` by value --- include/boost/locale/detail/encoding.hpp | 2 +- include/boost/locale/encoding.hpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/boost/locale/detail/encoding.hpp b/include/boost/locale/detail/encoding.hpp index ee669349..390c4bb9 100644 --- a/include/boost/locale/detail/encoding.hpp +++ b/include/boost/locale/detail/encoding.hpp @@ -24,7 +24,7 @@ namespace boost { namespace locale { namespace conv { namespace detail { virtual ~charset_converter() = default; virtual string_type convert(const CharIn* begin, const CharIn* end) = 0; - string_type convert(const boost::basic_string_view& text) + string_type convert(const boost::basic_string_view text) { return convert(text.data(), text.data() + text.length()); } diff --git a/include/boost/locale/encoding.hpp b/include/boost/locale/encoding.hpp index d0e1d9d0..52c8bc46 100644 --- a/include/boost/locale/encoding.hpp +++ b/include/boost/locale/encoding.hpp @@ -240,11 +240,11 @@ namespace boost { namespace locale { /// Convert \a text to UTF /// /// \throws conversion_error: Conversion failed - string_type convert(const boost::string_view& text) const { return impl_->convert(text); } + string_type convert(const boost::string_view text) const { return impl_->convert(text); } /// Convert \a text to UTF /// /// \throws conversion_error: Conversion failed - string_type operator()(const boost::string_view& text) const { return convert(text); } + string_type operator()(const boost::string_view text) const { return convert(text); } }; /// Converter class to decode an UTF string and encode it using a local encoding @@ -298,11 +298,11 @@ namespace boost { namespace locale { /// Convert \a text /// /// \throws conversion_error: Conversion failed - std::string convert(const boost::string_view& text) const { return impl_->convert(text); } + std::string convert(const boost::string_view text) const { return impl_->convert(text); } /// Convert \a text /// /// \throws conversion_error: Conversion failed - std::string operator()(const boost::string_view& text) const { return convert(text); } + std::string operator()(const boost::string_view text) const { return convert(text); } }; } // namespace conv }} // namespace boost::locale From f57605e8dcea43318e1f3b097a1b75ef583b194f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Dec 2024 11:59:42 +0100 Subject: [PATCH 44/67] Avoid superflous copy of time zone string --- include/boost/locale/formatting.hpp | 2 +- src/shared/formatting.cpp | 2 +- src/util/numeric.hpp | 15 ++++++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/include/boost/locale/formatting.hpp b/include/boost/locale/formatting.hpp index e3c8619e..1844398a 100644 --- a/include/boost/locale/formatting.hpp +++ b/include/boost/locale/formatting.hpp @@ -122,7 +122,7 @@ namespace boost { namespace locale { /// Set time zone for formatting dates and time void time_zone(const std::string&); /// Get time zone for formatting dates and time - std::string time_zone() const; + const std::string& time_zone() const; /// Set date/time pattern (strftime like) template diff --git a/src/shared/formatting.cpp b/src/shared/formatting.cpp index 1dc9b3a9..53f436a3 100644 --- a/src/shared/formatting.cpp +++ b/src/shared/formatting.cpp @@ -65,7 +65,7 @@ namespace boost { namespace locale { { time_zone_ = tz; } - std::string ios_info::time_zone() const + const std::string& ios_info::time_zone() const { return time_zone_; } diff --git a/src/util/numeric.hpp b/src/util/numeric.hpp index 146309e3..a67b611e 100644 --- a/src/util/numeric.hpp +++ b/src/util/numeric.hpp @@ -175,11 +175,11 @@ namespace boost { namespace locale { namespace util { iter_type format_time(iter_type out, std::ios_base& ios, CharType fill, std::time_t time, const string_type& format) const { - std::string tz = ios_info::get(ios).time_zone(); - std::tm tm; -#if BOOST_OS_LINUX || BOOST_OS_BSD_FREE || defined(__APPLE__) - std::vector tmp_buf(tz.c_str(), tz.c_str() + tz.size() + 1); + const std::string& tz = ios_info::get(ios).time_zone(); +#if BOOST_OS_BSD_FREE || defined(__APPLE__) + std::vector tz_nonconst; #endif + std::tm tm; if(tz.empty()) { #ifdef BOOST_WINDOWS // Windows uses TLS @@ -200,8 +200,13 @@ namespace boost { namespace locale { namespace util { #if BOOST_OS_LINUX || BOOST_OS_BSD_FREE || defined(__APPLE__) // These have extra fields to specify timezone if(gmtoff != 0) { +# if BOOST_OS_BSD_FREE || defined(__APPLE__) // bsd and apple want tm_zone be non-const - tm.tm_zone = tmp_buf.data(); + tz_nonconst.assign(tz.begin(), tz.end()); + tm.tm_zone = tz_nonconst.data(); +# else + tm.tm_zone = tz.data(); +# endif tm.tm_gmtoff = gmtoff; } #endif From b919614216d73e719dcb4835c89a97720c8d99eb Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Dec 2024 12:31:04 +0100 Subject: [PATCH 45/67] Move try_to_int to extra header --- CMakeLists.txt | 1 + include/boost/locale/format.hpp | 2 +- src/shared/format.cpp | 3 +- src/std/std_backend.cpp | 2 +- src/util/numeric.hpp | 19 ----------- src/util/numeric_conversion.hpp | 34 ++++++++++++++++++++ test/Jamfile.v2 | 1 + test/test_util.cpp | 41 ------------------------ test/test_util_numeric_convert.cpp | 51 ++++++++++++++++++++++++++++++ 9 files changed, 91 insertions(+), 63 deletions(-) create mode 100644 src/util/numeric_conversion.hpp create mode 100644 test/test_util_numeric_convert.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 32f10c9c..438f763d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ add_library(boost_locale src/util/locale_data.cpp src/util/make_std_unique.hpp src/util/numeric.hpp + src/util/numeric_conversion.hpp src/util/timezone.hpp ${headers} ) diff --git a/include/boost/locale/format.hpp b/include/boost/locale/format.hpp index 6adb8440..58a9a0ff 100644 --- a/include/boost/locale/format.hpp +++ b/include/boost/locale/format.hpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2021-2023 Alexander Grund +// Copyright (c) 2021-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt diff --git a/src/shared/format.cpp b/src/shared/format.cpp index ed2fb2ea..cceac60d 100644 --- a/src/shared/format.cpp +++ b/src/shared/format.cpp @@ -1,5 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -7,7 +8,7 @@ #include #include #include -#include "../util/numeric.hpp" +#include "../util/numeric_conversion.hpp" #include #include #include diff --git a/src/std/std_backend.cpp b/src/std/std_backend.cpp index 0bcd2800..1eadfd24 100644 --- a/src/std/std_backend.cpp +++ b/src/std/std_backend.cpp @@ -27,7 +27,7 @@ #include "../util/encoding.hpp" #include "../util/gregorian.hpp" #include "../util/make_std_unique.hpp" -#include "../util/numeric.hpp" +#include "../util/numeric_conversion.hpp" #include "all_generator.hpp" namespace { diff --git a/src/util/numeric.hpp b/src/util/numeric.hpp index a67b611e..e54c95c5 100644 --- a/src/util/numeric.hpp +++ b/src/util/numeric.hpp @@ -10,11 +10,8 @@ #include #include #include -#include -#include #include #include -#include #include #include #include @@ -23,22 +20,6 @@ #include "timezone.hpp" namespace boost { namespace locale { namespace util { - - inline bool try_to_int(const std::string& s, int& res) - { - if(s.empty()) - return false; - errno = 0; - char* end_char{}; - const auto v = std::strtol(s.c_str(), &end_char, 10); - if(errno == ERANGE || end_char != s.c_str() + s.size()) - return false; - if(v < std::numeric_limits::min() || v > std::numeric_limits::max()) - return false; - res = v; - return true; - } - template struct formatting_size_traits { static size_t size(const std::basic_string& s, const std::locale& /*l*/) { return s.size(); } diff --git a/src/util/numeric_conversion.hpp b/src/util/numeric_conversion.hpp new file mode 100644 index 00000000..37524e59 --- /dev/null +++ b/src/util/numeric_conversion.hpp @@ -0,0 +1,34 @@ +// +// Copyright (c) 2024 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#ifndef BOOST_LOCALE_IMPL_UTIL_NUMERIC_CONVERSIONS_HPP +#define BOOST_LOCALE_IMPL_UTIL_NUMERIC_CONVERSIONS_HPP + +#include +#include +#include +#include +#include + +namespace boost { namespace locale { namespace util { + + bool try_to_int(const string_view s, int value) + { + if(s.empty()) + return false; + errno = 0; + char* end_char{}; + const auto v = std::strtol(s.c_str(), &end_char, 10); + if(errno == ERANGE || end_char != s.c_str() + s.size()) + return false; + if(v < std::numeric_limits::min() || v > std::numeric_limits::max()) + return false; + res = v; + return true; + } +}}} // namespace boost::locale::util + +#endif diff --git a/test/Jamfile.v2 b/test/Jamfile.v2 index e3b85592..f673b9d6 100644 --- a/test/Jamfile.v2 +++ b/test/Jamfile.v2 @@ -37,6 +37,7 @@ run test_catalog.cpp ; run test_encoding.cpp ; run test_utf.cpp ; run test_util.cpp test_helpers.cpp ; +run test_util_numeric_convert.cpp ; run test_date_time.cpp ; run test_ios_info.cpp ; run test_ios_prop.cpp ; diff --git a/test/test_util.cpp b/test/test_util.cpp index caa5983e..f6a057fe 100644 --- a/test/test_util.cpp +++ b/test/test_util.cpp @@ -370,50 +370,9 @@ void test_locale_data() verify_against_icu(); } -#include "../src/util/numeric.hpp" -#include -#include -#include - -void test_try_to_int() -{ - using boost::locale::util::try_to_int; - - int v = 1337; - TEST(try_to_int("0", v)); - TEST_EQ(v, 0); - - TEST(try_to_int("42", v)); - TEST_EQ(v, 42); - - TEST(try_to_int("-1337", v)); - TEST_EQ(v, -1337); - - std::ostringstream ss; - ss.imbue(std::locale::classic()); - empty_stream(ss) << std::numeric_limits::min(); - TEST(try_to_int(ss.str(), v)); - TEST_EQ(v, std::numeric_limits::min()); - empty_stream(ss) << std::numeric_limits::max(); - TEST(try_to_int(ss.str(), v)); - TEST_EQ(v, std::numeric_limits::max()); - - TEST(!try_to_int("", v)); - TEST(!try_to_int("a", v)); - TEST(!try_to_int("1.", v)); - TEST(!try_to_int("1a", v)); - TEST(!try_to_int("a1", v)); - static_assert(sizeof(long long) > sizeof(int), "Value below under/overflows!"); - empty_stream(ss) << static_cast(std::numeric_limits::min()) - 1; - TEST(!try_to_int(ss.str(), v)); - empty_stream(ss) << static_cast(std::numeric_limits::max()) + 1; - TEST(!try_to_int(ss.str(), v)); -} - void test_main(int /*argc*/, char** /*argv*/) { test_hold_ptr(); test_get_system_locale(); test_locale_data(); - test_try_to_int(); } diff --git a/test/test_util_numeric_convert.cpp b/test/test_util_numeric_convert.cpp new file mode 100644 index 00000000..28613ca7 --- /dev/null +++ b/test/test_util_numeric_convert.cpp @@ -0,0 +1,51 @@ +// +// Copyright (c) 2022-2025 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#include "../src/util/numeric_conversion.hpp" +#include "boostLocale/test/tools.hpp" +#include +#include +#include + +void test_try_to_int() +{ + using boost::locale::util::try_to_int; + + int v = 1337; + TEST(try_to_int("0", v)); + TEST_EQ(v, 0); + + TEST(try_to_int("42", v)); + TEST_EQ(v, 42); + + TEST(try_to_int("-1337", v)); + TEST_EQ(v, -1337); + + std::ostringstream ss; + ss.imbue(std::locale::classic()); + empty_stream(ss) << std::numeric_limits::min(); + TEST(try_to_int(ss.str(), v)); + TEST_EQ(v, std::numeric_limits::min()); + empty_stream(ss) << std::numeric_limits::max(); + TEST(try_to_int(ss.str(), v)); + TEST_EQ(v, std::numeric_limits::max()); + + TEST(!try_to_int("", v)); + TEST(!try_to_int("a", v)); + TEST(!try_to_int("1.", v)); + TEST(!try_to_int("1a", v)); + TEST(!try_to_int("a1", v)); + static_assert(sizeof(long long) > sizeof(int), "Value below under/overflows!"); + empty_stream(ss) << static_cast(std::numeric_limits::min()) - 1; + TEST(!try_to_int(ss.str(), v)); + empty_stream(ss) << static_cast(std::numeric_limits::max()) + 1; + TEST(!try_to_int(ss.str(), v)); +} + +void test_main(int /*argc*/, char** /*argv*/) +{ + test_try_to_int(); +} From 258e95940248d9a113939f61b6ca2549c4c70e0b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 31 Dec 2024 13:40:03 +0100 Subject: [PATCH 46/67] Use Boost::charconv for try_to_int --- CMakeLists.txt | 1 + build/Jamfile.v2 | 1 + src/util/numeric_conversion.hpp | 15 +++------------ test/CMakeLists.txt | 3 ++- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 438f763d..500d7943 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,6 +53,7 @@ target_link_libraries(boost_locale Boost::iterator Boost::utility PRIVATE + Boost::charconv Boost::predef Boost::thread ) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 7a07e787..1f77dc19 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -16,6 +16,7 @@ import toolset ; path-constant TOP : .. ; constant boost_dependencies_private : + /boost/charconv//boost_charconv /boost/predef//boost_predef /boost/thread//boost_thread ; diff --git a/src/util/numeric_conversion.hpp b/src/util/numeric_conversion.hpp index 37524e59..4a333f8c 100644 --- a/src/util/numeric_conversion.hpp +++ b/src/util/numeric_conversion.hpp @@ -8,10 +8,8 @@ #define BOOST_LOCALE_IMPL_UTIL_NUMERIC_CONVERSIONS_HPP #include +#include #include -#include -#include -#include namespace boost { namespace locale { namespace util { @@ -19,15 +17,8 @@ namespace boost { namespace locale { namespace util { { if(s.empty()) return false; - errno = 0; - char* end_char{}; - const auto v = std::strtol(s.c_str(), &end_char, 10); - if(errno == ERANGE || end_char != s.c_str() + s.size()) - return false; - if(v < std::numeric_limits::min() || v > std::numeric_limits::max()) - return false; - res = v; - return true; + const auto res = boost::charconv::from_chars(s, value); + return res && res.ptr == (s.data() + s.size()); } }}} // namespace boost::locale::util diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 10284197..a976d009 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2022 Alexander Grund +# Copyright 2022-2024 Alexander Grund # Distributed under the Boost Software License, Version 1.0. # https://www.boost.org/LICENSE_1_0.txt @@ -32,6 +32,7 @@ if(NOT BOOST_LOCALE_ENABLE_POSIX) endif() boost_test_jamfile(FILE Jamfile.v2) +target_link_libraries(boost_locale-test_util_numeric_convert Boost::charconv) # Those require to be run in the test directory foreach(name test_formatting test_message) From 8cc6f42fd4267dd58e2a0a3cea5fa4f75532004d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Dec 2024 12:37:24 +0100 Subject: [PATCH 47/67] Make `try_to_int` a template --- src/shared/format.cpp | 7 +++---- src/util/numeric_conversion.hpp | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/shared/format.cpp b/src/shared/format.cpp index cceac60d..9853c7ff 100644 --- a/src/shared/format.cpp +++ b/src/shared/format.cpp @@ -64,10 +64,9 @@ namespace boost { namespace locale { namespace detail { { if(key.empty()) return; - int position; - if(util::try_to_int(key, position) && position > 0) { - static_assert(sizeof(unsigned) <= sizeof(decltype(d->position)), "Possible lossy conversion"); - d->position = static_cast(position - 1); + decltype(d->position) position; + if(util::try_to_int(key, position) && position > 0u) { + d->position = position - 1u; } else if(key == "num" || key == "number") { as::number(ios_); diff --git a/src/util/numeric_conversion.hpp b/src/util/numeric_conversion.hpp index 4a333f8c..9e882f6b 100644 --- a/src/util/numeric_conversion.hpp +++ b/src/util/numeric_conversion.hpp @@ -13,7 +13,8 @@ namespace boost { namespace locale { namespace util { - bool try_to_int(const string_view s, int value) + template + bool try_to_int(const string_view s, Integer& value) { if(s.empty()) return false; From 1abbbb3d6d103fb8fae9d209b48012df0096a573 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Dec 2024 20:12:05 +0100 Subject: [PATCH 48/67] Allow to check success of `TEST` macro at call site. Useful if next check(s) depend on previous ones. --- test/boostLocale/test/unit_test.hpp | 30 +++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/test/boostLocale/test/unit_test.hpp b/test/boostLocale/test/unit_test.hpp index cbfc8cd4..366fe838 100644 --- a/test/boostLocale/test/unit_test.hpp +++ b/test/boostLocale/test/unit_test.hpp @@ -88,28 +88,24 @@ namespace boost { namespace locale { namespace test { if(++boost::locale::test::results().error_counter > BOOST_LOCALE_ERROR_LIMIT) throw std::runtime_error("Error limits reached, stopping unit test"); } + + template + bool test_impl(const char* expr, const char* file, int line, const bool v) + { + boost::locale::test::results().test_counter++; + if(!v) { + boost::locale::test::report_error(expr, file, line); + throw std::runtime_error("Critical test " + std::string(expr) + " failed"); + } + return v; + } }}} // namespace boost::locale::test #define BOOST_LOCALE_TEST_REPORT_ERROR(expr) boost::locale::test::report_error(expr, __FILE__, __LINE__) -#define TEST(X) \ - do { \ - boost::locale::test::results().test_counter++; \ - if(X) \ - break; \ - BOOST_LOCALE_TEST_REPORT_ERROR(#X); \ - BOOST_LOCALE_START_CONST_CONDITION \ - } while(0) BOOST_LOCALE_END_CONST_CONDITION +#define TEST(X) (::boost::locale::test::test_impl(#X, __FILE__, __LINE__, (X) ? true : false)) -#define TEST_REQUIRE(X) \ - do { \ - boost::locale::test::results().test_counter++; \ - if(X) \ - break; \ - BOOST_LOCALE_TEST_REPORT_ERROR(#X); \ - throw std::runtime_error("Critical test " #X " failed"); \ - BOOST_LOCALE_START_CONST_CONDITION \ - } while(0) BOOST_LOCALE_END_CONST_CONDITION +#define TEST_REQUIRE(X) (::boost::locale::test::test_impl(#X, __FILE__, __LINE__, (X) ? true : false)) #define TEST_THROWS(X, E) \ do { \ From e8f9544352e2927c2b96e31c6a58b584788ccc0a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 31 Dec 2024 13:28:10 +0100 Subject: [PATCH 49/67] Replace some `TEST_REQUIRE` by `if TEST` --- test/formatting_common.hpp | 52 ++--- test/test_boundary.cpp | 40 ++-- test/test_codepage_converter.cpp | 296 ++++++++++++++--------------- test/test_posix_convert.cpp | 11 +- test/test_posix_formatting.cpp | 12 +- test/test_util.cpp | 52 ++--- test/test_util_numeric_convert.cpp | 20 +- 7 files changed, 248 insertions(+), 235 deletions(-) diff --git a/test/formatting_common.hpp b/test/formatting_common.hpp index 260bc20d..179efd3c 100644 --- a/test/formatting_common.hpp +++ b/test/formatting_common.hpp @@ -30,37 +30,43 @@ void test_parse_multi_number_by_char(const std::locale& locale) stream >> boost::locale::as::number; IntType value; - TEST_REQUIRE(stream >> value); - TEST_EQ(value, IntType(42)); - TEST_EQ(static_cast(stream.get()), '.'); - TEST_REQUIRE(stream >> value); - TEST_EQ(value, expectedInt); - TEST_REQUIRE(!(stream >> value)); - TEST(stream.eof()); + if TEST(stream >> value) { + TEST_EQ(value, IntType(42)); + TEST_EQ(static_cast(stream.get()), '.'); + if TEST(stream >> value) { + TEST_EQ(value, expectedInt); + if TEST(!(stream >> value)) + TEST(stream.eof()); + } + } stream.str(ascii_to("42.25,678")); stream.clear(); float fValue; - TEST_REQUIRE(stream >> fValue); - TEST_EQ(fValue, 42.25); - TEST_EQ(static_cast(stream.get()), ','); - TEST_REQUIRE(stream >> value); - TEST_EQ(value, IntType(678)); - TEST_REQUIRE(!(stream >> value)); - TEST(stream.eof()); + if TEST(stream >> fValue) { + TEST_EQ(fValue, 42.25); + TEST_EQ(static_cast(stream.get()), ','); + if TEST(stream >> value) { + TEST_EQ(value, IntType(678)); + if TEST(!(stream >> value)) + TEST(stream.eof()); + } + } // Parsing a floating point currency to integer truncates the floating point value but fully parses it stream.str(ascii_to("USD1,234.55,67.89")); stream.clear(); - TEST_REQUIRE(!(stream >> value)); - stream.clear(); - stream >> boost::locale::as::currency >> boost::locale::as::currency_iso; - if(stream >> value) { // Parsing currencies not fully supported by WinAPI backend - TEST_EQ(value, IntType(1234)); - TEST_EQ(static_cast(stream.get()), ','); - TEST_REQUIRE(stream >> boost::locale::as::number >> value); - TEST_EQ(value, IntType(67)); - TEST(!stream.eof()); + if TEST(!(stream >> value)) { + stream.clear(); + stream >> boost::locale::as::currency >> boost::locale::as::currency_iso; + if(stream >> value) { // Parsing currencies not fully supported by WinAPI backend + TEST_EQ(value, IntType(1234)); + TEST_EQ(static_cast(stream.get()), ','); + if TEST(stream >> boost::locale::as::number >> value) { + TEST_EQ(value, IntType(67)); + TEST(!stream.eof()); + } + } } } diff --git a/test/test_boundary.cpp b/test/test_boundary.cpp index e4e0ba4c..7a2715a4 100644 --- a/test/test_boundary.cpp +++ b/test/test_boundary.cpp @@ -37,9 +37,10 @@ void run_segment_iterator_test(const lb::segment_index& map, unsigned i = 0; typename lb::segment_index::iterator p; for(p = map.begin(); p != map.end(); ++p, i++) { - TEST_REQUIRE(i < masks.size()); - TEST_EQ(p->str(), chunks[i]); - TEST_EQ(p->rule(), masks[i]); + if TEST(i < masks.size()) { + TEST_EQ(p->str(), chunks[i]); + TEST_EQ(p->rule(), masks[i]); + } } TEST_EQ(i, chunks.size()); @@ -88,9 +89,10 @@ void run_break_iterator_test(const lb::boundary_point_index& map, unsigned i = 0; typename lb::boundary_point_index::iterator p; for(p = map.begin(); p != map.end(); ++p, i++) { - TEST_REQUIRE(i < masks.size()); - TEST(p->iterator() == iters[i]); - TEST_EQ(p->rule(), masks[i]); + if TEST(i < masks.size()) { + TEST(p->iterator() == iters[i]); + TEST_EQ(p->rule(), masks[i]); + } } TEST_EQ(i, iters.size()); @@ -117,12 +119,13 @@ void verify_index(const lb::boundary_point_index& map, const masks_t& masks) { BOOST_ASSERT(iters.size() == masks.size()); - TEST_REQUIRE(static_cast(std::distance(map.begin(), map.end())) == masks.size()); - size_t i = 0; - for(const auto& b_point : map) { - TEST(b_point.iterator() == iters[i]); - TEST_EQ(b_point.rule(), masks[i]); - ++i; + if TEST(static_cast(std::distance(map.begin(), map.end())) == masks.size()) { + size_t i = 0; + for(const auto& b_point : map) { + TEST(b_point.iterator() == iters[i]); + TEST_EQ(b_point.rule(), masks[i]); + ++i; + } } } @@ -130,12 +133,13 @@ template void verify_index(const lb::segment_index& map, const chunks_t& chunks, const masks_t& masks) { BOOST_ASSERT(chunks.size() == masks.size()); - TEST_REQUIRE(static_cast(std::distance(map.begin(), map.end())) == masks.size()); - size_t i = 0; - for(const auto& seg : map) { - TEST_EQ(seg.str(), chunks[i]); - TEST_EQ(seg.rule(), masks[i]); - ++i; + if TEST(static_cast(std::distance(map.begin(), map.end())) == masks.size()) { + size_t i = 0; + for(const auto& seg : map) { + TEST_EQ(seg.str(), chunks[i]); + TEST_EQ(seg.rule(), masks[i]); + ++i; + } } } diff --git a/test/test_codepage_converter.cpp b/test/test_codepage_converter.cpp index 458ab217..021a5b02 100644 --- a/test/test_codepage_converter.cpp +++ b/test/test_codepage_converter.cpp @@ -86,167 +86,167 @@ void test_main(int /*argc*/, char** /*argv*/) TEST(!create_simple_converter("UTF-8")); std::unique_ptr cvt = create_utf8_converter(); - TEST_REQUIRE(cvt); - TEST(cvt->is_thread_safe()); - TEST_EQ(cvt->max_len(), 4); - - std::cout << "-- Correct" << std::endl; - - TEST_TO("\x7f", 0x7f); - TEST_TO("\xC2\x80", 0x80); - TEST_TO("\xdf\xBF", 0x7FF); - TEST_TO("\xe0\xa0\x80", 0x800); - TEST_TO("\xef\xbf\xbf", 0xFFFF); - TEST_TO("\xf0\x90\x80\x80", 0x10000); - TEST_TO("\xf4\x8f\xbf\xbf", 0x10FFFF); - - std::cout << "-- Too big" << std::endl; - TEST_TO("\xf4\x9f\x80\x80", illegal); // 11 0000 - TEST_TO("\xfb\xbf\xbf\xbf", illegal); // 3FF FFFF - TEST_TO("\xf8\x90\x80\x80\x80", illegal); // 400 0000 - TEST_TO("\xfd\xbf\xbf\xbf\xbf\xbf", illegal); // 7fff ffff - - std::cout << "-- Invalid trail" << std::endl; - TEST_TO("\xC2\x7F", illegal); - TEST_TO("\xdf\x7F", illegal); - TEST_TO("\xe0\x7F\x80", illegal); - TEST_TO("\xef\xbf\x7F", illegal); - TEST_TO("\xe0\x7F\x80", illegal); - TEST_TO("\xef\xbf\x7F", illegal); - TEST_TO("\xf0\x7F\x80\x80", illegal); - TEST_TO("\xf4\x7f\xbf\xbf", illegal); - TEST_TO("\xf0\x90\x7F\x80", illegal); - TEST_TO("\xf4\x8f\x7F\xbf", illegal); - TEST_TO("\xf0\x90\x80\x7F", illegal); - TEST_TO("\xf4\x8f\xbf\x7F", illegal); - - std::cout << "-- Invalid length" << std::endl; - - // Test that this actually works - TEST_TO(make2(0x80), 0x80); - TEST_TO(make2(0x7ff), 0x7ff); - - TEST_TO(make3(0x800), 0x800); - TEST_TO(make3(0xffff), 0xffff); - - TEST_TO(make4(0x10000), 0x10000); - TEST_TO(make4(0x10ffff), 0x10ffff); - - TEST_TO(make4(0x110000), illegal); - TEST_TO(make4(0x1fffff), illegal); - - TEST_TO(make2(0), illegal); - TEST_TO(make3(0), illegal); - TEST_TO(make4(0), illegal); - TEST_TO(make2(0x7f), illegal); - TEST_TO(make3(0x7f), illegal); - TEST_TO(make4(0x7f), illegal); - - TEST_TO(make3(0x80), illegal); - TEST_TO(make4(0x80), illegal); - TEST_TO(make3(0x7ff), illegal); - TEST_TO(make4(0x7ff), illegal); - - TEST_TO(make4(0x8000), illegal); - TEST_TO(make4(0xffff), illegal); - - std::cout << "-- Invalid surrogate" << std::endl; - - TEST_TO(make3(0xD800), illegal); - TEST_TO(make3(0xDBFF), illegal); - TEST_TO(make3(0xDC00), illegal); - TEST_TO(make3(0xDFFF), illegal); - - TEST_TO(make4(0xD800), illegal); - TEST_TO(make4(0xDBFF), illegal); - TEST_TO(make4(0xDC00), illegal); - TEST_TO(make4(0xDFFF), illegal); - - std::cout << "-- Incomplete" << std::endl; - - TEST_TO("\x80", illegal); - TEST_TO("\xC2", incomplete); - - TEST_TO("\xdf", incomplete); - - TEST_TO("\xe0", incomplete); - TEST_TO("\xe0\xa0", incomplete); - - TEST_TO("\xef\xbf", incomplete); - TEST_TO("\xef", incomplete); - - TEST_TO("\xf0\x90\x80", incomplete); - TEST_TO("\xf0\x90", incomplete); - TEST_TO("\xf0", incomplete); - - TEST_TO("\xf4\x8f\xbf", incomplete); - TEST_TO("\xf4\x8f", incomplete); - TEST_TO("\xf4", incomplete); - - std::cout << "- To UTF-8\n"; - - std::cout << "-- Test correct" << std::endl; - - TEST_FROM("\x7f", 0x7f); - TEST_FROM("\xC2\x80", 0x80); - TEST_FROM("\xdf\xBF", 0x7FF); - TEST_INC(0x7FF, 1); - TEST_FROM("\xe0\xa0\x80", 0x800); - TEST_INC(0x800, 2); - TEST_INC(0x800, 1); - TEST_FROM("\xef\xbf\xbf", 0xFFFF); - TEST_INC(0x10000, 3); - TEST_INC(0x10000, 2); - TEST_INC(0x10000, 1); - TEST_FROM("\xf0\x90\x80\x80", 0x10000); - TEST_FROM("\xf4\x8f\xbf\xbf", 0x10FFFF); - - std::cout << "-- Test no surrogate " << std::endl; - - TEST_FROM(nullptr, 0xD800); - TEST_FROM(nullptr, 0xDBFF); - TEST_FROM(nullptr, 0xDC00); - TEST_FROM(nullptr, 0xDFFF); - - std::cout << "-- Test invalid " << std::endl; - - TEST_FROM(nullptr, 0x110000); - TEST_FROM(nullptr, 0x1FFFFF); - + if TEST(cvt) { + TEST(cvt->is_thread_safe()); + TEST_EQ(cvt->max_len(), 4); + + std::cout << "-- Correct" << std::endl; + + TEST_TO("\x7f", 0x7f); + TEST_TO("\xC2\x80", 0x80); + TEST_TO("\xdf\xBF", 0x7FF); + TEST_TO("\xe0\xa0\x80", 0x800); + TEST_TO("\xef\xbf\xbf", 0xFFFF); + TEST_TO("\xf0\x90\x80\x80", 0x10000); + TEST_TO("\xf4\x8f\xbf\xbf", 0x10FFFF); + + std::cout << "-- Too big" << std::endl; + TEST_TO("\xf4\x9f\x80\x80", illegal); // 11 0000 + TEST_TO("\xfb\xbf\xbf\xbf", illegal); // 3FF FFFF + TEST_TO("\xf8\x90\x80\x80\x80", illegal); // 400 0000 + TEST_TO("\xfd\xbf\xbf\xbf\xbf\xbf", illegal); // 7fff ffff + + std::cout << "-- Invalid trail" << std::endl; + TEST_TO("\xC2\x7F", illegal); + TEST_TO("\xdf\x7F", illegal); + TEST_TO("\xe0\x7F\x80", illegal); + TEST_TO("\xef\xbf\x7F", illegal); + TEST_TO("\xe0\x7F\x80", illegal); + TEST_TO("\xef\xbf\x7F", illegal); + TEST_TO("\xf0\x7F\x80\x80", illegal); + TEST_TO("\xf4\x7f\xbf\xbf", illegal); + TEST_TO("\xf0\x90\x7F\x80", illegal); + TEST_TO("\xf4\x8f\x7F\xbf", illegal); + TEST_TO("\xf0\x90\x80\x7F", illegal); + TEST_TO("\xf4\x8f\xbf\x7F", illegal); + + std::cout << "-- Invalid length" << std::endl; + + // Test that this actually works + TEST_TO(make2(0x80), 0x80); + TEST_TO(make2(0x7ff), 0x7ff); + + TEST_TO(make3(0x800), 0x800); + TEST_TO(make3(0xffff), 0xffff); + + TEST_TO(make4(0x10000), 0x10000); + TEST_TO(make4(0x10ffff), 0x10ffff); + + TEST_TO(make4(0x110000), illegal); + TEST_TO(make4(0x1fffff), illegal); + + TEST_TO(make2(0), illegal); + TEST_TO(make3(0), illegal); + TEST_TO(make4(0), illegal); + TEST_TO(make2(0x7f), illegal); + TEST_TO(make3(0x7f), illegal); + TEST_TO(make4(0x7f), illegal); + + TEST_TO(make3(0x80), illegal); + TEST_TO(make4(0x80), illegal); + TEST_TO(make3(0x7ff), illegal); + TEST_TO(make4(0x7ff), illegal); + + TEST_TO(make4(0x8000), illegal); + TEST_TO(make4(0xffff), illegal); + + std::cout << "-- Invalid surrogate" << std::endl; + + TEST_TO(make3(0xD800), illegal); + TEST_TO(make3(0xDBFF), illegal); + TEST_TO(make3(0xDC00), illegal); + TEST_TO(make3(0xDFFF), illegal); + + TEST_TO(make4(0xD800), illegal); + TEST_TO(make4(0xDBFF), illegal); + TEST_TO(make4(0xDC00), illegal); + TEST_TO(make4(0xDFFF), illegal); + + std::cout << "-- Incomplete" << std::endl; + + TEST_TO("\x80", illegal); + TEST_TO("\xC2", incomplete); + + TEST_TO("\xdf", incomplete); + + TEST_TO("\xe0", incomplete); + TEST_TO("\xe0\xa0", incomplete); + + TEST_TO("\xef\xbf", incomplete); + TEST_TO("\xef", incomplete); + + TEST_TO("\xf0\x90\x80", incomplete); + TEST_TO("\xf0\x90", incomplete); + TEST_TO("\xf0", incomplete); + + TEST_TO("\xf4\x8f\xbf", incomplete); + TEST_TO("\xf4\x8f", incomplete); + TEST_TO("\xf4", incomplete); + + std::cout << "- To UTF-8\n"; + + std::cout << "-- Test correct" << std::endl; + + TEST_FROM("\x7f", 0x7f); + TEST_FROM("\xC2\x80", 0x80); + TEST_FROM("\xdf\xBF", 0x7FF); + TEST_INC(0x7FF, 1); + TEST_FROM("\xe0\xa0\x80", 0x800); + TEST_INC(0x800, 2); + TEST_INC(0x800, 1); + TEST_FROM("\xef\xbf\xbf", 0xFFFF); + TEST_INC(0x10000, 3); + TEST_INC(0x10000, 2); + TEST_INC(0x10000, 1); + TEST_FROM("\xf0\x90\x80\x80", 0x10000); + TEST_FROM("\xf4\x8f\xbf\xbf", 0x10FFFF); + + std::cout << "-- Test no surrogate " << std::endl; + + TEST_FROM(nullptr, 0xD800); + TEST_FROM(nullptr, 0xDBFF); + TEST_FROM(nullptr, 0xDC00); + TEST_FROM(nullptr, 0xDFFF); + + std::cout << "-- Test invalid " << std::endl; + + TEST_FROM(nullptr, 0x110000); + TEST_FROM(nullptr, 0x1FFFFF); + } std::cout << "Test windows-1255" << std::endl; cvt = create_simple_converter("windows-1255"); - TEST_REQUIRE(cvt); - TEST(cvt->is_thread_safe()); - TEST_EQ(cvt->max_len(), 1); - - std::cout << "- From 1255" << std::endl; + if TEST(cvt) { + TEST(cvt->is_thread_safe()); + TEST_EQ(cvt->max_len(), 1); - TEST_TO("\xa4", 0x20aa); - TEST_TO("\xe0", 0x05d0); - TEST_TO("\xc4", 0x5b4); - TEST_TO("\xfb", illegal); - TEST_TO("\xdd", illegal); - TEST_TO("\xff", illegal); - TEST_TO("\xfe", 0x200f); + std::cout << "- From 1255" << std::endl; - std::cout << "- To 1255" << std::endl; + TEST_TO("\xa4", 0x20aa); + TEST_TO("\xe0", 0x05d0); + TEST_TO("\xc4", 0x5b4); + TEST_TO("\xfb", illegal); + TEST_TO("\xdd", illegal); + TEST_TO("\xff", illegal); + TEST_TO("\xfe", 0x200f); - TEST_FROM("\xa4", 0x20aa); - TEST_FROM("\xe0", 0x05d0); - TEST_FROM("\xc4", 0x5b4); - TEST_FROM("\xfe", 0x200f); + std::cout << "- To 1255" << std::endl; - TEST_FROM(nullptr, 0xe4); - TEST_FROM(nullptr, 0xd0); + TEST_FROM("\xa4", 0x20aa); + TEST_FROM("\xe0", 0x05d0); + TEST_FROM("\xc4", 0x5b4); + TEST_FROM("\xfe", 0x200f); + TEST_FROM(nullptr, 0xe4); + TEST_FROM(nullptr, 0xd0); + } #ifdef BOOST_LOCALE_WITH_ICU std::cout << "Testing Shift-JIS using ICU/uconv" << std::endl; cvt = boost::locale::impl_icu::create_uconv_converter("Shift-JIS"); - TEST_REQUIRE(cvt); - test_shiftjis(cvt); + if TEST(cvt) + test_shiftjis(cvt); #endif std::cout << "Testing Shift-JIS using POSIX/iconv" << std::endl; diff --git a/test/test_posix_convert.cpp b/test/test_posix_convert.cpp index 616ebbc7..9d2b3e7a 100644 --- a/test/test_posix_convert.cpp +++ b/test/test_posix_convert.cpp @@ -52,13 +52,14 @@ void test_char() else { std::cout << "Testing " << name << std::endl; locale_holder cl(newlocale(LC_ALL_MASK, name.c_str(), nullptr)); - TEST_REQUIRE(cl); + if TEST(cl) { #ifndef BOOST_LOCALE_NO_POSIX_BACKEND - if(towupper_l(L'i', cl) == 0x130) - test_one(gen(name), "i", "i", "İ"); - else - std::cout << " Turkish locale is not supported well" << std::endl; // LCOV_EXCL_LINE + if(towupper_l(L'i', cl) == 0x130) + test_one(gen(name), "i", "i", "İ"); + else + std::cout << " Turkish locale is not supported well" << std::endl; // LCOV_EXCL_LINE #endif + } } } diff --git a/test/test_posix_formatting.cpp b/test/test_posix_formatting.cpp index 15c4d6d2..e96a0e24 100644 --- a/test/test_posix_formatting.cpp +++ b/test/test_posix_formatting.cpp @@ -164,13 +164,13 @@ void test_main(int /*argc*/, char** /*argv*/) else { std::locale generated_locale = gen(locale_name); locale_holder real_locale(newlocale(LC_ALL_MASK, locale_name.c_str(), nullptr)); - TEST_REQUIRE(real_locale); + if TEST(real_locale) { + std::cout << "UTF-8" << std::endl; + test_by_char(generated_locale, real_locale); - std::cout << "UTF-8" << std::endl; - test_by_char(generated_locale, real_locale); - - std::cout << "Wide UTF-" << sizeof(wchar_t) * 8 << std::endl; - test_by_char(generated_locale, real_locale); + std::cout << "Wide UTF-" << sizeof(wchar_t) * 8 << std::endl; + test_by_char(generated_locale, real_locale); + } } } { diff --git a/test/test_util.cpp b/test/test_util.cpp index f6a057fe..1c4145ab 100644 --- a/test/test_util.cpp +++ b/test/test_util.cpp @@ -37,31 +37,33 @@ void test_hold_ptr() auto* raw = new Dummy(42); boost::locale::hold_ptr ptr(raw); const boost::locale::hold_ptr& const_ptr = ptr; - TEST_REQUIRE(ptr); - TEST(ptr.get() == raw); - TEST(const_ptr.get() == raw); - // const propagation - TEST_EQ((*ptr).foo(), raw->i_); - TEST_EQ((*const_ptr).foo(), -raw->i_); - TEST_EQ(ptr->foo(), raw->i_); - TEST_EQ(const_ptr->foo(), -raw->i_); - TEST_EQ(ptr.get()->foo(), raw->i_); - TEST_EQ(const_ptr.get()->foo(), -raw->i_); - // move construct - boost::locale::hold_ptr ptr2 = std::move(ptr); - TEST(!ptr); - TEST_REQUIRE(ptr2); - TEST(ptr2.get() == raw); - // move assign - ptr = std::move(ptr2); - TEST(ptr); - TEST_REQUIRE(!ptr2); - TEST(ptr.get() == raw); - // Swap - boost::locale::hold_ptr ptr3(new Dummy(1337)); - ptr.swap(ptr3); - TEST_EQ(ptr->foo(), 1337); - TEST_EQ(ptr3->foo(), 42); + if TEST(ptr) { + TEST(ptr.get() == raw); + TEST(const_ptr.get() == raw); + // const propagation + TEST_EQ((*ptr).foo(), raw->i_); + TEST_EQ((*const_ptr).foo(), -raw->i_); + TEST_EQ(ptr->foo(), raw->i_); + TEST_EQ(const_ptr->foo(), -raw->i_); + TEST_EQ(ptr.get()->foo(), raw->i_); + TEST_EQ(const_ptr.get()->foo(), -raw->i_); + // move construct + boost::locale::hold_ptr ptr2 = std::move(ptr); + TEST(!ptr); + if TEST(ptr2) { + TEST(ptr2.get() == raw); + // move assign + ptr = std::move(ptr2); + TEST(ptr); + TEST(!ptr2); + TEST(ptr.get() == raw); + // Swap + boost::locale::hold_ptr ptr3(new Dummy(1337)); + ptr.swap(ptr3); + TEST_EQ(ptr->foo(), 1337); + TEST_EQ(ptr3->foo(), 42); + } + } } TEST_EQ(Dummy::ctr, 0); auto* raw = new Dummy(42); diff --git a/test/test_util_numeric_convert.cpp b/test/test_util_numeric_convert.cpp index 28613ca7..cf24a5d3 100644 --- a/test/test_util_numeric_convert.cpp +++ b/test/test_util_numeric_convert.cpp @@ -15,23 +15,23 @@ void test_try_to_int() using boost::locale::util::try_to_int; int v = 1337; - TEST(try_to_int("0", v)); - TEST_EQ(v, 0); + if TEST(try_to_int("0", v)) + TEST_EQ(v, 0); - TEST(try_to_int("42", v)); - TEST_EQ(v, 42); + if TEST(try_to_int("42", v)) + TEST_EQ(v, 42); - TEST(try_to_int("-1337", v)); - TEST_EQ(v, -1337); + if TEST(try_to_int("-1337", v)) + TEST_EQ(v, -1337); std::ostringstream ss; ss.imbue(std::locale::classic()); empty_stream(ss) << std::numeric_limits::min(); - TEST(try_to_int(ss.str(), v)); - TEST_EQ(v, std::numeric_limits::min()); + if TEST(try_to_int(ss.str(), v)) + TEST_EQ(v, std::numeric_limits::min()); empty_stream(ss) << std::numeric_limits::max(); - TEST(try_to_int(ss.str(), v)); - TEST_EQ(v, std::numeric_limits::max()); + if TEST(try_to_int(ss.str(), v)) + TEST_EQ(v, std::numeric_limits::max()); TEST(!try_to_int("", v)); TEST(!try_to_int("a", v)); From ac069b6096d407613fa4673145e33b72ccd11438 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 31 Dec 2024 13:32:17 +0100 Subject: [PATCH 50/67] Fix handling of + prefix in try_to_int --- src/util/numeric_conversion.hpp | 9 ++++++--- test/test_util_numeric_convert.cpp | 9 +++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/util/numeric_conversion.hpp b/src/util/numeric_conversion.hpp index 9e882f6b..f4de06c0 100644 --- a/src/util/numeric_conversion.hpp +++ b/src/util/numeric_conversion.hpp @@ -14,10 +14,13 @@ namespace boost { namespace locale { namespace util { template - bool try_to_int(const string_view s, Integer& value) + bool try_to_int(string_view s, Integer& value) { - if(s.empty()) - return false; + if(s.size() >= 2 && s[0] == '+') { + if(s[1] == '-') // "+-" is not allowed, invalid "+" is detected by parser + return false; + s.remove_prefix(1); + } const auto res = boost::charconv::from_chars(s, value); return res && res.ptr == (s.data() + s.size()); } diff --git a/test/test_util_numeric_convert.cpp b/test/test_util_numeric_convert.cpp index cf24a5d3..c30cfaf0 100644 --- a/test/test_util_numeric_convert.cpp +++ b/test/test_util_numeric_convert.cpp @@ -24,6 +24,9 @@ void test_try_to_int() if TEST(try_to_int("-1337", v)) TEST_EQ(v, -1337); + if TEST(try_to_int("+1337", v)) + TEST_EQ(v, +1337); + std::ostringstream ss; ss.imbue(std::locale::classic()); empty_stream(ss) << std::numeric_limits::min(); @@ -34,6 +37,12 @@ void test_try_to_int() TEST_EQ(v, std::numeric_limits::max()); TEST(!try_to_int("", v)); + TEST(!try_to_int("+", v)); + TEST(!try_to_int("-", v)); + TEST(!try_to_int("++", v)); + TEST(!try_to_int("+-", v)); + TEST(!try_to_int("++1", v)); + TEST(!try_to_int("+-1", v)); TEST(!try_to_int("a", v)); TEST(!try_to_int("1.", v)); TEST(!try_to_int("1a", v)); From c2147f6486d7cb1db708650ccf07ad803edae5d1 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 2 Jan 2025 18:31:46 +0100 Subject: [PATCH 51/67] Switch to `boost::core::string_view` This is compatible to `std::string_view` and also to `boost::string_view` so hence the better option. --- CMakeLists.txt | 3 +-- build.jam | 3 ++- include/boost/locale/detail/any_string.hpp | 8 ++++---- include/boost/locale/detail/encoding.hpp | 6 +++--- include/boost/locale/encoding.hpp | 11 ++++++----- src/shared/message.cpp | 22 +++++++++++----------- src/util/encoding.cpp | 6 +++--- src/util/encoding.hpp | 10 +++++----- src/util/numeric_conversion.hpp | 6 +++--- 9 files changed, 38 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 500d7943..c0b4e693 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2020, 2021 Peter Dimov -# Copyright 2022-2024 Alexander Grund +# Copyright 2022-2025 Alexander Grund # Distributed under the Boost Software License, Version 1.0. # https://www.boost.org/LICENSE_1_0.txt @@ -51,7 +51,6 @@ target_link_libraries(boost_locale Boost::config Boost::core Boost::iterator - Boost::utility PRIVATE Boost::charconv Boost::predef diff --git a/build.jam b/build.jam index 573ca20a..a9b6ec74 100644 --- a/build.jam +++ b/build.jam @@ -1,4 +1,5 @@ # Copyright René Ferdinand Rivera Morell 2023-2024 +# Copyright(c) 2025 Alexander Grund # Distributed under the Boost Software License, Version 1.0. # (See accompanying file LICENSE_1_0.txt or copy at # http://www.boost.org/LICENSE_1_0.txt) @@ -20,7 +21,7 @@ constant boost_dependencies : /boost/config//boost_config /boost/core//boost_core /boost/iterator//boost_iterator - /boost/utility//boost_utility ; + ; project /boost/locale : common-requirements diff --git a/include/boost/locale/detail/any_string.hpp b/include/boost/locale/detail/any_string.hpp index c0cc7ffb..473d07e7 100644 --- a/include/boost/locale/detail/any_string.hpp +++ b/include/boost/locale/detail/any_string.hpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2023 Alexander Grund +// Copyright (c) 2023-2025 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include #include @@ -31,7 +31,7 @@ namespace boost { namespace locale { namespace detail { }; template struct BOOST_SYMBOL_VISIBLE impl : base { - explicit impl(const boost::basic_string_view value) : s(value) {} + explicit impl(const core::basic_string_view value) : s(value) {} impl* clone() const override { return new impl(*this); } std::basic_string s; }; @@ -49,7 +49,7 @@ namespace boost { namespace locale { namespace detail { } template - void set(const boost::basic_string_view s) + void set(const core::basic_string_view s) { BOOST_ASSERT(!s.empty()); s_.reset(new impl(s)); diff --git a/include/boost/locale/detail/encoding.hpp b/include/boost/locale/detail/encoding.hpp index 390c4bb9..2e37c308 100644 --- a/include/boost/locale/detail/encoding.hpp +++ b/include/boost/locale/detail/encoding.hpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2022-2023 Alexander Grund +// Copyright (c) 2022-2025 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include @@ -24,7 +24,7 @@ namespace boost { namespace locale { namespace conv { namespace detail { virtual ~charset_converter() = default; virtual string_type convert(const CharIn* begin, const CharIn* end) = 0; - string_type convert(const boost::basic_string_view text) + string_type convert(const core::basic_string_view text) { return convert(text.data(), text.data() + text.length()); } diff --git a/include/boost/locale/encoding.hpp b/include/boost/locale/encoding.hpp index 52c8bc46..abc68bac 100644 --- a/include/boost/locale/encoding.hpp +++ b/include/boost/locale/encoding.hpp @@ -1,5 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2025 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -240,11 +241,11 @@ namespace boost { namespace locale { /// Convert \a text to UTF /// /// \throws conversion_error: Conversion failed - string_type convert(const boost::string_view text) const { return impl_->convert(text); } + string_type convert(const core::string_view text) const { return impl_->convert(text); } /// Convert \a text to UTF /// /// \throws conversion_error: Conversion failed - string_type operator()(const boost::string_view text) const { return convert(text); } + string_type operator()(const core::string_view text) const { return convert(text); } }; /// Converter class to decode an UTF string and encode it using a local encoding @@ -254,7 +255,7 @@ namespace boost { namespace locale { public: using char_type = CharType; - using stringview_type = boost::basic_string_view; + using stringview_type = core::basic_string_view; /// Create an instance to convert UTF text to text encoded with \a charset according to policy \a how /// @@ -298,11 +299,11 @@ namespace boost { namespace locale { /// Convert \a text /// /// \throws conversion_error: Conversion failed - std::string convert(const boost::string_view text) const { return impl_->convert(text); } + std::string convert(const core::string_view text) const { return impl_->convert(text); } /// Convert \a text /// /// \throws conversion_error: Conversion failed - std::string operator()(const boost::string_view text) const { return convert(text); } + std::string operator()(const core::string_view text) const { return convert(text); } }; } // namespace conv }} // namespace boost::locale diff --git a/src/shared/message.cpp b/src/shared/message.cpp index 27aaa671..88cee09e 100644 --- a/src/shared/message.cpp +++ b/src/shared/message.cpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2015 Artyom Beilis (Tonkikh) -// Copyright (c) 2021-2023 Alexander Grund +// Copyright (c) 2021-2025 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -26,7 +26,7 @@ #include "mo_hash.hpp" #include "mo_lambda.hpp" #include -#include +#include #include #include #include @@ -141,7 +141,7 @@ namespace boost { namespace locale { namespace gnu_gettext { hash_offset_ = get(24); } - string_view find(const char* context_in, const char* key_in) const + core::string_view find(const char* context_in, const char* key_in) const { if(!has_hash()) return {}; @@ -192,13 +192,13 @@ namespace boost { namespace locale { namespace gnu_gettext { return data_.data() + off; } - string_view value(unsigned id) const + core::string_view value(unsigned id) const { const uint32_t len = get(translations_offset_ + id * 8); const uint32_t off = get(translations_offset_ + id * 8 + 4); if(len > data_.size() || off > data_.size() - len) throw std::runtime_error("Bad mo-file format"); - return string_view(&data_[off], len); + return core::string_view(&data_[off], len); } bool has_hash() const { return hash_size_ != 0; } @@ -233,7 +233,7 @@ namespace boost { namespace locale { namespace gnu_gettext { template struct mo_file_use_traits { static constexpr bool in_use = false; - using string_view_type = basic_string_view; + using string_view_type = core::basic_string_view; static string_view_type use(const mo_file&, const CharType*, const CharType*) { throw std::logic_error("Unexpected call"); // LCOV_EXCL_LINE @@ -243,7 +243,7 @@ namespace boost { namespace locale { namespace gnu_gettext { template<> struct mo_file_use_traits { static constexpr bool in_use = true; - using string_view_type = basic_string_view; + using string_view_type = core::basic_string_view; static string_view_type use(const mo_file& mo, const char* context, const char* key) { return mo.find(context, key); @@ -254,10 +254,10 @@ namespace boost { namespace locale { namespace gnu_gettext { template<> struct mo_file_use_traits { static constexpr bool in_use = true; - using string_view_type = basic_string_view; + using string_view_type = core::basic_string_view; static string_view_type use(const mo_file& mo, const char8_t* context, const char8_t* key) { - string_view res = mo.find(reinterpret_cast(context), reinterpret_cast(key)); + core::string_view res = mo.find(reinterpret_cast(context), reinterpret_cast(key)); return {reinterpret_cast(res.data()), res.size()}; } }; @@ -551,10 +551,10 @@ namespace boost { namespace locale { namespace gnu_gettext { return true; } - static std::string extract(boost::string_view meta, const std::string& key, const boost::string_view separators) + static std::string extract(core::string_view meta, const std::string& key, const core::string_view separators) { const size_t pos = meta.find(key); - if(pos == boost::string_view::npos) + if(pos == core::string_view::npos) return ""; meta.remove_prefix(pos + key.size()); const size_t end_pos = meta.find_first_of(separators); diff --git a/src/util/encoding.cpp b/src/util/encoding.cpp index 18b658fa..8b893f1c 100644 --- a/src/util/encoding.cpp +++ b/src/util/encoding.cpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2022-2023 Alexander Grund +// Copyright (c) 2022-2025 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -18,7 +18,7 @@ #include namespace boost { namespace locale { namespace util { - std::string normalize_encoding(const string_view encoding) + std::string normalize_encoding(const core::string_view encoding) { std::string result; result.reserve(encoding.length()); @@ -51,7 +51,7 @@ namespace boost { namespace locale { namespace util { return -1; } - int encoding_to_windows_codepage(const string_view encoding) + int encoding_to_windows_codepage(const core::string_view encoding) { return normalized_encoding_to_windows_codepage(normalize_encoding(encoding)); } diff --git a/src/util/encoding.hpp b/src/util/encoding.hpp index 0f4c3b56..2f73d49a 100644 --- a/src/util/encoding.hpp +++ b/src/util/encoding.hpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2022-2023 Alexander Grund +// Copyright (c) 2022-2025 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -9,7 +9,7 @@ #define BOOST_LOCALE_UTIL_ENCODING_HPP #include -#include +#include #include #include #include @@ -48,7 +48,7 @@ namespace boost { namespace locale { namespace util { #endif /// Make encoding lowercase and remove all non-alphanumeric characters - BOOST_LOCALE_DECL std::string normalize_encoding(string_view encoding); + BOOST_LOCALE_DECL std::string normalize_encoding(core::string_view encoding); /// True if the normalized encodings are equal inline bool are_encodings_equal(const std::string& l, const std::string& r) { @@ -58,10 +58,10 @@ namespace boost { namespace locale { namespace util { BOOST_LOCALE_DECL std::vector get_simple_encodings(); #if BOOST_LOCALE_USE_WIN32_API - int encoding_to_windows_codepage(string_view encoding); + int encoding_to_windows_codepage(core::string_view encoding); #else // Requires WinAPI -> Dummy returning invalid - inline int encoding_to_windows_codepage(string_view) // LCOV_EXCL_LINE + inline int encoding_to_windows_codepage(core::string_view) // LCOV_EXCL_LINE { return -1; // LCOV_EXCL_LINE } diff --git a/src/util/numeric_conversion.hpp b/src/util/numeric_conversion.hpp index f4de06c0..3efc50f5 100644 --- a/src/util/numeric_conversion.hpp +++ b/src/util/numeric_conversion.hpp @@ -1,5 +1,5 @@ // -// Copyright (c) 2024 Alexander Grund +// Copyright (c) 2024-2025 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -9,12 +9,12 @@ #include #include -#include +#include namespace boost { namespace locale { namespace util { template - bool try_to_int(string_view s, Integer& value) + bool try_to_int(core::string_view s, Integer& value) { if(s.size() >= 2 && s[0] == '+') { if(s[1] == '-') // "+-" is not allowed, invalid "+" is detected by parser From e540a63c0b3f6fd28e8ebb1389b19dbde65ea712 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 2 Jan 2025 19:35:44 +0100 Subject: [PATCH 52/67] Enable char8_t strings where available, remove Clang condition The issue was caused by using an incompatible libstdc++. For Clang 13 & 14 libstdc++ 11 works. --- include/boost/locale/boundary/boundary_point.hpp | 2 +- include/boost/locale/boundary/index.hpp | 4 ++-- include/boost/locale/boundary/segment.hpp | 2 +- include/boost/locale/config.hpp | 5 ----- include/boost/locale/format.hpp | 2 +- include/boost/locale/message.hpp | 2 +- src/encoding/codepage.cpp | 2 +- src/icu/conversion.cpp | 2 +- src/posix/converter.cpp | 2 +- src/shared/message.cpp | 2 +- src/std/converter.cpp | 2 +- src/util/foreach_char.hpp | 2 +- src/win32/converter.cpp | 2 +- test/boostLocale/test/tools.hpp | 2 +- test/show_config.cpp | 2 +- test/test_boundary.cpp | 4 ++-- test/test_convert.cpp | 4 ++-- test/test_encoding.cpp | 8 ++++---- test/test_generator.cpp | 2 +- test/test_ios_info.cpp | 2 +- test/test_message.cpp | 12 ++++++------ test/test_std_convert.cpp | 2 +- 22 files changed, 32 insertions(+), 37 deletions(-) diff --git a/include/boost/locale/boundary/boundary_point.hpp b/include/boost/locale/boundary/boundary_point.hpp index 370dd5f8..0743347c 100644 --- a/include/boost/locale/boundary/boundary_point.hpp +++ b/include/boost/locale/boundary/boundary_point.hpp @@ -100,7 +100,7 @@ namespace boost { namespace locale { namespace boundary { typedef boundary_point sboundary_point; ///< convenience typedef typedef boundary_point wsboundary_point; ///< convenience typedef -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t typedef boundary_point u8sboundary_point; ///< convenience typedef #endif #ifdef BOOST_LOCALE_ENABLE_CHAR16_T diff --git a/include/boost/locale/boundary/index.hpp b/include/boost/locale/boundary/index.hpp index 92b7613f..249ef877 100644 --- a/include/boost/locale/boundary/index.hpp +++ b/include/boost/locale/boundary/index.hpp @@ -868,7 +868,7 @@ namespace boost { namespace locale { namespace boundary { typedef segment_index ssegment_index; ///< convenience typedef typedef segment_index wssegment_index; ///< convenience typedef -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t typedef segment_index u8ssegment_index; ///< convenience typedef #endif #ifdef BOOST_LOCALE_ENABLE_CHAR16_T @@ -892,7 +892,7 @@ namespace boost { namespace locale { namespace boundary { typedef boundary_point_index sboundary_point_index; ///< convenience typedef typedef boundary_point_index wsboundary_point_index; ///< convenience typedef -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t typedef boundary_point_index u8sboundary_point_index; ///< convenience typedef #endif #ifdef BOOST_LOCALE_ENABLE_CHAR16_T diff --git a/include/boost/locale/boundary/segment.hpp b/include/boost/locale/boundary/segment.hpp index 24e129dc..84243abe 100644 --- a/include/boost/locale/boundary/segment.hpp +++ b/include/boost/locale/boundary/segment.hpp @@ -340,7 +340,7 @@ namespace boost { namespace locale { namespace boundary { typedef segment ssegment; ///< convenience typedef typedef segment wssegment; ///< convenience typedef -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t typedef segment u8ssegment; ///< convenience typedef #endif #ifdef BOOST_LOCALE_ENABLE_CHAR16_T diff --git a/include/boost/locale/config.hpp b/include/boost/locale/config.hpp index 1c69c92d..2064810d 100644 --- a/include/boost/locale/config.hpp +++ b/include/boost/locale/config.hpp @@ -89,11 +89,6 @@ # define BOOST_LOCALE_NO_SANITIZE(what) #endif -#if !defined(__cpp_lib_char8_t) || BOOST_WORKAROUND(BOOST_CLANG_VERSION, < 150000) -// No std::basic_string or bug in Clang: https://github.com/llvm/llvm-project/issues/55560 -# define BOOST_LOCALE_NO_CXX20_STRING8 -#endif - /// \endcond #endif // boost/locale/config.hpp diff --git a/include/boost/locale/format.hpp b/include/boost/locale/format.hpp index 58a9a0ff..68020a4c 100644 --- a/include/boost/locale/format.hpp +++ b/include/boost/locale/format.hpp @@ -418,7 +418,7 @@ namespace boost { namespace locale { typedef basic_format format; /// Definition of wchar_t based format typedef basic_format wformat; -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t /// Definition of char8_t based format typedef basic_format u8format; #endif diff --git a/include/boost/locale/message.hpp b/include/boost/locale/message.hpp index 88a4aefb..fafff385 100644 --- a/include/boost/locale/message.hpp +++ b/include/boost/locale/message.hpp @@ -341,7 +341,7 @@ namespace boost { namespace locale { typedef basic_message message; /// Convenience typedef for wchar_t typedef basic_message wmessage; -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t /// Convenience typedef for char8_t typedef basic_message u8message; #endif diff --git a/src/encoding/codepage.cpp b/src/encoding/codepage.cpp index 0f052b82..814a8035 100644 --- a/src/encoding/codepage.cpp +++ b/src/encoding/codepage.cpp @@ -224,7 +224,7 @@ namespace boost { namespace locale { namespace conv { BOOST_LOCALE_INSTANTIATE(char); BOOST_LOCALE_INSTANTIATE_NO_CHAR(wchar_t); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t BOOST_LOCALE_INSTANTIATE_NO_CHAR(char8_t); #endif diff --git a/src/icu/conversion.cpp b/src/icu/conversion.cpp index 130a0d4e..a7fddb47 100644 --- a/src/icu/conversion.cpp +++ b/src/icu/conversion.cpp @@ -198,7 +198,7 @@ namespace boost { namespace locale { namespace impl_icu { return std::locale(in, new utf8_converter_impl(cd)); return std::locale(in, new converter_impl(cd)); case char_facet_t::wchar_f: return std::locale(in, new converter_impl(cd)); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t case char_facet_t::char8_f: return std::locale(in, new utf8_converter_impl(cd)); #elif defined(__cpp_char8_t) case char_facet_t::char8_f: break; diff --git a/src/posix/converter.cpp b/src/posix/converter.cpp index 38a4bb69..13477aa0 100644 --- a/src/posix/converter.cpp +++ b/src/posix/converter.cpp @@ -125,7 +125,7 @@ namespace boost { namespace locale { namespace impl_posix { return std::locale(in, new std_converter(std::move(lc))); } case char_facet_t::wchar_f: return std::locale(in, new std_converter(std::move(lc))); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t case char_facet_t::char8_f: return std::locale(in, new utf8_converter(std::move(lc))); #elif defined(__cpp_char8_t) case char_facet_t::char8_f: break; diff --git a/src/shared/message.cpp b/src/shared/message.cpp index 88cee09e..70eafd22 100644 --- a/src/shared/message.cpp +++ b/src/shared/message.cpp @@ -620,7 +620,7 @@ namespace boost { namespace locale { namespace detail { case char_facet_t::nochar: break; case char_facet_t::char_f: return std::locale(in, gnu_gettext::create_messages_facet(minf)); case char_facet_t::wchar_f: return std::locale(in, gnu_gettext::create_messages_facet(minf)); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t case char_facet_t::char8_f: return std::locale(in, gnu_gettext::create_messages_facet(minf)); #elif defined(__cpp_char8_t) case char_facet_t::char8_f: break; diff --git a/src/std/converter.cpp b/src/std/converter.cpp index fcfb7df7..ec60a58d 100644 --- a/src/std/converter.cpp +++ b/src/std/converter.cpp @@ -104,7 +104,7 @@ namespace boost { namespace locale { namespace impl_std { else return std::locale(in, new std_converter(locale_name)); case char_facet_t::wchar_f: return std::locale(in, new std_converter(locale_name)); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t case char_facet_t::char8_f: return std::locale(in, new utf8_converter(locale_name)); #elif defined(__cpp_char8_t) case char_facet_t::char8_f: break; diff --git a/src/util/foreach_char.hpp b/src/util/foreach_char.hpp index 37328e12..1c0dc35f 100644 --- a/src/util/foreach_char.hpp +++ b/src/util/foreach_char.hpp @@ -9,7 +9,7 @@ #include -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t # define BOOST_LOCALE_FOREACH_CHAR_I_CHAR8_T(F) F(char8_t) # define BOOST_LOCALE_FOREACH_CHAR_I2_CHAR8_T(F) F(char8_t) #elif defined(__cpp_char8_t) diff --git a/src/win32/converter.cpp b/src/win32/converter.cpp index 455fef6c..fe168df7 100644 --- a/src/win32/converter.cpp +++ b/src/win32/converter.cpp @@ -62,7 +62,7 @@ namespace boost { namespace locale { namespace impl_win { case char_facet_t::nochar: break; case char_facet_t::char_f: return std::locale(in, new utf8_converter(lc)); case char_facet_t::wchar_f: return std::locale(in, new wide_converter(lc)); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t case char_facet_t::char8_f: return std::locale(in, new utf8_converter(lc)); #elif defined(__cpp_char8_t) case char_facet_t::char8_f: break; diff --git a/test/boostLocale/test/tools.hpp b/test/boostLocale/test/tools.hpp index 7e9a0453..6537aa5b 100644 --- a/test/boostLocale/test/tools.hpp +++ b/test/boostLocale/test/tools.hpp @@ -113,7 +113,7 @@ std::basic_string to(const std::string& utf8) return out; } -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t template<> std::basic_string to(const std::string& utf8) { diff --git a/test/show_config.cpp b/test/show_config.cpp index 226ec1cb..e2dd8541 100644 --- a/test/show_config.cpp +++ b/test/show_config.cpp @@ -65,7 +65,7 @@ void test_main(int /*argc*/, char** /*argv*/) std::cout << "no\n"; #endif std::cout << "- std::basic_string: "; -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << "yes\n"; #elif defined(__cpp_lib_char8_t) std::cout << "partial\n"; diff --git a/test/test_boundary.cpp b/test/test_boundary.cpp index 7a2715a4..64b1b766 100644 --- a/test/test_boundary.cpp +++ b/test/test_boundary.cpp @@ -310,7 +310,7 @@ void test_boundaries(std::string* all, int* first, int* second, lb::boundary_typ run_word(all, first, second, nullptr, nullptr, nullptr, g("he_IL.cp1255"), t); std::cout << " wchar_t" << std::endl; run_word(all, first, second, nullptr, nullptr, nullptr, g("he_IL.UTF-8"), t); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << " char8_t" << std::endl; run_word(all, first, second, nullptr, nullptr, nullptr, g("he_IL.UTF-8"), t); #endif @@ -373,7 +373,7 @@ void word_boundary() run_word(txt_simple, none_simple, zero, word_simple, zero, zero, utf8_en_locale); run_word(txt_all, none_all, num_all, word_all, kana_all, ideo_all, utf8_jp_locale); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << " char8_t" << std::endl; run_word(txt_empty, zero, zero, zero, zero, zero, g("ja_JP.UTF-8")); run_word(txt_simple, none_simple, zero, word_simple, zero, zero, utf8_en_locale); diff --git a/test/test_convert.cpp b/test/test_convert.cpp index 8752d747..d00568f1 100644 --- a/test/test_convert.cpp +++ b/test/test_convert.cpp @@ -25,7 +25,7 @@ void test_norm(std::string orig, std::string normal, boost::locale::norm_type ty { test_normc(orig, normal, type); test_normc(to(orig), to(normal), type); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t test_normc(to(orig), to(normal), type); #endif #ifdef BOOST_LOCALE_ENABLE_CHAR16_T @@ -105,7 +105,7 @@ void test_main(int /*argc*/, char** /*argv*/) TEST_ALL_CASES; #undef TEST_V -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t # define TEST_V(how, source_s, dest_s) TEST_A(char8_t, how, to(source_s), to(dest_s)) TEST_ALL_CASES; # undef TEST_V diff --git a/test/test_encoding.cpp b/test/test_encoding.cpp index 0a01d43d..6f1dcc60 100644 --- a/test/test_encoding.cpp +++ b/test/test_encoding.cpp @@ -425,7 +425,7 @@ void test_utf_to_utf_for() test_from_utf_for_impls(utf(utf8_string), utf8_string, "UTF-8"); std::cout << "---- wchar_t\n"; test_utf_to_utf_for(utf8_string); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << "---- char8_t\n"; test_utf_to_utf_for(utf8_string); #endif @@ -446,7 +446,7 @@ void test_utf_to_utf() test_utf_to_utf_for(); std::cout << "-- wchar_t\n"; test_utf_to_utf_for(); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << "-- char8_t\n"; test_utf_to_utf_for(); #endif @@ -643,7 +643,7 @@ void test_latin1_conversions() test_latin1_conversions_for(); std::cout << "-- wchar_t\n"; test_latin1_conversions_for(); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << "-- char8_t\n"; test_latin1_conversions_for(); #endif @@ -781,7 +781,7 @@ void test_main(int /*argc*/, char** /*argv*/) test_utf_for(); std::cout << " wchar_t" << std::endl; test_utf_for(); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << " char8_t" << std::endl; test_utf_for(); #endif diff --git a/test/test_generator.cpp b/test/test_generator.cpp index ed3ac3cb..88bfb876 100644 --- a/test/test_generator.cpp +++ b/test/test_generator.cpp @@ -329,7 +329,7 @@ void test_main(int /*argc*/, char** /*argv*/) #else # define TEST_FOR_CHAR8(check) (void)0 #endif -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t # define TEST_FOR_STRING8(check) TEST(check) #else # define TEST_FOR_STRING8(check) (void)0 diff --git a/test/test_ios_info.cpp b/test/test_ios_info.cpp index 79179a8f..bcffe3d6 100644 --- a/test/test_ios_info.cpp +++ b/test/test_ios_info.cpp @@ -222,7 +222,7 @@ void test_any_string() TEST_EQ(s.get(), ascii_to("Char32 Pattern")); TEST_THROWS(s.get(), std::bad_cast); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t s.set(ascii_to("Char8 Pattern")); TEST_EQ(s.get(), ascii_to("Char8 Pattern")); TEST_THROWS(s.get(), std::bad_cast); diff --git a/test/test_message.cpp b/test/test_message.cpp index 304d66c7..80ea7d77 100644 --- a/test/test_message.cpp +++ b/test/test_message.cpp @@ -84,7 +84,7 @@ std::wstring same_w(std::wstring s) return s; } -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::basic_string same_u8(std::basic_string s) { return s; @@ -376,7 +376,7 @@ void test_cntranslate(const std::string& c, impl::test_cntranslate(c, s, p, n, expected, l, domain); std::cout << " wchar_t" << std::endl; impl::test_cntranslate(c, s, p, n, expected, l, domain); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << " char8_t" << std::endl; impl::test_cntranslate(c, s, p, n, expected, l, domain); #endif @@ -401,7 +401,7 @@ void test_ntranslate(const std::string& s, impl::test_ntranslate(s, p, n, expected, l, domain); std::cout << " wchar_t" << std::endl; impl::test_ntranslate(s, p, n, expected, l, domain); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << " char8_t" << std::endl; impl::test_ntranslate(s, p, n, expected, l, domain); #endif @@ -425,7 +425,7 @@ void test_ctranslate(const std::string& c, impl::test_ctranslate(c, original, expected, l, domain); std::cout << " wchar_t" << std::endl; impl::test_ctranslate(c, original, expected, l, domain); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << " char8_t" << std::endl; impl::test_ctranslate(c, original, expected, l, domain); #endif @@ -448,7 +448,7 @@ void test_translate(const std::string& original, impl::test_translate(original, expected, l, domain); std::cout << " wchar_t" << std::endl; impl::test_translate(original, expected, l, domain); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << " char8_t" << std::endl; impl::test_translate(original, expected, l, domain); #endif @@ -570,7 +570,7 @@ void test_main(int argc, char** argv) TEST_EQ(same_s(bl::translate("hello")), "שלום"); TEST_EQ(same_w(bl::translate(to("hello"))), to("שלום")); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t TEST_EQ(same_u8(bl::translate(to("hello"))), to("שלום")); #endif #ifdef BOOST_LOCALE_ENABLE_CHAR16_T diff --git a/test/test_std_convert.cpp b/test/test_std_convert.cpp index bc8126e3..1b8537a2 100644 --- a/test/test_std_convert.cpp +++ b/test/test_std_convert.cpp @@ -84,7 +84,7 @@ void test_main(int /*argc*/, char** /*argv*/) test_char(); std::cout << "Testing wchar_t" << std::endl; test_char(); -#ifndef BOOST_LOCALE_NO_CXX20_STRING8 +#ifdef __cpp_lib_char8_t std::cout << "Testing char8_t" << std::endl; test_char(); #endif From 5f24abe113ad779360eeebbefa146bbe3a37d271 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 12 Jan 2025 18:38:35 +0100 Subject: [PATCH 53/67] Fix returning false from TEST macro --- test/boostLocale/test/unit_test.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/boostLocale/test/unit_test.hpp b/test/boostLocale/test/unit_test.hpp index 366fe838..9e17c21d 100644 --- a/test/boostLocale/test/unit_test.hpp +++ b/test/boostLocale/test/unit_test.hpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2021-2022 Alexander Grund +// Copyright (c) 2021-2025 Alexander Grund // Copyright (c) 2002, 2009, 2014 Peter Dimov // // Distributed under the Boost Software License, Version 1.0. @@ -95,7 +95,8 @@ namespace boost { namespace locale { namespace test { boost::locale::test::results().test_counter++; if(!v) { boost::locale::test::report_error(expr, file, line); - throw std::runtime_error("Critical test " + std::string(expr) + " failed"); + if(require) + throw std::runtime_error("Critical test " + std::string(expr) + " failed"); } return v; } From b01712623597333e11122763e32db62ca4f52d03 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 12 Jan 2025 15:13:29 +0100 Subject: [PATCH 54/67] GHA: Show output of all runs of binaries in test folder --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da2f0443..c0cc4765 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -317,7 +317,7 @@ jobs: - name: Test output if: '!matrix.coverity && always()' run: | - for f in $(find "$BOOST_ROOT/bin.v2/libs/$SELF/test" -type f -name 'test_*.run'); do + for f in $(find "$BOOST_ROOT/bin.v2/libs/$SELF/test" -type f -name '*.run'); do name=$(basename "$f") name=${name%.run} config=$(dirname "$f") From 211734c2c87105756d753edc1a0d3accb2ac2a74 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 8 Aug 2024 09:01:24 +0200 Subject: [PATCH 55/67] Add test for formatting large (u)int64 numbers As reported in #235 formatting the first number which doesn't fit into int64_t anymore fails to add the thousands separators. I.e.: `9223372036854775807` -> `9,223,372,036,854,775,807` `9223372036854775808` -> `9223372036854775808` Add a test reproducing that that for all backends. --- test/formatting_common.hpp | 38 ++++++++++++++++++++++++++++++++- test/test_formatting.cpp | 1 + test/test_posix_formatting.cpp | 1 + test/test_std_formatting.cpp | 4 +++- test/test_winapi_formatting.cpp | 1 + 5 files changed, 43 insertions(+), 2 deletions(-) diff --git a/test/formatting_common.hpp b/test/formatting_common.hpp index 179efd3c..2b0e5791 100644 --- a/test/formatting_common.hpp +++ b/test/formatting_common.hpp @@ -92,4 +92,40 @@ void test_parse_multi_number() BOOST_LOCALE_CALL(wchar_t); #undef BOOST_LOCALE_CALL #undef BOOST_LOCALE_CALL_I -} \ No newline at end of file +} + +template +void test_format_large_number_by_char(const std::locale& locale) +{ + std::basic_ostringstream output; + output.imbue(locale); + output << boost::locale::as::number; + + constexpr int64_t high_signed64 = 9223372036854775807; + static_assert(high_signed64 == std::numeric_limits::max(), "Value must match"); + + empty_stream(output) << high_signed64; + TEST_EQ(output.str(), ascii_to("9,223,372,036,854,775,807")); + empty_stream(output) << static_cast(high_signed64); + TEST_EQ(output.str(), ascii_to("9,223,372,036,854,775,807")); + empty_stream(output) << (static_cast(high_signed64) + 1u); + TEST_EQ(output.str(), ascii_to("9,223,372,036,854,775,808")); + empty_stream(output) << (static_cast(high_signed64) + 579u); + TEST_EQ(output.str(), ascii_to("9,223,372,036,854,776,386")); +} + +void test_format_large_number() +{ + const auto locale = boost::locale::generator{}("en_US.UTF-8"); + + std::cout << "Testing char" << std::endl; + test_format_large_number_by_char(locale); + + std::cout << "Testing wchar_t" << std::endl; + test_format_large_number_by_char(locale); + +#ifdef BOOST_LOCALE_ENABLE_CHAR16_T + std::cout << "Testing char16_t" << std::endl; + test_format_large_number_by_char(locale); +#endif +} diff --git a/test/test_formatting.cpp b/test/test_formatting.cpp index 4920e8c0..6a232ab8 100644 --- a/test/test_formatting.cpp +++ b/test/test_formatting.cpp @@ -891,6 +891,7 @@ void test_main(int argc, char** argv) test_format_class(); #endif + test_format_large_number(); test_parse_multi_number(); } diff --git a/test/test_posix_formatting.cpp b/test/test_posix_formatting.cpp index e96a0e24..c4eacf6a 100644 --- a/test/test_posix_formatting.cpp +++ b/test/test_posix_formatting.cpp @@ -186,6 +186,7 @@ void test_main(int /*argc*/, char** /*argv*/) TEST(v == "12345,45" || v == "12 345,45" || v == "12.345,45"); } } + test_format_large_number(); test_parse_multi_number(); } diff --git a/test/test_std_formatting.cpp b/test/test_std_formatting.cpp index 79156ad0..13ddd300 100644 --- a/test/test_std_formatting.cpp +++ b/test/test_std_formatting.cpp @@ -233,8 +233,10 @@ void test_main(int /*argc*/, char** /*argv*/) } // Std backend silently falls back to the C locale when the locale is not supported // which breaks the test assumptions - if(has_std_locale("en_US.UTF-8")) + if(has_std_locale("en_US.UTF-8")) { + test_format_large_number(); test_parse_multi_number(); + } } // boostinspect:noascii diff --git a/test/test_winapi_formatting.cpp b/test/test_winapi_formatting.cpp index 00a7216c..67d145ff 100644 --- a/test/test_winapi_formatting.cpp +++ b/test/test_winapi_formatting.cpp @@ -177,6 +177,7 @@ void test_main(int /*argc*/, char** /*argv*/) test_by_char(l, name, name_lcid.second); } } + test_format_large_number(); test_parse_multi_number(); std::cout << "- Testing strftime" << std::endl; test_date_time(gen("en_US.UTF-8")); From 42e65d0d3d4080481f29b3b49e2a017744ea6c6b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 5 Dec 2024 20:41:29 +0100 Subject: [PATCH 56/67] Implement support for uint64_t values in ICU backend ICU doesn't support uint64_t directly but provides access to formatting and parsing of decimal number strings. Use Boost.Charconv to interface with that. Fixes #235 --- build/Jamfile.v2 | 2 +- src/icu/formatter.cpp | 65 ++++++++++++++++++++++++++++++---------- src/icu/formatter.hpp | 6 ++++ src/icu/numeric.cpp | 41 ++++++++++--------------- test/test_formatting.cpp | 59 ++++++++++++++++++++++++++++++++++-- 5 files changed, 130 insertions(+), 43 deletions(-) diff --git a/build/Jamfile.v2 b/build/Jamfile.v2 index 1f77dc19..6eaf3871 100644 --- a/build/Jamfile.v2 +++ b/build/Jamfile.v2 @@ -1,6 +1,6 @@ # Copyright 2003 John Maddock # Copyright 2010 Artyom Beilis -# Copyright 2021 - 2022 Alexander Grund +# Copyright 2021 - 2024 Alexander Grund # # Distributed under the Boost Software License, Version 1.0. # https://www.boost.org/LICENSE_1_0.txt. diff --git a/src/icu/formatter.cpp b/src/icu/formatter.cpp index ccdab80c..978812ea 100644 --- a/src/icu/formatter.cpp +++ b/src/icu/formatter.cpp @@ -1,6 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) -// Copyright (c) 2021-2023 Alexander Grund +// Copyright (c) 2021-2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -13,8 +13,12 @@ #include "icu_util.hpp" #include "time_zone.hpp" #include "uconv.hpp" +#include +#include +#include #include #include +#include #ifdef BOOST_MSVC # pragma warning(push) # pragma warning(disable : 4251) // "identifier" : class "type" needs to have dll-interface... @@ -62,35 +66,69 @@ namespace boost { namespace locale { namespace impl_icu { string_type format(int64_t value, size_t& code_points) const override { return do_format(value, code_points); } string_type format(int32_t value, size_t& code_points) const override { return do_format(value, code_points); } size_t parse(const string_type& str, double& value) const override { return do_parse(str, value); } + size_t parse(const string_type& str, uint64_t& value) const override { return do_parse(str, value); } size_t parse(const string_type& str, int64_t& value) const override { return do_parse(str, value); } size_t parse(const string_type& str, int32_t& value) const override { return do_parse(str, value); } + string_type format(const uint64_t value, size_t& code_points) const override + { + // ICU only supports int64_t as the largest integer type + if(value <= static_cast(std::numeric_limits::max())) + return format(static_cast(value), code_points); + + // Fallback to using a StringPiece (decimal number) as input + char buffer[std::numeric_limits::digits10 + 2]; + auto res = boost::charconv::to_chars(buffer, std::end(buffer), value); + BOOST_ASSERT(res); + *res.ptr = '\0'; // ICU expects a NULL-terminated string even for the StringPiece + icu::UnicodeString tmp; + UErrorCode err = U_ZERO_ERROR; + icu_fmt_.format(icu::StringPiece(buffer, res.ptr - buffer), tmp, nullptr, err); + check_and_throw_icu_error(err); + code_points = tmp.countChar32(); + return cvt_.std(tmp); + } + private: bool get_value(double& v, icu::Formattable& fmt) const { UErrorCode err = U_ZERO_ERROR; v = fmt.getDouble(err); - if(U_FAILURE(err)) - return false; - return true; + return U_SUCCESS(err); } bool get_value(int64_t& v, icu::Formattable& fmt) const { UErrorCode err = U_ZERO_ERROR; v = fmt.getInt64(err); + return U_SUCCESS(err); + } + + bool get_value(uint64_t& v, icu::Formattable& fmt) const + { + UErrorCode err = U_ZERO_ERROR; + // ICU only supports int64_t as the largest integer type + const int64_t tmp = fmt.getInt64(err); + if(U_SUCCESS(err)) { + if(tmp < 0) + return false; + v = static_cast(tmp); + return true; + } + // Get value as a decimal number and parse that + err = U_ZERO_ERROR; + const auto decimals = fmt.getDecimalNumber(err); if(U_FAILURE(err)) - return false; - return true; + return false; // Not a number + const auto res = boost::charconv::from_chars({decimals.data(), static_cast(decimals.length())}, v); + return static_cast(res); } bool get_value(int32_t& v, icu::Formattable& fmt) const { UErrorCode err = U_ZERO_ERROR; v = fmt.getLong(err); - if(U_FAILURE(err)) - return false; - return true; + return U_SUCCESS(err); } template @@ -114,14 +152,11 @@ namespace boost { namespace locale { namespace impl_icu { icu_fmt_.setParseIntegerOnly(std::is_integral::value && isNumberOnly_); icu_fmt_.parse(tmp, val, pp); - ValueType tmp_v; - - if(pp.getIndex() == 0 || !get_value(tmp_v, val)) + if(pp.getIndex() == 0 || !get_value(v, val)) return 0; size_t cut = cvt_.cut(tmp, str.data(), str.data() + str.size(), pp.getIndex()); if(cut == 0) return 0; - v = tmp_v; return cut; } @@ -136,11 +171,11 @@ namespace boost { namespace locale { namespace impl_icu { typedef std::basic_string string_type; string_type format(double value, size_t& code_points) const override { return do_format(value, code_points); } + string_type format(uint64_t value, size_t& code_points) const override { return do_format(value, code_points); } string_type format(int64_t value, size_t& code_points) const override { return do_format(value, code_points); } - string_type format(int32_t value, size_t& code_points) const override { return do_format(value, code_points); } - size_t parse(const string_type& str, double& value) const override { return do_parse(str, value); } + size_t parse(const string_type& str, uint64_t& value) const override { return do_parse(str, value); } size_t parse(const string_type& str, int64_t& value) const override { return do_parse(str, value); } size_t parse(const string_type& str, int32_t& value) const override { return do_parse(str, value); } diff --git a/src/icu/formatter.hpp b/src/icu/formatter.hpp index 110a0315..bcda2788 100644 --- a/src/icu/formatter.hpp +++ b/src/icu/formatter.hpp @@ -1,5 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -31,6 +32,8 @@ namespace boost { namespace locale { namespace impl_icu { /// Format the value and return the number of Unicode code points virtual string_type format(double value, size_t& code_points) const = 0; /// Format the value and return the number of Unicode code points + virtual string_type format(uint64_t value, size_t& code_points) const = 0; + /// Format the value and return the number of Unicode code points virtual string_type format(int64_t value, size_t& code_points) const = 0; /// Format the value and return the number of Unicode code points virtual string_type format(int32_t value, size_t& code_points) const = 0; @@ -40,6 +43,9 @@ namespace boost { namespace locale { namespace impl_icu { virtual size_t parse(const string_type& str, double& value) const = 0; /// Parse the string and return the number of used characters. If it returns 0 /// then parsing failed. + virtual size_t parse(const string_type& str, uint64_t& value) const = 0; + /// Parse the string and return the number of used characters. If it returns 0 + /// then parsing failed. virtual size_t parse(const string_type& str, int64_t& value) const = 0; /// Parse the string and return the number of used characters. If it returns 0 /// then parsing failed. diff --git a/src/icu/numeric.cpp b/src/icu/numeric.cpp index 1fcf044a..e54ab1c4 100644 --- a/src/icu/numeric.cpp +++ b/src/icu/numeric.cpp @@ -1,5 +1,6 @@ // // Copyright (c) 2009-2011 Artyom Beilis (Tonkikh) +// Copyright (c) 2024 Alexander Grund // // Distributed under the Boost Software License, Version 1.0. // https://www.boost.org/LICENSE_1_0.txt @@ -19,41 +20,31 @@ namespace boost { namespace locale { namespace impl_icu { namespace detail { - template::is_integer> - struct icu_format_type; + template + struct choose_type_by_digits + : std::conditional::digits <= std::numeric_limits::digits, + PreferredType, + AlternativeType> {}; - template - struct icu_format_type { - // ICU supports 32 and 64 bit ints, use the former as long as it fits, else the latter - typedef typename std::conditional::digits <= 31, int32_t, int64_t>::type type; + template::is_integer> + struct icu_format_type { + static_assert(sizeof(T) <= sizeof(int64_t), "Only up to 64 bit integer types are supported by ICU"); + // ICU supports (only) int32_t and int64_t, use the former as long as it fits, else the latter + using large_type = typename choose_type_by_digits::type; + using type = typename choose_type_by_digits::type; }; template struct icu_format_type { // Only float type ICU supports is double - typedef double type; - }; - - // ICU does not support uint64_t values so fall back to the parent/std formatting - // if the number is to large to fit into an int64_t - template::is_signed && std::numeric_limits::is_integer - && (sizeof(T) >= sizeof(uint64_t))> - struct use_parent_traits { - static bool use(T /*v*/) { return false; } - }; - template - struct use_parent_traits { - static bool use(T v) { return v > static_cast(std::numeric_limits::max()); } + using type = double; }; template - static bool use_parent(std::ios_base& ios, ValueType v) + static bool use_parent(std::ios_base& ios) { const uint64_t flg = ios_info::get(ios).display_flags(); if(flg == flags::posix) return true; - if(use_parent_traits::use(v)) - return true; if(!std::numeric_limits::is_integer) return false; @@ -105,7 +96,7 @@ namespace boost { namespace locale { namespace impl_icu { template iter_type do_real_put(iter_type out, std::ios_base& ios, CharType fill, ValueType val) const { - if(detail::use_parent(ios, val)) + if(detail::use_parent(ios)) return std::num_put::do_put(out, ios, fill, val); const std::unique_ptr formatter = formatter_type::create(ios, loc_, enc_); @@ -240,7 +231,7 @@ namespace boost { namespace locale { namespace impl_icu { do_real_get(iter_type in, iter_type end, std::ios_base& ios, std::ios_base::iostate& err, ValueType& val) const { stream_type* stream_ptr = dynamic_cast(&ios); - if(!stream_ptr || detail::use_parent(ios, ValueType(0))) + if(!stream_ptr || detail::use_parent(ios)) return std::num_get::do_get(in, end, ios, err, val); const std::unique_ptr formatter = formatter_type::create(ios, loc_, enc_); diff --git a/test/test_formatting.cpp b/test/test_formatting.cpp index 6a232ab8..c05ad6ea 100644 --- a/test/test_formatting.cpp +++ b/test/test_formatting.cpp @@ -380,18 +380,22 @@ void test_manip(std::string e_charset = "UTF-8") TEST_MIN_MAX(int16_t, "-32,768", "32,767"); TEST_MIN_MAX(uint16_t, "0", "65,535"); TEST_PARSE_FAILS(as::number, "-1", uint16_t); + TEST_PARSE_FAILS(as::number, "-32,767", uint16_t); if(stdlib_correctly_errors_on_out_of_range_int16()) TEST_PARSE_FAILS(as::number, "65,535", int16_t); TEST_MIN_MAX(int32_t, "-2,147,483,648", "2,147,483,647"); TEST_MIN_MAX(uint32_t, "0", "4,294,967,295"); TEST_PARSE_FAILS(as::number, "-1", uint32_t); + TEST_PARSE_FAILS(as::number, "-2,147,483,647", uint32_t); TEST_PARSE_FAILS(as::number, "4,294,967,295", int32_t); TEST_MIN_MAX(int64_t, "-9,223,372,036,854,775,808", "9,223,372,036,854,775,807"); - // ICU does not support uint64, but we have a fallback to format it at least - TEST_MIN_MAX_FMT(as::number, uint64_t, "0", "18446744073709551615"); + TEST_MIN_MAX(uint64_t, "0", "18,446,744,073,709,551,615"); TEST_PARSE_FAILS(as::number, "-1", uint64_t); + TEST_PARSE_FAILS(as::number, "-9,223,372,036,854,775,807", uint64_t); + TEST_PARSE_FAILS(as::number, "18,446,744,073,709,551,615", int64_t); + TEST_PARSE_FAILS(as::number, "18,446,744,073,709,551,616", uint64_t); TEST_FMT_PARSE_3(as::number, std::left, std::setw(3), 15, "15 "); TEST_FMT_PARSE_3(as::number, std::right, std::setw(3), 15, " 15"); @@ -857,6 +861,55 @@ void test_format_class(std::string charset = "UTF-8") TEST_FORMAT_CLS("{1,gmt,ftime='%D'}", a_datetime, "12/31/13"); } +/// Test formatting and parsing of uint64_t values that are not natively supported by ICU. +/// They use a custom code path which gets exercised by this. +void test_uint64_format() +{ +#ifdef BOOST_LOCALE_WITH_ICU + std::set tested_langs; + int32_t count; + auto* cur_locale = icu::Locale::getAvailableLocales(count); + constexpr uint64_t value = std::numeric_limits::max() + uint64_t(3); + const std::string posix_value = as_posix_string(value); + constexpr int32_t short_value = std::numeric_limits::max(); + const std::string posix_short_value = as_posix_string(short_value); + boost::locale::generator g; + const std::string utf8 = ".UTF-8"; + // Test with each language supported by ICU to ensure the implementation really + // is independent of the language and doesn't fail e.g. for different separators. + for(int i = 0; i < count; i++, cur_locale++) { + if(!tested_langs.insert(cur_locale->getLanguage()).second) + continue; + TEST_CONTEXT(cur_locale->getName()); + UErrorCode err{}; + std::unique_ptr fmt{icu::NumberFormat::createInstance(*cur_locale, err)}; + icu::UnicodeString s; + fmt->format(short_value, s, nullptr, err); + if(U_FAILURE(err)) + continue; + const std::string icu_value = boost::locale::conv::utf_to_utf(s.getBuffer(), s.getBuffer() + s.length()); + std::stringstream ss; + ss.imbue(g(cur_locale->getName() + utf8)); + ss << boost::locale::as::number; + // Sanity check + ss << short_value; + TEST_EQ(ss.str(), icu_value); + + // Assumption: Either both the int32 and uint64 values are in POSIX format, or neither are + // This is the case if separators are used and/or numbers are not ASCII + empty_stream(ss) << value; + if(icu_value == posix_short_value) + TEST_EQ(ss.str(), posix_value); + else + TEST_NE(ss.str(), posix_value); + + uint64_t parsed_value{}; + TEST(ss >> parsed_value); + TEST_EQ(parsed_value, value); + } +#endif +} + BOOST_LOCALE_DISABLE_UNREACHABLE_CODE_WARNING void test_main(int argc, char** argv) { @@ -867,6 +920,8 @@ void test_main(int argc, char** argv) std::cout << "ICU is not build... Skipping\n"; return; #endif + test_uint64_format(); + boost::locale::time_zone::global("GMT+4:00"); std::cout << "Testing char, UTF-8" << std::endl; test_manip(); From 3536525cff07c25e58f681ee3a623ffe14b6ea32 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 29 Dec 2024 14:26:10 +0100 Subject: [PATCH 57/67] Handle ICU version that keep parsed number in scientific format ICU might return 9223372036854775810 as 9.22337203685477581E+18 Use the internal parser of Boost.Charconv to handle this. --- src/icu/formatter.cpp | 50 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/src/icu/formatter.cpp b/src/icu/formatter.cpp index 978812ea..e60c58e5 100644 --- a/src/icu/formatter.cpp +++ b/src/icu/formatter.cpp @@ -14,8 +14,9 @@ #include "time_zone.hpp" #include "uconv.hpp" #include -#include +#include #include +#include #include #include #include @@ -37,6 +38,30 @@ namespace boost { namespace locale { namespace impl_icu { namespace { + + // Create lookup table where: powers_of_10[i] == 10**i + constexpr uint64_t pow10(unsigned exponent) + { + return (exponent == 0) ? 1 : pow10(exponent - 1) * 10u; + } + template + using array_if_true = typename std::enable_if>::type; + + template + constexpr array_if_true make_powers_of_10(Values... values) + { + return {values...}; + } + template + constexpr array_if_true make_powers_of_10(Values... values) + { + return make_powers_of_10(values..., pow10(sizeof...(Values))); + } + constexpr auto powers_of_10 = make_powers_of_10::digits10 + 1>(); + static_assert(powers_of_10[0] == 1u, "!"); + static_assert(powers_of_10[1] == 10u, "!"); + static_assert(powers_of_10[5] == 100000u, "!"); + // Set the min/max fraction digits for the NumberFormat void set_fraction_digits(icu::NumberFormat& nf, const std::ios_base::fmtflags how, std::streamsize precision) { @@ -80,6 +105,7 @@ namespace boost { namespace locale { namespace impl_icu { char buffer[std::numeric_limits::digits10 + 2]; auto res = boost::charconv::to_chars(buffer, std::end(buffer), value); BOOST_ASSERT(res); + BOOST_ASSERT(res.ptr < std::end(buffer)); *res.ptr = '\0'; // ICU expects a NULL-terminated string even for the StringPiece icu::UnicodeString tmp; UErrorCode err = U_ZERO_ERROR; @@ -120,8 +146,26 @@ namespace boost { namespace locale { namespace impl_icu { const auto decimals = fmt.getDecimalNumber(err); if(U_FAILURE(err)) return false; // Not a number - const auto res = boost::charconv::from_chars({decimals.data(), static_cast(decimals.length())}, v); - return static_cast(res); + + // Parse raw value as ICU might return 9223372036854775810 as 9.22337203685477581E+18 + uint64_t sig; + int32_t exp; + bool sign; + const char* end_decimal = decimals.data() + decimals.length(); + const auto res = boost::charconv::detail::parser(decimals.data(), end_decimal, sign, sig, exp); + // The ICU value should always be valid + BOOST_ASSERT_MSG(res, "Failed to parse integer representation of ICU"); + BOOST_ASSERT_MSG(res.ptr == end_decimal, "Did not fully parse integer representation of ICU"); + BOOST_ASSERT_MSG(exp >= 0, "ICU truncated parsed integer"); + // Gracefully handle if it is not the case, i.e. likely a bug in ICU + if(BOOST_UNLIKELY(!res || res.ptr != end_decimal || exp < 0)) + return false; // LCOV_EXCL_LINE + if(sign) // Negative value + return false; + if(exp >= powers_of_10.size()) + return false; // Out of range + v = sig * powers_of_10[exp]; + return true; } bool get_value(int32_t& v, icu::Formattable& fmt) const From ae0da496131883ba5e07900ad405399db5bba94f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Dec 2024 20:52:15 +0100 Subject: [PATCH 58/67] Avoid corner cases with ICUs scientific format parsing `boost::charconv::detail::parser` is not made for parsing (large) integers in exponential notation. It is mainly tested for parsing floating point numbers in hexadecimal format. Given we know ICU will output either an integer string or a number in "E notation" (1.2E2) we can convert that rather easily to a "regular" integer string by "moving" the dot to the right according to the exponent. The trailing gap is filled with zeros before passing it to `from_chars` which is now able to handle the range checks for us. This avoids overflows that can happen when multiplying the significant by the exponent which, due to integer arithmetic, would be cumbersome to guard against. Any situation that could yield a fractional or a too large value can be caught early. --- src/icu/formatter.cpp | 58 +------------ src/util/numeric_conversion.hpp | 80 +++++++++++++++++ test/test_util_numeric_convert.cpp | 134 ++++++++++++++++++++++++++++- 3 files changed, 216 insertions(+), 56 deletions(-) diff --git a/src/icu/formatter.cpp b/src/icu/formatter.cpp index e60c58e5..64f8e1e4 100644 --- a/src/icu/formatter.cpp +++ b/src/icu/formatter.cpp @@ -9,23 +9,21 @@ #include #include #include "../util/foreach_char.hpp" +#include "../util/numeric_conversion.hpp" #include "formatters_cache.hpp" #include "icu_util.hpp" #include "time_zone.hpp" #include "uconv.hpp" #include -#include +#include #include -#include #include #include -#include #ifdef BOOST_MSVC # pragma warning(push) # pragma warning(disable : 4251) // "identifier" : class "type" needs to have dll-interface... #endif #include -#include #include #include #include @@ -38,30 +36,6 @@ namespace boost { namespace locale { namespace impl_icu { namespace { - - // Create lookup table where: powers_of_10[i] == 10**i - constexpr uint64_t pow10(unsigned exponent) - { - return (exponent == 0) ? 1 : pow10(exponent - 1) * 10u; - } - template - using array_if_true = typename std::enable_if>::type; - - template - constexpr array_if_true make_powers_of_10(Values... values) - { - return {values...}; - } - template - constexpr array_if_true make_powers_of_10(Values... values) - { - return make_powers_of_10(values..., pow10(sizeof...(Values))); - } - constexpr auto powers_of_10 = make_powers_of_10::digits10 + 1>(); - static_assert(powers_of_10[0] == 1u, "!"); - static_assert(powers_of_10[1] == 10u, "!"); - static_assert(powers_of_10[5] == 100000u, "!"); - // Set the min/max fraction digits for the NumberFormat void set_fraction_digits(icu::NumberFormat& nf, const std::ios_base::fmtflags how, std::streamsize precision) { @@ -102,7 +76,7 @@ namespace boost { namespace locale { namespace impl_icu { return format(static_cast(value), code_points); // Fallback to using a StringPiece (decimal number) as input - char buffer[std::numeric_limits::digits10 + 2]; + char buffer[boost::charconv::limits::max_chars10 + 1]; auto res = boost::charconv::to_chars(buffer, std::end(buffer), value); BOOST_ASSERT(res); BOOST_ASSERT(res.ptr < std::end(buffer)); @@ -141,31 +115,7 @@ namespace boost { namespace locale { namespace impl_icu { v = static_cast(tmp); return true; } - // Get value as a decimal number and parse that - err = U_ZERO_ERROR; - const auto decimals = fmt.getDecimalNumber(err); - if(U_FAILURE(err)) - return false; // Not a number - - // Parse raw value as ICU might return 9223372036854775810 as 9.22337203685477581E+18 - uint64_t sig; - int32_t exp; - bool sign; - const char* end_decimal = decimals.data() + decimals.length(); - const auto res = boost::charconv::detail::parser(decimals.data(), end_decimal, sign, sig, exp); - // The ICU value should always be valid - BOOST_ASSERT_MSG(res, "Failed to parse integer representation of ICU"); - BOOST_ASSERT_MSG(res.ptr == end_decimal, "Did not fully parse integer representation of ICU"); - BOOST_ASSERT_MSG(exp >= 0, "ICU truncated parsed integer"); - // Gracefully handle if it is not the case, i.e. likely a bug in ICU - if(BOOST_UNLIKELY(!res || res.ptr != end_decimal || exp < 0)) - return false; // LCOV_EXCL_LINE - if(sign) // Negative value - return false; - if(exp >= powers_of_10.size()) - return false; // Out of range - v = sig * powers_of_10[exp]; - return true; + return util::try_parse_icu(fmt, v); } bool get_value(int32_t& v, icu::Formattable& fmt) const diff --git a/src/util/numeric_conversion.hpp b/src/util/numeric_conversion.hpp index 3efc50f5..7344aff3 100644 --- a/src/util/numeric_conversion.hpp +++ b/src/util/numeric_conversion.hpp @@ -10,6 +10,12 @@ #include #include #include +#include +#include +#include +#ifdef BOOST_LOCALE_WITH_ICU +# include +#endif namespace boost { namespace locale { namespace util { @@ -24,6 +30,80 @@ namespace boost { namespace locale { namespace util { const auto res = boost::charconv::from_chars(s, value); return res && res.ptr == (s.data() + s.size()); } + + /// Parse a string in scientific format to an integer. + /// In particular the "E notation" is used. + /// I.e. "\d.\d+E\d+", e.g. 5.12E3 == 5120; 5E2 == 500; 2E+1 == 20) + /// Additionally plain integers are recognized. + template + bool try_scientific_to_int(const core::string_view s, Integer& value) + { + static_assert(std::is_integral::value && std::is_unsigned::value, + "Must be an unsigned integer"); + if(s.size() < 3) // At least: iEj for E notation + return try_to_int(s, value); + if(s[0] == '-') + return false; + constexpr auto maxDigits = std::numeric_limits::digits10 + 1; + std::array buffer; + // Convert to a regular integer string without exponent or fractional + core::string_view string_value; + + const auto expPos = s.find('E', 1); + if(s[1] == '.') { // "Shift" the dot to right according to exponent + int8_t exponent; + if(BOOST_UNLIKELY(expPos == core::string_view::npos || !try_to_int(s.substr(expPos + 1), exponent))) + return false; + const auto numSignificantDigits = expPos - 1; // Exclude dot + const auto numDigits = exponent + 1u; // E0 -> 1 digit + if(BOOST_UNLIKELY(exponent < 0 || numDigits < numSignificantDigits)) + return false; // Fractional + else if(BOOST_UNLIKELY(numDigits > maxDigits)) + return false; // Too large + + // Copy to buffer excluding dot + buffer[0] = s[0]; + const auto bufPos = std::copy(s.begin() + 2, s.begin() + expPos, buffer.begin() + 1); + std::fill(bufPos, buffer.begin() + numDigits, '0'); + string_value = core::string_view(buffer.data(), numDigits); + } else { // Pad with zeros according to exponent + if(expPos == core::string_view::npos) + return try_to_int(s, value); // Shortcut: Regular integer + const core::string_view significant = s.substr(0, expPos); + int8_t exponent; + if(BOOST_UNLIKELY(!try_to_int(s.substr(expPos + 1), exponent))) + return false; + else if(BOOST_UNLIKELY(exponent < 0)) + return false; // Fractional + else if(BOOST_UNLIKELY(exponent == 0)) + string_value = significant; + else { + const auto numDigits = significant.size() + exponent; + if(BOOST_UNLIKELY(numDigits > maxDigits)) + return false; // Too large + else { + const auto bufPos = std::copy(significant.begin(), significant.end(), buffer.begin()); + std::fill(bufPos, buffer.begin() + numDigits, '0'); + string_value = core::string_view(buffer.data(), numDigits); + } + } + } + return try_to_int(string_value, value); + } + +#ifdef BOOST_LOCALE_WITH_ICU + template + bool try_parse_icu(icu::Formattable& fmt, Integer& value) + { + // Get value as a decimal number and parse that + UErrorCode err = U_ZERO_ERROR; + const auto decimals = fmt.getDecimalNumber(err); + if(U_FAILURE(err)) + return false; // Not a number + const core::string_view s(decimals.data(), decimals.length()); + return try_scientific_to_int(s, value); + } +#endif }}} // namespace boost::locale::util #endif diff --git a/test/test_util_numeric_convert.cpp b/test/test_util_numeric_convert.cpp index c30cfaf0..309a68e7 100644 --- a/test/test_util_numeric_convert.cpp +++ b/test/test_util_numeric_convert.cpp @@ -6,9 +6,14 @@ #include "../src/util/numeric_conversion.hpp" #include "boostLocale/test/tools.hpp" +#include +#include +#include #include -#include -#include +#ifdef BOOST_LOCALE_WITH_ICU +# include +# include +#endif void test_try_to_int() { @@ -54,7 +59,132 @@ void test_try_to_int() TEST(!try_to_int(ss.str(), v)); } +void test_try_scientific_to_int() +{ + using boost::locale::util::try_scientific_to_int; + + for(const auto s : {"255", "255E0", "2.55E2", "255E+0", "2.55E+2"}) { + TEST_CONTEXT(s); + uint8_t v; + if TEST(try_scientific_to_int(s, v)) + TEST_EQ(v, 255); + } + for(const auto s : {"0", "0E0", "0.0E1", "0.0E2", "0.00E2"}) { + TEST_CONTEXT(s); + uint8_t v; + if TEST(try_scientific_to_int(s, v)) + TEST_EQ(v, 0); + } + for(const auto s : {"3", "3E0", "0.3E1", "0.03E2"}) { + TEST_CONTEXT(s); + uint8_t v; + if TEST(try_scientific_to_int(s, v)) + TEST_EQ(v, 3); + } + for(const auto s : {"50", "5E1", "0.5E2"}) { + TEST_CONTEXT(s); + uint8_t v; + if TEST(try_scientific_to_int(s, v)) + TEST_EQ(v, 50); + } + for(const auto s : { + // clang-format off + "-1.1E1", + // Too large + "256", "256E0", "2.56E2", "3.0E2", + "1000","1000E0", "100E1", "10E2", "1E3","1.0E3", + // negative + "-1", "-1.1E1", + // Fractional + "1.1", "1.1E0", "1.01E1", "1.001E2", "1E-1", "1.1E-1", "0.1E-1", + // Possibly fractional: non-normalized E notation + "1.0", "1.0E0", "1.00E1", "1.000E2", + "0.0", "0.0E0", "0.00E1", "0.000E2", + "10E-1", "10.0E-1", "0.0E-1", + // Possibly fractional: Trailing zeros not covered by exponent + "1.10E1", "1.010E2", "0.10E1", "0.010E2", "0.0010E3", "0.00010E4", + // Bogus characters + "a", "aE0", "1.aE1", "a.1E1", "1Ea", "1.1Ea", + "1a", "1aE0", "1.1aE1", "1a.1E1", "1E1a", "1.1E1a", + "a1", "a1E0", "1.a1E1", "a1.1E1", "1Ea1", "1.1Ea1", + "1a1", "1a1E0", "1.1a1E1", "1a1.1E1", "1E1a1", "1.1E1a1", + // clang-format on + }) + { + TEST_CONTEXT(s); + uint8_t v; + TEST(!try_scientific_to_int(s, v)); + } +} + +#ifdef BOOST_LOCALE_WITH_ICU +template +void spotcheck_icu_parsing(icu::NumberFormat& parser, const T val) +{ + TEST_CONTEXT(val); + char buf[boost::charconv::limits::max_chars10]; + const auto r = boost::charconv::to_chars(buf, std::end(buf), val); + if(!TEST(r)) + return; // LCOV_EXCL_LINE + UChar u_buf[boost::charconv::limits::max_chars10]; + std::copy(buf, r.ptr, u_buf); + icu::UnicodeString tmp(false, u_buf, static_cast(r.ptr - buf)); + + icu::Formattable parsedByICU; + UErrorCode err = U_ZERO_ERROR; + parser.parse(tmp, parsedByICU, err); + if TEST(U_SUCCESS(err)) { + T parsedVal{}; + if TEST(boost::locale::util::try_parse_icu(parsedByICU, parsedVal)) + TEST_EQ(parsedVal, val); + } +} + +void test_try_parse_icu() +{ + UErrorCode err = U_ZERO_ERROR; + std::unique_ptr parser(icu::NumberFormat::createInstance(icu::Locale::getEnglish(), err)); + TEST_REQUIRE(U_SUCCESS(err)); + using limits = std::numeric_limits; + for(const uint64_t v : {limits::min(), limits::max(), limits::max() - 1, limits::max() / 2}) + spotcheck_icu_parsing(*parser, v); + for(uint64_t v = 10, c = 1; c < limits::max() / 10; c = v, v *= 10) { + spotcheck_icu_parsing(*parser, v); // 100[...]0 + const auto v1 = limits::max() / v; + const auto v2 = v1 * v; // Large with trailing zeros + spotcheck_icu_parsing(*parser, v1); + spotcheck_icu_parsing(*parser, v2); + } + // Unable to parse to uint64_t + const char16_t* bigVal1 = u"18446744073709551616"; // UINT64_MAX + 1 + const char16_t* bigVal2 = u"184467440737095516150"; // UINT64_MAX * 10 + const char16_t* signedVal1 = u"-9223372036854775808"; // INT64_MIN + const char16_t* signedVal2 = u"-9223372036854775809"; // INT64_MIN - 1 + for(const auto* v : {bigVal1, bigVal2, signedVal1, signedVal2}) { + TEST_CONTEXT(v); + // ICU < 59 doesn't use char16_t but another 2 byte type + icu::UnicodeString tmp(true, reinterpret_cast(v), -1); + + icu::Formattable parsedByICU; + parser->parse(tmp, parsedByICU, err); + if TEST(U_SUCCESS(err)) { + uint64_t parsedVal{}; + TEST(!boost::locale::util::try_parse_icu(parsedByICU, parsedVal)); + } + } + // Test some numbers randomly to find missed cases + std::mt19937_64 mt{std::random_device{}()}; + std::uniform_int_distribution distr; + for(int i = 0; i < 42; ++i) + spotcheck_icu_parsing(*parser, distr(mt)); +} +#endif + void test_main(int /*argc*/, char** /*argv*/) { test_try_to_int(); + test_try_scientific_to_int(); +#ifdef BOOST_LOCALE_WITH_ICU + test_try_parse_icu(); +#endif } From 2a23d33ca08c36f0cd2d3baf054638e92a69c36f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Sun, 5 Jan 2025 20:14:53 +0100 Subject: [PATCH 59/67] Avoid copy when parsing ICU number Instead of filling a temporary buffer we can decompose a number like "x.yyyEz" to "(x * 10^3 + yyy) * 10^(z - 3)" I.e. we subtract from the exponent what is required as an exponent to make the fractional into an integer significant. For the simple case of "xEz" we just do "x * 10^z". This requires additional range checks before multiplying but avoids extra memory accesses. --- src/util/numeric_conversion.hpp | 110 +++++++++++++++++++++----------- 1 file changed, 73 insertions(+), 37 deletions(-) diff --git a/src/util/numeric_conversion.hpp b/src/util/numeric_conversion.hpp index 7344aff3..b0f53460 100644 --- a/src/util/numeric_conversion.hpp +++ b/src/util/numeric_conversion.hpp @@ -8,16 +8,45 @@ #define BOOST_LOCALE_IMPL_UTIL_NUMERIC_CONVERSIONS_HPP #include +#include #include #include #include #include +#include #include #ifdef BOOST_LOCALE_WITH_ICU # include #endif namespace boost { namespace locale { namespace util { + namespace { + + // Create lookup table where: powers_of_10[i] == 10**i + constexpr uint64_t pow10(unsigned exponent) + { + return (exponent == 0) ? 1 : pow10(exponent - 1) * 10u; + } + template + using array_if_true = typename std::enable_if>::type; + + template + constexpr array_if_true make_powers_of_10(Values... values) + { + return {{values...}}; + } + template + constexpr array_if_true make_powers_of_10(Values... values) + { + return make_powers_of_10(values..., pow10(sizeof...(Values))); + } + constexpr auto powers_of_10 = make_powers_of_10::digits10 + 1>(); +#ifndef BOOST_NO_CXX14_CONSTEXPR + static_assert(powers_of_10[0] == 1u, "!"); + static_assert(powers_of_10[1] == 10u, "!"); + static_assert(powers_of_10[5] == 100000u, "!"); +#endif + } // namespace template bool try_to_int(core::string_view s, Integer& value) @@ -45,50 +74,57 @@ namespace boost { namespace locale { namespace util { if(s[0] == '-') return false; constexpr auto maxDigits = std::numeric_limits::digits10 + 1; - std::array buffer; - // Convert to a regular integer string without exponent or fractional - core::string_view string_value; const auto expPos = s.find('E', 1); - if(s[1] == '.') { // "Shift" the dot to right according to exponent - int8_t exponent; - if(BOOST_UNLIKELY(expPos == core::string_view::npos || !try_to_int(s.substr(expPos + 1), exponent))) - return false; - const auto numSignificantDigits = expPos - 1; // Exclude dot - const auto numDigits = exponent + 1u; // E0 -> 1 digit - if(BOOST_UNLIKELY(exponent < 0 || numDigits < numSignificantDigits)) + if(expPos == core::string_view::npos) + return (s[1] != '.') && try_to_int(s, value); // Shortcut: Regular integer + uint8_t exponent; // Negative exponent would be a fractional + if(BOOST_UNLIKELY(!try_to_int(s.substr(expPos + 1), exponent))) + return false; + + core::string_view significant = s.substr(0, expPos); + Integer significant_value; + if(s[1] == '.') { + const auto numSignificantDigits = significant.size() - 1u; // Exclude dot + const auto numDigits = exponent + 1u; // E0 -> 1 digit + if(BOOST_UNLIKELY(numDigits < numSignificantDigits)) return false; // Fractional else if(BOOST_UNLIKELY(numDigits > maxDigits)) return false; // Too large + // Factor to get from the fractional number to an integer + BOOST_ASSERT(numSignificantDigits - 1u < powers_of_10.size()); + const auto factor = static_cast(powers_of_10[numSignificantDigits - 1]); + exponent = static_cast(numDigits - numSignificantDigits); - // Copy to buffer excluding dot - buffer[0] = s[0]; - const auto bufPos = std::copy(s.begin() + 2, s.begin() + expPos, buffer.begin() + 1); - std::fill(bufPos, buffer.begin() + numDigits, '0'); - string_value = core::string_view(buffer.data(), numDigits); - } else { // Pad with zeros according to exponent - if(expPos == core::string_view::npos) - return try_to_int(s, value); // Shortcut: Regular integer - const core::string_view significant = s.substr(0, expPos); - int8_t exponent; - if(BOOST_UNLIKELY(!try_to_int(s.substr(expPos + 1), exponent))) - return false; - else if(BOOST_UNLIKELY(exponent < 0)) - return false; // Fractional - else if(BOOST_UNLIKELY(exponent == 0)) - string_value = significant; - else { - const auto numDigits = significant.size() + exponent; - if(BOOST_UNLIKELY(numDigits > maxDigits)) - return false; // Too large - else { - const auto bufPos = std::copy(significant.begin(), significant.end(), buffer.begin()); - std::fill(bufPos, buffer.begin() + numDigits, '0'); - string_value = core::string_view(buffer.data(), numDigits); - } + const unsigned firstDigit = significant[0] - '0'; + if(firstDigit > 9u) + return false; // Not a digit + if(numSignificantDigits == maxDigits) { + const auto maxFirstDigit = std::numeric_limits::max() / powers_of_10[maxDigits - 1]; + if(firstDigit > maxFirstDigit) + return false; } - } - return try_to_int(string_value, value); + significant.remove_prefix(2); + if(BOOST_UNLIKELY(!try_to_int(significant, significant_value))) + return false; + // firstDigit * factor + significant_value <= max + if(static_cast(firstDigit) > (std::numeric_limits::max() - significant_value) / factor) + return false; + significant_value += static_cast(firstDigit * factor); + } else if(BOOST_UNLIKELY(significant.size() + exponent > maxDigits)) + return false; + else if(BOOST_UNLIKELY(!try_to_int(significant, significant_value))) + return false; + // Add zeros if necessary + if(exponent > 0u) { + BOOST_ASSERT(exponent < powers_of_10.size()); + const auto factor = static_cast(powers_of_10[exponent]); + if(significant_value > std::numeric_limits::max() / factor) + return false; + value = significant_value * factor; + } else + value = significant_value; + return true; } #ifdef BOOST_LOCALE_WITH_ICU From 8019e889c47c7af45c6b85251f161cdcf1269407 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 Jan 2025 19:30:08 +0100 Subject: [PATCH 60/67] Remove left-over condition --- test/show_config.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/show_config.cpp b/test/show_config.cpp index e2dd8541..48721cc5 100644 --- a/test/show_config.cpp +++ b/test/show_config.cpp @@ -67,8 +67,6 @@ void test_main(int /*argc*/, char** /*argv*/) std::cout << "- std::basic_string: "; #ifdef __cpp_lib_char8_t std::cout << "yes\n"; -#elif defined(__cpp_lib_char8_t) - std::cout << "partial\n"; #else std::cout << "no\n"; #endif From d03c4ed3960005a3a108e651cc1a0fef5eebe61e Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 Jan 2025 19:29:15 +0100 Subject: [PATCH 61/67] Fix test failure for POSIX formatting when locale is not available Falls back to C locale where output doesn't match. --- test/test_posix_formatting.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_posix_formatting.cpp b/test/test_posix_formatting.cpp index c4eacf6a..cc50e238 100644 --- a/test/test_posix_formatting.cpp +++ b/test/test_posix_formatting.cpp @@ -186,8 +186,10 @@ void test_main(int /*argc*/, char** /*argv*/) TEST(v == "12345,45" || v == "12 345,45" || v == "12.345,45"); } } - test_format_large_number(); - test_parse_multi_number(); + if(has_posix_locale("en_US.UTF-8")) { + test_format_large_number(); + test_parse_multi_number(); + } } // boostinspect:noascii From 86e59b6c03d840349176285487fc42d838813c80 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 Jan 2025 19:35:27 +0100 Subject: [PATCH 62/67] Reduce verbosity of formatting_common test --- test/formatting_common.hpp | 44 ++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/test/formatting_common.hpp b/test/formatting_common.hpp index 2b0e5791..f76b1381 100644 --- a/test/formatting_common.hpp +++ b/test/formatting_common.hpp @@ -75,18 +75,19 @@ void test_parse_multi_number() { const auto locale = boost::locale::generator{}("en_US.UTF-8"); -#define BOOST_LOCALE_CALL_I(T, I) \ - std::cout << "\t" #I << std::endl; \ - test_parse_multi_number_by_char(locale); - -#define BOOST_LOCALE_CALL(T) \ - std::cout << "test_parse_multi_number " #T << std::endl; \ - BOOST_LOCALE_CALL_I(T, int16_t); \ - BOOST_LOCALE_CALL_I(T, uint16_t); \ - BOOST_LOCALE_CALL_I(T, int32_t); \ - BOOST_LOCALE_CALL_I(T, uint32_t); \ - BOOST_LOCALE_CALL_I(T, int64_t); \ - BOOST_LOCALE_CALL_I(T, uint64_t); +#define BOOST_LOCALE_CALL_I(T, I) \ + { \ + TEST_CONTEXT("type" #T << ':' << #I); \ + test_parse_multi_number_by_char(locale); \ + } + +#define BOOST_LOCALE_CALL(T) \ + BOOST_LOCALE_CALL_I(T, int16_t) \ + BOOST_LOCALE_CALL_I(T, uint16_t) \ + BOOST_LOCALE_CALL_I(T, int32_t) \ + BOOST_LOCALE_CALL_I(T, uint32_t) \ + BOOST_LOCALE_CALL_I(T, int64_t) \ + BOOST_LOCALE_CALL_I(T, uint64_t) BOOST_LOCALE_CALL(char); BOOST_LOCALE_CALL(wchar_t); @@ -118,14 +119,19 @@ void test_format_large_number() { const auto locale = boost::locale::generator{}("en_US.UTF-8"); - std::cout << "Testing char" << std::endl; - test_format_large_number_by_char(locale); - - std::cout << "Testing wchar_t" << std::endl; - test_format_large_number_by_char(locale); +#define BOOST_LOCALE_CALL(T) \ + { \ + TEST_CONTEXT("type " << #T); \ + test_format_large_number_by_char(locale); \ + } + BOOST_LOCALE_CALL(char); + BOOST_LOCALE_CALL(wchar_t); #ifdef BOOST_LOCALE_ENABLE_CHAR16_T - std::cout << "Testing char16_t" << std::endl; - test_format_large_number_by_char(locale); + BOOST_LOCALE_CALL(char16_t); #endif +#ifdef BOOST_LOCALE_ENABLE_CHAR32_T + BOOST_LOCALE_CALL(char32_t); +#endif +#undef BOOST_LOCALE_CALL } From d56bad6d6963b5b174da8c2fffef7e3e3fa62ff8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 15 Jan 2025 14:40:44 +0100 Subject: [PATCH 63/67] Add some tests for missed cases - Date formatting for UInt64 - Error cases - Practically unreachable cases --- src/util/numeric_conversion.hpp | 4 +++- test/test_formatting.cpp | 7 +++++-- test/test_util_numeric_convert.cpp | 7 ++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/util/numeric_conversion.hpp b/src/util/numeric_conversion.hpp index b0f53460..5cc15bf9 100644 --- a/src/util/numeric_conversion.hpp +++ b/src/util/numeric_conversion.hpp @@ -131,11 +131,13 @@ namespace boost { namespace locale { namespace util { template bool try_parse_icu(icu::Formattable& fmt, Integer& value) { + if(!fmt.isNumeric()) + return false; // Get value as a decimal number and parse that UErrorCode err = U_ZERO_ERROR; const auto decimals = fmt.getDecimalNumber(err); if(U_FAILURE(err)) - return false; // Not a number + return false; // Memory error LCOV_EXCL_LINE const core::string_view s(decimals.data(), decimals.length()); return try_scientific_to_int(s, value); } diff --git a/test/test_formatting.cpp b/test/test_formatting.cpp index c05ad6ea..b847a50a 100644 --- a/test/test_formatting.cpp +++ b/test/test_formatting.cpp @@ -412,6 +412,7 @@ void test_manip(std::string e_charset = "UTF-8") TEST_PARSE_FAILS(as::percent, "1", double); TEST_FMT_PARSE_1(as::currency, 1345, "$1,345.00"); + TEST_FMT_PARSE_1(as::currency, uint64_t(1345), "$1,345.00"); TEST_FMT_PARSE_1(as::currency, 1345.34, "$1,345.34"); TEST_PARSE_FAILS(as::currency, "$", double); @@ -434,6 +435,7 @@ void test_manip(std::string e_charset = "UTF-8") TEST_FMT_PARSE_3_2(as::date, as::date_medium, as::gmt, a_datetime, "Feb 5, 1970", a_date); TEST_FMT_PARSE_3_2(as::date, as::date_long, as::gmt, a_datetime, "February 5, 1970", a_date); TEST_FMT_PARSE_3_2(as::date, as::date_full, as::gmt, a_datetime, "Thursday, February 5, 1970", a_date); + TEST_FMT_PARSE_2_2(as::date, as::gmt, uint64_t(a_datetime), "Feb 5, 1970", uint64_t(a_date)); TEST_PARSE_FAILS(as::date >> as::date_short, "aa/bb/cc", double); @@ -886,7 +888,7 @@ void test_uint64_format() icu::UnicodeString s; fmt->format(short_value, s, nullptr, err); if(U_FAILURE(err)) - continue; + continue; // LCOV_EXCL_LINE const std::string icu_value = boost::locale::conv::utf_to_utf(s.getBuffer(), s.getBuffer() + s.length()); std::stringstream ss; ss.imbue(g(cur_locale->getName() + utf8)); @@ -897,9 +899,10 @@ void test_uint64_format() // Assumption: Either both the int32 and uint64 values are in POSIX format, or neither are // This is the case if separators are used and/or numbers are not ASCII + // All languages likely use separators so not running into the POSIX case is OK. empty_stream(ss) << value; if(icu_value == posix_short_value) - TEST_EQ(ss.str(), posix_value); + TEST_EQ(ss.str(), posix_value); // LCOV_EXCL_LINE else TEST_NE(ss.str(), posix_value); diff --git a/test/test_util_numeric_convert.cpp b/test/test_util_numeric_convert.cpp index 309a68e7..f09ccb75 100644 --- a/test/test_util_numeric_convert.cpp +++ b/test/test_util_numeric_convert.cpp @@ -91,7 +91,7 @@ void test_try_scientific_to_int() // clang-format off "-1.1E1", // Too large - "256", "256E0", "2.56E2", "3.0E2", + "256", "256E0", "2.56E2", "3.0E2", "3.00E2", "300", "399", "3.99E2", "999", "9.99E2", "1000","1000E0", "100E1", "10E2", "1E3","1.0E3", // negative "-1", "-1.1E1", @@ -172,6 +172,11 @@ void test_try_parse_icu() TEST(!boost::locale::util::try_parse_icu(parsedByICU, parsedVal)); } } + { + icu::Formattable invalidNumber{"Str"}; + uint64_t parsedVal{}; + TEST(!boost::locale::util::try_parse_icu(invalidNumber, parsedVal)); + } // Test some numbers randomly to find missed cases std::mt19937_64 mt{std::random_device{}()}; std::uniform_int_distribution distr; From 06a8affd2bb32e4211c3d09ed70d0ad8c60689cc Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 14 Jan 2025 20:04:52 +0100 Subject: [PATCH 64/67] Add fuzzing of try_scientific_to_int to CI --- .github/workflows/ci.yml | 10 +- fuzzing/Jamfile | 54 ++++++ fuzzing/fuzz_scientific_to_int.cpp | 37 ++++ .../scientific_to_int.txt | 173 ++++++++++++++++++ 4 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 fuzzing/Jamfile create mode 100644 fuzzing/fuzz_scientific_to_int.cpp create mode 100644 fuzzing/seedcorpus/fuzz_scientific_to_int/scientific_to_int.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da2f0443..c5aacb8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,6 +128,8 @@ jobs: # https://github.com/llvm/llvm-project/issues/59827: disabled 2b/23 for clang-17 with libstdc++13 in 24.04 - { compiler: clang-17, cxxstd: '11,14,17,20', os: ubuntu-24.04 } - { compiler: clang-18, cxxstd: '11,14,17,20,23,2c', os: ubuntu-24.04 } + - { name: Run code fuzzer, fuzzing: yes, + compiler: clang-18, cxxstd: '20', os: ubuntu-24.04, variant: debug, link: static } # libc++ - { compiler: clang-6.0, cxxstd: '11,14', os: ubuntu-22.04, container: 'ubuntu:18.04', stdlib: libc++, install: 'clang-6.0 libc++-dev libc++abi-dev' } @@ -296,7 +298,8 @@ jobs: # More entries can be added in the same way, see the B2_ARGS assignment in ci/enforce.sh for the possible keys. # B2_DEFINES: ${{matrix.defines}} # Variables set here (to non-empty) will override the top-level environment variables, e.g. - # B2_VARIANT: ${{matrix.variant}} + B2_VARIANT: ${{matrix.variant}} + B2_LINK: ${{matrix.link}} B2_UBSAN: ${{matrix.ubsan}} run: source ci/github/install.sh @@ -368,6 +371,11 @@ jobs: COVERITY_SCAN_NOTIFICATION_EMAIL: ${{ secrets.COVERITY_SCAN_NOTIFICATION_EMAIL }} COVERITY_SCAN_TOKEN: ${{ secrets.COVERITY_SCAN_TOKEN }} + - name: Run fuzzing + if: matrix.fuzzing + run: B2_TARGETS="libs/$SELF/fuzzing" ci/build.sh + env: {B2_FLAGS: -a} + windows: defaults: run: diff --git a/fuzzing/Jamfile b/fuzzing/Jamfile new file mode 100644 index 00000000..f003cf55 --- /dev/null +++ b/fuzzing/Jamfile @@ -0,0 +1,54 @@ +# Copyright (c) 2024 Matt Borland +# Copyright (c) 2025 Alexander Grund +# +# Distributed under the Boost Software License, Version 1.0. +# https://www.boost.org/LICENSE_1_0.txt. + + +import common ; +import regex ; + +local all_fuzzers = [ regex.replace-list + [ glob "fuzz_*.cpp" ] : ".cpp" : "" +] ; + +for local fuzzer in $(all_fuzzers) +{ + local fuzz_time = 60 ; + + # Create the output corpus directories + make /tmp/corpus/$(fuzzer) : : common.MkDir ; + make /tmp/mincorpus/$(fuzzer) : : common.MkDir ; + + # Build the fuzzer + exe $(fuzzer) + : + $(fuzzer).cpp + : requirements + on + speed + on + norecover + -fsanitize=fuzzer + -fsanitize=fuzzer + /boost/locale//boost_locale + ; + + # Run the fuzzer for a short while + run $(fuzzer) + : "seedcorpus/$(fuzzer) -max_total_time=$(fuzz_time)" + : target-name $(fuzzer)-fuzzing + : requirements + /tmp/corpus/$(fuzzer) + ; + + # Minimize the corpus + run $(fuzzer) + : "/tmp/mincorpus/$(fuzzer) /tmp/corpus/$(fuzzer) -merge=1" + : target-name $(fuzzer)-minimize-corpus + : requirements + $(fuzzer)-fuzzing + /tmp/corpus/$(fuzzer) + /tmp/mincorpus/$(fuzzer) + ; +} \ No newline at end of file diff --git a/fuzzing/fuzz_scientific_to_int.cpp b/fuzzing/fuzz_scientific_to_int.cpp new file mode 100644 index 00000000..e3a8d19f --- /dev/null +++ b/fuzzing/fuzz_scientific_to_int.cpp @@ -0,0 +1,37 @@ +// +// Copyright (c) 2025 Alexander Grund +// +// Distributed under the Boost Software License, Version 1.0. +// https://www.boost.org/LICENSE_1_0.txt + +#include "../src/util/numeric_conversion.hpp" + +#include +#include +#include +#include + +extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size) +{ + using boost::locale::util::try_scientific_to_int; + try { + const boost::core::string_view sv{reinterpret_cast(data), size}; + + uint8_t u8{}; + try_scientific_to_int(sv, u8); + + uint16_t u16{}; + try_scientific_to_int(sv, u16); + + uint32_t u32{}; + try_scientific_to_int(sv, u32); + + uint8_t u64{}; + try_scientific_to_int(sv, u64); + } catch(...) { + std::cerr << "Error with '" << data << "' (size " << size << ')' << std::endl; + std::terminate(); + } + + return 0; +} diff --git a/fuzzing/seedcorpus/fuzz_scientific_to_int/scientific_to_int.txt b/fuzzing/seedcorpus/fuzz_scientific_to_int/scientific_to_int.txt new file mode 100644 index 00000000..edf0cd6b --- /dev/null +++ b/fuzzing/seedcorpus/fuzz_scientific_to_int/scientific_to_int.txt @@ -0,0 +1,173 @@ +1 +1E0 +0.1E1 +0.01E2 +0.001E3 +10 +10E0 +1E1 +0.1E2 +0.01E3 +100 +1E2 +10E1 +0.1E3 +0.01E4 +123 +1.23E2 +12.3E1 +123E0 +0.123E3 +0 +0E0 +0.0E1 +0.000E3 +255 +255E0 +2.55E2 +25.5E1 +0.255E3 +50 +5E1 +0.5E2 +0.05E3 +450 +4.5E2 +45E1 +0.45E3 +-50 +-5E1 +-0.5E2 +-0.05E3 +250 +2.5E2 +25E1 +250E0 +0.25E3 +2.5E+2 +-700 +-7E2 +-70E1 +-700E0 +-0.7E3 +1234 +1.234E3 +123.4E1 +1234E0 +12.34E2 +0.01234E5 +123000 +123E3 +1.23E5 +12.3E4 +0.000123E9 +-0.0005E0 +-0.005E1 +-0.05E2 +-5E-4 +9999 +9.999E3 +99.99E2 +999.9E1 +9999E0 +1.5 +1.5E0 +0.15E1 +0.015E2 +-1 +-1E0 +-0.1E1 +-0.01E2 +-10 +-10E0 +-1E1 +-0.1E2 +-100 +-1E2 +-10E1 +-100E0 +45 +4.5E1 +0.45E2 +5678 +5.678E3 +56.78E2 +567.8E1 +5678E0 +999000 +9.99E5 +99.9E4 +999E3 +9990E2 +200 +2E2 +20E1 +200E0 +2.0E+2 +2147483648 +2.147483648E9 +4294967296 +4.294967296E9 +10000000000 +1E10 +18446744074.073709551615E9 +9223372036854775808 +9.223372036854775808E18 +100000000000 +1E11 +1000000000000 +1E12 +1234567890123 +1.234567890123E12 +9876543210000 +9.87654321E12 +9999999999999 +9.999999999999E12 +99999999999999 +9.9999999999999E13 +100000000000000 +1E14 +123456789012345 +1.23456789012345E14 +987654321098765 +9.87654321098765E14 +1000000000000000 +1E15 +1844674400000000000 +1.8446744E18 +9000000000000000000 +9E18 +9990000000000000000 +9.99E18 +10000000000000000000 +1E19 +12345678900000000000 +1.23456789E19 +9876543210000000000 +9.87654321E18 +18440000000000000000 +1.844E19 +900000000000000 +9E14 +1844674400000000 +1.8446744E15 +1844674407000000000 +1.844674407E18 +12300000000000000000 +1.23E19 +18446744073700000000 +1.84467440737E19 +9999999999999999999 +9.999999999999999999E18 +1844674400000000000 +1.8446744E15 +10.5E18 +100.0000E18 +5E +15E +225E +5E- +15E- +225E- +5e1 +5.5e1 From f6dce8fe0c268b5d5e11cad399f8780648681a41 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 15 Jan 2025 11:10:06 +0100 Subject: [PATCH 65/67] Fix corpus Pass absolute paths to corpus. The corpus needs to be single files with one sample each. Add a Python script that converts files with multiple samples into single files. --- fuzzing/Jamfile | 43 +++++++++++++++---- fuzzing/make-corpus.py | 29 +++++++++++++ .../{scientific_to_int.txt => input.txt} | 0 3 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 fuzzing/make-corpus.py rename fuzzing/seedcorpus/fuzz_scientific_to_int/{scientific_to_int.txt => input.txt} (100%) diff --git a/fuzzing/Jamfile b/fuzzing/Jamfile index f003cf55..4a1dc263 100644 --- a/fuzzing/Jamfile +++ b/fuzzing/Jamfile @@ -6,19 +6,45 @@ import common ; +import path ; +import python ; import regex ; +import toolset ; + +path-constant HERE : . ; local all_fuzzers = [ regex.replace-list [ glob "fuzz_*.cpp" ] : ".cpp" : "" ] ; +if ! [ python.configured ] +{ + using python ; +} + +.make-corpus-script = $(HERE)/make-corpus.py ; + +rule make-corpus ( target : sources + : properties * ) +{ + RUNNER on $(target) = [ path.native $(.make-corpus-script) ] ; +} +actions make-corpus +{ + "$(PYTHON:E=python)" "$(RUNNER)" $(<) $(>) +} +toolset.flags $(__name__).make-corpus PYTHON ; + for local fuzzer in $(all_fuzzers) { local fuzz_time = 60 ; + local corpus = /tmp/corpus/$(fuzzer) ; + local min_corpus = /tmp/mincorpus/$(fuzzer) ; + local seed_corpus = $(HERE)/seedcorpus/$(fuzzer) ; + local seed_files = [ glob "$(seed_corpus)/*" ] ; # Create the output corpus directories - make /tmp/corpus/$(fuzzer) : : common.MkDir ; - make /tmp/mincorpus/$(fuzzer) : : common.MkDir ; + make $(corpus) : $(seed_files) : make-corpus ; + make $(min_corpus) : : common.MkDir ; # Build the fuzzer exe $(fuzzer) @@ -36,19 +62,20 @@ for local fuzzer in $(all_fuzzers) # Run the fuzzer for a short while run $(fuzzer) - : "seedcorpus/$(fuzzer) -max_total_time=$(fuzz_time)" + : "$(corpus) -max_total_time=$(fuzz_time)" : target-name $(fuzzer)-fuzzing : requirements - /tmp/corpus/$(fuzzer) + $(corpus) ; # Minimize the corpus run $(fuzzer) - : "/tmp/mincorpus/$(fuzzer) /tmp/corpus/$(fuzzer) -merge=1" + : "$(min_corpus) $(corpus) -merge=1" : target-name $(fuzzer)-minimize-corpus : requirements $(fuzzer)-fuzzing - /tmp/corpus/$(fuzzer) - /tmp/mincorpus/$(fuzzer) + $(corpus) + $(min_corpus) ; -} \ No newline at end of file +} + diff --git a/fuzzing/make-corpus.py b/fuzzing/make-corpus.py new file mode 100644 index 00000000..6ac7d035 --- /dev/null +++ b/fuzzing/make-corpus.py @@ -0,0 +1,29 @@ +#!/bin/env python + +import os +import sys + + +def get_samples(input_files): + for file_name in input_files: + if not os.path.isfile(file_name): + raise RuntimeError("Not a file: " + file_name) + with open(file_name, 'r') as input_file: + yield from input_file + + +def process_files(output_folder, input_files): + if not os.path.exists(output_folder): + os.makedirs(output_folder) + + for i, sample in enumerate(get_samples(input_files)): + with open(os.path.join(output_folder, str(i) + ".txt"), 'w') as output_file: + output_file.write(sample) + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python script.py [ ...]") + sys.exit(1) + + process_files(output_folder=sys.argv[1], input_files=sys.argv[2:]) diff --git a/fuzzing/seedcorpus/fuzz_scientific_to_int/scientific_to_int.txt b/fuzzing/seedcorpus/fuzz_scientific_to_int/input.txt similarity index 100% rename from fuzzing/seedcorpus/fuzz_scientific_to_int/scientific_to_int.txt rename to fuzzing/seedcorpus/fuzz_scientific_to_int/input.txt From 325da03250e2470e9998786a9d805ebd083d7d10 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 15 Jan 2025 11:28:49 +0100 Subject: [PATCH 66/67] Show input data as string on failure --- fuzzing/fuzz_scientific_to_int.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fuzzing/fuzz_scientific_to_int.cpp b/fuzzing/fuzz_scientific_to_int.cpp index e3a8d19f..133ecf88 100644 --- a/fuzzing/fuzz_scientific_to_int.cpp +++ b/fuzzing/fuzz_scientific_to_int.cpp @@ -13,10 +13,9 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size) { + const boost::core::string_view sv{reinterpret_cast(data), size}; using boost::locale::util::try_scientific_to_int; try { - const boost::core::string_view sv{reinterpret_cast(data), size}; - uint8_t u8{}; try_scientific_to_int(sv, u8); @@ -29,7 +28,7 @@ extern "C" int LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size uint8_t u64{}; try_scientific_to_int(sv, u64); } catch(...) { - std::cerr << "Error with '" << data << "' (size " << size << ')' << std::endl; + std::cerr << "Error with '" << sv << "' (size " << size << ')' << std::endl; std::terminate(); } From bd9eab28702c5e199e688c7af682a574d62a9e4a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 15 Jan 2025 16:09:02 +0100 Subject: [PATCH 67/67] Fuzzing: Add quoting and dependency on script --- fuzzing/Jamfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fuzzing/Jamfile b/fuzzing/Jamfile index 4a1dc263..c18db719 100644 --- a/fuzzing/Jamfile +++ b/fuzzing/Jamfile @@ -30,7 +30,7 @@ rule make-corpus ( target : sources + : properties * ) } actions make-corpus { - "$(PYTHON:E=python)" "$(RUNNER)" $(<) $(>) + "$(PYTHON:E=python)" "$(RUNNER)" "$(<)" "$(>)" } toolset.flags $(__name__).make-corpus PYTHON ; @@ -43,7 +43,7 @@ for local fuzzer in $(all_fuzzers) local seed_files = [ glob "$(seed_corpus)/*" ] ; # Create the output corpus directories - make $(corpus) : $(seed_files) : make-corpus ; + make $(corpus) : $(seed_files) : make-corpus : $(.make-corpus-script) ; make $(min_corpus) : : common.MkDir ; # Build the fuzzer