diff --git a/.gitmodules b/.gitmodules index b85a7cf22..dc10059cc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -37,3 +37,6 @@ [submodule "3rdparty/any"] path = 3rdparty/any url = https://github.com/thelink2012/any.git +[submodule "3rdparty/utfcpp"] + path = 3rdparty/utfcpp + url = https://github.com/nemtrif/utfcpp.git diff --git a/3rdparty/LICENSE.3rdparty b/3rdparty/LICENSE.3rdparty index 2300a7b7a..13d417803 100644 --- a/3rdparty/LICENSE.3rdparty +++ b/3rdparty/LICENSE.3rdparty @@ -352,6 +352,34 @@ permanent authorization for you to choose that version for the Library. +================================================================================ +utfcpp +================================================================================ + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + ================================================================================ utf8proc ================================================================================ diff --git a/3rdparty/utfcpp b/3rdparty/utfcpp new file mode 160000 index 000000000..b26a5f718 --- /dev/null +++ b/3rdparty/utfcpp @@ -0,0 +1 @@ +Subproject commit b26a5f718f4f370af1852a0d5c6ae8fa031ba7d0 diff --git a/doc/autogen/spicy-types.spicy b/doc/autogen/spicy-types.spicy index bda2e2c68..b5a6dfb82 100644 --- a/doc/autogen/spicy-types.spicy +++ b/doc/autogen/spicy-types.spicy @@ -54,8 +54,10 @@ Specifies the character set for bytes encoding/decoding. .. spicy-code:: type Charset = enum { - ASCII, - UTF8 + ASCII, # ASCII encoding + UTF8, # UTF8 encoding + UTF16LE, # UTF16 little endian encoding + UTF16BE, # UTF16 big endian encoding }; .. _spicy_decodeerrorstrategy: @@ -67,9 +69,9 @@ Specifies how data is handled that's not representable in a specified character .. spicy-code:: type DecodeErrorStrategy = enum { - IGNORE, # data is skipped but processing continues - REPLACE, # data is replaced with a valid place-holder and processing continues - STRICT # runtime error is triggered + IGNORE, # data is skipped but processing continues + REPLACE, # data is replaced with a valid place-holder and processing continues + STRICT # runtime error is triggered }; .. _spicy_matchstate: diff --git a/doc/autogen/types/string.rst b/doc/autogen/types/string.rst index b3624ef8a..5d9b7c7f0 100644 --- a/doc/autogen/types/string.rst +++ b/doc/autogen/types/string.rst @@ -1,6 +1,6 @@ .. rubric:: Methods -.. spicy:method:: string::encode string encode False bytes ([ charset: spicy::Charset = spicy::Charset::UTF8 ]) +.. spicy:method:: string::encode string encode False bytes ([ charset: spicy::Charset = spicy::Charset::UTF8 ], [ errors: spicy::DecodeErrorStrategy = spicy::DecodeErrorStrategy::REPLACE ]) Converts the string into a binary representation encoded with the given character set. diff --git a/hilti/lib/hilti.hlt b/hilti/lib/hilti.hlt index 021352d22..ba176bfac 100644 --- a/hilti/lib/hilti.hlt +++ b/hilti/lib/hilti.hlt @@ -11,8 +11,8 @@ public type Side = enum { Left, Right, Both } &cxxname="hilti::rt::bytes::Side"; public type AddressFamily = enum { IPv4, IPv6 } &cxxname="hilti::rt::AddressFamily"; public type RealType = enum { IEEE754_Single, IEEE754_Double } &cxxname="hilti::rt::real::Type"; public type Protocol = enum { TCP, UDP, ICMP } &cxxname="hilti::rt::Protocol"; -public type Charset = enum { ASCII, UTF8 } &cxxname="hilti::rt::bytes::Charset"; -public type DecodeErrorStrategy = enum { IGNORE, REPLACE, STRICT } &cxxname="hilti::rt::bytes::DecodeErrorStrategy"; +public type Charset = enum { ASCII, UTF8, UTF16LE, UTF16BE } &cxxname="hilti::rt::unicode::Charset"; +public type DecodeErrorStrategy = enum { IGNORE, REPLACE, STRICT } &cxxname="hilti::rt::unicode::DecodeErrorStrategy"; public type Captures = vector; public type Profiler = __library_type("hilti::rt::Profiler"); diff --git a/hilti/runtime/CMakeLists.txt b/hilti/runtime/CMakeLists.txt index cdcec680a..dee8b7836 100644 --- a/hilti/runtime/CMakeLists.txt +++ b/hilti/runtime/CMakeLists.txt @@ -46,6 +46,7 @@ set(SOURCES src/types/stream.cc src/types/string.cc src/types/time.cc + src/unicode.cc src/util.cc src/version.cc ${PROJECT_SOURCE_DIR}/3rdparty/utf8proc/utf8proc.c) @@ -67,8 +68,10 @@ foreach (lib hilti-rt hilti-rt-debug) add_dependencies(${lib}-objects fiber) target_include_directories( - ${lib}-objects PRIVATE ${PROJECT_SOURCE_DIR}/3rdparty/fiber/include - ${PROJECT_SOURCE_DIR}/3rdparty/fiber/deps/cxx-header-utils/include) + ${lib}-objects + PRIVATE ${PROJECT_SOURCE_DIR}/3rdparty/fiber/include + ${PROJECT_SOURCE_DIR}/3rdparty/fiber/deps/cxx-header-utils/include + ${PROJECT_SOURCE_DIR}/3rdparty/utfcpp/source) add_library(${lib} STATIC) target_link_libraries(${lib} ${lib}-objects) diff --git a/hilti/runtime/include/types/bytes.h b/hilti/runtime/include/types/bytes.h index 0699485df..58594f1fa 100644 --- a/hilti/runtime/include/types/bytes.h +++ b/hilti/runtime/include/types/bytes.h @@ -15,9 +15,9 @@ #include #include #include -#include #include #include +#include #include namespace hilti::rt { @@ -38,12 +38,6 @@ HILTI_RT_ENUM_WITH_DEFAULT(Side, Left, Both // left and right side ); -/** For bytes decoding, which character set to use. */ -HILTI_RT_ENUM(Charset, Undef, UTF8, ASCII); - -/** For bytes decoding, how to handle decoding errors. */ -using DecodeErrorStrategy = string::DecodeErrorStrategy; - /** * Safe bytes iterator traversing the content of an instance. * @@ -257,17 +251,10 @@ class Bytes : protected std::string { using C = std::shared_ptr; /** - * Creates a bytes instance from a raw string representation - * encoded in a specified character set. - * - * @param s raw data - * @param cs character set the raw data is assumed to be encoded in - * @param errors how to handle errors when decoding the data - * @return bytes instances encoding *s* in character set *cs* + * Creates a bytes instance from a raw string representation. */ - Bytes(std::string s, bytes::Charset cs, bytes::DecodeErrorStrategy errors = bytes::DecodeErrorStrategy::REPLACE); + Bytes(Base s) : Base(std::move(s)) {} - Bytes(Base&& str) : Base(std::move(str)) {} Bytes(const Bytes& xs) : Base(xs) {} Bytes(Bytes&& xs) noexcept : Base(std::move(xs)) {} @@ -442,8 +429,8 @@ class Bytes : protected std::string { * @param errors how to handle errors when decoding the data * @return UTF8 string */ - std::string decode(bytes::Charset cs, - bytes::DecodeErrorStrategy errors = bytes::DecodeErrorStrategy::REPLACE) const; + std::string decode(unicode::Charset cs, + unicode::DecodeErrorStrategy errors = unicode::DecodeErrorStrategy::REPLACE) const; /** Returns true if the data begins with a given, other bytes instance. */ bool startsWith(const Bytes& b) const { return hilti::rt::startsWith(*this, b); } @@ -457,9 +444,7 @@ class Bytes : protected std::string { * @param errors how to handle errors when decoding/encoding the data * @return an upper case version of the instance */ - Bytes upper(bytes::Charset cs, bytes::DecodeErrorStrategy errors = bytes::DecodeErrorStrategy::REPLACE) const { - return Bytes(hilti::rt::string::upper(decode(cs, errors), errors), cs, errors); - } + Bytes upper(unicode::Charset cs, unicode::DecodeErrorStrategy errors = unicode::DecodeErrorStrategy::REPLACE) const; /** * Returns an upper-case version of the instance. @@ -468,9 +453,7 @@ class Bytes : protected std::string { * @param errors how to handle errors when decoding/encoding the data * @return a lower case version of the instance */ - Bytes lower(bytes::Charset cs, bytes::DecodeErrorStrategy errors = bytes::DecodeErrorStrategy::REPLACE) const { - return Bytes(hilti::rt::string::lower(decode(cs, errors), errors), cs, errors); - } + Bytes lower(unicode::Charset cs, unicode::DecodeErrorStrategy errors = unicode::DecodeErrorStrategy::REPLACE) const; /** * Removes leading and/or trailing sequences of all characters of a set @@ -685,8 +668,6 @@ inline std::string detail::to_string_for_print(const Bytes& x) { namespace detail::adl { std::string to_string(const Bytes& x, adl::tag /*unused*/); std::string to_string(const bytes::Side& x, adl::tag /*unused*/); -std::string to_string(const bytes::Charset& x, adl::tag /*unused*/); -std::string to_string(const bytes::DecodeErrorStrategy& x, adl::tag /*unused*/); } // namespace detail::adl } // namespace hilti::rt diff --git a/hilti/runtime/include/types/string.h b/hilti/runtime/include/types/string.h index 69e840a3b..f4b960b3b 100644 --- a/hilti/runtime/include/types/string.h +++ b/hilti/runtime/include/types/string.h @@ -9,18 +9,14 @@ #include #include #include +#include #include namespace hilti::rt { -namespace string { +class Bytes; -/* When processing UTF8, how to handle invalid data not representing UTF8 codepoints. */ -HILTI_RT_ENUM_WITH_DEFAULT(DecodeErrorStrategy, IGNORE, - IGNORE, // skip data - REPLACE, // replace with a place-holder - STRICT // throw a runtime error -); +namespace string { /** * Computes the length of a UTF8 string in number of codepoints. @@ -30,7 +26,8 @@ HILTI_RT_ENUM_WITH_DEFAULT(DecodeErrorStrategy, IGNORE, * @return the length of the input string * @throws RuntimeError if the input is not a valid UTF8 string */ -integer::safe size(const std::string& s, DecodeErrorStrategy errors = DecodeErrorStrategy::REPLACE); +integer::safe size(const std::string& s, + unicode::DecodeErrorStrategy errors = unicode::DecodeErrorStrategy::REPLACE); /** * Computes a lower-case version of an UTF8 string. @@ -40,7 +37,7 @@ integer::safe size(const std::string& s, DecodeErrorStrategy errors = * @return a lower-case version of the input string * @throws RuntimeError if the input is not a valid UTF8 string */ -std::string lower(std::string_view s, DecodeErrorStrategy errors = DecodeErrorStrategy::REPLACE); +std::string lower(std::string_view s, unicode::DecodeErrorStrategy errors = unicode::DecodeErrorStrategy::REPLACE); /** * Computes a upper-case version of an UTF8 string. @@ -50,7 +47,7 @@ std::string lower(std::string_view s, DecodeErrorStrategy errors = DecodeErrorSt * @return a upper-case version of the input string * @throws RuntimeError if the input is not a valid UTF8 string */ -std::string upper(std::string_view s, DecodeErrorStrategy errors = DecodeErrorStrategy::REPLACE); +std::string upper(std::string_view s, unicode::DecodeErrorStrategy errors = unicode::DecodeErrorStrategy::REPLACE); /** * Splits the string at sequences of whitespace. @@ -88,6 +85,18 @@ std::tuple split1(const std::string& s); */ std::tuple split1(const std::string& s, const std::string& sep); +/** + * Creates a bytes instance from a raw string representation + * encoded in a specified character set. + * + * @param s raw data + * @param cs character set the raw data is assumed to be encoded in + * @param errors how to handle errors when decoding the data + * @return bytes instances encoding *s* in character set *cs* + */ +rt::Bytes encode(std::string s, unicode::Charset cs, + unicode::DecodeErrorStrategy errors = unicode::DecodeErrorStrategy::REPLACE); + } // namespace string namespace detail::adl { diff --git a/hilti/runtime/include/unicode.h b/hilti/runtime/include/unicode.h new file mode 100644 index 000000000..73349a3a3 --- /dev/null +++ b/hilti/runtime/include/unicode.h @@ -0,0 +1,33 @@ +// Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details. + +#pragma once + +#include + +#include +#include + +namespace hilti::rt { + +namespace unicode { + +/* When processing unicode, how to handle invalid data not representing unicode codepoints. */ +HILTI_RT_ENUM_WITH_DEFAULT(DecodeErrorStrategy, IGNORE, + IGNORE, // skip data + REPLACE, // replace with a place-holder + STRICT // throw a runtime error +); + +/** For bytes decoding, which character set to use. */ +HILTI_RT_ENUM(Charset, Undef, UTF8, UTF16LE, UTF16BE, ASCII); + +constexpr uint32_t REPLACEMENT_CHARACTER = 0x0000FFFD; + +} // namespace unicode + +namespace detail::adl { +std::string to_string(const unicode::DecodeErrorStrategy& x, adl::tag /*unused*/); +std::string to_string(const unicode::Charset& x, adl::tag /*unused*/); +} // namespace detail::adl + +} // namespace hilti::rt diff --git a/hilti/runtime/include/util.h b/hilti/runtime/include/util.h index 263ec46e2..0f9ee4874 100644 --- a/hilti/runtime/include/util.h +++ b/hilti/runtime/include/util.h @@ -240,7 +240,7 @@ std::string replace(std::string s, std::string_view o, std::string_view n); * * \note This function is not UTF8-aware. */ -bool startsWith(const std::string& s, const std::string& prefix); +bool startsWith(std::string_view s, std::string_view prefix); /** * Python-style enumerate() that returns an iterable yielding pairs `(index, diff --git a/hilti/runtime/src/tests/bytes.cc b/hilti/runtime/src/tests/bytes.cc index 954d9891c..189e7b89f 100644 --- a/hilti/runtime/src/tests/bytes.cc +++ b/hilti/runtime/src/tests/bytes.cc @@ -10,6 +10,7 @@ #include #include +using namespace std::string_literals; using namespace hilti::rt; using namespace hilti::rt::bytes; @@ -39,46 +40,51 @@ TEST_CASE("at") { CHECK_THROWS_WITH_AS(*b.at(5), "index 5 out of bounds", const IndexError&); } -TEST_CASE("construct") { - CHECK_EQ(Bytes("123", Enum(bytes::Charset::ASCII)).str(), "123"); - CHECK_EQ(Bytes("abc", Enum(bytes::Charset::ASCII)).str(), "abc"); - CHECK_EQ(Bytes("abc", Enum(bytes::Charset::UTF8)).str(), "abc"); - - CHECK_EQ(Bytes("\xF0\x9F\x98\x85", Enum(bytes::Charset::UTF8)).str(), "\xF0\x9F\x98\x85"); - CHECK_EQ(Bytes("\xc3\x28", Enum(bytes::Charset::UTF8), bytes::DecodeErrorStrategy::REPLACE).str(), "\ufffd("); - CHECK_EQ(Bytes("\xc3\x28", Enum(bytes::Charset::UTF8), bytes::DecodeErrorStrategy::IGNORE).str(), "("); - CHECK_THROWS_WITH_AS(Bytes("\xc3\x28", Enum(bytes::Charset::UTF8), bytes::DecodeErrorStrategy::STRICT).str(), - "illegal UTF8 sequence in string", const RuntimeError&); - - CHECK_EQ(Bytes("\xF0\x9F\x98\x85", Enum(bytes::Charset::ASCII), bytes::DecodeErrorStrategy::REPLACE).str(), "????"); - CHECK_EQ(Bytes("\xF0\x9F\x98\x85", Enum(bytes::Charset::ASCII), bytes::DecodeErrorStrategy::IGNORE).str(), ""); - CHECK_THROWS_WITH_AS(Bytes("\xF0\x9F\x98\x85", Enum(bytes::Charset::ASCII), bytes::DecodeErrorStrategy::STRICT) - .str(), - "illegal ASCII character in string", const RuntimeError&); - - // NOLINTNEXTLINE(bugprone-throw-keyword-missing) - CHECK_THROWS_WITH_AS(Bytes("123", Enum(bytes::Charset::Undef)), "unknown character set for encoding", - const RuntimeError&); -} - TEST_CASE("decode") { - CHECK_EQ("123"_b.decode(bytes::Charset::ASCII), "123"); - CHECK_EQ("abc"_b.decode(bytes::Charset::ASCII), "abc"); - CHECK_EQ("abc"_b.decode(bytes::Charset::UTF8), "abc"); - CHECK_EQ("\xF0\x9F\x98\x85"_b.decode(bytes::Charset::UTF8), "\xF0\x9F\x98\x85"); - CHECK_EQ("\xF0\x9F\x98\x85"_b.decode(bytes::Charset::ASCII), "????"); - - CHECK_EQ("€100"_b.decode(bytes::Charset::ASCII, bytes::DecodeErrorStrategy::REPLACE), "???100"); - CHECK_EQ("€100"_b.decode(bytes::Charset::ASCII, bytes::DecodeErrorStrategy::IGNORE), "100"); - CHECK_THROWS_WITH_AS("123ä4"_b.decode(bytes::Charset::ASCII, bytes::DecodeErrorStrategy::STRICT), + CHECK_EQ("123"_b.decode(unicode::Charset::ASCII), "123"); + CHECK_EQ("abc"_b.decode(unicode::Charset::ASCII), "abc"); + CHECK_EQ("abc"_b.decode(unicode::Charset::UTF8), "abc"); + CHECK_EQ("\xF0\x9F\x98\x85"_b.decode(unicode::Charset::UTF8), "\xF0\x9F\x98\x85"); + CHECK_EQ("\xF0\x9F\x98\x85"_b.decode(unicode::Charset::ASCII), "????"); + + CHECK_EQ("€100"_b.decode(unicode::Charset::ASCII, unicode::DecodeErrorStrategy::REPLACE), "???100"); + CHECK_EQ("€100"_b.decode(unicode::Charset::ASCII, unicode::DecodeErrorStrategy::IGNORE), "100"); + CHECK_THROWS_WITH_AS("123ä4"_b.decode(unicode::Charset::ASCII, unicode::DecodeErrorStrategy::STRICT), "illegal ASCII character in string", const RuntimeError&); - CHECK_EQ("\xc3\x28"_b.decode(bytes::Charset::UTF8, bytes::DecodeErrorStrategy::REPLACE), "\ufffd("); - CHECK_EQ("\xc3\x28"_b.decode(bytes::Charset::UTF8, bytes::DecodeErrorStrategy::IGNORE), "("); - CHECK_THROWS_WITH_AS("\xc3\x28"_b.decode(bytes::Charset::UTF8, bytes::DecodeErrorStrategy::STRICT), + CHECK_EQ("\xc3\x28"_b.decode(unicode::Charset::UTF8, unicode::DecodeErrorStrategy::REPLACE), "\ufffd("); + CHECK_EQ("\xc3\x28"_b.decode(unicode::Charset::UTF8, unicode::DecodeErrorStrategy::IGNORE), "("); + CHECK_THROWS_WITH_AS("\xc3\x28"_b.decode(unicode::Charset::UTF8, unicode::DecodeErrorStrategy::STRICT), "illegal UTF8 sequence in string", const RuntimeError&); - CHECK_THROWS_WITH_AS("123"_b.decode(bytes::Charset::Undef), "unknown character set for decoding", + CHECK_EQ(Bytes("\0a\0b\0c"s).decode(unicode::Charset::UTF16BE, unicode::DecodeErrorStrategy::STRICT), "abc"); + CHECK_EQ(Bytes("a\0b\0c\0"s).decode(unicode::Charset::UTF16LE, unicode::DecodeErrorStrategy::STRICT), "abc"); + + // Our `decode` of UTF-16 bytes returns UTF8 string with BOM if they do not fit into ASCII, see e.g., + // https://stackoverflow.com/questions/2223882/whats-the-difference-between-utf-8-and-utf-8-with-bom. + // To compute the expected results in Python encode with `utf_8_sig` encoding. + // + // LHS is an UTF16 encoding of '東京', RHS UTF8 with BOM. + CHECK_EQ("\xff\xfeqg\xacN"_b.decode(unicode::Charset ::UTF16LE, unicode::DecodeErrorStrategy::STRICT), + "\ufeff東京"); + + // Decoding of UTF16 with BOM. The byte order in the charset is just a hint, but we still decode as UTF16. + CHECK_EQ("\xff\xfeqg\xacN"_b.decode(unicode::Charset ::UTF16BE, unicode::DecodeErrorStrategy::STRICT), + "\ufeff東京"); + + // Decoding of too few bytes for UTF16 (expected even number, provided uneven). + CHECK_THROWS_WITH_AS(Bytes("\0a\0b\0"s).decode(unicode::Charset::UTF16BE, unicode::DecodeErrorStrategy::STRICT), + "illegal UTF16 character in string", const RuntimeError&); + CHECK_EQ(Bytes("\0a\0b\0"s).decode(unicode::Charset::UTF16BE, unicode::DecodeErrorStrategy::IGNORE), "ab"); + CHECK_EQ(Bytes("\0a\0b\0"s).decode(unicode::Charset::UTF16BE, unicode::DecodeErrorStrategy::REPLACE), "ab\ufffd"); + + // Our UTF16 implementation seems to differ in what it considers invalid encodings, e.g., `\x00\xd8` is rejected by + // python-3.1[1-3], but accepted by us. + // + // TODO(bbannier): Test rejection of invalid UTF16 (but with even length). + CHECK_EQ(Bytes("\x00\xd8").decode(unicode::Charset::UTF16LE, unicode::DecodeErrorStrategy::STRICT), ""); + + CHECK_THROWS_WITH_AS("123"_b.decode(unicode::Charset::Undef), "unknown character set for decoding", const RuntimeError&); } @@ -206,14 +212,21 @@ TEST_CASE("join") { } TEST_CASE("lower") { - CHECK_EQ("ABC123"_b.lower(bytes::Charset::UTF8).str(), "abc123"); - CHECK_EQ("ABC123"_b.lower(bytes::Charset::ASCII).str(), "abc123"); - CHECK_EQ("Gänsefüßchen"_b.lower(bytes::Charset::UTF8).str(), "gänsefüßchen"); - CHECK_EQ("Gänsefüßchen"_b.lower(bytes::Charset::ASCII).str(), "g??nsef????chen"); + CHECK_EQ("ABC123"_b.lower(unicode::Charset::UTF8).str(), "abc123"); + CHECK_EQ("ABC123"_b.lower(unicode::Charset::ASCII).str(), "abc123"); + CHECK_EQ("Gänsefüßchen"_b.lower(unicode::Charset::UTF8).str(), "gänsefüßchen"); + CHECK_EQ("Gänsefüßchen"_b.lower(unicode::Charset::ASCII).str(), "g??nsef????chen"); // NOLINTNEXTLINE(bugprone-throw-keyword-missing) - CHECK_THROWS_WITH_AS("123"_b.lower(bytes::Charset::Undef), "unknown character set for decoding", + CHECK_THROWS_WITH_AS("123"_b.lower(unicode::Charset::Undef), "unknown character set for decoding", const RuntimeError&); + + // No case change expected for these Japanese codepoints. + const auto tokio8 = "東京"_b; + CHECK_EQ(tokio8.lower(unicode::Charset::UTF16LE, unicode::DecodeErrorStrategy::STRICT), tokio8); + + const auto tokio16 = "\xff\xfeqg\xacN"_b; // 東京 in UTF16LE. + CHECK_EQ(tokio16.lower(unicode::Charset::UTF16LE, unicode::DecodeErrorStrategy::STRICT), tokio16); } TEST_CASE("match") { @@ -505,14 +518,24 @@ TEST_CASE("toTime") { } TEST_CASE("upper") { - CHECK_EQ("abc123"_b.upper(bytes::Charset::UTF8).str(), "ABC123"); - CHECK_EQ("abc123"_b.upper(bytes::Charset::ASCII).str(), "ABC123"); - CHECK_EQ("Gänsefüßchen"_b.upper(bytes::Charset::UTF8).str(), "GÄNSEFÜẞCHEN"); - CHECK_EQ("Gänsefüßchen"_b.upper(bytes::Charset::ASCII).str(), "G??NSEF????CHEN"); + CHECK_EQ("abc123"_b.upper(unicode::Charset::UTF8).str(), "ABC123"); + CHECK_EQ("abc123"_b.upper(unicode::Charset::ASCII).str(), "ABC123"); + CHECK_EQ("Gänsefüßchen"_b.upper(unicode::Charset::UTF8).str(), "GÄNSEFÜẞCHEN"); + CHECK_EQ("Gänsefüßchen"_b.upper(unicode::Charset::ASCII).str(), "G??NSEF????CHEN"); + + CHECK_EQ(Bytes("a\0b\0c\0"s).upper(unicode::Charset::UTF16LE, unicode::DecodeErrorStrategy::STRICT), + Bytes("A\0B\0C\0"s).upper(unicode::Charset::UTF16LE, unicode::DecodeErrorStrategy::STRICT)); // NOLINTNEXTLINE(bugprone-throw-keyword-missing) - CHECK_THROWS_WITH_AS("123"_b.upper(bytes::Charset::Undef), "unknown character set for decoding", + CHECK_THROWS_WITH_AS("123"_b.upper(unicode::Charset::Undef), "unknown character set for decoding", const RuntimeError&); + + // No case change expected for these Japanese codepoints. + const auto tokio8 = "東京"_b; + CHECK_EQ(tokio8.upper(unicode::Charset::UTF16LE, unicode::DecodeErrorStrategy::STRICT), tokio8); + + const auto tokio16 = "\xff\xfeqg\xacN"_b; // 東京 in UTF16LE. + CHECK_EQ(tokio16.upper(unicode::Charset::UTF16LE, unicode::DecodeErrorStrategy::STRICT), tokio16); } TEST_CASE("append") { diff --git a/hilti/runtime/src/tests/integer.cc b/hilti/runtime/src/tests/integer.cc index c186b25be..a1885adb4 100644 --- a/hilti/runtime/src/tests/integer.cc +++ b/hilti/runtime/src/tests/integer.cc @@ -10,6 +10,7 @@ #include #include #include +#include #include using namespace hilti::rt; diff --git a/hilti/runtime/src/tests/string.cc b/hilti/runtime/src/tests/string.cc index 71e56e2c2..479fa8a35 100644 --- a/hilti/runtime/src/tests/string.cc +++ b/hilti/runtime/src/tests/string.cc @@ -1,28 +1,61 @@ // Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details. #include +#include #include #include +#include using namespace hilti::rt; +using namespace hilti::rt::bytes::literals; TEST_SUITE_BEGIN("string"); +TEST_CASE("encode") { + CHECK_EQ(string::encode("", unicode::Charset::ASCII), ""_b); + CHECK_EQ(string::encode("123", unicode::Charset::ASCII), "123"_b); + CHECK_EQ(string::encode("abc", unicode::Charset::ASCII), "abc"_b); + CHECK_EQ(string::encode("abc", unicode::Charset::UTF8), "abc"_b); + + CHECK_EQ(string::encode("\xF0\x9F\x98\x85", unicode::Charset::UTF8), "\xF0\x9F\x98\x85"_b); + CHECK_EQ(string::encode("\xc3\x28", unicode::Charset::UTF8), "\ufffd("_b); + CHECK_EQ(string::encode("\xc3\x28", unicode::Charset::UTF8, unicode::DecodeErrorStrategy::IGNORE), "("_b); + CHECK_THROWS_WITH_AS(string::encode("\xc3\x28", unicode::Charset::UTF8, unicode::DecodeErrorStrategy::STRICT), + "illegal UTF8 sequence in string", const RuntimeError&); + + + CHECK_EQ(string::encode("\xF0\x9F\x98\x85", unicode::Charset::ASCII, unicode::DecodeErrorStrategy::REPLACE), + "????"_b); + CHECK_EQ(string::encode("\xF0\x9F\x98\x85", unicode::Charset::ASCII, unicode::DecodeErrorStrategy::IGNORE), ""_b); + CHECK_THROWS_WITH_AS(string::encode("\xF0\x9F\x98\x85", unicode::Charset::ASCII, + unicode::DecodeErrorStrategy::STRICT), + "illegal ASCII character in string", const RuntimeError&); + + CHECK_EQ(string::encode("abc", unicode::Charset::UTF16LE, unicode::DecodeErrorStrategy::STRICT), "a\0b\0c\0"_b); + CHECK_EQ(string::encode("abc", unicode::Charset::UTF16BE, unicode::DecodeErrorStrategy::STRICT), "\0a\0b\0c"_b); + CHECK_EQ(string::encode("東京", unicode::Charset::UTF16LE, unicode::DecodeErrorStrategy::STRICT), "qg\xacN"_b); + CHECK_EQ(string::encode("東京", unicode::Charset::UTF16BE, unicode::DecodeErrorStrategy::STRICT), "gqN\xac"_b); + + // NOLINTNEXTLINE(bugprone-throw-keyword-missing) + CHECK_THROWS_WITH_AS(string::encode("123", unicode::Charset::Undef), "unknown character set for encoding", + const RuntimeError&); +} + TEST_CASE("lower") { CHECK_EQ(string::lower(""), ""); CHECK_EQ(string::lower("123Abc"), "123abc"); CHECK_EQ(string::lower("GÄNSEFÜẞCHEN"), "gänsefüßchen"); CHECK_EQ(string::lower("\xc3\x28" "aBcD", - string::DecodeErrorStrategy::REPLACE), + unicode::DecodeErrorStrategy::REPLACE), "\ufffd(abcd"); CHECK_EQ(string::lower("\xc3\x28" "aBcD", - string::DecodeErrorStrategy::IGNORE), + unicode::DecodeErrorStrategy::IGNORE), "(abcd"); CHECK_THROWS_WITH_AS(string::lower("\xc3\x28" "aBcD", - string::DecodeErrorStrategy::STRICT), + unicode::DecodeErrorStrategy::STRICT), "illegal UTF8 sequence in string", const RuntimeError&); } @@ -32,15 +65,15 @@ TEST_CASE("size") { CHECK_EQ(string::size("Gänsefüßchen"), 12U); CHECK_EQ(string::size("\xc3\x28" "aBcD", - string::DecodeErrorStrategy::REPLACE), + unicode::DecodeErrorStrategy::REPLACE), 6U); CHECK_EQ(string::size("\xc3\x28" "aBcD", - string::DecodeErrorStrategy::IGNORE), + unicode::DecodeErrorStrategy::IGNORE), 5U); CHECK_THROWS_WITH_AS(string::size("\xc3\x28" "aBcD", - string::DecodeErrorStrategy::STRICT), + unicode::DecodeErrorStrategy::STRICT), "illegal UTF8 sequence in string", const RuntimeError&); } @@ -50,15 +83,15 @@ TEST_CASE("upper") { CHECK_EQ(string::upper("Gänsefüßchen"), "GÄNSEFÜẞCHEN"); CHECK_EQ(string::upper("\xc3\x28" "aBcD", - string::DecodeErrorStrategy::REPLACE), + unicode::DecodeErrorStrategy::REPLACE), "\ufffd(ABCD"); CHECK_EQ(string::upper("\xc3\x28" "aBcD", - string::DecodeErrorStrategy::IGNORE), + unicode::DecodeErrorStrategy::IGNORE), "(ABCD"); CHECK_THROWS_WITH_AS(string::upper("\xc3\x28" "aBcD", - string::DecodeErrorStrategy::STRICT), + unicode::DecodeErrorStrategy::STRICT), "illegal UTF8 sequence in string", const RuntimeError&); } diff --git a/hilti/runtime/src/tests/to_string.cc b/hilti/runtime/src/tests/to_string.cc index d5c692d96..00b37a561 100644 --- a/hilti/runtime/src/tests/to_string.cc +++ b/hilti/runtime/src/tests/to_string.cc @@ -108,15 +108,17 @@ TEST_CASE("integer::BitOrder") { } TEST_CASE("bytes::Charset") { - CHECK_EQ(to_string(Enum(bytes::Charset::ASCII)), "Charset::ASCII"); - CHECK_EQ(to_string(Enum(bytes::Charset::UTF8)), "Charset::UTF8"); - CHECK_EQ(to_string(Enum(bytes::Charset::Undef)), "Charset::Undef"); + CHECK_EQ(to_string(Enum(unicode::Charset::ASCII)), "Charset::ASCII"); + CHECK_EQ(to_string(Enum(unicode::Charset::UTF8)), "Charset::UTF8"); + CHECK_EQ(to_string(Enum(unicode::Charset::UTF16BE)), "Charset::UTF16BE"); + CHECK_EQ(to_string(Enum(unicode::Charset::UTF16LE)), "Charset::UTF16LE"); + CHECK_EQ(to_string(Enum(unicode::Charset::Undef)), "Charset::Undef"); } -TEST_CASE("bytes::DecodeErrorStrategy") { - CHECK_EQ(to_string(Enum(bytes::DecodeErrorStrategy::IGNORE)), "DecodeErrorStrategy::IGNORE"); - CHECK_EQ(to_string(Enum(bytes::DecodeErrorStrategy::REPLACE)), "DecodeErrorStrategy::REPLACE"); - CHECK_EQ(to_string(Enum(bytes::DecodeErrorStrategy::STRICT)), "DecodeErrorStrategy::STRICT"); +TEST_CASE("unicode::DecodeErrorStrategy") { + CHECK_EQ(to_string(Enum(unicode::DecodeErrorStrategy::IGNORE)), "DecodeErrorStrategy::IGNORE"); + CHECK_EQ(to_string(Enum(unicode::DecodeErrorStrategy::REPLACE)), "DecodeErrorStrategy::REPLACE"); + CHECK_EQ(to_string(Enum(unicode::DecodeErrorStrategy::STRICT)), "DecodeErrorStrategy::STRICT"); } TEST_CASE("bytes::Side") { diff --git a/hilti/runtime/src/tests/union.cc b/hilti/runtime/src/tests/union.cc index 7b76a134b..d21a953c1 100644 --- a/hilti/runtime/src/tests/union.cc +++ b/hilti/runtime/src/tests/union.cc @@ -6,6 +6,7 @@ #include #include #include +#include #include using namespace hilti::rt; diff --git a/hilti/runtime/src/tests/util.cc b/hilti/runtime/src/tests/util.cc index 13bce427e..9710322ba 100644 --- a/hilti/runtime/src/tests/util.cc +++ b/hilti/runtime/src/tests/util.cc @@ -148,7 +148,7 @@ TEST_CASE("createTemporaryFile") { SUBCASE("custom prefix") { auto prefix = "1234567890"; tmp = createTemporaryFile(prefix).valueOrThrow(); - CHECK(startsWith(tmp.filename(), prefix)); + CHECK(startsWith(tmp.filename().string(), prefix)); } CAPTURE(tmp); diff --git a/hilti/runtime/src/types/bytes.cc b/hilti/runtime/src/types/bytes.cc index 728e3d461..23cfdf346 100644 --- a/hilti/runtime/src/types/bytes.cc +++ b/hilti/runtime/src/types/bytes.cc @@ -1,19 +1,78 @@ // Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details. -#include +#include +#include #include #include +#include +#include +#include #include #include #include #include +#include #include using namespace hilti::rt; using namespace hilti::rt::bytes; +namespace { + +// An iterator over `char16_t` which can adjust the byte order. +struct U16Iterator { + // Most of this is boilerplate. + using iterator_category = std::forward_iterator_tag; + using difference_type = std::ptrdiff_t; + using value_type = const char16_t; + using pointer = value_type*; + using reference = value_type&; + + pointer cur = nullptr; + + U16Iterator& operator++() { + ++cur; + return *this; + } + + U16Iterator operator++(int) { + auto tmp = *this; + ++(*this); + return tmp; + } + + friend bool operator==(const U16Iterator& a, const U16Iterator& b) { return a.cur == b.cur; }; + friend bool operator!=(const U16Iterator& a, const U16Iterator& b) { return ! (a == b); }; + + // Implementation of custom behavior below. + enum Order { LE, BE, Detected }; + + U16Iterator(pointer ptr, Order order) : cur(ptr), order(order) {} + + Order order; + + auto operator*() const { + switch ( order ) { + case Detected: [[fallthrough]]; + case LE: return *cur; + case BE: { + auto r = *cur; + + char* xs = reinterpret_cast(&r); + std::swap(xs[0], xs[1]); + + return r; + } + } + + cannot_be_reached(); + } +}; + +} // namespace + std::tuple Bytes::find(const Bytes& needle, const const_iterator& start) const { auto b = begin(); @@ -46,78 +105,104 @@ std::tuple Bytes::find(const Bytes& needle, const c } } -Bytes::Bytes(std::string s, bytes::Charset cs, DecodeErrorStrategy errors) { +std::string Bytes::decode(unicode::Charset cs, unicode::DecodeErrorStrategy errors) const try { + if ( Base::empty() ) + return ""; + switch ( cs.value() ) { - case bytes::Charset::UTF8: { - // Data supposedly is already in UTF-8, but let's validate it. + case unicode::Charset::UTF8: { std::string t; - auto p = reinterpret_cast(s.data()); - auto e = p + s.size(); + auto p = Base::begin(); + auto e = Base::end(); while ( p < e ) { - utf8proc_int32_t cp; - auto n = utf8proc_iterate(p, e - p, &cp); - - if ( n < 0 ) { + try { + auto cp = utf8::next(p, e); + utf8::append(cp, t); + } catch ( const utf8::invalid_utf8& ) { switch ( errors.value() ) { - case DecodeErrorStrategy::IGNORE: break; - case DecodeErrorStrategy::REPLACE: t += "\ufffd"; break; - case DecodeErrorStrategy::STRICT: throw RuntimeError("illegal UTF8 sequence in string"); + case unicode::DecodeErrorStrategy::IGNORE: break; + case unicode::DecodeErrorStrategy::REPLACE: { + utf8::append(unicode::REPLACEMENT_CHARACTER, t); + break; + } + case unicode::DecodeErrorStrategy::STRICT: + throw RuntimeError("illegal UTF8 sequence in string"); } - p += 1; - continue; + ++p; } - - t += std::string(reinterpret_cast(p), n); - p += n; } - *this = std::move(t); - return; + return t; } - case bytes::Charset::ASCII: { - std::string t; - for ( const auto& c : s ) { - if ( c >= 32 && c < 0x7f ) - t += static_cast(c); - else { - switch ( errors.value() ) { - case DecodeErrorStrategy::IGNORE: break; - case DecodeErrorStrategy::REPLACE: t += '?'; break; - case DecodeErrorStrategy::STRICT: throw RuntimeError("illegal ASCII character in string"); + case unicode::Charset::UTF16BE: [[fallthrough]]; + case unicode::Charset::UTF16LE: { + if ( Base::size() % 2 != 0 ) { + switch ( errors.value() ) { + case unicode::DecodeErrorStrategy::STRICT: throw RuntimeError("illegal UTF16 character in string"); + case unicode::DecodeErrorStrategy::IGNORE: { + // Ignore the last byte. + return Bytes(str().substr(0, Base::size() / 2 * 2)).decode(cs, errors); + } + case unicode::DecodeErrorStrategy::REPLACE: { + // Convert everything but the last byte, and append replacement. + auto dec = Bytes(str().substr(0, Base::size() / 2 * 2)).decode(cs, errors); + utf8::append(unicode::REPLACEMENT_CHARACTER, dec); + return dec; } } } - *this = std::move(t); - return; - } + // We can assume an even number of bytes. - case bytes::Charset::Undef: throw RuntimeError("unknown character set for encoding"); - } + std::u16string t; - cannot_be_reached(); -} + // utfcpp expects to iterate a `u16string` or `u16string_view`. + auto v16 = std::u16string_view{reinterpret_cast(Base::data()), Base::size() / 2}; -std::string Bytes::decode(bytes::Charset cs, bytes::DecodeErrorStrategy errors) const { - switch ( cs.value() ) { - case bytes::Charset::UTF8: - // Data is already in UTF-8, but let's validate it. - return Bytes(str(), cs, errors).str(); + // We prefer to use the byte order from a BOM if present. If none is found use the passed byte order. + U16Iterator::Order order = U16Iterator::Detected; + if ( ! startsWith("\xFF\xFE") && ! startsWith("\xFE\xFF") ) + order = (cs.value() == unicode::Charset::UTF16LE ? U16Iterator::LE : U16Iterator::BE); + + auto p = U16Iterator(v16.begin(), order); + auto e = U16Iterator(v16.end(), order); + + while ( p != e ) { + try { + auto cp = utf8::next16(p, e); + utf8::append16(cp, t); + } catch ( const utf8::invalid_utf16& ) { + switch ( errors.value() ) { + case unicode::DecodeErrorStrategy::IGNORE: break; + case unicode::DecodeErrorStrategy::REPLACE: + utf8::append16(unicode::REPLACEMENT_CHARACTER, t); + break; + case unicode::DecodeErrorStrategy::STRICT: + throw RuntimeError("illegal UTF16 character in string"); + } - case bytes::Charset::ASCII: { + ++p; + } + } + + return {utf8::utf16to8(t)}; + } + + case unicode::Charset::ASCII: { std::string s; for ( auto c : str() ) { if ( c >= 32 && c < 0x7f ) s += c; else { switch ( errors.value() ) { - case DecodeErrorStrategy::IGNORE: break; - case DecodeErrorStrategy::REPLACE: s += "?"; break; - case DecodeErrorStrategy::STRICT: throw RuntimeError("illegal ASCII character in string"); + case unicode::DecodeErrorStrategy::IGNORE: break; + case unicode::DecodeErrorStrategy::REPLACE: s += "?"; break; + case unicode::DecodeErrorStrategy::STRICT: + throw RuntimeError("illegal ASCII character in string"); } } } @@ -125,10 +210,16 @@ std::string Bytes::decode(bytes::Charset cs, bytes::DecodeErrorStrategy errors) return s; } - case bytes::Charset::Undef: throw RuntimeError("unknown character set for decoding"); + case unicode::Charset::Undef: throw RuntimeError("unknown character set for decoding"); } cannot_be_reached(); +} catch ( const RuntimeError& ) { + // Directly propagate already correctly wrapped exceptions. + throw; +} catch ( ... ) { + // Throw a new `RuntimeError` for any other exception which has made it out of the function. + throw RuntimeError("could not decode bytes"); } Bytes Bytes::strip(const Bytes& set, bytes::Side side) const { @@ -155,6 +246,10 @@ Bytes Bytes::strip(bytes::Side side) const { cannot_be_reached(); } +Bytes Bytes::upper(unicode::Charset cs, unicode::DecodeErrorStrategy errors) const { + return string::encode(string::upper(decode(cs, errors), errors), cs, errors); +} + integer::safe Bytes::toInt(uint64_t base) const { int64_t x = 0; if ( hilti::rt::atoi_n(str().begin(), str().end(), base, &x) == str().end() ) @@ -229,6 +324,10 @@ double Bytes::toReal() const { return d; } +Bytes Bytes::lower(unicode::Charset cs, unicode::DecodeErrorStrategy errors) const { + return string::encode(string::lower(decode(cs, errors), errors), cs, errors); +} + Result Bytes::match(const RegExp& re, unsigned int group) const { auto groups = re.matchGroups(*this); @@ -249,26 +348,6 @@ std::string to_string(const Bytes& x, tag /*unused*/) { return fmt("b\"%s\"", escapeBytes(x.str(), render_style::Bytes::EscapeQuotes)); } -std::string to_string(const bytes::Charset& x, tag /*unused*/) { - switch ( x.value() ) { - case bytes::Charset::ASCII: return "Charset::ASCII"; - case bytes::Charset::UTF8: return "Charset::UTF8"; - case bytes::Charset::Undef: return "Charset::Undef"; - } - - cannot_be_reached(); -} - -std::string to_string(const bytes::DecodeErrorStrategy& x, tag /*unused*/) { - switch ( x.value() ) { - case bytes::DecodeErrorStrategy::IGNORE: return "DecodeErrorStrategy::IGNORE"; - case bytes::DecodeErrorStrategy::REPLACE: return "DecodeErrorStrategy::REPLACE"; - case bytes::DecodeErrorStrategy::STRICT: return "DecodeErrorStrategy::STRICT"; - } - - cannot_be_reached(); -} - std::string to_string(const bytes::Side& x, tag /*unused*/) { switch ( x.value() ) { case bytes::Side::Left: return "Side::Left"; diff --git a/hilti/runtime/src/types/string.cc b/hilti/runtime/src/types/string.cc index 8c4b1b27e..5b970d89b 100644 --- a/hilti/runtime/src/types/string.cc +++ b/hilti/runtime/src/types/string.cc @@ -1,41 +1,47 @@ // Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details. +#include #include +#include + #include +#include #include +#include using namespace hilti::rt; -integer::safe string::size(const std::string& s, DecodeErrorStrategy errors) { - auto p = reinterpret_cast(s.data()); - auto e = p + s.size(); +integer::safe string::size(const std::string& s, unicode::DecodeErrorStrategy errors) { + auto p = s.begin(); + auto e = s.end(); uint64_t len = 0; while ( p < e ) { - utf8proc_int32_t cp; - auto n = utf8proc_iterate(p, e - p, &cp); - - if ( n < 0 ) { + try { + // `utf8::next` is for iterating UTF-8 strings. + utf8::next(p, s.end()); + ++len; + } catch ( const utf8::invalid_utf8& ) { switch ( errors.value() ) { - case DecodeErrorStrategy::IGNORE: break; - case DecodeErrorStrategy::REPLACE: ++len; break; - case DecodeErrorStrategy::STRICT: throw RuntimeError("illegal UTF8 sequence in string"); + case unicode::DecodeErrorStrategy::STRICT: throw RuntimeError("illegal UTF8 sequence in string"); + case unicode::DecodeErrorStrategy::REPLACE: { + ++len; + } + [[fallthrough]]; + case unicode::DecodeErrorStrategy::IGNORE: { + ++p; + break; + } } - - p += 1; - continue; } - - ++len; - p += n; } return len; } -std::string string::upper(std::string_view s, DecodeErrorStrategy errors) { +std::string string::upper(std::string_view s, unicode::DecodeErrorStrategy errors) { auto p = reinterpret_cast(s.data()); auto e = p + s.size(); @@ -48,9 +54,9 @@ std::string string::upper(std::string_view s, DecodeErrorStrategy errors) { if ( n < 0 ) { switch ( errors.value() ) { - case DecodeErrorStrategy::IGNORE: break; - case DecodeErrorStrategy::REPLACE: rval += "\ufffd"; break; - case DecodeErrorStrategy::STRICT: throw RuntimeError("illegal UTF8 sequence in string"); + case unicode::DecodeErrorStrategy::IGNORE: break; + case unicode::DecodeErrorStrategy::REPLACE: utf8::append(unicode::REPLACEMENT_CHARACTER, rval); break; + case unicode::DecodeErrorStrategy::STRICT: throw RuntimeError("illegal UTF8 sequence in string"); } p += 1; @@ -65,7 +71,7 @@ std::string string::upper(std::string_view s, DecodeErrorStrategy errors) { return rval; } -std::string string::lower(std::string_view s, DecodeErrorStrategy errors) { +std::string string::lower(std::string_view s, unicode::DecodeErrorStrategy errors) { auto p = reinterpret_cast(s.data()); auto e = p + s.size(); @@ -78,9 +84,9 @@ std::string string::lower(std::string_view s, DecodeErrorStrategy errors) { if ( n < 0 ) { switch ( errors.value() ) { - case DecodeErrorStrategy::IGNORE: break; - case DecodeErrorStrategy::REPLACE: rval += "\ufffd"; break; - case DecodeErrorStrategy::STRICT: throw RuntimeError("illegal UTF8 sequence in string"); + case unicode::DecodeErrorStrategy::IGNORE: break; + case unicode::DecodeErrorStrategy::REPLACE: utf8::append(unicode::REPLACEMENT_CHARACTER, rval); break; + case unicode::DecodeErrorStrategy::STRICT: throw RuntimeError("illegal UTF8 sequence in string"); } p += 1; @@ -128,3 +134,96 @@ std::tuple string::split1(const std::string& s, const auto pair = hilti::rt::split1(s, sep); return {pair.first, pair.second}; } + +Bytes string::encode(std::string s, unicode::Charset cs, unicode::DecodeErrorStrategy errors) try { + if ( s.empty() ) + return {std::move(s)}; + + switch ( cs.value() ) { + case unicode::Charset::UTF8: { + // HILTI `string` is always UTF-8, but we could be invoked with raw bags of bytes here as well, so validate. + std::string t; + + auto p = s.begin(); + auto e = s.end(); + + while ( p < e ) { + try { + auto cp = utf8::next(p, e); + utf8::append(cp, t); + } catch ( const utf8::invalid_utf8& ) { + switch ( errors.value() ) { + case unicode::DecodeErrorStrategy::IGNORE: break; + case unicode::DecodeErrorStrategy::REPLACE: { + utf8::append(unicode::REPLACEMENT_CHARACTER, t); + break; + } + case unicode::DecodeErrorStrategy::STRICT: + throw RuntimeError("illegal UTF8 sequence in string"); + } + + ++p; + } + } + + return Bytes(std::move(t)); + } + + case unicode::Charset::UTF16BE: [[fallthrough]]; + case unicode::Charset::UTF16LE: { + // HILTI `string` is always UTF-8, but we could be invoked with raw bags of bytes here as well, so validate. + auto t8 = string::encode(s, unicode::Charset::UTF8, errors).str(); + + auto t = utf8::utf8to16(t8); + + std::string result; + result.reserve(t.size() * 2); + for ( auto c : t ) { + auto* xs = reinterpret_cast(&c); + + switch ( cs.value() ) { + case unicode::Charset::UTF16LE: { + result += xs[0]; + result += xs[1]; + break; + } + case unicode::Charset::UTF16BE: { + result += xs[1]; + result += xs[0]; + break; + } + } + } + + return {std::move(result)}; + } + + case unicode::Charset::ASCII: { + std::string t; + for ( const auto& c : s ) { + if ( c >= 32 && c < 0x7f ) + t += static_cast(c); + else { + switch ( errors.value() ) { + case unicode::DecodeErrorStrategy::IGNORE: break; + case unicode::DecodeErrorStrategy::REPLACE: t += '?'; break; + case unicode::DecodeErrorStrategy::STRICT: + throw RuntimeError("illegal ASCII character in string"); + } + } + } + + return Bytes(std::move(t)); + } + + case unicode::Charset::Undef: throw RuntimeError("unknown character set for encoding"); + } + + cannot_be_reached(); +} catch ( const RuntimeError& ) { + // Directly propagate already correctly wrapped exceptions. + throw; +} catch ( ... ) { + // Throw a new `RuntimeError` for any other exception which has made it out of the function. + throw RuntimeError("could not encode string"); +} diff --git a/hilti/runtime/src/unicode.cc b/hilti/runtime/src/unicode.cc new file mode 100644 index 000000000..82d820cc8 --- /dev/null +++ b/hilti/runtime/src/unicode.cc @@ -0,0 +1,29 @@ +// Copyright (c) 2020-2023 by the Zeek Project. See LICENSE for details. + +#include "hilti/rt/unicode.h" + +namespace hilti::rt::detail::adl { + +std::string to_string(const unicode::DecodeErrorStrategy& x, tag /*unused*/) { + switch ( x.value() ) { + case unicode::DecodeErrorStrategy::IGNORE: return "DecodeErrorStrategy::IGNORE"; + case unicode::DecodeErrorStrategy::REPLACE: return "DecodeErrorStrategy::REPLACE"; + case unicode::DecodeErrorStrategy::STRICT: return "DecodeErrorStrategy::STRICT"; + } + + cannot_be_reached(); +} + +std::string to_string(const unicode::Charset& x, tag /*unused*/) { + switch ( x.value() ) { + case unicode::Charset::ASCII: return "Charset::ASCII"; + case unicode::Charset::UTF8: return "Charset::UTF8"; + case unicode::Charset::UTF16BE: return "Charset::UTF16BE"; + case unicode::Charset::UTF16LE: return "Charset::UTF16LE"; + case unicode::Charset::Undef: return "Charset::Undef"; + } + + cannot_be_reached(); +} + +} // namespace hilti::rt::detail::adl diff --git a/hilti/runtime/src/util.cc b/hilti/runtime/src/util.cc index 1f0ad0fee..946cffd8f 100644 --- a/hilti/runtime/src/util.cc +++ b/hilti/runtime/src/util.cc @@ -381,17 +381,7 @@ std::string hilti::rt::replace(std::string s, std::string_view o, std::string_vi return s; } -bool hilti::rt::startsWith(const std::string& s, const std::string& prefix) { - if ( s.size() < prefix.size() ) - return false; - - for ( size_t i = 0; i < prefix.size(); ++i ) { - if ( s[i] != prefix[i] ) - return false; - } - - return true; -} +bool hilti::rt::startsWith(std::string_view s, std::string_view prefix) { return s.substr(0, prefix.size()) == prefix; } hilti::rt::ByteOrder hilti::rt::systemByteOrder() { #ifdef LITTLE_ENDIAN diff --git a/hilti/toolchain/src/ast/operators/string.cc b/hilti/toolchain/src/ast/operators/string.cc index 277228869..fecde656e 100644 --- a/hilti/toolchain/src/ast/operators/string.cc +++ b/hilti/toolchain/src/ast/operators/string.cc @@ -116,6 +116,9 @@ class Encode : public BuiltInMemberCall { .param0 = {.name = "charset", .type = {parameter::Kind::In, builder->typeName("hilti::Charset")}, .default_ = builder->expressionName("hilti::Charset::UTF8")}, + .param1 = {.name = "errors", + .type = {parameter::Kind::In, builder->typeName("hilti::DecodeErrorStrategy")}, + .default_ = builder->expressionName("hilti::DecodeErrorStrategy::REPLACE")}, .result = {Constness::Const, builder->typeBytes()}, .ns = "string", .doc = diff --git a/hilti/toolchain/src/compiler/codegen/operators.cc b/hilti/toolchain/src/compiler/codegen/operators.cc index 5ed1da0fe..b6ea2bb40 100644 --- a/hilti/toolchain/src/compiler/codegen/operators.cc +++ b/hilti/toolchain/src/compiler/codegen/operators.cc @@ -774,7 +774,7 @@ struct Visitor : hilti::visitor::PreOrder { void operator()(operator_::string::Encode* n) final { auto [self, args] = methodArguments(n); - result = fmt("::hilti::rt::Bytes(%s, %s)", self, args[0]); + result = fmt("::hilti::rt::string::encode(%s, %s, %s)", self, args[0], args[1]); } void operator()(operator_::string::Modulo* n) final { diff --git a/spicy/lib/spicy.spicy b/spicy/lib/spicy.spicy index 4d878e9a8..547be1da8 100644 --- a/spicy/lib/spicy.spicy +++ b/spicy/lib/spicy.spicy @@ -32,16 +32,18 @@ public type ByteOrder = enum { ## Specifies the character set for bytes encoding/decoding. public type Charset = enum { - ASCII, - UTF8 -} &cxxname="hilti::rt::bytes::Charset"; + ASCII, # ASCII encoding + UTF8, # UTF8 encoding + UTF16LE, # UTF16 little endian encoding + UTF16BE, # UTF16 big endian encoding +} &cxxname="hilti::rt::unicode::Charset"; ## Specifies how data is handled that's not representable in a specified character set. public type DecodeErrorStrategy = enum { - IGNORE, # data is skipped but processing continues - REPLACE, # data is replaced with a valid place-holder and processing continues - STRICT # runtime error is triggered -} &cxxname="hilti::rt::bytes::DecodeErrorStrategy"; + IGNORE, # data is skipped but processing continues + REPLACE, # data is replaced with a valid place-holder and processing continues + STRICT # runtime error is triggered +} &cxxname="hilti::rt::unicode::DecodeErrorStrategy"; ## Represents the error value of ``result`` instances. Use ``error"My error message"`` ## to create an error value of this type. diff --git a/tests/Baseline/hilti.ast.basic-module/debug.log b/tests/Baseline/hilti.ast.basic-module/debug.log index 98ea77b22..337b63fc5 100644 --- a/tests/Baseline/hilti.ast.basic-module/debug.log +++ b/tests/Baseline/hilti.ast.basic-module/debug.log @@ -323,43 +323,61 @@ [debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] -[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:14:1-14:80) [@d:XXX] +[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:14:1-14:100) [@d:XXX] [debug/ast-final] | ASCII -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16BE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16LE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | UTF8 -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | Undef -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) -[debug/ast-final] - QualifiedType [parent @d:XXX] (hilti.hlt:14:23-14:42) [@q:XXX] -[debug/ast-final] - type::Enum [parent @q:XXX] (hilti.hlt:14:23-14:42) (resolved) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @d:XXX] (hilti.hlt:14:23-14:60) [@q:XXX] +[debug/ast-final] - type::Enum [parent @q:XXX] (hilti.hlt:14:23-14:60) (resolved) [@t:XXX] [debug/ast-final] | ASCII -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16BE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16LE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | UTF8 -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | Undef -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:30-14:34) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:37-14:40) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] -[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:23-14:42) [@t:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:43-14:49) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:14:43-14:79) [@a:XXX] -[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:14:43-14:79) [@a:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] +[debug/ast-final] - +[debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] +[debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:52-14:58) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] +[debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] +[debug/ast-final] - +[debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] +[debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:23-14:60) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] +[debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] +[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:14:61-14:99) [@a:XXX] +[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:14:61-14:99) [@a:XXX] [debug/ast-final] - expression::Ctor [parent @a:XXX] (const) (resolved) [@e:XXX] -[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] +[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] -[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:15:1-15:116) [@d:XXX] +[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:15:1-15:118) [@d:XXX] [debug/ast-final] | IGNORE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | REPLACE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | STRICT -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) @@ -398,10 +416,10 @@ [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:15:35-15:66) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:15:42-15:47) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:15:67-15:115) [@a:XXX] -[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:15:67-15:115) [@a:XXX] +[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:15:67-15:117) [@a:XXX] +[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:15:67-15:117) [@a:XXX] [debug/ast-final] - expression::Ctor [parent @a:XXX] (const) (resolved) [@e:XXX] -[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] +[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] [debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:16:1-16:37) [@d:XXX] @@ -928,8 +946,8 @@ [debug/ast-final] [D6] hilti::AddressFamily [declaration::Type] (hilti.hlt:11:1-11:84) [debug/ast-final] [D7] hilti::RealType [declaration::Type] (hilti.hlt:12:1-12:96) [debug/ast-final] [D8] hilti::Protocol [declaration::Type] (hilti.hlt:13:1-13:78) -[debug/ast-final] [D9] hilti::Charset [declaration::Type] (hilti.hlt:14:1-14:80) -[debug/ast-final] [D10] hilti::DecodeErrorStrategy [declaration::Type] (hilti.hlt:15:1-15:116) +[debug/ast-final] [D9] hilti::Charset [declaration::Type] (hilti.hlt:14:1-14:100) +[debug/ast-final] [D10] hilti::DecodeErrorStrategy [declaration::Type] (hilti.hlt:15:1-15:118) [debug/ast-final] [D11] hilti::Captures [declaration::Type] (hilti.hlt:16:1-16:37) [debug/ast-final] [D12] hilti::Profiler [declaration::Type] (hilti.hlt:17:1-17:61) [debug/ast-final] [D13] hilti::MatchState [declaration::Type] (hilti.hlt:19:1-21:43) @@ -940,7 +958,7 @@ [debug/ast-final] [D18] hilti::RecoverableFailure [declaration::Type] (hilti.hlt:56:1-56:98) [debug/ast-final] [D19] hilti::MissingData [declaration::Type] (hilti.hlt:59:1-59:84) [debug/ast-final] [D20] hilti [declaration::Module] (hilti.hlt:3:1-68:1) -[debug/ast-final] [D21] hilti::UTF8 [declaration::Constant] (hilti.hlt:14:23-14:42) +[debug/ast-final] [D21] hilti::UTF8 [declaration::Constant] (hilti.hlt:14:23-14:60) [debug/ast-final] [D22] hilti::REPLACE [declaration::Constant] (hilti.hlt:15:35-15:66) [debug/ast-final] [T1] hilti::BitOrder [type::Enum] (hilti.hlt:8:24-8:42) [debug/ast-final] [T2] hilti::ByteOrder [type::Enum] (hilti.hlt:9:25-9:59) @@ -948,7 +966,7 @@ [debug/ast-final] [T4] hilti::AddressFamily [type::Enum] (hilti.hlt:11:29-11:47) [debug/ast-final] [T5] hilti::RealType [type::Enum] (hilti.hlt:12:24-12:62) [debug/ast-final] [T6] hilti::Protocol [type::Enum] (hilti.hlt:13:24-13:46) -[debug/ast-final] [T7] hilti::Charset [type::Enum] (hilti.hlt:14:23-14:42) +[debug/ast-final] [T7] hilti::Charset [type::Enum] (hilti.hlt:14:23-14:60) [debug/ast-final] [T8] hilti::DecodeErrorStrategy [type::Enum] (hilti.hlt:15:35-15:66) [debug/ast-final] [T9] hilti::MatchState [type::Struct] (hilti.hlt:19:26-21:1) [debug/ast-final] [T10] hilti::StreamStatistics [type::Struct] (hilti.hlt:23:32-28:1) diff --git a/tests/Baseline/hilti.ast.coercion/output b/tests/Baseline/hilti.ast.coercion/output index 28ed71bd8..ee8689168 100644 --- a/tests/Baseline/hilti.ast.coercion/output +++ b/tests/Baseline/hilti.ast.coercion/output @@ -488,43 +488,61 @@ [debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] -[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:14:1-14:80) [@d:XXX] +[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:14:1-14:100) [@d:XXX] [debug/ast-final] | ASCII -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16BE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16LE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | UTF8 -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | Undef -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) -[debug/ast-final] - QualifiedType [parent @d:XXX] (hilti.hlt:14:23-14:42) [@q:XXX] -[debug/ast-final] - type::Enum [parent @q:XXX] (hilti.hlt:14:23-14:42) (resolved) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @d:XXX] (hilti.hlt:14:23-14:60) [@q:XXX] +[debug/ast-final] - type::Enum [parent @q:XXX] (hilti.hlt:14:23-14:60) (resolved) [@t:XXX] [debug/ast-final] | ASCII -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16BE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16LE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | UTF8 -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | Undef -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:30-14:34) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:37-14:40) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] -[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:23-14:42) [@t:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:43-14:49) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:14:43-14:79) [@a:XXX] -[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:14:43-14:79) [@a:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] +[debug/ast-final] - +[debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] +[debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:52-14:58) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] +[debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] +[debug/ast-final] - +[debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] +[debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:23-14:60) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] +[debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] +[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:14:61-14:99) [@a:XXX] +[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:14:61-14:99) [@a:XXX] [debug/ast-final] - expression::Ctor [parent @a:XXX] (const) (resolved) [@e:XXX] -[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] +[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] -[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:15:1-15:116) [@d:XXX] +[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:15:1-15:118) [@d:XXX] [debug/ast-final] | IGNORE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | REPLACE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | STRICT -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) @@ -563,10 +581,10 @@ [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:15:35-15:66) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:15:42-15:47) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:15:67-15:115) [@a:XXX] -[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:15:67-15:115) [@a:XXX] +[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:15:67-15:117) [@a:XXX] +[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:15:67-15:117) [@a:XXX] [debug/ast-final] - expression::Ctor [parent @a:XXX] (const) (resolved) [@e:XXX] -[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] +[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] [debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:16:1-16:37) [@d:XXX] @@ -1093,8 +1111,8 @@ [debug/ast-final] [D6] hilti::AddressFamily [declaration::Type] (hilti.hlt:11:1-11:84) [debug/ast-final] [D7] hilti::RealType [declaration::Type] (hilti.hlt:12:1-12:96) [debug/ast-final] [D8] hilti::Protocol [declaration::Type] (hilti.hlt:13:1-13:78) -[debug/ast-final] [D9] hilti::Charset [declaration::Type] (hilti.hlt:14:1-14:80) -[debug/ast-final] [D10] hilti::DecodeErrorStrategy [declaration::Type] (hilti.hlt:15:1-15:116) +[debug/ast-final] [D9] hilti::Charset [declaration::Type] (hilti.hlt:14:1-14:100) +[debug/ast-final] [D10] hilti::DecodeErrorStrategy [declaration::Type] (hilti.hlt:15:1-15:118) [debug/ast-final] [D11] hilti::Captures [declaration::Type] (hilti.hlt:16:1-16:37) [debug/ast-final] [D12] hilti::Profiler [declaration::Type] (hilti.hlt:17:1-17:61) [debug/ast-final] [D13] hilti::MatchState [declaration::Type] (hilti.hlt:19:1-21:43) @@ -1105,7 +1123,7 @@ [debug/ast-final] [D18] hilti::RecoverableFailure [declaration::Type] (hilti.hlt:56:1-56:98) [debug/ast-final] [D19] hilti::MissingData [declaration::Type] (hilti.hlt:59:1-59:84) [debug/ast-final] [D20] hilti [declaration::Module] (hilti.hlt:3:1-68:1) -[debug/ast-final] [D21] hilti::UTF8 [declaration::Constant] (hilti.hlt:14:23-14:42) +[debug/ast-final] [D21] hilti::UTF8 [declaration::Constant] (hilti.hlt:14:23-14:60) [debug/ast-final] [D22] hilti::REPLACE [declaration::Constant] (hilti.hlt:15:35-15:66) [debug/ast-final] [T1] hilti::BitOrder [type::Enum] (hilti.hlt:8:24-8:42) [debug/ast-final] [T2] hilti::ByteOrder [type::Enum] (hilti.hlt:9:25-9:59) @@ -1113,7 +1131,7 @@ [debug/ast-final] [T4] hilti::AddressFamily [type::Enum] (hilti.hlt:11:29-11:47) [debug/ast-final] [T5] hilti::RealType [type::Enum] (hilti.hlt:12:24-12:62) [debug/ast-final] [T6] hilti::Protocol [type::Enum] (hilti.hlt:13:24-13:46) -[debug/ast-final] [T7] hilti::Charset [type::Enum] (hilti.hlt:14:23-14:42) +[debug/ast-final] [T7] hilti::Charset [type::Enum] (hilti.hlt:14:23-14:60) [debug/ast-final] [T8] hilti::DecodeErrorStrategy [type::Enum] (hilti.hlt:15:35-15:66) [debug/ast-final] [T9] hilti::MatchState [type::Struct] (hilti.hlt:19:26-21:1) [debug/ast-final] [T10] hilti::StreamStatistics [type::Struct] (hilti.hlt:23:32-28:1) diff --git a/tests/Baseline/hilti.ast.imported-id/output b/tests/Baseline/hilti.ast.imported-id/output index f595de462..ff10fa1d6 100644 --- a/tests/Baseline/hilti.ast.imported-id/output +++ b/tests/Baseline/hilti.ast.imported-id/output @@ -10,7 +10,7 @@ [debug/resolver] -> [T4] type::Enum | enum { } (hilti.hlt:11:29-11:47) [debug/resolver] -> [T5] type::Enum | enum { } (hilti.hlt:12:24-12:62) [debug/resolver] -> [T6] type::Enum | enum { } (hilti.hlt:13:24-13:46) -[debug/resolver] -> [T7] type::Enum | enum { } (hilti.hlt:14:23-14:42) +[debug/resolver] -> [T7] type::Enum | enum { } (hilti.hlt:14:23-14:60) [debug/resolver] -> [T8] type::Enum | enum { } (hilti.hlt:15:35-15:66) [debug/resolver] -> [T9] type::Struct | struct { method Captures captures(view data); } (hilti.hlt:19:26-21:1) [debug/resolver] -> [T10] type::Struct | struct { uint<64> num_data_bytes; uint<64> num_data_chunks; uint<64> num_gap_bytes; uint<64> num_gap_chunks; } (hilti.hlt:23:32-28:1) @@ -111,22 +111,24 @@ [debug/resolver] [hilti.hlt:13:1-13:78] declaration::Type "public type Protocol = enum { TCP = 0, UDP = 1, ICMP = 2 } &cxxname="::hilti::rt::Protocol";" -> set declaration's fully qualified ID to hilti::Protocol [debug/resolver] -> [D8] declaration::Type hilti::Protocol | public type Protocol = enum { TCP = 0, UDP = 1, ICMP = 2 } &cxxname="::hilti::rt::Protocol"; (hilti.hlt:13:1-13:78) [debug/resolver] [hilti.hlt:13:24-13:46] type::Enum "hilti::Protocol" -> set type's declaration to D8 -[debug/resolver] [hilti.hlt:14:23-14:42] declaration::Constant "const enum { ASCII = 0, UTF8 = 1 } ASCII = ::ASCII;" -> set declaration's canonical ID to hilti::ASCII -[debug/resolver] [hilti.hlt:14:23-14:42] declaration::Constant "const enum { ASCII = 0, UTF8 = 1 } UTF8 = ::UTF8;" -> set declaration's canonical ID to hilti::UTF8 -[debug/resolver] [hilti.hlt:14:23-14:42] declaration::Constant "const enum { ASCII = 0, UTF8 = 1 } Undef = ::Undef;" -> set declaration's canonical ID to hilti::Undef_7 -[debug/resolver] [hilti.hlt:14:44-14:79] Attribute "&cxxname="hilti::rt::bytes::Charset"" -> Attribute "&cxxname="::hilti::rt::bytes::Charset"" -[debug/resolver] [hilti.hlt:14:1-14:80] declaration::Type "public type Charset = enum { ASCII = 0, UTF8 = 1 } &cxxname="::hilti::rt::bytes::Charset";" -> set declaration's canonical ID to hilti::Charset -[debug/resolver] [hilti.hlt:14:1-14:80] declaration::Type "public type Charset = enum { ASCII = 0, UTF8 = 1 } &cxxname="::hilti::rt::bytes::Charset";" -> set declaration's fully qualified ID to hilti::Charset -[debug/resolver] -> [D9] declaration::Type hilti::Charset | public type Charset = enum { ASCII = 0, UTF8 = 1 } &cxxname="::hilti::rt::bytes::Charset"; (hilti.hlt:14:1-14:80) -[debug/resolver] [hilti.hlt:14:23-14:42] type::Enum "hilti::Charset" -> set type's declaration to D9 +[debug/resolver] [hilti.hlt:14:23-14:60] declaration::Constant "const enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } ASCII = ::ASCII;" -> set declaration's canonical ID to hilti::ASCII +[debug/resolver] [hilti.hlt:14:23-14:60] declaration::Constant "const enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } UTF8 = ::UTF8;" -> set declaration's canonical ID to hilti::UTF8 +[debug/resolver] [hilti.hlt:14:23-14:60] declaration::Constant "const enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } UTF16LE = ::UTF16LE;" -> set declaration's canonical ID to hilti::UTF16LE +[debug/resolver] [hilti.hlt:14:23-14:60] declaration::Constant "const enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } UTF16BE = ::UTF16BE;" -> set declaration's canonical ID to hilti::UTF16BE +[debug/resolver] [hilti.hlt:14:23-14:60] declaration::Constant "const enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } Undef = ::Undef;" -> set declaration's canonical ID to hilti::Undef_7 +[debug/resolver] [hilti.hlt:14:62-14:99] Attribute "&cxxname="hilti::rt::unicode::Charset"" -> Attribute "&cxxname="::hilti::rt::unicode::Charset"" +[debug/resolver] [hilti.hlt:14:1-14:100] declaration::Type "public type Charset = enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } &cxxname="::hilti::rt::unicode::Charset";" -> set declaration's canonical ID to hilti::Charset +[debug/resolver] [hilti.hlt:14:1-14:100] declaration::Type "public type Charset = enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } &cxxname="::hilti::rt::unicode::Charset";" -> set declaration's fully qualified ID to hilti::Charset +[debug/resolver] -> [D9] declaration::Type hilti::Charset | public type Charset = enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } &cxxname="::hilti::rt::unicode::Charset"; (hilti.hlt:14:1-14:100) +[debug/resolver] [hilti.hlt:14:23-14:60] type::Enum "hilti::Charset" -> set type's declaration to D9 [debug/resolver] [hilti.hlt:15:35-15:66] declaration::Constant "const enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } IGNORE = ::IGNORE;" -> set declaration's canonical ID to hilti::IGNORE [debug/resolver] [hilti.hlt:15:35-15:66] declaration::Constant "const enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } REPLACE = ::REPLACE;" -> set declaration's canonical ID to hilti::REPLACE [debug/resolver] [hilti.hlt:15:35-15:66] declaration::Constant "const enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } STRICT = ::STRICT;" -> set declaration's canonical ID to hilti::STRICT [debug/resolver] [hilti.hlt:15:35-15:66] declaration::Constant "const enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } Undef = ::Undef;" -> set declaration's canonical ID to hilti::Undef_8 -[debug/resolver] [hilti.hlt:15:68-15:115] Attribute "&cxxname="hilti::rt::bytes::DecodeErrorStrategy"" -> Attribute "&cxxname="::hilti::rt::bytes::DecodeErrorStrategy"" -[debug/resolver] [hilti.hlt:15:1-15:116] declaration::Type "public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::bytes::DecodeErrorStrategy";" -> set declaration's canonical ID to hilti::DecodeErrorStrategy -[debug/resolver] [hilti.hlt:15:1-15:116] declaration::Type "public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::bytes::DecodeErrorStrategy";" -> set declaration's fully qualified ID to hilti::DecodeErrorStrategy -[debug/resolver] -> [D10] declaration::Type hilti::DecodeErrorStrategy | public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::bytes::DecodeErrorStrategy"; (hilti.hlt:15:1-15:116) +[debug/resolver] [hilti.hlt:15:68-15:117] Attribute "&cxxname="hilti::rt::unicode::DecodeErrorStrategy"" -> Attribute "&cxxname="::hilti::rt::unicode::DecodeErrorStrategy"" +[debug/resolver] [hilti.hlt:15:1-15:118] declaration::Type "public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::unicode::DecodeErrorStrategy";" -> set declaration's canonical ID to hilti::DecodeErrorStrategy +[debug/resolver] [hilti.hlt:15:1-15:118] declaration::Type "public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::unicode::DecodeErrorStrategy";" -> set declaration's fully qualified ID to hilti::DecodeErrorStrategy +[debug/resolver] -> [D10] declaration::Type hilti::DecodeErrorStrategy | public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::unicode::DecodeErrorStrategy"; (hilti.hlt:15:1-15:118) [debug/resolver] [hilti.hlt:15:35-15:66] type::Enum "hilti::DecodeErrorStrategy" -> set type's declaration to D10 [debug/resolver] [hilti.hlt:16:1-16:37] declaration::Type "public type Captures = vector;" -> set declaration's canonical ID to hilti::Captures [debug/resolver] [hilti.hlt:16:1-16:37] declaration::Type "public type Captures = vector;" -> set declaration's fully qualified ID to hilti::Captures @@ -289,10 +291,10 @@ [debug/resolver] [hilti.hlt:67:1-67:119] declaration::Function "declare public function string exception_where(hilti::RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype;" -> set declaration's canonical ID to hilti::exception_where_2 [debug/resolver] [hilti.hlt:67:1-67:119] declaration::Function "declare public function string exception_where(hilti::RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype;" -> set declaration's fully qualified ID to hilti::exception_where [debug/resolver] [hilti.hlt:67:1-67:119] declaration::Function "declare public function string exception_where(hilti::RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype;" -> creating function call operator -[debug/resolver] [hilti.hlt:3:1-68:1] declaration::Module "module hilti { public type BitOrder = enum { LSB0 = 0, MSB0 = 1 } &cxxname="::hilti::rt::integer::BitOrder"; public type ByteOrder = enum { Little = 0, Big = 1, Network = 2, Host = 3 } &cxxname="::hilti::rt::ByteOrder"; public type Side = enum { Left = 0, Right = 1, Both = 2 } &cxxname="::hilti::rt::bytes::Side"; public type AddressFamily = enum { IPv4 = 0, IPv6 = 1 } &cxxname="::hilti::rt::AddressFamily"; public type RealType = enum { IEEE754_Single = 0, IEEE754_Double = 1 } &cxxname="::hilti::rt::real::Type"; public type Protocol = enum { TCP = 0, UDP = 1, ICMP = 2 } &cxxname="::hilti::rt::Protocol"; public type Charset = enum { ASCII = 0, UTF8 = 1 } &cxxname="::hilti::rt::bytes::Charset"; public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::bytes::DecodeErrorStrategy"; public type Captures = vector; public type Profiler = Profiler &cxxname="::hilti::rt::Profiler"; public type MatchState = struct { method Captures captures(view data); } &cxxname="::hilti::rt::regexp::MatchState"; public type StreamStatistics = struct { uint<64> num_data_bytes; uint<64> num_data_chunks; uint<64> num_gap_bytes; uint<64> num_gap_chunks; } &cxxname="::hilti::rt::stream::Statistics"; public type Exception = exception &cxxname="::hilti::rt::Exception"; public type SystemException = [exception :Exception &cxxname="::std::exception"; public type RuntimeError = [exception :Exception &cxxname="::hilti::rt::RuntimeError"; public type RecoverableFailure = [exception :Exception &cxxname="::hilti::rt::RecoverableFailure"; public type MissingData = [exception :Exception &cxxname="::hilti::rt::MissingData"; declare public function void print(any obj, bool newline = True) &cxxname="::hilti::rt::print" &have_prototype; declare public function void printValues(tuple<*> t, bool newline = True) &cxxname="::hilti::rt::printValues" &have_prototype; declare public function void debug(string dbg_stream, any obj) &cxxname="::hilti::rt::debug::print" &have_prototype; declare public function void debugIndent(string dbg_stream) &cxxname="::hilti::rt::debug::indent" &have_prototype; declare public function void debugDedent(string dbg_stream) &cxxname="::hilti::rt::debug::dedent" &have_prototype; declare public function time current_time() &cxxname="::hilti::rt::time::current_time" &have_prototype; declare public function time mktime(uint<64> y, uint<64> m, uint<64> d, uint<64> H, uint<64> M, uint<64> S) &cxxname="::hilti::rt::time::mktime" &have_prototype; declare public function void abort() &cxxname="::hilti::rt::abort_with_backtrace" &have_prototype; declare public function optional profiler_start(string name, optional> size = Null) &cxxname="::hilti::rt::profiler::start" &have_prototype; declare public function void profiler_stop(optional p, optional> size = Null) &cxxname="::hilti::rt::profiler::stop" &have_prototype; declare public function string exception_what(SystemException excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_what(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_where(SystemException excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; declare public function string exception_where(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; }" -> set declaration's fully qualified ID to hilti -[debug/resolver] [hilti.hlt:3:1-68:1] declaration::Module "module hilti { public type BitOrder = enum { LSB0 = 0, MSB0 = 1 } &cxxname="::hilti::rt::integer::BitOrder"; public type ByteOrder = enum { Little = 0, Big = 1, Network = 2, Host = 3 } &cxxname="::hilti::rt::ByteOrder"; public type Side = enum { Left = 0, Right = 1, Both = 2 } &cxxname="::hilti::rt::bytes::Side"; public type AddressFamily = enum { IPv4 = 0, IPv6 = 1 } &cxxname="::hilti::rt::AddressFamily"; public type RealType = enum { IEEE754_Single = 0, IEEE754_Double = 1 } &cxxname="::hilti::rt::real::Type"; public type Protocol = enum { TCP = 0, UDP = 1, ICMP = 2 } &cxxname="::hilti::rt::Protocol"; public type Charset = enum { ASCII = 0, UTF8 = 1 } &cxxname="::hilti::rt::bytes::Charset"; public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::bytes::DecodeErrorStrategy"; public type Captures = vector; public type Profiler = Profiler &cxxname="::hilti::rt::Profiler"; public type MatchState = struct { method Captures captures(view data); } &cxxname="::hilti::rt::regexp::MatchState"; public type StreamStatistics = struct { uint<64> num_data_bytes; uint<64> num_data_chunks; uint<64> num_gap_bytes; uint<64> num_gap_chunks; } &cxxname="::hilti::rt::stream::Statistics"; public type Exception = exception &cxxname="::hilti::rt::Exception"; public type SystemException = [exception :Exception &cxxname="::std::exception"; public type RuntimeError = [exception :Exception &cxxname="::hilti::rt::RuntimeError"; public type RecoverableFailure = [exception :Exception &cxxname="::hilti::rt::RecoverableFailure"; public type MissingData = [exception :Exception &cxxname="::hilti::rt::MissingData"; declare public function void print(any obj, bool newline = True) &cxxname="::hilti::rt::print" &have_prototype; declare public function void printValues(tuple<*> t, bool newline = True) &cxxname="::hilti::rt::printValues" &have_prototype; declare public function void debug(string dbg_stream, any obj) &cxxname="::hilti::rt::debug::print" &have_prototype; declare public function void debugIndent(string dbg_stream) &cxxname="::hilti::rt::debug::indent" &have_prototype; declare public function void debugDedent(string dbg_stream) &cxxname="::hilti::rt::debug::dedent" &have_prototype; declare public function time current_time() &cxxname="::hilti::rt::time::current_time" &have_prototype; declare public function time mktime(uint<64> y, uint<64> m, uint<64> d, uint<64> H, uint<64> M, uint<64> S) &cxxname="::hilti::rt::time::mktime" &have_prototype; declare public function void abort() &cxxname="::hilti::rt::abort_with_backtrace" &have_prototype; declare public function optional profiler_start(string name, optional> size = Null) &cxxname="::hilti::rt::profiler::start" &have_prototype; declare public function void profiler_stop(optional p, optional> size = Null) &cxxname="::hilti::rt::profiler::stop" &have_prototype; declare public function string exception_what(SystemException excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_what(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_where(SystemException excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; declare public function string exception_where(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; }" -> set module's canonical ID to hilti -[debug/resolver] -> [D20] declaration::Module hilti | module hilti { public type BitOrder = enum { LSB0 = 0, MSB0 = 1 } &cxxname="::hilti::rt::integer::BitOrder"; public type ByteOrder = enum { Little = 0, Big = 1, Network = 2, Host = 3 } &cxxname="::hilti::rt::ByteOrder"; public type Side = enum { Left = 0, Right = 1, Both = 2 } &cxxname="::hilti::rt::bytes::Side"; public type AddressFamily = enum { IPv4 = 0, IPv6 = 1 } &cxxname="::hilti::rt::AddressFamily"; public type RealType = enum { IEEE754_Single = 0, IEEE754_Double = 1 } &cxxname="::hilti::rt::real::Type"; public type Protocol = enum { TCP = 0, UDP = 1, ICMP = 2 } &cxxname="::hilti::rt::Protocol"; public type Charset = enum { ASCII = 0, UTF8 = 1 } &cxxname="::hilti::rt::bytes::Charset"; public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::bytes::DecodeErrorStrategy"; public type Captures = vector; public type Profiler = Profiler &cxxname="::hilti::rt::Profiler"; public type MatchState = struct { method Captures captures(view data); } &cxxname="::hilti::rt::regexp::MatchState"; public type StreamStatistics = struct { uint<64> num_data_bytes; uint<64> num_data_chunks; uint<64> num_gap_bytes; uint<64> num_gap_chunks; } &cxxname="::hilti::rt::stream::Statistics"; public type Exception = exception &cxxname="::hilti::rt::Exception"; public type SystemException = [exception :Exception &cxxname="::std::exception"; public type RuntimeError = [exception :Exception &cxxname="::hilti::rt::RuntimeError"; public type RecoverableFailure = [exception :Exception &cxxname="::hilti::rt::RecoverableFailure"; public type MissingData = [exception :Exception &cxxname="::hilti::rt::MissingData"; declare public function void print(any obj, bool newline = True) &cxxname="::hilti::rt::print" &have_prototype; declare public function void printValues(tuple<*> t, bool newline = True) &cxxname="::hilti::rt::printValues" &have_prototype; declare public function void debug(string dbg_stream, any obj) &cxxname="::hilti::rt::debug::print" &have_prototype; declare public function void debugIndent(string dbg_stream) &cxxname="::hilti::rt::debug::indent" &have_prototype; declare public function void debugDedent(string dbg_stream) &cxxname="::hilti::rt::debug::dedent" &have_prototype; declare public function time current_time() &cxxname="::hilti::rt::time::current_time" &have_prototype; declare public function time mktime(uint<64> y, uint<64> m, uint<64> d, uint<64> H, uint<64> M, uint<64> S) &cxxname="::hilti::rt::time::mktime" &have_prototype; declare public function void abort() &cxxname="::hilti::rt::abort_with_backtrace" &have_prototype; declare public function optional profiler_start(string name, optional> size = Null) &cxxname="::hilti::rt::profiler::start" &have_prototype; declare public function void profiler_stop(optional p, optional> size = Null) &cxxname="::hilti::rt::profiler::stop" &have_prototype; declare public function string exception_what(SystemException excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_what(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_where(SystemException excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; declare public function string exception_where(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; } (hilti.hlt:3:1-68:1) -[debug/resolver] [hilti.hlt:3:1-68:1] declaration::Module "module hilti { public type BitOrder = enum { LSB0 = 0, MSB0 = 1 } &cxxname="::hilti::rt::integer::BitOrder"; public type ByteOrder = enum { Little = 0, Big = 1, Network = 2, Host = 3 } &cxxname="::hilti::rt::ByteOrder"; public type Side = enum { Left = 0, Right = 1, Both = 2 } &cxxname="::hilti::rt::bytes::Side"; public type AddressFamily = enum { IPv4 = 0, IPv6 = 1 } &cxxname="::hilti::rt::AddressFamily"; public type RealType = enum { IEEE754_Single = 0, IEEE754_Double = 1 } &cxxname="::hilti::rt::real::Type"; public type Protocol = enum { TCP = 0, UDP = 1, ICMP = 2 } &cxxname="::hilti::rt::Protocol"; public type Charset = enum { ASCII = 0, UTF8 = 1 } &cxxname="::hilti::rt::bytes::Charset"; public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::bytes::DecodeErrorStrategy"; public type Captures = vector; public type Profiler = Profiler &cxxname="::hilti::rt::Profiler"; public type MatchState = struct { method Captures captures(view data); } &cxxname="::hilti::rt::regexp::MatchState"; public type StreamStatistics = struct { uint<64> num_data_bytes; uint<64> num_data_chunks; uint<64> num_gap_bytes; uint<64> num_gap_chunks; } &cxxname="::hilti::rt::stream::Statistics"; public type Exception = exception &cxxname="::hilti::rt::Exception"; public type SystemException = [exception :Exception &cxxname="::std::exception"; public type RuntimeError = [exception :Exception &cxxname="::hilti::rt::RuntimeError"; public type RecoverableFailure = [exception :Exception &cxxname="::hilti::rt::RecoverableFailure"; public type MissingData = [exception :Exception &cxxname="::hilti::rt::MissingData"; declare public function void print(any obj, bool newline = True) &cxxname="::hilti::rt::print" &have_prototype; declare public function void printValues(tuple<*> t, bool newline = True) &cxxname="::hilti::rt::printValues" &have_prototype; declare public function void debug(string dbg_stream, any obj) &cxxname="::hilti::rt::debug::print" &have_prototype; declare public function void debugIndent(string dbg_stream) &cxxname="::hilti::rt::debug::indent" &have_prototype; declare public function void debugDedent(string dbg_stream) &cxxname="::hilti::rt::debug::dedent" &have_prototype; declare public function time current_time() &cxxname="::hilti::rt::time::current_time" &have_prototype; declare public function time mktime(uint<64> y, uint<64> m, uint<64> d, uint<64> H, uint<64> M, uint<64> S) &cxxname="::hilti::rt::time::mktime" &have_prototype; declare public function void abort() &cxxname="::hilti::rt::abort_with_backtrace" &have_prototype; declare public function optional profiler_start(string name, optional> size = Null) &cxxname="::hilti::rt::profiler::start" &have_prototype; declare public function void profiler_stop(optional p, optional> size = Null) &cxxname="::hilti::rt::profiler::stop" &have_prototype; declare public function string exception_what(SystemException excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_what(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_where(SystemException excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; declare public function string exception_where(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; }" -> set module's declaration index to D20 +[debug/resolver] [hilti.hlt:3:1-68:1] declaration::Module "module hilti { public type BitOrder = enum { LSB0 = 0, MSB0 = 1 } &cxxname="::hilti::rt::integer::BitOrder"; public type ByteOrder = enum { Little = 0, Big = 1, Network = 2, Host = 3 } &cxxname="::hilti::rt::ByteOrder"; public type Side = enum { Left = 0, Right = 1, Both = 2 } &cxxname="::hilti::rt::bytes::Side"; public type AddressFamily = enum { IPv4 = 0, IPv6 = 1 } &cxxname="::hilti::rt::AddressFamily"; public type RealType = enum { IEEE754_Single = 0, IEEE754_Double = 1 } &cxxname="::hilti::rt::real::Type"; public type Protocol = enum { TCP = 0, UDP = 1, ICMP = 2 } &cxxname="::hilti::rt::Protocol"; public type Charset = enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } &cxxname="::hilti::rt::unicode::Charset"; public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::unicode::DecodeErrorStrategy"; public type Captures = vector; public type Profiler = Profiler &cxxname="::hilti::rt::Profiler"; public type MatchState = struct { method Captures captures(view data); } &cxxname="::hilti::rt::regexp::MatchState"; public type StreamStatistics = struct { uint<64> num_data_bytes; uint<64> num_data_chunks; uint<64> num_gap_bytes; uint<64> num_gap_chunks; } &cxxname="::hilti::rt::stream::Statistics"; public type Exception = exception &cxxname="::hilti::rt::Exception"; public type SystemException = [exception :Exception &cxxname="::std::exception"; public type RuntimeError = [exception :Exception &cxxname="::hilti::rt::RuntimeError"; public type RecoverableFailure = [exception :Exception &cxxname="::hilti::rt::RecoverableFailure"; public type MissingData = [exception :Exception &cxxname="::hilti::rt::MissingData"; declare public function void print(any obj, bool newline = True) &cxxname="::hilti::rt::print" &have_prototype; declare public function void printValues(tuple<*> t, bool newline = True) &cxxname="::hilti::rt::printValues" &have_prototype; declare public function void debug(string dbg_stream, any obj) &cxxname="::hilti::rt::debug::print" &have_prototype; declare public function void debugIndent(string dbg_stream) &cxxname="::hilti::rt::debug::indent" &have_prototype; declare public function void debugDedent(string dbg_stream) &cxxname="::hilti::rt::debug::dedent" &have_prototype; declare public function time current_time() &cxxname="::hilti::rt::time::current_time" &have_prototype; declare public function time mktime(uint<64> y, uint<64> m, uint<64> d, uint<64> H, uint<64> M, uint<64> S) &cxxname="::hilti::rt::time::mktime" &have_prototype; declare public function void abort() &cxxname="::hilti::rt::abort_with_backtrace" &have_prototype; declare public function optional profiler_start(string name, optional> size = Null) &cxxname="::hilti::rt::profiler::start" &have_prototype; declare public function void profiler_stop(optional p, optional> size = Null) &cxxname="::hilti::rt::profiler::stop" &have_prototype; declare public function string exception_what(SystemException excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_what(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_where(SystemException excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; declare public function string exception_where(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; }" -> set declaration's fully qualified ID to hilti +[debug/resolver] [hilti.hlt:3:1-68:1] declaration::Module "module hilti { public type BitOrder = enum { LSB0 = 0, MSB0 = 1 } &cxxname="::hilti::rt::integer::BitOrder"; public type ByteOrder = enum { Little = 0, Big = 1, Network = 2, Host = 3 } &cxxname="::hilti::rt::ByteOrder"; public type Side = enum { Left = 0, Right = 1, Both = 2 } &cxxname="::hilti::rt::bytes::Side"; public type AddressFamily = enum { IPv4 = 0, IPv6 = 1 } &cxxname="::hilti::rt::AddressFamily"; public type RealType = enum { IEEE754_Single = 0, IEEE754_Double = 1 } &cxxname="::hilti::rt::real::Type"; public type Protocol = enum { TCP = 0, UDP = 1, ICMP = 2 } &cxxname="::hilti::rt::Protocol"; public type Charset = enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } &cxxname="::hilti::rt::unicode::Charset"; public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::unicode::DecodeErrorStrategy"; public type Captures = vector; public type Profiler = Profiler &cxxname="::hilti::rt::Profiler"; public type MatchState = struct { method Captures captures(view data); } &cxxname="::hilti::rt::regexp::MatchState"; public type StreamStatistics = struct { uint<64> num_data_bytes; uint<64> num_data_chunks; uint<64> num_gap_bytes; uint<64> num_gap_chunks; } &cxxname="::hilti::rt::stream::Statistics"; public type Exception = exception &cxxname="::hilti::rt::Exception"; public type SystemException = [exception :Exception &cxxname="::std::exception"; public type RuntimeError = [exception :Exception &cxxname="::hilti::rt::RuntimeError"; public type RecoverableFailure = [exception :Exception &cxxname="::hilti::rt::RecoverableFailure"; public type MissingData = [exception :Exception &cxxname="::hilti::rt::MissingData"; declare public function void print(any obj, bool newline = True) &cxxname="::hilti::rt::print" &have_prototype; declare public function void printValues(tuple<*> t, bool newline = True) &cxxname="::hilti::rt::printValues" &have_prototype; declare public function void debug(string dbg_stream, any obj) &cxxname="::hilti::rt::debug::print" &have_prototype; declare public function void debugIndent(string dbg_stream) &cxxname="::hilti::rt::debug::indent" &have_prototype; declare public function void debugDedent(string dbg_stream) &cxxname="::hilti::rt::debug::dedent" &have_prototype; declare public function time current_time() &cxxname="::hilti::rt::time::current_time" &have_prototype; declare public function time mktime(uint<64> y, uint<64> m, uint<64> d, uint<64> H, uint<64> M, uint<64> S) &cxxname="::hilti::rt::time::mktime" &have_prototype; declare public function void abort() &cxxname="::hilti::rt::abort_with_backtrace" &have_prototype; declare public function optional profiler_start(string name, optional> size = Null) &cxxname="::hilti::rt::profiler::start" &have_prototype; declare public function void profiler_stop(optional p, optional> size = Null) &cxxname="::hilti::rt::profiler::stop" &have_prototype; declare public function string exception_what(SystemException excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_what(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_where(SystemException excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; declare public function string exception_where(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; }" -> set module's canonical ID to hilti +[debug/resolver] -> [D20] declaration::Module hilti | module hilti { public type BitOrder = enum { LSB0 = 0, MSB0 = 1 } &cxxname="::hilti::rt::integer::BitOrder"; public type ByteOrder = enum { Little = 0, Big = 1, Network = 2, Host = 3 } &cxxname="::hilti::rt::ByteOrder"; public type Side = enum { Left = 0, Right = 1, Both = 2 } &cxxname="::hilti::rt::bytes::Side"; public type AddressFamily = enum { IPv4 = 0, IPv6 = 1 } &cxxname="::hilti::rt::AddressFamily"; public type RealType = enum { IEEE754_Single = 0, IEEE754_Double = 1 } &cxxname="::hilti::rt::real::Type"; public type Protocol = enum { TCP = 0, UDP = 1, ICMP = 2 } &cxxname="::hilti::rt::Protocol"; public type Charset = enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } &cxxname="::hilti::rt::unicode::Charset"; public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::unicode::DecodeErrorStrategy"; public type Captures = vector; public type Profiler = Profiler &cxxname="::hilti::rt::Profiler"; public type MatchState = struct { method Captures captures(view data); } &cxxname="::hilti::rt::regexp::MatchState"; public type StreamStatistics = struct { uint<64> num_data_bytes; uint<64> num_data_chunks; uint<64> num_gap_bytes; uint<64> num_gap_chunks; } &cxxname="::hilti::rt::stream::Statistics"; public type Exception = exception &cxxname="::hilti::rt::Exception"; public type SystemException = [exception :Exception &cxxname="::std::exception"; public type RuntimeError = [exception :Exception &cxxname="::hilti::rt::RuntimeError"; public type RecoverableFailure = [exception :Exception &cxxname="::hilti::rt::RecoverableFailure"; public type MissingData = [exception :Exception &cxxname="::hilti::rt::MissingData"; declare public function void print(any obj, bool newline = True) &cxxname="::hilti::rt::print" &have_prototype; declare public function void printValues(tuple<*> t, bool newline = True) &cxxname="::hilti::rt::printValues" &have_prototype; declare public function void debug(string dbg_stream, any obj) &cxxname="::hilti::rt::debug::print" &have_prototype; declare public function void debugIndent(string dbg_stream) &cxxname="::hilti::rt::debug::indent" &have_prototype; declare public function void debugDedent(string dbg_stream) &cxxname="::hilti::rt::debug::dedent" &have_prototype; declare public function time current_time() &cxxname="::hilti::rt::time::current_time" &have_prototype; declare public function time mktime(uint<64> y, uint<64> m, uint<64> d, uint<64> H, uint<64> M, uint<64> S) &cxxname="::hilti::rt::time::mktime" &have_prototype; declare public function void abort() &cxxname="::hilti::rt::abort_with_backtrace" &have_prototype; declare public function optional profiler_start(string name, optional> size = Null) &cxxname="::hilti::rt::profiler::start" &have_prototype; declare public function void profiler_stop(optional p, optional> size = Null) &cxxname="::hilti::rt::profiler::stop" &have_prototype; declare public function string exception_what(SystemException excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_what(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_where(SystemException excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; declare public function string exception_where(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; } (hilti.hlt:3:1-68:1) +[debug/resolver] [hilti.hlt:3:1-68:1] declaration::Module "module hilti { public type BitOrder = enum { LSB0 = 0, MSB0 = 1 } &cxxname="::hilti::rt::integer::BitOrder"; public type ByteOrder = enum { Little = 0, Big = 1, Network = 2, Host = 3 } &cxxname="::hilti::rt::ByteOrder"; public type Side = enum { Left = 0, Right = 1, Both = 2 } &cxxname="::hilti::rt::bytes::Side"; public type AddressFamily = enum { IPv4 = 0, IPv6 = 1 } &cxxname="::hilti::rt::AddressFamily"; public type RealType = enum { IEEE754_Single = 0, IEEE754_Double = 1 } &cxxname="::hilti::rt::real::Type"; public type Protocol = enum { TCP = 0, UDP = 1, ICMP = 2 } &cxxname="::hilti::rt::Protocol"; public type Charset = enum { ASCII = 0, UTF8 = 1, UTF16LE = 2, UTF16BE = 3 } &cxxname="::hilti::rt::unicode::Charset"; public type DecodeErrorStrategy = enum { IGNORE = 0, REPLACE = 1, STRICT = 2 } &cxxname="::hilti::rt::unicode::DecodeErrorStrategy"; public type Captures = vector; public type Profiler = Profiler &cxxname="::hilti::rt::Profiler"; public type MatchState = struct { method Captures captures(view data); } &cxxname="::hilti::rt::regexp::MatchState"; public type StreamStatistics = struct { uint<64> num_data_bytes; uint<64> num_data_chunks; uint<64> num_gap_bytes; uint<64> num_gap_chunks; } &cxxname="::hilti::rt::stream::Statistics"; public type Exception = exception &cxxname="::hilti::rt::Exception"; public type SystemException = [exception :Exception &cxxname="::std::exception"; public type RuntimeError = [exception :Exception &cxxname="::hilti::rt::RuntimeError"; public type RecoverableFailure = [exception :Exception &cxxname="::hilti::rt::RecoverableFailure"; public type MissingData = [exception :Exception &cxxname="::hilti::rt::MissingData"; declare public function void print(any obj, bool newline = True) &cxxname="::hilti::rt::print" &have_prototype; declare public function void printValues(tuple<*> t, bool newline = True) &cxxname="::hilti::rt::printValues" &have_prototype; declare public function void debug(string dbg_stream, any obj) &cxxname="::hilti::rt::debug::print" &have_prototype; declare public function void debugIndent(string dbg_stream) &cxxname="::hilti::rt::debug::indent" &have_prototype; declare public function void debugDedent(string dbg_stream) &cxxname="::hilti::rt::debug::dedent" &have_prototype; declare public function time current_time() &cxxname="::hilti::rt::time::current_time" &have_prototype; declare public function time mktime(uint<64> y, uint<64> m, uint<64> d, uint<64> H, uint<64> M, uint<64> S) &cxxname="::hilti::rt::time::mktime" &have_prototype; declare public function void abort() &cxxname="::hilti::rt::abort_with_backtrace" &have_prototype; declare public function optional profiler_start(string name, optional> size = Null) &cxxname="::hilti::rt::profiler::start" &have_prototype; declare public function void profiler_stop(optional p, optional> size = Null) &cxxname="::hilti::rt::profiler::stop" &have_prototype; declare public function string exception_what(SystemException excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_what(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::what" &have_prototype; declare public function string exception_where(SystemException excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; declare public function string exception_where(RecoverableFailure excpt) &cxxname="::hilti::rt::exception::where" &have_prototype; }" -> set module's declaration index to D20 [debug/resolver] [bar.hlt:4:1-4:11] declaration::ImportedModule "import Foo;" -> set declaration's canonical ID to Bar::Foo [debug/resolver] [bar.hlt:4:1-4:11] declaration::ImportedModule "import Foo;" -> set declaration's fully qualified ID to Bar::Foo [debug/resolver] [bar.hlt:4:1-4:11] declaration::ImportedModule "import Foo;" -> imported module Foo @@ -315,7 +317,7 @@ [debug/compiler] -> modified [debug/compiler] processing ASTs, round 2 [debug/compiler] [HILTI] building scopes -[debug/resolver] -> [D23] declaration::Constant hilti::UTF8 | const hilti::Charset UTF8 = hilti::Charset::UTF8; (hilti.hlt:14:23-14:42) +[debug/resolver] -> [D23] declaration::Constant hilti::UTF8 | const hilti::Charset UTF8 = hilti::Charset::UTF8; (hilti.hlt:14:23-14:60) [debug/resolver] -> [D24] declaration::Constant hilti::REPLACE | const hilti::DecodeErrorStrategy REPLACE = hilti::DecodeErrorStrategy::REPLACE; (hilti.hlt:15:35-15:66) [debug/compiler] [HILTI] resolving AST [debug/resolver] -> [T17] type::String Bar::Bar1 | string (bar.hlt:6:20-6:25) @@ -347,9 +349,11 @@ [debug/resolver] [hilti.hlt:13:24-13:46] declaration::Constant "const hilti::Protocol UDP = hilti::Protocol::UDP;" -> set declaration's fully qualified ID to hilti::Protocol::UDP [debug/resolver] [hilti.hlt:13:24-13:46] declaration::Constant "const hilti::Protocol ICMP = hilti::Protocol::ICMP;" -> set declaration's fully qualified ID to hilti::Protocol::ICMP [debug/resolver] [hilti.hlt:13:24-13:46] declaration::Constant "const hilti::Protocol Undef = hilti::Protocol::Undef;" -> set declaration's fully qualified ID to hilti::Protocol::Undef -[debug/resolver] [hilti.hlt:14:23-14:42] declaration::Constant "const hilti::Charset ASCII = hilti::Charset::ASCII;" -> set declaration's fully qualified ID to hilti::Charset::ASCII -[debug/resolver] [hilti.hlt:14:23-14:42] declaration::Constant "const hilti::Charset UTF8 = hilti::Charset::UTF8;" -> set declaration's fully qualified ID to hilti::Charset::UTF8 -[debug/resolver] [hilti.hlt:14:23-14:42] declaration::Constant "const hilti::Charset Undef = hilti::Charset::Undef;" -> set declaration's fully qualified ID to hilti::Charset::Undef +[debug/resolver] [hilti.hlt:14:23-14:60] declaration::Constant "const hilti::Charset ASCII = hilti::Charset::ASCII;" -> set declaration's fully qualified ID to hilti::Charset::ASCII +[debug/resolver] [hilti.hlt:14:23-14:60] declaration::Constant "const hilti::Charset UTF8 = hilti::Charset::UTF8;" -> set declaration's fully qualified ID to hilti::Charset::UTF8 +[debug/resolver] [hilti.hlt:14:23-14:60] declaration::Constant "const hilti::Charset UTF16LE = hilti::Charset::UTF16LE;" -> set declaration's fully qualified ID to hilti::Charset::UTF16LE +[debug/resolver] [hilti.hlt:14:23-14:60] declaration::Constant "const hilti::Charset UTF16BE = hilti::Charset::UTF16BE;" -> set declaration's fully qualified ID to hilti::Charset::UTF16BE +[debug/resolver] [hilti.hlt:14:23-14:60] declaration::Constant "const hilti::Charset Undef = hilti::Charset::Undef;" -> set declaration's fully qualified ID to hilti::Charset::Undef [debug/resolver] [hilti.hlt:15:35-15:66] declaration::Constant "const hilti::DecodeErrorStrategy IGNORE = hilti::DecodeErrorStrategy::IGNORE;" -> set declaration's fully qualified ID to hilti::DecodeErrorStrategy::IGNORE [debug/resolver] [hilti.hlt:15:35-15:66] declaration::Constant "const hilti::DecodeErrorStrategy REPLACE = hilti::DecodeErrorStrategy::REPLACE;" -> set declaration's fully qualified ID to hilti::DecodeErrorStrategy::REPLACE [debug/resolver] [hilti.hlt:15:35-15:66] declaration::Constant "const hilti::DecodeErrorStrategy STRICT = hilti::DecodeErrorStrategy::STRICT;" -> set declaration's fully qualified ID to hilti::DecodeErrorStrategy::STRICT @@ -681,43 +685,61 @@ [debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] -[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:14:1-14:80) [@d:XXX] +[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:14:1-14:100) [@d:XXX] [debug/ast-final] | ASCII -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16BE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16LE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | UTF8 -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | Undef -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) -[debug/ast-final] - QualifiedType [parent @d:XXX] (hilti.hlt:14:23-14:42) [@q:XXX] -[debug/ast-final] - type::Enum [parent @q:XXX] (hilti.hlt:14:23-14:42) (resolved) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @d:XXX] (hilti.hlt:14:23-14:60) [@q:XXX] +[debug/ast-final] - type::Enum [parent @q:XXX] (hilti.hlt:14:23-14:60) (resolved) [@t:XXX] [debug/ast-final] | ASCII -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16BE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16LE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | UTF8 -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | Undef -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:30-14:34) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:37-14:40) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] -[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:23-14:42) [@t:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:43-14:49) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:14:43-14:79) [@a:XXX] -[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:14:43-14:79) [@a:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] +[debug/ast-final] - +[debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] +[debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:52-14:58) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] +[debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] +[debug/ast-final] - +[debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] +[debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:23-14:60) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] +[debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] +[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:14:61-14:99) [@a:XXX] +[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:14:61-14:99) [@a:XXX] [debug/ast-final] - expression::Ctor [parent @a:XXX] (const) (resolved) [@e:XXX] -[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] +[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] -[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:15:1-15:116) [@d:XXX] +[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:15:1-15:118) [@d:XXX] [debug/ast-final] | IGNORE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | REPLACE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | STRICT -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) @@ -756,10 +778,10 @@ [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:15:35-15:66) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:15:42-15:47) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:15:67-15:115) [@a:XXX] -[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:15:67-15:115) [@a:XXX] +[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:15:67-15:117) [@a:XXX] +[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:15:67-15:117) [@a:XXX] [debug/ast-final] - expression::Ctor [parent @a:XXX] (const) (resolved) [@e:XXX] -[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] +[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] [debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:16:1-16:37) [@d:XXX] @@ -1302,8 +1324,8 @@ [debug/ast-final] [D6] hilti::AddressFamily [declaration::Type] (hilti.hlt:11:1-11:84) [debug/ast-final] [D7] hilti::RealType [declaration::Type] (hilti.hlt:12:1-12:96) [debug/ast-final] [D8] hilti::Protocol [declaration::Type] (hilti.hlt:13:1-13:78) -[debug/ast-final] [D9] hilti::Charset [declaration::Type] (hilti.hlt:14:1-14:80) -[debug/ast-final] [D10] hilti::DecodeErrorStrategy [declaration::Type] (hilti.hlt:15:1-15:116) +[debug/ast-final] [D9] hilti::Charset [declaration::Type] (hilti.hlt:14:1-14:100) +[debug/ast-final] [D10] hilti::DecodeErrorStrategy [declaration::Type] (hilti.hlt:15:1-15:118) [debug/ast-final] [D11] hilti::Captures [declaration::Type] (hilti.hlt:16:1-16:37) [debug/ast-final] [D12] hilti::Profiler [declaration::Type] (hilti.hlt:17:1-17:61) [debug/ast-final] [D13] hilti::MatchState [declaration::Type] (hilti.hlt:19:1-21:43) @@ -1316,7 +1338,7 @@ [debug/ast-final] [D20] hilti [declaration::Module] (hilti.hlt:3:1-68:1) [debug/ast-final] [D21] Bar::Bar1 [declaration::Type] (bar.hlt:6:1-6:26) [debug/ast-final] [D22] Bar [declaration::Module] (bar.hlt:2:1-11:1) -[debug/ast-final] [D23] hilti::UTF8 [declaration::Constant] (hilti.hlt:14:23-14:42) +[debug/ast-final] [D23] hilti::UTF8 [declaration::Constant] (hilti.hlt:14:23-14:60) [debug/ast-final] [D24] hilti::REPLACE [declaration::Constant] (hilti.hlt:15:35-15:66) [debug/ast-final] [T1] hilti::BitOrder [type::Enum] (hilti.hlt:8:24-8:42) [debug/ast-final] [T2] hilti::ByteOrder [type::Enum] (hilti.hlt:9:25-9:59) @@ -1324,7 +1346,7 @@ [debug/ast-final] [T4] hilti::AddressFamily [type::Enum] (hilti.hlt:11:29-11:47) [debug/ast-final] [T5] hilti::RealType [type::Enum] (hilti.hlt:12:24-12:62) [debug/ast-final] [T6] hilti::Protocol [type::Enum] (hilti.hlt:13:24-13:46) -[debug/ast-final] [T7] hilti::Charset [type::Enum] (hilti.hlt:14:23-14:42) +[debug/ast-final] [T7] hilti::Charset [type::Enum] (hilti.hlt:14:23-14:60) [debug/ast-final] [T8] hilti::DecodeErrorStrategy [type::Enum] (hilti.hlt:15:35-15:66) [debug/ast-final] [T9] hilti::MatchState [type::Struct] (hilti.hlt:19:26-21:1) [debug/ast-final] [T10] hilti::StreamStatistics [type::Struct] (hilti.hlt:23:32-28:1) diff --git a/tests/Baseline/hilti.ast.types/output b/tests/Baseline/hilti.ast.types/output index ebab88eda..24b130957 100644 --- a/tests/Baseline/hilti.ast.types/output +++ b/tests/Baseline/hilti.ast.types/output @@ -361,43 +361,61 @@ [debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] -[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:14:1-14:80) [@d:XXX] +[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:14:1-14:100) [@d:XXX] [debug/ast-final] | ASCII -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16BE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16LE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | UTF8 -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | Undef -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) -[debug/ast-final] - QualifiedType [parent @d:XXX] (hilti.hlt:14:23-14:42) [@q:XXX] -[debug/ast-final] - type::Enum [parent @q:XXX] (hilti.hlt:14:23-14:42) (resolved) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @d:XXX] (hilti.hlt:14:23-14:60) [@q:XXX] +[debug/ast-final] - type::Enum [parent @q:XXX] (hilti.hlt:14:23-14:60) (resolved) [@t:XXX] [debug/ast-final] | ASCII -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16BE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16LE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | UTF8 -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | Undef -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:30-14:34) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:37-14:40) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] -[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:23-14:42) [@t:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:43-14:49) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:14:43-14:79) [@a:XXX] -[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:14:43-14:79) [@a:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] +[debug/ast-final] - +[debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] +[debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:52-14:58) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] +[debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] +[debug/ast-final] - +[debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] +[debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:23-14:60) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] +[debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] +[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:14:61-14:99) [@a:XXX] +[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:14:61-14:99) [@a:XXX] [debug/ast-final] - expression::Ctor [parent @a:XXX] (const) (resolved) [@e:XXX] -[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] +[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] -[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:15:1-15:116) [@d:XXX] +[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:15:1-15:118) [@d:XXX] [debug/ast-final] | IGNORE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | REPLACE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | STRICT -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) @@ -436,10 +454,10 @@ [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:15:35-15:66) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:15:42-15:47) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:15:67-15:115) [@a:XXX] -[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:15:67-15:115) [@a:XXX] +[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:15:67-15:117) [@a:XXX] +[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:15:67-15:117) [@a:XXX] [debug/ast-final] - expression::Ctor [parent @a:XXX] (const) (resolved) [@e:XXX] -[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] +[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] [debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:16:1-16:37) [@d:XXX] @@ -965,8 +983,8 @@ [debug/ast-final] [D5] hilti::AddressFamily [declaration::Type] (hilti.hlt:11:1-11:84) [debug/ast-final] [D6] hilti::RealType [declaration::Type] (hilti.hlt:12:1-12:96) [debug/ast-final] [D7] hilti::Protocol [declaration::Type] (hilti.hlt:13:1-13:78) -[debug/ast-final] [D8] hilti::Charset [declaration::Type] (hilti.hlt:14:1-14:80) -[debug/ast-final] [D9] hilti::DecodeErrorStrategy [declaration::Type] (hilti.hlt:15:1-15:116) +[debug/ast-final] [D8] hilti::Charset [declaration::Type] (hilti.hlt:14:1-14:100) +[debug/ast-final] [D9] hilti::DecodeErrorStrategy [declaration::Type] (hilti.hlt:15:1-15:118) [debug/ast-final] [D10] hilti::Captures [declaration::Type] (hilti.hlt:16:1-16:37) [debug/ast-final] [D11] hilti::Profiler [declaration::Type] (hilti.hlt:17:1-17:61) [debug/ast-final] [D12] hilti::MatchState [declaration::Type] (hilti.hlt:19:1-21:43) @@ -977,7 +995,7 @@ [debug/ast-final] [D17] hilti::RecoverableFailure [declaration::Type] (hilti.hlt:56:1-56:98) [debug/ast-final] [D18] hilti::MissingData [declaration::Type] (hilti.hlt:59:1-59:84) [debug/ast-final] [D19] hilti [declaration::Module] (hilti.hlt:3:1-68:1) -[debug/ast-final] [D20] hilti::UTF8 [declaration::Constant] (hilti.hlt:14:23-14:42) +[debug/ast-final] [D20] hilti::UTF8 [declaration::Constant] (hilti.hlt:14:23-14:60) [debug/ast-final] [D21] hilti::REPLACE [declaration::Constant] (hilti.hlt:15:35-15:66) [debug/ast-final] [T1] hilti::BitOrder [type::Enum] (hilti.hlt:8:24-8:42) [debug/ast-final] [T2] hilti::ByteOrder [type::Enum] (hilti.hlt:9:25-9:59) @@ -985,7 +1003,7 @@ [debug/ast-final] [T4] hilti::AddressFamily [type::Enum] (hilti.hlt:11:29-11:47) [debug/ast-final] [T5] hilti::RealType [type::Enum] (hilti.hlt:12:24-12:62) [debug/ast-final] [T6] hilti::Protocol [type::Enum] (hilti.hlt:13:24-13:46) -[debug/ast-final] [T7] hilti::Charset [type::Enum] (hilti.hlt:14:23-14:42) +[debug/ast-final] [T7] hilti::Charset [type::Enum] (hilti.hlt:14:23-14:60) [debug/ast-final] [T8] hilti::DecodeErrorStrategy [type::Enum] (hilti.hlt:15:35-15:66) [debug/ast-final] [T9] hilti::MatchState [type::Struct] (hilti.hlt:19:26-21:1) [debug/ast-final] [T10] hilti::StreamStatistics [type::Struct] (hilti.hlt:23:32-28:1) diff --git a/tests/Baseline/hilti.expressions.ctor-replacement/output b/tests/Baseline/hilti.expressions.ctor-replacement/output index 91eee317b..0040fc4a2 100644 --- a/tests/Baseline/hilti.expressions.ctor-replacement/output +++ b/tests/Baseline/hilti.expressions.ctor-replacement/output @@ -317,43 +317,61 @@ [debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] -[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:14:1-14:80) [@d:XXX] +[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:14:1-14:100) [@d:XXX] [debug/ast-final] | ASCII -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16BE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16LE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | UTF8 -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | Undef -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) -[debug/ast-final] - QualifiedType [parent @d:XXX] (hilti.hlt:14:23-14:42) [@q:XXX] -[debug/ast-final] - type::Enum [parent @q:XXX] (hilti.hlt:14:23-14:42) (resolved) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @d:XXX] (hilti.hlt:14:23-14:60) [@q:XXX] +[debug/ast-final] - type::Enum [parent @q:XXX] (hilti.hlt:14:23-14:60) (resolved) [@t:XXX] [debug/ast-final] | ASCII -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16BE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) +[debug/ast-final] | UTF16LE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | UTF8 -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | Undef -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:30-14:34) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:37-14:40) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:42) [@d:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] [debug/ast-final] - [debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] [debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] -[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:23-14:42) [@t:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:43-14:49) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:14:43-14:79) [@a:XXX] -[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:14:43-14:79) [@a:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] +[debug/ast-final] - +[debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] +[debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:52-14:58) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] +[debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] +[debug/ast-final] - declaration::Constant [parent @t:XXX] (hilti.hlt:14:23-14:60) [@d:XXX] +[debug/ast-final] - +[debug/ast-final] - expression::Ctor [parent @d:XXX] (const) (resolved) [@e:XXX] +[debug/ast-final] - ctor::Enum [parent @e:XXX] [@c:XXX] +[debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:14:23-14:60) [@t:XXX] +[debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:14:30-14:34) [@q:XXX] +[debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] +[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:14:61-14:99) [@a:XXX] +[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:14:61-14:99) [@a:XXX] [debug/ast-final] - expression::Ctor [parent @a:XXX] (const) (resolved) [@e:XXX] -[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] +[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] -[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:15:1-15:116) [@d:XXX] +[debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:15:1-15:118) [@d:XXX] [debug/ast-final] | IGNORE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | REPLACE -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) [debug/ast-final] | STRICT -> declaration::Constant [parent @t:XXX] [@d:XXX] ([@d:XXX]) @@ -392,10 +410,10 @@ [debug/ast-final] - type::enum_::Label [parent @c:XXX] (hilti.hlt:15:35-15:66) [@t:XXX] [debug/ast-final] - QualifiedType [parent @t:XXX] (hilti.hlt:15:42-15:47) [@q:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] -[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:15:67-15:115) [@a:XXX] -[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:15:67-15:115) [@a:XXX] +[debug/ast-final] - AttributeSet [parent @d:XXX] (hilti.hlt:15:67-15:117) [@a:XXX] +[debug/ast-final] - Attribute [parent @a:XXX] (hilti.hlt:15:67-15:117) [@a:XXX] [debug/ast-final] - expression::Ctor [parent @a:XXX] (const) (resolved) [@e:XXX] -[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] +[debug/ast-final] - ctor::String [parent @e:XXX] [@c:XXX] [debug/ast-final] - QualifiedType [parent @c:XXX] [@q:XXX] [debug/ast-final] - type::String [parent @q:XXX] (resolved) [@t:XXX] [debug/ast-final] - declaration::Type [parent @d:XXX] (hilti.hlt:16:1-16:37) [@d:XXX] @@ -921,8 +939,8 @@ [debug/ast-final] [D5] hilti::AddressFamily [declaration::Type] (hilti.hlt:11:1-11:84) [debug/ast-final] [D6] hilti::RealType [declaration::Type] (hilti.hlt:12:1-12:96) [debug/ast-final] [D7] hilti::Protocol [declaration::Type] (hilti.hlt:13:1-13:78) -[debug/ast-final] [D8] hilti::Charset [declaration::Type] (hilti.hlt:14:1-14:80) -[debug/ast-final] [D9] hilti::DecodeErrorStrategy [declaration::Type] (hilti.hlt:15:1-15:116) +[debug/ast-final] [D8] hilti::Charset [declaration::Type] (hilti.hlt:14:1-14:100) +[debug/ast-final] [D9] hilti::DecodeErrorStrategy [declaration::Type] (hilti.hlt:15:1-15:118) [debug/ast-final] [D10] hilti::Captures [declaration::Type] (hilti.hlt:16:1-16:37) [debug/ast-final] [D11] hilti::Profiler [declaration::Type] (hilti.hlt:17:1-17:61) [debug/ast-final] [D12] hilti::MatchState [declaration::Type] (hilti.hlt:19:1-21:43) @@ -933,7 +951,7 @@ [debug/ast-final] [D17] hilti::RecoverableFailure [declaration::Type] (hilti.hlt:56:1-56:98) [debug/ast-final] [D18] hilti::MissingData [declaration::Type] (hilti.hlt:59:1-59:84) [debug/ast-final] [D19] hilti [declaration::Module] (hilti.hlt:3:1-68:1) -[debug/ast-final] [D20] hilti::UTF8 [declaration::Constant] (hilti.hlt:14:23-14:42) +[debug/ast-final] [D20] hilti::UTF8 [declaration::Constant] (hilti.hlt:14:23-14:60) [debug/ast-final] [D21] hilti::REPLACE [declaration::Constant] (hilti.hlt:15:35-15:66) [debug/ast-final] [T1] hilti::BitOrder [type::Enum] (hilti.hlt:8:24-8:42) [debug/ast-final] [T2] hilti::ByteOrder [type::Enum] (hilti.hlt:9:25-9:59) @@ -941,7 +959,7 @@ [debug/ast-final] [T4] hilti::AddressFamily [type::Enum] (hilti.hlt:11:29-11:47) [debug/ast-final] [T5] hilti::RealType [type::Enum] (hilti.hlt:12:24-12:62) [debug/ast-final] [T6] hilti::Protocol [type::Enum] (hilti.hlt:13:24-13:46) -[debug/ast-final] [T7] hilti::Charset [type::Enum] (hilti.hlt:14:23-14:42) +[debug/ast-final] [T7] hilti::Charset [type::Enum] (hilti.hlt:14:23-14:60) [debug/ast-final] [T8] hilti::DecodeErrorStrategy [type::Enum] (hilti.hlt:15:35-15:66) [debug/ast-final] [T9] hilti::MatchState [type::Struct] (hilti.hlt:19:26-21:1) [debug/ast-final] [T10] hilti::StreamStatistics [type::Struct] (hilti.hlt:23:32-28:1) diff --git a/tests/Baseline/hilti.types.struct.canonical-ids/output b/tests/Baseline/hilti.types.struct.canonical-ids/output index 379f66a6e..207e02c3b 100644 --- a/tests/Baseline/hilti.types.struct.canonical-ids/output +++ b/tests/Baseline/hilti.types.struct.canonical-ids/output @@ -46,6 +46,8 @@ [debug/ast-declarations] - Type "Charset" (hilti::Charset) [debug/ast-declarations] - Constant "ASCII" (hilti::ASCII) [debug/ast-declarations] - Constant "UTF8" (hilti::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (hilti::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (hilti::UTF16BE) [debug/ast-declarations] - Constant "Undef" (hilti::Undef_7) [debug/ast-declarations] - Type "DecodeErrorStrategy" (hilti::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (hilti::IGNORE) @@ -152,6 +154,8 @@ [debug/ast-declarations] - Type "Charset" (hilti::Charset) [debug/ast-declarations] - Constant "ASCII" (hilti::ASCII) [debug/ast-declarations] - Constant "UTF8" (hilti::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (hilti::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (hilti::UTF16BE) [debug/ast-declarations] - Constant "Undef" (hilti::Undef_7) [debug/ast-declarations] - Type "DecodeErrorStrategy" (hilti::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (hilti::IGNORE) diff --git a/tests/Baseline/spicy.types.function.cxxname-normalization/output b/tests/Baseline/spicy.types.function.cxxname-normalization/output index fe7ed1c66..271c991a0 100644 --- a/tests/Baseline/spicy.types.function.cxxname-normalization/output +++ b/tests/Baseline/spicy.types.function.cxxname-normalization/output @@ -7,8 +7,8 @@ [debug/resolver] [hilti.hlt:11:49-11:83] Attribute "&cxxname="hilti::rt::AddressFamily"" -> Attribute "&cxxname="::hilti::rt::AddressFamily"" [debug/resolver] [hilti.hlt:12:64-12:95] Attribute "&cxxname="hilti::rt::real::Type"" -> Attribute "&cxxname="::hilti::rt::real::Type"" [debug/resolver] [hilti.hlt:13:48-13:77] Attribute "&cxxname="hilti::rt::Protocol"" -> Attribute "&cxxname="::hilti::rt::Protocol"" -[debug/resolver] [hilti.hlt:14:44-14:79] Attribute "&cxxname="hilti::rt::bytes::Charset"" -> Attribute "&cxxname="::hilti::rt::bytes::Charset"" -[debug/resolver] [hilti.hlt:15:68-15:115] Attribute "&cxxname="hilti::rt::bytes::DecodeErrorStrategy"" -> Attribute "&cxxname="::hilti::rt::bytes::DecodeErrorStrategy"" +[debug/resolver] [hilti.hlt:14:62-14:99] Attribute "&cxxname="hilti::rt::unicode::Charset"" -> Attribute "&cxxname="::hilti::rt::unicode::Charset"" +[debug/resolver] [hilti.hlt:15:68-15:117] Attribute "&cxxname="hilti::rt::unicode::DecodeErrorStrategy"" -> Attribute "&cxxname="::hilti::rt::unicode::DecodeErrorStrategy"" [debug/resolver] [hilti.hlt:21:3-21:42] Attribute "&cxxname="hilti::rt::regexp::MatchState"" -> Attribute "&cxxname="::hilti::rt::regexp::MatchState"" [debug/resolver] [hilti.hlt:28:3-28:44] Attribute "&cxxname="hilti::rt::stream::Statistics"" -> Attribute "&cxxname="::hilti::rt::stream::Statistics"" [debug/resolver] [hilti.hlt:30:57-30:83] Attribute "&cxxname="hilti::rt::print"" -> Attribute "&cxxname="::hilti::rt::print"" @@ -63,31 +63,31 @@ [debug/resolver] [spicy.spicy:14:3-14:37] Attribute "&cxxname="hilti::rt::AddressFamily"" -> Attribute "&cxxname="::hilti::rt::AddressFamily"" [debug/resolver] [spicy.spicy:23:3-23:41] Attribute "&cxxname="hilti::rt::integer::BitOrder"" -> Attribute "&cxxname="::hilti::rt::integer::BitOrder"" [debug/resolver] [spicy.spicy:31:3-31:33] Attribute "&cxxname="hilti::rt::ByteOrder"" -> Attribute "&cxxname="::hilti::rt::ByteOrder"" -[debug/resolver] [spicy.spicy:37:3-37:38] Attribute "&cxxname="hilti::rt::bytes::Charset"" -> Attribute "&cxxname="::hilti::rt::bytes::Charset"" -[debug/resolver] [spicy.spicy:44:3-44:50] Attribute "&cxxname="hilti::rt::bytes::DecodeErrorStrategy"" -> Attribute "&cxxname="::hilti::rt::bytes::DecodeErrorStrategy"" -[debug/resolver] [spicy.spicy:52:3-52:42] Attribute "&cxxname="hilti::rt::regexp::MatchState"" -> Attribute "&cxxname="::hilti::rt::regexp::MatchState"" -[debug/resolver] [spicy.spicy:60:3-60:44] Attribute "&cxxname="hilti::rt::stream::Statistics"" -> Attribute "&cxxname="::hilti::rt::stream::Statistics"" -[debug/resolver] [spicy.spicy:67:3-67:32] Attribute "&cxxname="hilti::rt::Protocol"" -> Attribute "&cxxname="::hilti::rt::Protocol"" -[debug/resolver] [spicy.spicy:73:3-73:34] Attribute "&cxxname="hilti::rt::real::Type"" -> Attribute "&cxxname="::hilti::rt::real::Type"" -[debug/resolver] [spicy.spicy:78:3-78:47] Attribute "&cxxname="spicy::rt::sink::ReassemblerPolicy"" -> Attribute "&cxxname="::spicy::rt::sink::ReassemblerPolicy"" -[debug/resolver] [spicy.spicy:85:3-85:35] Attribute "&cxxname="hilti::rt::bytes::Side"" -> Attribute "&cxxname="::hilti::rt::bytes::Side"" -[debug/resolver] [spicy.spicy:91:3-91:41] Attribute "&cxxname="hilti::rt::stream::Direction"" -> Attribute "&cxxname="::hilti::rt::stream::Direction"" -[debug/resolver] [spicy.spicy:102:60-102:91] Attribute "&cxxname="spicy::rt::zlib::init"" -> Attribute "&cxxname="::spicy::rt::zlib::init"" -[debug/resolver] [spicy.spicy:105:81-105:118] Attribute "&cxxname="spicy::rt::zlib::decompress"" -> Attribute "&cxxname="::spicy::rt::zlib::decompress"" -[debug/resolver] [spicy.spicy:108:64-108:97] Attribute "&cxxname="spicy::rt::zlib::finish"" -> Attribute "&cxxname="::spicy::rt::zlib::finish"" -[debug/resolver] [spicy.spicy:111:81-111:116] Attribute "&cxxname="spicy::rt::base64::encode"" -> Attribute "&cxxname="::spicy::rt::base64::encode"" -[debug/resolver] [spicy.spicy:114:81-114:116] Attribute "&cxxname="spicy::rt::base64::decode"" -> Attribute "&cxxname="::spicy::rt::base64::decode"" -[debug/resolver] [spicy.spicy:117:68-117:103] Attribute "&cxxname="spicy::rt::base64::finish"" -> Attribute "&cxxname="::spicy::rt::base64::finish"" -[debug/resolver] [spicy.spicy:120:39-120:76] Attribute "&cxxname="spicy::rt::zlib::crc32_init"" -> Attribute "&cxxname="::spicy::rt::zlib::crc32_init"" -[debug/resolver] [spicy.spicy:123:62-123:98] Attribute "&cxxname="spicy::rt::zlib::crc32_add"" -> Attribute "&cxxname="::spicy::rt::zlib::crc32_add"" -[debug/resolver] [spicy.spicy:126:39-126:78] Attribute "&cxxname="hilti::rt::time::current_time"" -> Attribute "&cxxname="::hilti::rt::time::current_time"" -[debug/resolver] [spicy.spicy:136:97-136:130] Attribute "&cxxname="hilti::rt::time::mktime"" -> Attribute "&cxxname="::hilti::rt::time::mktime"" -[debug/resolver] [spicy.spicy:139:59-139:98] Attribute "&cxxname="spicy::rt::bytes_to_hexstring"" -> Attribute "&cxxname="::spicy::rt::bytes_to_hexstring"" -[debug/resolver] [spicy.spicy:142:53-142:86] Attribute "&cxxname="spicy::rt::bytes_to_mac"" -> Attribute "&cxxname="::spicy::rt::bytes_to_mac"" -[debug/resolver] [spicy.spicy:145:57-145:84] Attribute "&cxxname="hilti::rt::getenv"" -> Attribute "&cxxname="::hilti::rt::getenv"" -[debug/resolver] [spicy.spicy:157:68-157:97] Attribute "&cxxname="hilti::rt::strftime"" -> Attribute "&cxxname="::hilti::rt::strftime"" -[debug/resolver] [spicy.spicy:169:62-169:91] Attribute "&cxxname="hilti::rt::strptime"" -> Attribute "&cxxname="::hilti::rt::strptime"" -[debug/resolver] [spicy.spicy:174:49-174:84] Attribute "&cxxname="hilti::rt::address::parse"" -> Attribute "&cxxname="::hilti::rt::address::parse"" -[debug/resolver] [spicy.spicy:179:48-179:83] Attribute "&cxxname="hilti::rt::address::parse"" -> Attribute "&cxxname="::hilti::rt::address::parse"" -[debug/resolver] [spicy.spicy:184:39-184:72] Attribute "&cxxname="spicy::rt::accept_input"" -> Attribute "&cxxname="::spicy::rt::accept_input"" -[debug/resolver] [spicy.spicy:195:54-195:88] Attribute "&cxxname="spicy::rt::decline_input"" -> Attribute "&cxxname="::spicy::rt::decline_input"" +[debug/resolver] [spicy.spicy:39:3-39:40] Attribute "&cxxname="hilti::rt::unicode::Charset"" -> Attribute "&cxxname="::hilti::rt::unicode::Charset"" +[debug/resolver] [spicy.spicy:46:3-46:52] Attribute "&cxxname="hilti::rt::unicode::DecodeErrorStrategy"" -> Attribute "&cxxname="::hilti::rt::unicode::DecodeErrorStrategy"" +[debug/resolver] [spicy.spicy:54:3-54:42] Attribute "&cxxname="hilti::rt::regexp::MatchState"" -> Attribute "&cxxname="::hilti::rt::regexp::MatchState"" +[debug/resolver] [spicy.spicy:62:3-62:44] Attribute "&cxxname="hilti::rt::stream::Statistics"" -> Attribute "&cxxname="::hilti::rt::stream::Statistics"" +[debug/resolver] [spicy.spicy:69:3-69:32] Attribute "&cxxname="hilti::rt::Protocol"" -> Attribute "&cxxname="::hilti::rt::Protocol"" +[debug/resolver] [spicy.spicy:75:3-75:34] Attribute "&cxxname="hilti::rt::real::Type"" -> Attribute "&cxxname="::hilti::rt::real::Type"" +[debug/resolver] [spicy.spicy:80:3-80:47] Attribute "&cxxname="spicy::rt::sink::ReassemblerPolicy"" -> Attribute "&cxxname="::spicy::rt::sink::ReassemblerPolicy"" +[debug/resolver] [spicy.spicy:87:3-87:35] Attribute "&cxxname="hilti::rt::bytes::Side"" -> Attribute "&cxxname="::hilti::rt::bytes::Side"" +[debug/resolver] [spicy.spicy:93:3-93:41] Attribute "&cxxname="hilti::rt::stream::Direction"" -> Attribute "&cxxname="::hilti::rt::stream::Direction"" +[debug/resolver] [spicy.spicy:104:60-104:91] Attribute "&cxxname="spicy::rt::zlib::init"" -> Attribute "&cxxname="::spicy::rt::zlib::init"" +[debug/resolver] [spicy.spicy:107:81-107:118] Attribute "&cxxname="spicy::rt::zlib::decompress"" -> Attribute "&cxxname="::spicy::rt::zlib::decompress"" +[debug/resolver] [spicy.spicy:110:64-110:97] Attribute "&cxxname="spicy::rt::zlib::finish"" -> Attribute "&cxxname="::spicy::rt::zlib::finish"" +[debug/resolver] [spicy.spicy:113:81-113:116] Attribute "&cxxname="spicy::rt::base64::encode"" -> Attribute "&cxxname="::spicy::rt::base64::encode"" +[debug/resolver] [spicy.spicy:116:81-116:116] Attribute "&cxxname="spicy::rt::base64::decode"" -> Attribute "&cxxname="::spicy::rt::base64::decode"" +[debug/resolver] [spicy.spicy:119:68-119:103] Attribute "&cxxname="spicy::rt::base64::finish"" -> Attribute "&cxxname="::spicy::rt::base64::finish"" +[debug/resolver] [spicy.spicy:122:39-122:76] Attribute "&cxxname="spicy::rt::zlib::crc32_init"" -> Attribute "&cxxname="::spicy::rt::zlib::crc32_init"" +[debug/resolver] [spicy.spicy:125:62-125:98] Attribute "&cxxname="spicy::rt::zlib::crc32_add"" -> Attribute "&cxxname="::spicy::rt::zlib::crc32_add"" +[debug/resolver] [spicy.spicy:128:39-128:78] Attribute "&cxxname="hilti::rt::time::current_time"" -> Attribute "&cxxname="::hilti::rt::time::current_time"" +[debug/resolver] [spicy.spicy:138:97-138:130] Attribute "&cxxname="hilti::rt::time::mktime"" -> Attribute "&cxxname="::hilti::rt::time::mktime"" +[debug/resolver] [spicy.spicy:141:59-141:98] Attribute "&cxxname="spicy::rt::bytes_to_hexstring"" -> Attribute "&cxxname="::spicy::rt::bytes_to_hexstring"" +[debug/resolver] [spicy.spicy:144:53-144:86] Attribute "&cxxname="spicy::rt::bytes_to_mac"" -> Attribute "&cxxname="::spicy::rt::bytes_to_mac"" +[debug/resolver] [spicy.spicy:147:57-147:84] Attribute "&cxxname="hilti::rt::getenv"" -> Attribute "&cxxname="::hilti::rt::getenv"" +[debug/resolver] [spicy.spicy:159:68-159:97] Attribute "&cxxname="hilti::rt::strftime"" -> Attribute "&cxxname="::hilti::rt::strftime"" +[debug/resolver] [spicy.spicy:171:62-171:91] Attribute "&cxxname="hilti::rt::strptime"" -> Attribute "&cxxname="::hilti::rt::strptime"" +[debug/resolver] [spicy.spicy:176:49-176:84] Attribute "&cxxname="hilti::rt::address::parse"" -> Attribute "&cxxname="::hilti::rt::address::parse"" +[debug/resolver] [spicy.spicy:181:48-181:83] Attribute "&cxxname="hilti::rt::address::parse"" -> Attribute "&cxxname="::hilti::rt::address::parse"" +[debug/resolver] [spicy.spicy:186:39-186:72] Attribute "&cxxname="spicy::rt::accept_input"" -> Attribute "&cxxname="::spicy::rt::accept_input"" +[debug/resolver] [spicy.spicy:197:54-197:88] Attribute "&cxxname="spicy::rt::decline_input"" -> Attribute "&cxxname="::spicy::rt::decline_input"" diff --git a/tests/Baseline/spicy.types.unit.canonical-ids-with-import/output b/tests/Baseline/spicy.types.unit.canonical-ids-with-import/output index 5a9225617..48f49d0eb 100644 --- a/tests/Baseline/spicy.types.unit.canonical-ids-with-import/output +++ b/tests/Baseline/spicy.types.unit.canonical-ids-with-import/output @@ -43,6 +43,8 @@ [debug/ast-declarations] - Type "Charset" (hilti::Charset) [debug/ast-declarations] - Constant "ASCII" (hilti::ASCII) [debug/ast-declarations] - Constant "UTF8" (hilti::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (hilti::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (hilti::UTF16BE) [debug/ast-declarations] - Constant "Undef" (hilti::Undef_7) [debug/ast-declarations] - Type "DecodeErrorStrategy" (hilti::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (hilti::IGNORE) @@ -289,6 +291,8 @@ [debug/ast-declarations] - Type "Charset" (spicy::Charset) [debug/ast-declarations] - Constant "ASCII" (spicy::ASCII) [debug/ast-declarations] - Constant "UTF8" (spicy::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (spicy::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (spicy::UTF16BE) [debug/ast-declarations] - Constant "Undef" (spicy::Undef_4) [debug/ast-declarations] - Type "DecodeErrorStrategy" (spicy::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (spicy::IGNORE) @@ -604,6 +608,8 @@ [debug/ast-declarations] - Type "Charset" (hilti::Charset) [debug/ast-declarations] - Constant "ASCII" (hilti::ASCII) [debug/ast-declarations] - Constant "UTF8" (hilti::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (hilti::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (hilti::UTF16BE) [debug/ast-declarations] - Constant "Undef" (hilti::Undef_7) [debug/ast-declarations] - Type "DecodeErrorStrategy" (hilti::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (hilti::IGNORE) @@ -850,6 +856,8 @@ [debug/ast-declarations] - Type "Charset" (spicy::Charset) [debug/ast-declarations] - Constant "ASCII" (spicy::ASCII) [debug/ast-declarations] - Constant "UTF8" (spicy::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (spicy::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (spicy::UTF16BE) [debug/ast-declarations] - Constant "Undef" (spicy::Undef_4) [debug/ast-declarations] - Type "DecodeErrorStrategy" (spicy::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (spicy::IGNORE) @@ -1080,6 +1088,8 @@ [debug/ast-declarations] - Type "Charset" (hilti::Charset) [debug/ast-declarations] - Constant "ASCII" (hilti::ASCII) [debug/ast-declarations] - Constant "UTF8" (hilti::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (hilti::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (hilti::UTF16BE) [debug/ast-declarations] - Constant "Undef" (hilti::Undef_7) [debug/ast-declarations] - Type "DecodeErrorStrategy" (hilti::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (hilti::IGNORE) @@ -1320,6 +1330,8 @@ [debug/ast-declarations] - Type "Charset" (spicy::Charset) [debug/ast-declarations] - Constant "ASCII" (spicy::ASCII) [debug/ast-declarations] - Constant "UTF8" (spicy::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (spicy::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (spicy::UTF16BE) [debug/ast-declarations] - Constant "Undef" (spicy::Undef_4) [debug/ast-declarations] - Type "DecodeErrorStrategy" (spicy::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (spicy::IGNORE) diff --git a/tests/Baseline/spicy.types.unit.canonical-ids/output b/tests/Baseline/spicy.types.unit.canonical-ids/output index f024199aa..75351438c 100644 --- a/tests/Baseline/spicy.types.unit.canonical-ids/output +++ b/tests/Baseline/spicy.types.unit.canonical-ids/output @@ -42,6 +42,8 @@ [debug/ast-declarations] - Type "Charset" (hilti::Charset) [debug/ast-declarations] - Constant "ASCII" (hilti::ASCII) [debug/ast-declarations] - Constant "UTF8" (hilti::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (hilti::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (hilti::UTF16BE) [debug/ast-declarations] - Constant "Undef" (hilti::Undef_7) [debug/ast-declarations] - Type "DecodeErrorStrategy" (hilti::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (hilti::IGNORE) @@ -288,6 +290,8 @@ [debug/ast-declarations] - Type "Charset" (spicy::Charset) [debug/ast-declarations] - Constant "ASCII" (spicy::ASCII) [debug/ast-declarations] - Constant "UTF8" (spicy::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (spicy::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (spicy::UTF16BE) [debug/ast-declarations] - Constant "Undef" (spicy::Undef_4) [debug/ast-declarations] - Type "DecodeErrorStrategy" (spicy::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (spicy::IGNORE) @@ -751,6 +755,8 @@ [debug/ast-declarations] - Type "Charset" (hilti::Charset) [debug/ast-declarations] - Constant "ASCII" (hilti::ASCII) [debug/ast-declarations] - Constant "UTF8" (hilti::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (hilti::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (hilti::UTF16BE) [debug/ast-declarations] - Constant "Undef" (hilti::Undef_7) [debug/ast-declarations] - Type "DecodeErrorStrategy" (hilti::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (hilti::IGNORE) @@ -997,6 +1003,8 @@ [debug/ast-declarations] - Type "Charset" (spicy::Charset) [debug/ast-declarations] - Constant "ASCII" (spicy::ASCII) [debug/ast-declarations] - Constant "UTF8" (spicy::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (spicy::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (spicy::UTF16BE) [debug/ast-declarations] - Constant "Undef" (spicy::Undef_4) [debug/ast-declarations] - Type "DecodeErrorStrategy" (spicy::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (spicy::IGNORE) @@ -1217,6 +1225,8 @@ [debug/ast-declarations] - Type "Charset" (hilti::Charset) [debug/ast-declarations] - Constant "ASCII" (hilti::ASCII) [debug/ast-declarations] - Constant "UTF8" (hilti::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (hilti::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (hilti::UTF16BE) [debug/ast-declarations] - Constant "Undef" (hilti::Undef_7) [debug/ast-declarations] - Type "DecodeErrorStrategy" (hilti::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (hilti::IGNORE) @@ -1457,6 +1467,8 @@ [debug/ast-declarations] - Type "Charset" (spicy::Charset) [debug/ast-declarations] - Constant "ASCII" (spicy::ASCII) [debug/ast-declarations] - Constant "UTF8" (spicy::UTF8) +[debug/ast-declarations] - Constant "UTF16LE" (spicy::UTF16LE) +[debug/ast-declarations] - Constant "UTF16BE" (spicy::UTF16BE) [debug/ast-declarations] - Constant "Undef" (spicy::Undef_4) [debug/ast-declarations] - Type "DecodeErrorStrategy" (spicy::DecodeErrorStrategy) [debug/ast-declarations] - Constant "IGNORE" (spicy::IGNORE) diff --git a/tests/hilti/types/bytes/decode.hlt b/tests/hilti/types/bytes/decode.hlt index 2ae8a9b24..5829eedc6 100644 --- a/tests/hilti/types/bytes/decode.hlt +++ b/tests/hilti/types/bytes/decode.hlt @@ -10,4 +10,7 @@ hilti::print(b"testüng\n".decode(hilti::Charset::ASCII)); hilti::print(b"testüng".decode(hilti::Charset::UTF8)); assert b"testüng".decode(hilti::Charset::UTF8) == b"testüng".decode(); # Check default hilti::print(b" 123\x01 ".strip().decode(hilti::Charset::ASCII)); + +assert b"f\x00o\x00o\x00".decode(hilti::Charset::UTF16LE) == "foo"; +assert b"\x00f\x00o\x00o".decode(hilti::Charset::UTF16BE) == "foo"; } diff --git a/tests/hilti/types/string/encode.hlt b/tests/hilti/types/string/encode.hlt index 1b8e876cf..1057732cb 100644 --- a/tests/hilti/types/string/encode.hlt +++ b/tests/hilti/types/string/encode.hlt @@ -9,9 +9,13 @@ import hilti; hilti::print("testing".encode(hilti::Charset::ASCII)); hilti::print("testüng\n".encode(hilti::Charset::ASCII)); + hilti::print("testüng".encode(hilti::Charset::UTF8)); assert "testüng".encode(hilti::Charset::UTF8) == "testüng".encode(); # Check default +assert "foo".encode(hilti::Charset::UTF16LE) == b"f\x00o\x00o\x00"; +assert "foo".encode(hilti::Charset::UTF16BE) == b"\x00f\x00o\x00o"; + assert b"testüng".decode().encode() == b"testüng"; # Check round-trip. }