diff --git a/CHANGES b/CHANGES index 47a7b02fa..8dfd434ae 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,19 @@ -1.12.0-dev.242 | 2024-12-13 09:26:36 +0100 +1.13.0-dev.2 | 2024-12-19 17:08:09 +0100 + + * GH-1949: Fix codegen for string literals containing null bytes. (Benjamin Bannier, Corelight) + + Our strings are UTF8 which can contain literal null bytes. We were + previously generating incorrect C++ for such literals. This was due to + us constructing C++ value via constructors taking `const char*` which is + expected to be null-terminated, i.e., everything after the terminator + was ignored even though it was emitted. + + With this patch we switch to C++ literals for generating literal and + runtime strings. + + Closes #1949. + +1.13.0-dev.0 | 2024-12-13 09:26:36 +0100 * GH-1901: Add consistent validation for attributes. (Evan Typanski, Corelight) diff --git a/VERSION b/VERSION index 39b42bcc7..dd2b91fce 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.12.0-dev.242 +1.13.0-dev.2 diff --git a/hilti/runtime/include/libhilti.h b/hilti/runtime/include/libhilti.h index 1c4e4d364..0cd72412c 100644 --- a/hilti/runtime/include/libhilti.h +++ b/hilti/runtime/include/libhilti.h @@ -26,6 +26,8 @@ #include #include -using namespace hilti::rt::bytes::literals; // NOLINT (google-global-names-in-headers) +using namespace hilti::rt::bytes::literals; // NOLINT(google-global-names-in-headers) +using namespace std::string_literals; // NOLINT(google-global-names-in-headers) +using namespace std::string_view_literals; // NOLINT(google-global-names-in-headers) #endif diff --git a/hilti/toolchain/src/compiler/codegen/ctors.cc b/hilti/toolchain/src/compiler/codegen/ctors.cc index d13adb7f8..45c7d5ef8 100644 --- a/hilti/toolchain/src/compiler/codegen/ctors.cc +++ b/hilti/toolchain/src/compiler/codegen/ctors.cc @@ -241,12 +241,13 @@ struct Visitor : hilti::visitor::PreOrder { } void operator()(ctor::String* n) final { + // Generate C++ string literals for both literal and non-literals + // strings to ensure correct C++ code for strings containing literal + // null bytes. if ( n->isLiteral() ) - result = fmt("std::string_view(\"%s\")", - util::escapeUTF8(n->value(), hilti::rt::render_style::UTF8::EscapeQuotes)); + result = fmt("\"%s\"sv", util::escapeUTF8(n->value(), hilti::rt::render_style::UTF8::EscapeQuotes)); else - result = - fmt("std::string(\"%s\")", util::escapeUTF8(n->value(), hilti::rt::render_style::UTF8::EscapeQuotes)); + result = fmt("\"%s\"s", util::escapeUTF8(n->value(), hilti::rt::render_style::UTF8::EscapeQuotes)); } void operator()(ctor::Tuple* n) final { diff --git a/tests/Baseline/hilti.hiltic.print.globals/output b/tests/Baseline/hilti.hiltic.print.globals/output index e0cecf755..02e3b4df3 100644 --- a/tests/Baseline/hilti.hiltic.print.globals/output +++ b/tests/Baseline/hilti.hiltic.print.globals/output @@ -26,7 +26,7 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module) extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) { ::hilti::rt::detail::initModuleGlobals<__globals_t>(__globals_index); - __globals()->X = std::string("Hello, world!"); + __globals()->X = "Hello, world!"s; } extern void __hlt::Foo::__init_module() { diff --git a/tests/Baseline/hilti.hiltic.print.globals/output2 b/tests/Baseline/hilti.hiltic.print.globals/output2 index 5491c2dd2..4c82215bd 100644 --- a/tests/Baseline/hilti.hiltic.print.globals/output2 +++ b/tests/Baseline/hilti.hiltic.print.globals/output2 @@ -20,7 +20,7 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module) extern void __hlt::Foo::__destroy_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::X.reset();; } -extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::X = std::string("Hello, world!"); } +extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::X = "Hello, world!"s; } extern void __hlt::Foo::__init_module() { __location__("<...>/globals.hlt:16:1-16:15"); diff --git a/tests/Baseline/hilti.hiltic.print.hello-world/output b/tests/Baseline/hilti.hiltic.print.hello-world/output index ffc345252..98b4a1930 100644 --- a/tests/Baseline/hilti.hiltic.print.hello-world/output +++ b/tests/Baseline/hilti.hiltic.print.hello-world/output @@ -17,7 +17,7 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module) extern void __hlt::Foo::__init_module() { __location__("<...>/hello-world.hlt:10:1-10:29"); - ::hilti::rt::print(std::string("Hello, world!"), ::hilti::rt::Bool(true)); + ::hilti::rt::print("Hello, world!"s, ::hilti::rt::Bool(true)); } extern void __hlt::Foo::__register_module() { ::hilti::rt::detail::registerModule({ "Foo", __hlt_hlto_scope, &__init_module, nullptr, nullptr, nullptr}); } diff --git a/tests/Baseline/hilti.hiltic.print.import/output b/tests/Baseline/hilti.hiltic.print.import/output index 086ce3521..a0332142f 100644 --- a/tests/Baseline/hilti.hiltic.print.import/output +++ b/tests/Baseline/hilti.hiltic.print.import/output @@ -26,12 +26,12 @@ HILTI_PRE_INIT(__hlt::Bar::__register_module) extern void __hlt::Bar::__init_globals(::hilti::rt::Context* ctx) { ::hilti::rt::detail::initModuleGlobals<__globals_t>(__globals_index); - __globals()->bar = std::string("Bar!"); + __globals()->bar = "Bar!"s; } extern void __hlt::Bar::__init_module() { __location__("bar.hlt:10:1-10:38"); - ::hilti::rt::print(std::string("Hello, world from Bar!"), ::hilti::rt::Bool(true)); + ::hilti::rt::print("Hello, world from Bar!"s, ::hilti::rt::Bool(true)); __location__("bar.hlt:11:1-11:22"); ::hilti::rt::print(Foo::__globals()->foo, ::hilti::rt::Bool(true)); __location__("bar.hlt:12:1-12:17"); @@ -66,12 +66,12 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module) extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) { ::hilti::rt::detail::initModuleGlobals<__globals_t>(__globals_index); - __globals()->foo = std::string("Foo!"); + __globals()->foo = "Foo!"s; } extern void __hlt::Foo::__init_module() { __location__("foo.hlt:10:1-10:38"); - ::hilti::rt::print(std::string("Hello, world from Foo!"), ::hilti::rt::Bool(true)); + ::hilti::rt::print("Hello, world from Foo!"s, ::hilti::rt::Bool(true)); __location__("foo.hlt:11:1-11:17"); ::hilti::rt::print(Foo::__globals()->foo, ::hilti::rt::Bool(true)); __location__("foo.hlt:12:1-12:22"); diff --git a/tests/Baseline/hilti.hiltic.print.import/output2 b/tests/Baseline/hilti.hiltic.print.import/output2 index 67685f517..84cc17597 100644 --- a/tests/Baseline/hilti.hiltic.print.import/output2 +++ b/tests/Baseline/hilti.hiltic.print.import/output2 @@ -27,11 +27,11 @@ HILTI_PRE_INIT(__hlt::Bar::__register_module) extern void __hlt::Bar::__destroy_globals(::hilti::rt::Context* ctx) { ::__hlt::Bar::bar.reset();; } -extern void __hlt::Bar::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Bar::bar = std::string("Bar!"); } +extern void __hlt::Bar::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Bar::bar = "Bar!"s; } extern void __hlt::Bar::__init_module() { __location__("bar.hlt:10:1-10:38"); - ::hilti::rt::print(std::string("Hello, world from Bar!"), ::hilti::rt::Bool(true)); + ::hilti::rt::print("Hello, world from Bar!"s, ::hilti::rt::Bool(true)); __location__("bar.hlt:11:1-11:22"); ::hilti::rt::print((*__hlt::Foo::foo), ::hilti::rt::Bool(true)); __location__("bar.hlt:12:1-12:17"); @@ -64,11 +64,11 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module) extern void __hlt::Foo::__destroy_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::foo.reset();; } -extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::foo = std::string("Foo!"); } +extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::foo = "Foo!"s; } extern void __hlt::Foo::__init_module() { __location__("foo.hlt:10:1-10:38"); - ::hilti::rt::print(std::string("Hello, world from Foo!"), ::hilti::rt::Bool(true)); + ::hilti::rt::print("Hello, world from Foo!"s, ::hilti::rt::Bool(true)); __location__("foo.hlt:11:1-11:17"); ::hilti::rt::print((*__hlt::Foo::foo), ::hilti::rt::Bool(true)); __location__("foo.hlt:12:1-12:22"); diff --git a/tests/Baseline/hilti.hiltic.print.yield/output b/tests/Baseline/hilti.hiltic.print.yield/output index 0bf812431..985b6edd0 100644 --- a/tests/Baseline/hilti.hiltic.print.yield/output +++ b/tests/Baseline/hilti.hiltic.print.yield/output @@ -27,23 +27,23 @@ extern void __hlt::Foo::__register_module() { ::hilti::rt::detail::registerModul extern auto __hlt::Foo::test(const std::string& x) -> std::string { ::hilti::rt::detail::checkStack(); __location__("<...>/yield.hlt:12:5-12:49"); - ::hilti::rt::print(std::string("HILTI - 1 - argument: "), ::hilti::rt::Bool(false)); + ::hilti::rt::print("HILTI - 1 - argument: "s, ::hilti::rt::Bool(false)); __location__("<...>/yield.hlt:13:5-13:19"); ::hilti::rt::print(x, ::hilti::rt::Bool(true)); __location__("<...>/yield.hlt:15:5-15:10"); ::hilti::rt::detail::yield(); __location__("<...>/yield.hlt:16:5-16:29"); - ::hilti::rt::print(std::string("HILTI - 2"), ::hilti::rt::Bool(true)); + ::hilti::rt::print("HILTI - 2"s, ::hilti::rt::Bool(true)); __location__("<...>/yield.hlt:17:5-17:10"); ::hilti::rt::detail::yield(); __location__("<...>/yield.hlt:18:5-18:29"); - ::hilti::rt::print(std::string("HILTI - 3"), ::hilti::rt::Bool(true)); + ::hilti::rt::print("HILTI - 3"s, ::hilti::rt::Bool(true)); __location__("<...>/yield.hlt:19:5-19:10"); ::hilti::rt::detail::yield(); __location__("<...>/yield.hlt:20:5-20:32"); - ::hilti::rt::print(std::string("HILTI - Done"), ::hilti::rt::Bool(true)); + ::hilti::rt::print("HILTI - Done"s, ::hilti::rt::Bool(true)); __location__("<...>/yield.hlt:22:5-22:25"); - return std::string("test-result"); + return "test-result"s; } extern auto hlt::Foo::test(const std::string& x) -> ::hilti::rt::Resumable { diff --git a/tests/Baseline/hilti.types.tuple.coercion/output b/tests/Baseline/hilti.types.tuple.coercion/output index 1f77fc988..36a600882 100644 --- a/tests/Baseline/hilti.types.tuple.coercion/output +++ b/tests/Baseline/hilti.types.tuple.coercion/output @@ -26,6 +26,6 @@ extern void __hlt::Foo::__register_module() { ::hilti::rt::detail::registerModul static auto __hlt::Foo::f() -> std::tuple<::hilti::rt::integer::safe, std::string> { ::hilti::rt::detail::checkStack(); __location__("<...>/coercion.hlt:11:5-11:24"); - return std::make_tuple(::hilti::rt::integer::safe{123U}, std::string("abc")); + return std::make_tuple(::hilti::rt::integer::safe{123U}, "abc"s); } diff --git a/tests/Baseline/spicy.tools.preprocessor/output b/tests/Baseline/spicy.tools.preprocessor/output index 5194d5ba1..3f8f08e9e 100644 --- a/tests/Baseline/spicy.tools.preprocessor/output +++ b/tests/Baseline/spicy.tools.preprocessor/output @@ -1,5 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. == spicyc - ::hilti::rt::print(std::string("have spicy version"), ::hilti::rt::Bool(true)); - ::hilti::rt::print(std::string("have spicy version >= 0.4"), ::hilti::rt::Bool(true)); - ::hilti::rt::print(std::string("not have spicy version >= 4"), ::hilti::rt::Bool(true)); + ::hilti::rt::print("have spicy version"s, ::hilti::rt::Bool(true)); + ::hilti::rt::print("have spicy version >= 0.4"s, ::hilti::rt::Bool(true)); + ::hilti::rt::print("not have spicy version >= 4"s, ::hilti::rt::Bool(true)); diff --git a/tests/Baseline/spicy.tools.spicyc-hello-world/test.hlt b/tests/Baseline/spicy.tools.spicyc-hello-world/test.hlt index bfd592e27..426437aaf 100644 --- a/tests/Baseline/spicy.tools.spicyc-hello-world/test.hlt +++ b/tests/Baseline/spicy.tools.spicyc-hello-world/test.hlt @@ -18,9 +18,9 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module) extern void __hlt::Foo::__init_module() { __location__("<...>/spicyc-hello-world.spicy:8:1-8:22"); - ::hilti::rt::print(std::string("Hello, world!"), ::hilti::rt::Bool(true)); + ::hilti::rt::print("Hello, world!"s, ::hilti::rt::Bool(true)); __location__("<...>/spicyc-hello-world.spicy:9:1-9:24"); - ::hilti::rt::printValues(std::make_tuple(std::string("Hello"), std::string("world!")), ::hilti::rt::Bool(true)); + ::hilti::rt::printValues(std::make_tuple("Hello"s, "world!"s), ::hilti::rt::Bool(true)); } extern void __hlt::Foo::__register_module() { ::hilti::rt::detail::registerModule({ "Foo", __hlt_hlto_scope, &__init_module, nullptr, nullptr, nullptr}); } diff --git a/tests/hilti/types/string/operators.hlt b/tests/hilti/types/string/operators.hlt index 488223c22..d11157458 100644 --- a/tests/hilti/types/string/operators.hlt +++ b/tests/hilti/types/string/operators.hlt @@ -44,4 +44,7 @@ assert "Γœπ”«π”¦π” π”¬π”‘π”’".lower() == "ü𝔫𝔦𝔠𝔬𝔑𝔒"; assert "abc123DEF".upper() == "ABC123DEF"; assert "ü𝔫𝔦𝔠𝔬𝔑𝔒".upper() == "Γœπ”«π”¦π” π”¬π”‘π”’"; +# Strings can contain null bytes. +assert |"\x00ABC"| == 4; + }