Merge remote-tracking branch 'origin/topic/bbannier/issue-1949'

zeek · Dec 19, 2024 · 4c2cd8c · 4c2cd8c
2 parents 2d0ce53 + 607d897
commit 4c2cd8c
Show file tree

Hide file tree

Showing 14 changed files with 50 additions and 29 deletions.
diff --git a/CHANGES b/CHANGES
@@ -1,4 +1,19 @@
-1.12.0-dev.242 | 2024-12-13 09:26:36 +0100
+1.13.0-dev.2 | 2024-12-19 17:08:09 +0100
+
+  * GH-1949: Fix codegen for string literals containing null bytes. (Benjamin Bannier, Corelight)
+
+    Our strings are UTF8 which can contain literal null bytes. We were
+    previously generating incorrect C++ for such literals. This was due to
+    us constructing C++ value via constructors taking `const char*` which is
+    expected to be null-terminated, i.e., everything after the terminator
+    was ignored even though it was emitted.
+
+    With this patch we switch to C++ literals for generating literal and
+    runtime strings.
+
+    Closes #1949.
+
+1.13.0-dev.0 | 2024-12-13 09:26:36 +0100
 
   * GH-1901: Add consistent validation for attributes. (Evan Typanski, Corelight)
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.12.0-dev.242
+1.13.0-dev.2
diff --git a/hilti/runtime/include/libhilti.h b/hilti/runtime/include/libhilti.h
@@ -26,6 +26,8 @@
 #include <hilti/rt/types/all.h>
 #include <hilti/rt/util.h>
 
-using namespace hilti::rt::bytes::literals; // NOLINT (google-global-names-in-headers)
+using namespace hilti::rt::bytes::literals; // NOLINT(google-global-names-in-headers)
+using namespace std::string_literals;       // NOLINT(google-global-names-in-headers)
+using namespace std::string_view_literals;  // NOLINT(google-global-names-in-headers)
 
 #endif
diff --git a/hilti/toolchain/src/compiler/codegen/ctors.cc b/hilti/toolchain/src/compiler/codegen/ctors.cc
@@ -241,12 +241,13 @@ struct Visitor : hilti::visitor::PreOrder {
     }
 
     void operator()(ctor::String* n) final {
+        // Generate C++ string literals for both literal and non-literals
+        // strings to ensure correct C++ code for strings containing literal
+        // null bytes.
         if ( n->isLiteral() )
-            result = fmt("std::string_view(\"%s\")",
-                         util::escapeUTF8(n->value(), hilti::rt::render_style::UTF8::EscapeQuotes));
+            result = fmt("\"%s\"sv", util::escapeUTF8(n->value(), hilti::rt::render_style::UTF8::EscapeQuotes));
         else
-            result =
-                fmt("std::string(\"%s\")", util::escapeUTF8(n->value(), hilti::rt::render_style::UTF8::EscapeQuotes));
+            result = fmt("\"%s\"s", util::escapeUTF8(n->value(), hilti::rt::render_style::UTF8::EscapeQuotes));
     }
 
     void operator()(ctor::Tuple* n) final {

diff --git a/tests/Baseline/hilti.hiltic.print.globals/output b/tests/Baseline/hilti.hiltic.print.globals/output
@@ -26,7 +26,7 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module)
 
 extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) {
     ::hilti::rt::detail::initModuleGlobals<__globals_t>(__globals_index);
-    __globals()->X = std::string("Hello, world!");
+    __globals()->X = "Hello, world!"s;
 }
 
 extern void __hlt::Foo::__init_module() {

diff --git a/tests/Baseline/hilti.hiltic.print.globals/output2 b/tests/Baseline/hilti.hiltic.print.globals/output2
@@ -20,7 +20,7 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module)
 
 extern void __hlt::Foo::__destroy_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::X.reset();; }
 
-extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::X = std::string("Hello, world!"); }
+extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::X = "Hello, world!"s; }
 
 extern void __hlt::Foo::__init_module() {
       __location__("<...>/globals.hlt:16:1-16:15");

diff --git a/tests/Baseline/hilti.hiltic.print.hello-world/output b/tests/Baseline/hilti.hiltic.print.hello-world/output
@@ -17,7 +17,7 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module)
 
 extern void __hlt::Foo::__init_module() {
       __location__("<...>/hello-world.hlt:10:1-10:29");
-    ::hilti::rt::print(std::string("Hello, world!"), ::hilti::rt::Bool(true));
+    ::hilti::rt::print("Hello, world!"s, ::hilti::rt::Bool(true));
 }
 
 extern void __hlt::Foo::__register_module() { ::hilti::rt::detail::registerModule({ "Foo", __hlt_hlto_scope, &__init_module, nullptr, nullptr, nullptr}); }

diff --git a/tests/Baseline/hilti.hiltic.print.import/output b/tests/Baseline/hilti.hiltic.print.import/output
@@ -26,12 +26,12 @@ HILTI_PRE_INIT(__hlt::Bar::__register_module)
 
 extern void __hlt::Bar::__init_globals(::hilti::rt::Context* ctx) {
     ::hilti::rt::detail::initModuleGlobals<__globals_t>(__globals_index);
-    __globals()->bar = std::string("Bar!");
+    __globals()->bar = "Bar!"s;
 }
 
 extern void __hlt::Bar::__init_module() {
       __location__("bar.hlt:10:1-10:38");
-    ::hilti::rt::print(std::string("Hello, world from Bar!"), ::hilti::rt::Bool(true));
+    ::hilti::rt::print("Hello, world from Bar!"s, ::hilti::rt::Bool(true));
       __location__("bar.hlt:11:1-11:22");
     ::hilti::rt::print(Foo::__globals()->foo, ::hilti::rt::Bool(true));
       __location__("bar.hlt:12:1-12:17");
@@ -66,12 +66,12 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module)
 
 extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) {
     ::hilti::rt::detail::initModuleGlobals<__globals_t>(__globals_index);
-    __globals()->foo = std::string("Foo!");
+    __globals()->foo = "Foo!"s;
 }
 
 extern void __hlt::Foo::__init_module() {
       __location__("foo.hlt:10:1-10:38");
-    ::hilti::rt::print(std::string("Hello, world from Foo!"), ::hilti::rt::Bool(true));
+    ::hilti::rt::print("Hello, world from Foo!"s, ::hilti::rt::Bool(true));
       __location__("foo.hlt:11:1-11:17");
     ::hilti::rt::print(Foo::__globals()->foo, ::hilti::rt::Bool(true));
       __location__("foo.hlt:12:1-12:22");

diff --git a/tests/Baseline/hilti.hiltic.print.import/output2 b/tests/Baseline/hilti.hiltic.print.import/output2
@@ -27,11 +27,11 @@ HILTI_PRE_INIT(__hlt::Bar::__register_module)
 
 extern void __hlt::Bar::__destroy_globals(::hilti::rt::Context* ctx) { ::__hlt::Bar::bar.reset();; }
 
-extern void __hlt::Bar::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Bar::bar = std::string("Bar!"); }
+extern void __hlt::Bar::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Bar::bar = "Bar!"s; }
 
 extern void __hlt::Bar::__init_module() {
       __location__("bar.hlt:10:1-10:38");
-    ::hilti::rt::print(std::string("Hello, world from Bar!"), ::hilti::rt::Bool(true));
+    ::hilti::rt::print("Hello, world from Bar!"s, ::hilti::rt::Bool(true));
       __location__("bar.hlt:11:1-11:22");
     ::hilti::rt::print((*__hlt::Foo::foo), ::hilti::rt::Bool(true));
       __location__("bar.hlt:12:1-12:17");
@@ -64,11 +64,11 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module)
 
 extern void __hlt::Foo::__destroy_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::foo.reset();; }
 
-extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::foo = std::string("Foo!"); }
+extern void __hlt::Foo::__init_globals(::hilti::rt::Context* ctx) { ::__hlt::Foo::foo = "Foo!"s; }
 
 extern void __hlt::Foo::__init_module() {
       __location__("foo.hlt:10:1-10:38");
-    ::hilti::rt::print(std::string("Hello, world from Foo!"), ::hilti::rt::Bool(true));
+    ::hilti::rt::print("Hello, world from Foo!"s, ::hilti::rt::Bool(true));
       __location__("foo.hlt:11:1-11:17");
     ::hilti::rt::print((*__hlt::Foo::foo), ::hilti::rt::Bool(true));
       __location__("foo.hlt:12:1-12:22");

diff --git a/tests/Baseline/hilti.hiltic.print.yield/output b/tests/Baseline/hilti.hiltic.print.yield/output
@@ -27,23 +27,23 @@ extern void __hlt::Foo::__register_module() { ::hilti::rt::detail::registerModul
 extern auto __hlt::Foo::test(const std::string& x) -> std::string {
     ::hilti::rt::detail::checkStack();
       __location__("<...>/yield.hlt:12:5-12:49");
-    ::hilti::rt::print(std::string("HILTI - 1 - argument: "), ::hilti::rt::Bool(false));
+    ::hilti::rt::print("HILTI - 1 - argument: "s, ::hilti::rt::Bool(false));
       __location__("<...>/yield.hlt:13:5-13:19");
     ::hilti::rt::print(x, ::hilti::rt::Bool(true));
       __location__("<...>/yield.hlt:15:5-15:10");
     ::hilti::rt::detail::yield();
       __location__("<...>/yield.hlt:16:5-16:29");
-    ::hilti::rt::print(std::string("HILTI - 2"), ::hilti::rt::Bool(true));
+    ::hilti::rt::print("HILTI - 2"s, ::hilti::rt::Bool(true));
       __location__("<...>/yield.hlt:17:5-17:10");
     ::hilti::rt::detail::yield();
       __location__("<...>/yield.hlt:18:5-18:29");
-    ::hilti::rt::print(std::string("HILTI - 3"), ::hilti::rt::Bool(true));
+    ::hilti::rt::print("HILTI - 3"s, ::hilti::rt::Bool(true));
       __location__("<...>/yield.hlt:19:5-19:10");
     ::hilti::rt::detail::yield();
       __location__("<...>/yield.hlt:20:5-20:32");
-    ::hilti::rt::print(std::string("HILTI - Done"), ::hilti::rt::Bool(true));
+    ::hilti::rt::print("HILTI - Done"s, ::hilti::rt::Bool(true));
       __location__("<...>/yield.hlt:22:5-22:25");
-    return std::string("test-result");
+    return "test-result"s;
 }
 
 extern auto hlt::Foo::test(const std::string& x) -> ::hilti::rt::Resumable {

diff --git a/tests/Baseline/hilti.types.tuple.coercion/output b/tests/Baseline/hilti.types.tuple.coercion/output
@@ -26,6 +26,6 @@ extern void __hlt::Foo::__register_module() { ::hilti::rt::detail::registerModul
 static auto __hlt::Foo::f() -> std::tuple<::hilti::rt::integer::safe<uint64_t>, std::string> {
     ::hilti::rt::detail::checkStack();
       __location__("<...>/coercion.hlt:11:5-11:24");
-    return std::make_tuple(::hilti::rt::integer::safe<std::uint64_t>{123U}, std::string("abc"));
+    return std::make_tuple(::hilti::rt::integer::safe<std::uint64_t>{123U}, "abc"s);
 }
 
diff --git a/tests/Baseline/spicy.tools.preprocessor/output b/tests/Baseline/spicy.tools.preprocessor/output
@@ -1,5 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 == spicyc
-    ::hilti::rt::print(std::string("have spicy version"), ::hilti::rt::Bool(true));
-    ::hilti::rt::print(std::string("have spicy version >= 0.4"), ::hilti::rt::Bool(true));
-    ::hilti::rt::print(std::string("not have spicy version >= 4"), ::hilti::rt::Bool(true));
+    ::hilti::rt::print("have spicy version"s, ::hilti::rt::Bool(true));
+    ::hilti::rt::print("have spicy version >= 0.4"s, ::hilti::rt::Bool(true));
+    ::hilti::rt::print("not have spicy version >= 4"s, ::hilti::rt::Bool(true));
diff --git a/tests/Baseline/spicy.tools.spicyc-hello-world/test.hlt b/tests/Baseline/spicy.tools.spicyc-hello-world/test.hlt
@@ -18,9 +18,9 @@ HILTI_PRE_INIT(__hlt::Foo::__register_module)
 
 extern void __hlt::Foo::__init_module() {
       __location__("<...>/spicyc-hello-world.spicy:8:1-8:22");
-    ::hilti::rt::print(std::string("Hello, world!"), ::hilti::rt::Bool(true));
+    ::hilti::rt::print("Hello, world!"s, ::hilti::rt::Bool(true));
       __location__("<...>/spicyc-hello-world.spicy:9:1-9:24");
-    ::hilti::rt::printValues(std::make_tuple(std::string("Hello"), std::string("world!")), ::hilti::rt::Bool(true));
+    ::hilti::rt::printValues(std::make_tuple("Hello"s, "world!"s), ::hilti::rt::Bool(true));
 }
 
 extern void __hlt::Foo::__register_module() { ::hilti::rt::detail::registerModule({ "Foo", __hlt_hlto_scope, &__init_module, nullptr, nullptr, nullptr}); }
diff --git a/tests/hilti/types/string/operators.hlt b/tests/hilti/types/string/operators.hlt
@@ -44,4 +44,7 @@ assert "Ü𝔫𝔦𝔠𝔬𝔡𝔢".lower() == "ü𝔫𝔦𝔠𝔬𝔡𝔢";
 assert "abc123DEF".upper() == "ABC123DEF";
 assert "ü𝔫𝔦𝔠𝔬𝔡𝔢".upper() == "Ü𝔫𝔦𝔠𝔬𝔡𝔢";
 
+# Strings can contain null bytes.
+assert |"\x00ABC"| == 4;
+
 }