[GR-54697] Implement debuginfo generation at image-runtime.

substratevm/CHANGELOG.md

@@ -9,6 +9,7 @@ This changelog summarizes major changes to GraalVM Native Image.
 * (GR-59864) Added JVM version check to the Native Image agent. The agent will abort execution if the JVM major version does not match the version it was built with, and warn if the full JVM version is different.
 * (GR-59135) Verify if hosted options passed to `native-image` exist prior to starting the builder. Provide suggestions how to fix unknown options early on.
 * (GR-61492) The experimental JDWP option is now present in standard GraalVM builds.
+* (GR-54697) Parallelize debug info generation and add support for run-time debug info generation. `-H:+RuntimeDebugInfo` adds a run-time debug info generator into a native image for use with GDB.
 
 ## GraalVM for JDK 24 (Internal Version 24.2.0)
 * (GR-59717) Added `DuringSetupAccess.registerObjectReachabilityHandler` to allow registering a callback that is executed when an object of a specified type is marked as reachable during heap scanning.

substratevm/debug/include/gdb_jit_compilation_interface.h

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SVM_NATIVE_GDBJITCOMPILATIONINTERFACE_H
+#define SVM_NATIVE_GDBJITCOMPILATIONINTERFACE_H
+
+#include <stdint.h>
+
+typedef enum
+{
+  JIT_NOACTION = 0,
+  JIT_REGISTER,
+  JIT_UNREGISTER
+} jit_actions_t;
+
+struct jit_code_entry
+{
+  struct jit_code_entry *next_entry;
+  struct jit_code_entry *prev_entry;
+  const char *symfile_addr;
+  uint64_t symfile_size;
+};
+
+struct jit_descriptor
+{
+  uint32_t version;
+  /* This type should be jit_actions_t, but we use uint32_t
+     to be explicit about the bitwidth.  */
+  uint32_t action_flag;
+  struct jit_code_entry *relevant_entry;
+  struct jit_code_entry *first_entry;
+};
+
+#endif

substratevm/mx.substratevm/gdb_utils.py

@@ -110,6 +110,8 @@ def check(self, text, skip_fails=True):
         for i in range(0, num_rexps):
             rexp = rexps[i]
             match = None
+            if skip_fails:
+                line_idx = 0
             while line_idx < num_lines and match is None:
                 line = lines[line_idx]
                 match = rexp.match(line)

substratevm/mx.substratevm/mx_substratevm.py

@@ -1122,7 +1122,7 @@ def run_debug_test(image_name: str, testfile: str, source_path: str, with_isolat
             else:
                 build_args += ['-o', join(build_dir, image_name)]
 
-            mx.log(f"native_image {' '.join(build_args)}")
+            mx.log(f"native-image {' '.join(build_args)}")
             native_image(build_args)
 
             if build_cinterfacetutorial:
@@ -1139,14 +1139,14 @@ def run_debug_test(image_name: str, testfile: str, source_path: str, with_isolat
                 mx.run(c_command, cwd=build_dir)
         if mx.get_os() == 'linux':
             logfile = join(path, pathlib.Path(testfile).stem + ('' if with_isolates else '_no_isolates') + '.log')
-            os.environ.update({'gdbdebughelperstest_logfile': logfile})
+            os.environ.update({'gdb_logfile': logfile})
             gdb_command = gdb_args + [
                 '-iex', f"set logging file {logfile}",
                 '-iex', 'set logging enabled on',
                 '-iex', f"set auto-load safe-path {join(build_dir, 'gdb-debughelpers.py')}",
                 '-x', testfile, join(build_dir, image_name)
             ]
-            mx.log(' '.join(gdb_command))
+            log_gdb_command(gdb_command)
             # unittest may result in different exit code, nonZeroIsFatal ensures that we can go on with other test
             return mx.run(gdb_command, cwd=build_dir, nonZeroIsFatal=False)
         return 0
@@ -1173,6 +1173,97 @@ def run_debug_test(image_name: str, testfile: str, source_path: str, with_isolat
         mx.abort(status)
 
 
+def log_gdb_command(gdb_command):
+    mx.log(' '.join([(f"'{c}'" if ' ' in c else c) for c in gdb_command]))
+
+
+def _runtimedebuginfotest(native_image, output_path, args=None):
+    """Build and run the runtimedebuginfotest"""
+
+    args = [] if args is None else args
+
+    test_proj = mx.dependency('com.oracle.svm.test')
+    test_source_path = test_proj.source_dirs()[0]
+
+    test_python_source_dir = join(test_source_path, 'com', 'oracle', 'svm', 'test', 'debug', 'helper')
+    test_runtime_compilation_py = join(test_python_source_dir, 'test_runtime_compilation.py')
+    test_runtime_deopt_py = join(test_python_source_dir, 'test_runtime_deopt.py')
+    testdeopt_js = join(suite.dir, 'mx.substratevm', 'testdeopt.js')
+
+    gdb_args = [
+        os.environ.get('GDB_BIN', 'gdb'),
+        '--nx',
+        '-q',  # do not print the introductory and copyright messages
+        '-iex', "set pagination off",  # messages from enabling logging could already cause pagination, so this must be done first
+        '-iex', "set logging redirect on",
+        '-iex', "set logging overwrite off",
+    ]
+
+    # clean / create output directory
+    if exists(output_path):
+        mx.rmtree(output_path)
+    mx_util.ensure_dir_exists(output_path)
+
+    # Build the native image from Java code
+    build_args = [
+        '-g', '-O0',
+        '-o', join(output_path, 'runtimedebuginfotest'),
+        '-cp', classpath('com.oracle.svm.test'),
+        # We do not want to step into class initializer, so initialize everything at build time.
+        '--initialize-at-build-time=com.oracle.svm.test.debug.helper',
+        '--features=com.oracle.svm.test.debug.helper.RuntimeCompileDebugInfoTest$RegisterMethodsFeature',
+        'com.oracle.svm.test.debug.helper.RuntimeCompileDebugInfoTest',
+    ] + svm_experimental_options([
+        '-H:DebugInfoSourceSearchPath=' + test_source_path,
+        '-H:+SourceLevelDebug',
+        '-H:+RuntimeDebugInfo',
+    ]) + args
+
+    mx.log(f"native-image {' '.join(build_args)}")
+    runtime_compile_binary = native_image(build_args)
+
+    logfile = join(output_path, 'test_runtime_compilation.log')
+    os.environ.update({'gdb_logfile': logfile})
+    gdb_command = gdb_args + [
+        '-iex', f"set logging file {logfile}",
+        '-iex', "set logging enabled on",
+        '-iex', f"set auto-load safe-path {join(output_path, 'gdb-debughelpers.py')}",
+        '-x', test_runtime_compilation_py, runtime_compile_binary
+    ]
+    log_gdb_command(gdb_command)
+    # unittest may result in different exit code, nonZeroIsFatal ensures that we can go on with other test
+    status = mx.run(gdb_command, cwd=output_path, nonZeroIsFatal=False)
+
+    def run_js_test(eager: bool = False):
+        jslib = mx.add_lib_suffix(native_image(
+            args +
+            svm_experimental_options([
+                '-H:+SourceLevelDebug',
+                '-H:+RuntimeDebugInfo',
+                '-H:+LazyDeoptimization' if eager else '-H:-LazyDeoptimization',
+            ]) +
+            ['-g', '-O0', '--macro:jsvm-library']
+        ))
+        js_launcher = get_js_launcher(jslib)
+        logfile = join(output_path, 'test_runtime_deopt_' + ('eager' if eager else 'lazy') + '.log')
+        os.environ.update({'gdb_logfile': logfile})
+        gdb_command = gdb_args + [
+            '-iex', f"set logging file {logfile}",
+            '-iex', "set logging enabled on",
+            '-iex', f"set auto-load safe-path {join(output_path, 'gdb-debughelpers.py')}",
+            '-x', test_runtime_deopt_py, '--args', js_launcher, testdeopt_js
+        ]
+        log_gdb_command(gdb_command)
+        # unittest may result in different exit code, nonZeroIsFatal ensures that we can go on with other test
+        return mx.run(gdb_command, cwd=output_path, nonZeroIsFatal=False)
+
+    status |= run_js_test()
+    status |= run_js_test(True)
+
+    if status != 0:
+        mx.abort(status)
+
+
 def _javac_image(native_image, path, args=None):
     args = [] if args is None else args
     mx_util.ensure_dir_exists(path)
@@ -1763,6 +1854,26 @@ def gdbdebughelperstest(args, config=None):
         config=config
     )
 
+
+@mx.command(suite.name, 'runtimedebuginfotest', 'Runs the runtime debuginfo generation test')
+def runtimedebuginfotest(args, config=None):
+    """
+    runs a native image that compiles code and creates debug info at runtime.
+    """
+    parser = ArgumentParser(prog='mx runtimedebuginfotest')
+    all_args = ['--output-path', '--with-isolates-only']
+    masked_args = [_mask(arg, all_args) for arg in args]
+    parser.add_argument(all_args[0], metavar='<output-path>', nargs=1, help='Path of the generated image', default=[join(svmbuild_dir(), "runtimedebuginfotest")])
+    parser.add_argument('image_args', nargs='*', default=[])
+    parsed = parser.parse_args(masked_args)
+    output_path = unmask(parsed.output_path)[0]
+    native_image_context_run(
+        lambda native_image, a:
+        _runtimedebuginfotest(native_image, output_path, a), unmask(parsed.image_args),
+        config=config
+    )
+
+
 @mx.command(suite_name=suite.name, command_name='helloworld', usage_msg='[options]')
 def helloworld(args, config=None):
     """
@@ -1855,6 +1966,7 @@ def build_and_test_java_agent_image(native_image, args):
 
     native_image_context_run(build_and_test_java_agent_image, args)
 
+
 @mx.command(suite.name, 'clinittest', 'Runs the ')
 def clinittest(args):
     def build_and_test_clinittest_image(native_image, args):

substratevm/mx.substratevm/suite.py

@@ -309,6 +309,7 @@
             "dependencies": [
                 "com.oracle.svm.common",
                 "com.oracle.svm.shaded.org.objectweb.asm",
+                "com.oracle.objectfile",
             ],
             "requires" : [
                 "java.compiler",
@@ -680,7 +681,6 @@
             "subDir": "src",
             "sourceDirs": ["src"],
             "dependencies": [
-                "com.oracle.objectfile",
                 "com.oracle.graal.reachability",
                 "com.oracle.svm.core.graal.amd64",
             ],
@@ -1077,6 +1077,10 @@
                     "jdk.internal.misc",
                     "sun.security.jca",
                 ],
+                "jdk.internal.vm.ci" : [
+                    "jdk.vm.ci.code",
+                    "jdk.vm.ci.meta",
+                ],
             },
             "checkstyle": "com.oracle.svm.test",
             "checkstyleVersion" : "10.21.0",
@@ -1864,6 +1868,8 @@
                 "com.oracle.objectfile",
                 "com.oracle.objectfile.io",
                 "com.oracle.objectfile.debuginfo",
+                "com.oracle.objectfile.debugentry",
+                "com.oracle.objectfile.debugentry.range",
                 "com.oracle.objectfile.macho",
               ],
 
@@ -1994,6 +2000,7 @@
                             "dependency:com.oracle.svm.native.libchelper/*",
                             "dependency:com.oracle.svm.native.jvm.posix/*",
                             "dependency:com.oracle.svm.native.libcontainer/*",
+                            "file:debug/include",
                         ],
                     },
                 },
@@ -2002,6 +2009,7 @@
                         # on all other os's we don't want libc specific subdirectories
                         "include/": [
                             "dependency:com.oracle.svm.native.libchelper/include/*",
+                            "file:debug/include/*",
                         ],
                         "<os>-<arch>/": [
                             "dependency:com.oracle.svm.native.libchelper/<os>-<arch>/default/*",

substratevm/mx.substratevm/testdeopt.js

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+function add(a, b, test) {
+    if (test) {
+        a += b;
+    }
+    return a + b;
+}
+
+// trigger compilation add for ints and test = true
+for (let i = 0; i < 1000 * 1000; i++) {
+    add(i, i, true);
+}
+
+// deoptimize with failed assumption in compiled method
+// then trigger compilation again
+console.log("deopt1")
+for (let i = 0; i < 1000 * 1000; i++) {
+    add(i, i, false);
+}
+
+// deoptimize with different parameter types
+console.log("deopt2");
+add({f1: "test1", f2: 2}, {x: "x", y: {test: 42}}, false);

substratevm/mx.substratevm/testhello.py

@@ -147,25 +147,25 @@ def test():
 
     # check incoming parameters are bound to sensible values
     exec_string = execute("info args")
-    rexp = [fr"__0 = {digits_pattern}",
-            fr"__1 = 0x{hex_digits_pattern}"]
+    rexp = [fr"__int0 = {digits_pattern}",
+            fr"__long1 = 0x{hex_digits_pattern}"]
     checker = Checker(f"info args : {method_name}", rexp)
     checker.check(exec_string)
 
-    exec_string = execute("p __0")
+    exec_string = execute("p __int0")
     rexp = [fr"\${digits_pattern} = 1"]
-    checker = Checker("p __0", rexp)
+    checker = Checker("p __int0", rexp)
     checker.check(exec_string)
 
-    exec_string = execute("p __1")
+    exec_string = execute("p __long1")
     rexp = [fr"\${digits_pattern} = \(org\.graalvm\.nativeimage\.c\.type\.CCharPointerPointer\) 0x{hex_digits_pattern}"]
-    checker = Checker("p __1", rexp)
+    checker = Checker("p __long1", rexp)
     checker.check(exec_string)
 
-    exec_string = execute("p __1[0]")
+    exec_string = execute("p __long1[0]")
     rexp = [
         fr'\${digits_pattern} = \(org\.graalvm\.nativeimage\.c\.type\.CCharPointer\) 0x{hex_digits_pattern} "{wildcard_pattern}/hello_image"']
-    checker = Checker("p __1[0]", rexp)
+    checker = Checker("p __long1[0]", rexp)
     checker.check(exec_string)
 
     # set a break point at hello.Hello::main

substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/ObjectFile.java

@@ -219,16 +219,12 @@ public static Format getNativeFormat() {
     }
 
     private static ObjectFile getNativeObjectFile(int pageSize, boolean runtimeDebugInfoGeneration) {
-        switch (ObjectFile.getNativeFormat()) {
-            case ELF:
-                return new ELFObjectFile(pageSize, runtimeDebugInfoGeneration);
-            case MACH_O:
-                return new MachOObjectFile(pageSize);
-            case PECOFF:
-                return new PECoffObjectFile(pageSize);
-            default:
-                throw new AssertionError("Unreachable");
-        }
+        return switch (ObjectFile.getNativeFormat()) {
+            case ELF -> new ELFObjectFile(pageSize, runtimeDebugInfoGeneration);
+            case MACH_O -> new MachOObjectFile(pageSize);
+            case PECOFF -> new PECoffObjectFile(pageSize);
+            case LLVM -> throw new AssertionError("Unsupported NativeObjectFile for format " + ObjectFile.getNativeFormat());
+        };
     }
 
     public static ObjectFile getNativeObjectFile(int pageSize) {
@@ -1828,7 +1824,7 @@ public final SymbolTable getOrCreateSymbolTable() {
      * Temporary storage for a debug context installed in a nested scope under a call. to
      * {@link #withDebugContext}
      */
-    private DebugContext debugContext = null;
+    protected DebugContext debugContext = DebugContext.disabled(null);
 
     /**
      * Allows a task to be executed with a debug context in a named subscope bound to the object