From afca052cb6b99bf77d3da52ad9b5306c1949ced2 Mon Sep 17 00:00:00 2001 From: Martin Entlicher Date: Tue, 17 Oct 2023 18:00:07 +0200 Subject: [PATCH 1/3] Repaired DAP tests on Windows and updated weekly jobs. (GR-49254) --- tools/ci/ci.jsonnet | 21 +++++------ .../truffle/tools/dap/test/DAPTester.java | 36 +++++++++++++++++-- .../tools/dap/test/RelativeSourceDAPTest.java | 8 +++-- .../tools/dap/test/SimpleLanguageDAPTest.java | 14 ++------ .../dap/server/LoadedSourcesHandler.java | 5 +-- 5 files changed, 55 insertions(+), 29 deletions(-) diff --git a/tools/ci/ci.jsonnet b/tools/ci/ci.jsonnet index 15a94b77e505..acdfb668078a 100644 --- a/tools/ci/ci.jsonnet +++ b/tools/ci/ci.jsonnet @@ -7,7 +7,7 @@ setup+: [ ["cd", "./tools"], ], - timelimit: "30:00", + timelimit: "45:00", }, local common_guard = { @@ -30,8 +30,8 @@ } }, - local tools_gate_lite = tools_common + { - name: 'gate-tools-lite-oraclejdk' + self.jdk_version + '-' + self.os + '-' + self.arch, + local tools_weekly = tools_common + { + name: 'weekly-tools-oraclejdk' + self.jdk_version + '-' + self.os + '-' + self.arch, run: [ ["mx", "build"], ["mx", "unittest", "--verbose"], @@ -81,17 +81,18 @@ }, builds: [ - common.linux_amd64 + common.oraclejdk20 + tools_gate, + common.linux_amd64 + common.oraclejdk21 + tools_gate, common.linux_amd64 + common.oraclejdk17 + tools_gate, - common.linux_amd64 + common.oraclejdk20 + tools_javadoc, + common.linux_amd64 + common.oraclejdk21 + tools_javadoc, common.linux_amd64 + common.oraclejdk17 + tools_coverage_weekly, - common.linux_aarch64 + common.labsjdk17 + tools_gate_lite, + common.linux_aarch64 + common.labsjdk21 + tools_weekly, + common.linux_aarch64 + common.labsjdk17 + tools_weekly, - common.windows_amd64 + common.oraclejdk20 + tools_gate_lite + devkits["windows-jdk20"], - common.windows_amd64 + common.oraclejdk17 + tools_gate_lite + devkits["windows-jdk17"], + common.windows_amd64 + common.oraclejdk21 + tools_weekly + devkits["windows-jdk21"], + common.windows_amd64 + common.oraclejdk17 + tools_weekly + devkits["windows-jdk17"], - common.darwin_amd64 + common.oraclejdk20 + tools_gate_lite, - common.darwin_amd64 + common.oraclejdk17 + tools_gate_lite, + common.darwin_amd64 + common.oraclejdk21 + tools_weekly, + common.darwin_amd64 + common.oraclejdk17 + tools_weekly, ], } diff --git a/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/DAPTester.java b/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/DAPTester.java index 4f4f10f74de6..18ba19a55397 100644 --- a/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/DAPTester.java +++ b/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/DAPTester.java @@ -24,6 +24,7 @@ */ package com.oracle.truffle.tools.dap.test; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -47,6 +48,7 @@ import org.graalvm.polyglot.Source; import org.graalvm.polyglot.Value; import org.graalvm.shadowed.org.json.JSONArray; +import org.graalvm.shadowed.org.json.JSONException; import org.graalvm.shadowed.org.json.JSONObject; import org.junit.Assert; @@ -103,6 +105,7 @@ public void finish() throws IOException { throw new AssertionError("Last eval(...) has not finished yet", ex); } catch (ExecutionException ex) { // Guest language execution failed + throw new AssertionError(ex); } } if (handler.getInputStream().available() > 0) { @@ -112,7 +115,16 @@ public void finish() throws IOException { public Future eval(Source source) { lastValue = CompletableFuture.supplyAsync(() -> { - return context.eval(source); + try { + return context.eval(source); + } catch (Throwable t) { + // Async exceptions are not visible till we check the future. + // We might never check the future + // if we're blocked by waiting for an expected output, + // which does not come due to the exception. + t.printStackTrace(System.err); + throw t; + } }, executor); return lastValue; } @@ -171,7 +183,15 @@ private static byte[] readMessageBytes(InputStream in) throws IOException { } public boolean compareReceivedMessages(String... messages) throws Exception { - List expectedObjects = Arrays.stream(messages).map(message -> new JSONObject(message)).collect(Collectors.toList()); + List expectedObjects = Arrays.stream(messages).map(message -> { + JSONObject json; + try { + json = new JSONObject(message); + } catch (JSONException jex) { + throw new RuntimeException(message, jex); + } + return json; + }).collect(Collectors.toList()); int size = expectedObjects.size(); while (size > 0) { final String receivedMessage = getMessage(); @@ -235,6 +255,18 @@ private static boolean compare(JSONObject expectedObject, JSONObject receivedObj return true; } + public static String getFilePath(File file) { + String path; + try { + path = file.getCanonicalPath(); + } catch (IOException ex) { + path = file.getAbsolutePath(); + } + // We need to escape backlash for correct JSON: + path = path.replace("\\", "\\\\"); + return path; + } + private static final class ProxyOutputStream extends OutputStream { OutputStream delegate; diff --git a/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/RelativeSourceDAPTest.java b/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/RelativeSourceDAPTest.java index 7ce2d007fa31..530092504055 100644 --- a/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/RelativeSourceDAPTest.java +++ b/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/RelativeSourceDAPTest.java @@ -44,6 +44,8 @@ import static org.junit.Assert.assertTrue; import org.junit.Test; +import static com.oracle.truffle.tools.dap.test.DAPTester.getFilePath; + /** * Test of SourcePath option. */ @@ -131,7 +133,7 @@ public void testSourcePath() throws Exception { tester.compareReceivedMessages("{\"event\":\"thread\",\"body\":{\"threadId\":1,\"reason\":\"started\"},\"type\":\"event\",\"seq\":" + (seq++) + "}"); } // Suspend at the beginning of the script: - String sourceJson = (i == 0) ? "{\"name\":\"" + new File(resolvedURI[i].getPath()).getName() + "\",\"path\":\"" + resolvedURI[i].getPath() + "\"}" + String sourceJson = (i == 0) ? "{\"name\":\"" + new File(resolvedURI[i].getPath()).getName() + "\",\"path\":\"" + getFilePath(new File(resolvedURI[i])) + "\"}" : "{\"sourceReference\":" + (i + 1) + ",\"path\":\"" + resolvedURI[i].getSchemeSpecificPart() + "\",\"name\":\"test" + (i + 1) + ".file\"}"; tester.compareReceivedMessages( "{\"event\":\"loadedSource\",\"body\":{\"reason\":\"new\",\"source\":" + sourceJson + "},\"type\":\"event\",\"seq\":" + (seq++) + "}"); @@ -175,7 +177,7 @@ public void testNonExistingSourcePath() throws Exception { tester.sendMessage("{\"command\":\"configurationDone\",\"type\":\"request\",\"seq\":5}"); tester.compareReceivedMessages("{\"success\":true,\"type\":\"response\",\"request_seq\":5,\"command\":\"configurationDone\",\"seq\":5}"); - String resolvedPath = new File("relative/path").toPath().toAbsolutePath().toString(); + String resolvedPath = getFilePath(new File("relative/path")); tester.eval(source); String sourceJson = "{\"sourceReference\":1,\"path\":\"" + resolvedPath + "\",\"name\":\"path\"}"; @@ -219,7 +221,7 @@ private static void testBreakpoints(int sectionLine, int sectionColumn, int bpLi TestDebugNoContentLanguage language = new TestDebugNoContentLanguage(relativePath, true, true); ProxyLanguage.setDelegate(language); Source source = Source.create(ProxyLanguage.ID, sourceContent); - String sourceJson = "{\"name\":\"" + filePath.getFileName() + "\",\"path\":\"" + filePath + "\"}"; + String sourceJson = "{\"name\":\"" + filePath.getFileName() + "\",\"path\":\"" + getFilePath(filePath.toFile()) + "\"}"; DAPTester tester = DAPTester.start(false, null, Collections.singletonList(sourcePathURI)); tester.sendMessage( diff --git a/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/SimpleLanguageDAPTest.java b/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/SimpleLanguageDAPTest.java index 479bcd5214c8..5efa853bf190 100644 --- a/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/SimpleLanguageDAPTest.java +++ b/tools/src/com.oracle.truffle.tools.dap.test/src/com/oracle/truffle/tools/dap/test/SimpleLanguageDAPTest.java @@ -36,6 +36,8 @@ import org.junit.Ignore; import org.junit.Test; +import static com.oracle.truffle.tools.dap.test.DAPTester.getFilePath; + @Ignore("GR-43473") public final class SimpleLanguageDAPTest { @@ -1196,18 +1198,6 @@ private static File createTemporaryFile(String content) throws IOException { return file; } - private static String getFilePath(File file) { - String path; - try { - path = file.getCanonicalPath(); - } catch (IOException ex) { - path = file.getAbsolutePath(); - } - // We need to escape backlash for correct JSON: - path = path.replace("\\", "\\\\"); - return path; - } - private static String replaceNewLines(String nl) { return nl.replace("\n", "\\n").replace("\r", "\\r"); } diff --git a/tools/src/com.oracle.truffle.tools.dap/src/com/oracle/truffle/tools/dap/server/LoadedSourcesHandler.java b/tools/src/com.oracle.truffle.tools.dap/src/com/oracle/truffle/tools/dap/server/LoadedSourcesHandler.java index 0214feea716e..1ad2e853f645 100644 --- a/tools/src/com.oracle.truffle.tools.dap/src/com/oracle/truffle/tools/dap/server/LoadedSourcesHandler.java +++ b/tools/src/com.oracle.truffle.tools.dap/src/com/oracle/truffle/tools/dap/server/LoadedSourcesHandler.java @@ -34,6 +34,7 @@ import com.oracle.truffle.tools.dap.types.LoadedSourceEvent; import java.net.URI; +import java.nio.file.InvalidPathException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -204,7 +205,7 @@ private Pair getPath(Source source, TruffleContext truffleConte tFile = context.getEnv().getTruffleFile(truffleContext, path); } } - } catch (UnsupportedOperationException ex) { + } catch (UnsupportedOperationException | InvalidPathException ex) { // Unsupported URI/path } if (tFile != null) { @@ -217,7 +218,7 @@ private Pair getPath(Source source, TruffleContext truffleConte } else if (path != null) { try { srcRef = !context.getEnv().getTruffleFile(truffleContext, path).isReadable(); - } catch (SecurityException ex) { + } catch (SecurityException | InvalidPathException ex) { // Can not verify readability srcRef = true; } From d60447d838d5ce3dbfe2f29c05bafff1aefc59c0 Mon Sep 17 00:00:00 2001 From: Tom Shull Date: Fri, 6 Oct 2023 11:49:36 +0200 Subject: [PATCH 2/3] Add option to ensure no direct relocations are emitted in the text section. --- .../src/com/oracle/objectfile/ObjectFile.java | 27 +++++++++++++++++++ .../com/oracle/svm/core/SubstrateOptions.java | 3 +++ .../oracle/svm/hosted/image/NativeImage.java | 17 ++++++++++++ 3 files changed, 47 insertions(+) diff --git a/substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/ObjectFile.java b/substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/ObjectFile.java index 519fa8734266..45befb0a0fd0 100644 --- a/substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/ObjectFile.java +++ b/substratevm/src/com.oracle.objectfile/src/com/oracle/objectfile/ObjectFile.java @@ -329,6 +329,33 @@ public static boolean isPCRelative(RelocationKind kind) { case PC_RELATIVE_2: case PC_RELATIVE_4: case PC_RELATIVE_8: + case AARCH64_R_AARCH64_ADR_PREL_PG_HI21: + case AARCH64_R_AARCH64_ADD_ABS_LO12_NC: + case AARCH64_R_LD_PREL_LO19: + case AARCH64_R_GOT_LD_PREL19: + case AARCH64_R_AARCH64_LDST128_ABS_LO12_NC: + case AARCH64_R_AARCH64_LDST64_ABS_LO12_NC: + case AARCH64_R_AARCH64_LDST32_ABS_LO12_NC: + case AARCH64_R_AARCH64_LDST16_ABS_LO12_NC: + case AARCH64_R_AARCH64_LDST8_ABS_LO12_NC: + return true; + } + return false; + } + + public static boolean isDirect(RelocationKind kind) { + switch (kind) { + case DIRECT_1: + case DIRECT_2: + case DIRECT_4: + case DIRECT_8: + case AARCH64_R_MOVW_UABS_G0: + case AARCH64_R_MOVW_UABS_G0_NC: + case AARCH64_R_MOVW_UABS_G1: + case AARCH64_R_MOVW_UABS_G1_NC: + case AARCH64_R_MOVW_UABS_G2: + case AARCH64_R_MOVW_UABS_G2_NC: + case AARCH64_R_MOVW_UABS_G3: return true; } return false; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 2948aacaa928..f06cf70dc159 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -409,6 +409,9 @@ public static void updateMaxJavaStackTraceDepth(EconomicMap, Object @Option(help = "Use only a writable native image heap (requires ld.gold linker)")// public static final HostedOptionKey ForceNoROSectionRelocations = new HostedOptionKey<>(false); + @Option(help = "Force no direct relocations to be present in the text section of the generated image", type = OptionType.Debug) // + public static final HostedOptionKey NoDirectRelocationsInText = new HostedOptionKey<>(true); + @Option(help = "Support multiple isolates.") // public static final HostedOptionKey SpawnIsolates = new HostedOptionKey<>(true); diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java index 68bb406bb11a..d2cc85366598 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java @@ -97,7 +97,9 @@ import com.oracle.svm.core.image.ImageHeapLayoutInfo; import com.oracle.svm.core.image.ImageHeapPartition; import com.oracle.svm.core.meta.MethodPointer; +import com.oracle.svm.core.option.SubstrateOptionsParser; import com.oracle.svm.core.util.UserError; +import com.oracle.svm.core.util.VMError; import com.oracle.svm.hosted.FeatureImpl; import com.oracle.svm.hosted.NativeImageOptions; import com.oracle.svm.hosted.c.CGlobalDataFeature; @@ -566,9 +568,20 @@ private static boolean checkEmbeddedOffset(ProgbitsSectionImpl sectionImpl, fina return true; } + private void validateNoDirectRelocationsInTextSection(RelocatableBuffer.Info info) { + if (SubstrateOptions.NoDirectRelocationsInText.getValue() && RelocationKind.isDirect(info.getRelocationKind())) { + String message = "%nFound direct relocation in text section. This means that the resulting generated image will have relocations present within the text section. If this is okay, you can skip this check by setting the flag %s"; + throw VMError.shouldNotReachHere(message, SubstrateOptionsParser.commandArgument(SubstrateOptions.NoDirectRelocationsInText, "-")); + } + } + private void markFunctionRelocationSite(final ProgbitsSectionImpl sectionImpl, final int offset, final RelocatableBuffer.Info info) { assert info.getTargetObject() instanceof CFunctionPointer : "Wrong type for FunctionPointer relocation: " + info.getTargetObject().toString(); + if (sectionImpl.getElement() == textSection) { + validateNoDirectRelocationsInTextSection(info); + } + // References to functions are via relocations to the symbol for the function. MethodPointer methodPointer = (MethodPointer) info.getTargetObject(); ResolvedJavaMethod method = methodPointer.getMethod(); @@ -622,10 +635,14 @@ private void markDataRelocationSiteFromText(RelocatableBuffer buffer, final Prog info.getRelocationSize(); Object target = info.getTargetObject(); if (target instanceof DataSectionReference) { + validateNoDirectRelocationsInTextSection(info); + long addend = ((DataSectionReference) target).getOffset() - info.getAddend(); assert isAddendAligned(arch, addend, info.getRelocationKind()) : "improper addend alignment"; sectionImpl.markRelocationSite(offset, info.getRelocationKind(), roDataSection.getName(), addend); } else if (target instanceof CGlobalDataReference) { + validateNoDirectRelocationsInTextSection(info); + CGlobalDataReference ref = (CGlobalDataReference) target; CGlobalDataInfo dataInfo = ref.getDataInfo(); CGlobalDataImpl data = dataInfo.getData(); From fccb23b827f6ef6512025d00af4635a0341e53f5 Mon Sep 17 00:00:00 2001 From: Olya Gupalo Date: Tue, 24 Oct 2023 12:45:32 +0200 Subject: [PATCH 3/3] Include syntax require two parameters --- docs/reference-manual/native-image/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference-manual/native-image/README.md b/docs/reference-manual/native-image/README.md index 9a699efc27fc..c83853fc478b 100644 --- a/docs/reference-manual/native-image/README.md +++ b/docs/reference-manual/native-image/README.md @@ -41,9 +41,9 @@ Choose your operating system to find instructions to meet the prerequisites. {% include snippet-tabs -tab1type="markdown" tab1id="Linux" tab1name="Linux" -tab2type="markdown" tab2id="macOS" tab2name="macOS" -tab3type="markdown" tab3id="Windows" tab3name="Windows" +tab1type="markdown" tab1id="Linux" tab1name="Linux" tab1path="native_image/linux.md" +tab2type="markdown" tab2id="macOS" tab2name="macOS" tab2path="native_image/macos.md" +tab3type="markdown" tab3id="Windows" tab3name="Windows" tab3path="native_image/windows.md" %}