Shenandoah support

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/aarch64/AArch64ReadBarrierSetLIRGenerator.java

@@ -28,6 +28,7 @@
 import jdk.graal.compiler.core.common.memory.BarrierType;
 import jdk.graal.compiler.core.common.memory.MemoryOrderMode;
 import jdk.graal.compiler.lir.Variable;
+import jdk.graal.compiler.lir.gen.BarrierSetLIRGeneratorTool;
 import jdk.graal.compiler.lir.gen.LIRGeneratorTool;
 import jdk.graal.compiler.lir.gen.ReadBarrierSetLIRGeneratorTool;
 import jdk.vm.ci.aarch64.AArch64Kind;
@@ -37,7 +38,7 @@
 /**
  * AArch64 specific LIR generation for GC barriers.
  */
-public interface AArch64ReadBarrierSetLIRGenerator extends ReadBarrierSetLIRGeneratorTool {
+public interface AArch64ReadBarrierSetLIRGenerator extends BarrierSetLIRGeneratorTool {
 
     /**
      * Emit an atomic read-and-write instruction with any required GC barriers.

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyMidTier.java

@@ -44,6 +44,5 @@ public EconomyMidTier() {
         appendPhase(new MidTierLoweringPhase(canonicalizer));
         appendPhase(new FrameStateAssignmentPhase());
         appendPhase(canonicalizer);
-        appendPhase(new WriteBarrierAdditionPhase());
     }
 }

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java

@@ -47,6 +47,7 @@
 import jdk.graal.compiler.phases.common.PropagateDeoptimizeProbabilityPhase;
 import jdk.graal.compiler.phases.common.RemoveOpaqueValuePhase;
 import jdk.graal.compiler.phases.common.TransplantGraphsPhase;
+import jdk.graal.compiler.phases.common.WriteBarrierAdditionPhase;
 import jdk.graal.compiler.phases.schedule.SchedulePhase;
 import jdk.graal.compiler.phases.schedule.SchedulePhase.SchedulingStrategy;
 import jdk.graal.compiler.phases.tiers.LowTierContext;
@@ -87,6 +88,8 @@ public LowTier(OptionValues options) {
         appendPhase(new FixReadsPhase(true,
                         new SchedulePhase(GraalOptions.StressTestEarlyReads.getValue(options) ? SchedulingStrategy.EARLIEST : SchedulingStrategy.LATEST_OUT_OF_LOOPS_IMPLICIT_NULL_CHECKS)));
 
+        appendPhase(new WriteBarrierAdditionPhase());
+
         appendPhase(canonicalizerWithoutGVN);
 
         /*

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/MidTier.java

@@ -122,8 +122,6 @@ public MidTier(OptionValues options) {
         }
 
         appendPhase(canonicalizer);
-
-        appendPhase(new WriteBarrierAdditionPhase());
     }
 
     @Override

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/graph/Node.java

@@ -1166,13 +1166,17 @@ private void replaceAtUsagePos(Node replacement, Node usage, Position pos) {
         }
     }
 
+    public void replaceAtUsages(Node replacement, InputType inputType) {
+        replaceAtUsages(replacement, inputType, (NodePredicate) n -> true);
+    }
+
     /**
      * For each use of {@code this} in another node, {@code n}, replace it with {@code replacement}
      * if the type of the use is {@code inputType}.
      *
      * @see #replaceAtUsages(Node)
      */
-    public void replaceAtUsages(Node replacement, InputType inputType) {
+    public void replaceAtUsages(Node replacement, InputType inputType, Predicate<Node> filter) {
         checkReplaceWith(replacement);
         int i = 0;
         int usageCount = this.getUsageCount();
@@ -1182,7 +1186,7 @@ public void replaceAtUsages(Node replacement, InputType inputType) {
         usages: while (i < usageCount) {
             Node usage = this.getUsageAt(i);
             for (Position pos : usage.inputPositions()) {
-                if (pos.getInputType() == inputType && pos.get(usage) == this) {
+                if (pos.getInputType() == inputType && pos.get(usage) == this && filter.test(usage)) {
                     replaceAtUsagePos(replacement, usage, pos);
                     this.movUsageFromEndTo(i);
                     usageCount--;

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java

@@ -400,8 +400,8 @@ public int threadLastJavaFpOffset() {
     // This field has no type in vmStructs.cpp
     public final int objectMonitorOwner = getFieldOffset("ObjectMonitor::_owner", Integer.class, JDK > 21 ? "int64_t" : null);
     public final int objectMonitorRecursions = getFieldOffset("ObjectMonitor::_recursions", Integer.class, "intptr_t");
-    public final int objectMonitorCxq = getFieldOffset("ObjectMonitor::_cxq", Integer.class, "ObjectWaiter*");
-    public final int objectMonitorEntryList = getFieldOffset("ObjectMonitor::_EntryList", Integer.class, "ObjectWaiter*");
+    public final int objectMonitorCxq = getFieldOffset("ObjectMonitor::_cxq", Integer.class, "ObjectWaiter*", -1, JDK < 25);
+    public final int objectMonitorEntryList = getFieldOffset("ObjectMonitor::_EntryList", Integer.class, "ObjectWaiter*", -1, JDK < 25);
     public final int objectMonitorSucc = getFieldOffset("ObjectMonitor::_succ", Integer.class, JDK > 21 ? "int64_t" : "JavaThread*");
 
     public final int contEntryOffset = getFieldOffset("JavaThread::_cont_entry", Integer.class, "ContinuationEntry*", -1, JDK >= 24);
@@ -455,6 +455,13 @@ public int threadLastJavaFpOffset() {
     public final int g1CardQueueBufferOffset = getConstant("G1ThreadLocalData::dirty_card_queue_buffer_offset", Integer.class, -1, !g1LowLatencyPostWriteBarrierSupport);
     public final int g1CardTableBaseOffset = getConstant("G1ThreadLocalData::card_table_base_offset", Integer.class, -1, g1LowLatencyPostWriteBarrierSupport);
 
+    public final int shenandoahGCStateOffset = getConstant("ShenandoahThreadLocalData::gc_state_offset", Integer.class);
+    public final int shenandoahSATBIndexOffset = getConstant("ShenandoahThreadLocalData::satb_mark_queue_index_offset", Integer.class);
+    public final int shenandoahSATBBufferOffset = getConstant("ShenandoahThreadLocalData::satb_mark_queue_buffer_offset", Integer.class);
+    public final int shenandoahCardTableOffset = getConstant("ShenandoahThreadLocalData::card_table_offset", Integer.class);
+    public final int shenandoahGCRegionSizeBytesShift = getFieldValue("CompilerToVM::Data::shenandoah_region_size_bytes_shift", Integer.class, "int");
+    public final long shenandoahGCCSetFastTestAddress = getFieldValue("CompilerToVM::Data::shenandoah_in_cset_fast_test_addr", Long.class, "address");
+
     public final int klassOffset = getFieldValue("java_lang_Class::_klass_offset", Integer.class, "int");
     public final int arrayKlassOffset = getFieldValue("java_lang_Class::_array_klass_offset", Integer.class, "int");
 
@@ -625,6 +632,14 @@ private long getZGCAddressField(String name) {
     public final long zBarrierSetRuntimeLoadBarrierOnOopArray = getZGCAddressField("ZBarrierSetRuntime::load_barrier_on_oop_array");
     public final int zPointerLoadShift = getConstant("ZPointerLoadShift", Integer.class, -1, osArch.equals("aarch64") && zgcSupport);
 
+    public final long shenandoahLoadBarrierStrong = getAddress("ShenandoahRuntime::load_reference_barrier_strong");
+    public final long shenandoahLoadBarrierStrongNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_strong_narrow");
+    public final long shenandoahLoadBarrierWeak = getAddress("ShenandoahRuntime::load_reference_barrier_weak");
+    public final long shenandoahLoadBarrierWeakNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_weak_narrow");
+    public final long shenandoahLoadBarrierPhantom = getAddress("ShenandoahRuntime::load_reference_barrier_phantom");
+    public final long shenandoahLoadBarrierPhantomNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_phantom_narrow");
+    public final long shenandoahPreBarrier = getAddress("ShenandoahRuntime::pre_barrier");
+
     // aarch64 specific nmethod entry barrier support
     // @formatter:off
     public final int BarrierSetAssembler_nmethod_patching_type = getFieldValue("CompilerToVM::Data::BarrierSetAssembler_nmethod_patching_type", Integer.class, "int", -1, osArch.equals("aarch64"));

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfigAccess.java

@@ -190,7 +190,7 @@ protected void reportErrors() {
             List<String> messages = new ArrayList<>();
             if (!missing.isEmpty()) {
                 messages.add(String.format("VM config values missing that should be present in %s:%n    %s", runtime,
-                                missing.stream().sorted().collect(Collectors.joining(System.lineSeparator() + "    "))));
+                                missing.stream().sorted().collect(Collectors.joining(  " /   "))));
             }
             if (!unexpected.isEmpty()) {
                 messages.add(String.format("VM config values not expected to be present in %s:%n    %s", runtime,

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java

@@ -259,6 +259,8 @@ private BarrierSet createBarrierSet(GraalHotSpotVMConfig config, MetaAccessProvi
         ResolvedJavaField referentField = HotSpotReplacementsUtil.referentField(metaAccess);
         if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) {
             return new HotSpotZBarrierSet(objectArrayType, referentField);
+        } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) {
+            return new HotSpotShenandoahBarrierSet(objectArrayType, referentField, config);
         } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Epsilon) {
             return new NoBarrierSet();
         } else if (config.useG1GC()) {

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java

@@ -210,9 +210,7 @@ public enum HotSpotGC {
         G1("UseG1GC"),
         Z(JavaVersionUtil.JAVA_SPEC > 21, true, flagIsSet("UseZGC")),
         Epsilon(true, true, flagIsSet("UseEpsilonGC")),
-
-        // Unsupported GCs
-        Shenandoah(false, true, flagIsSet("UseShenandoahGC"));
+        Shenandoah(JavaVersionUtil.JAVA_SPEC > 24, true, flagIsSet("UseShenandoahGC"));
 
         HotSpotGC(String flag) {
             this(true, true, flagIsSet(flag));

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotShenandoahBarrierSet.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package jdk.graal.compiler.hotspot;
+
+import jdk.graal.compiler.core.common.CompressEncoding;
+import jdk.graal.compiler.core.common.type.ObjectStamp;
+import jdk.graal.compiler.hotspot.nodes.HotSpotCompressionNode;
+import jdk.graal.compiler.nodes.NodeView;
+import jdk.graal.compiler.nodes.type.NarrowOopStamp;
+import org.graalvm.word.LocationIdentity;
+
+import jdk.graal.compiler.core.common.memory.BarrierType;
+import jdk.graal.compiler.core.common.type.AbstractObjectStamp;
+import jdk.graal.compiler.core.common.type.Stamp;
+import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil;
+import jdk.graal.compiler.nodes.ValueNode;
+import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahBarrierSet;
+import jdk.vm.ci.meta.JavaKind;
+import jdk.vm.ci.meta.ResolvedJavaField;
+import jdk.vm.ci.meta.ResolvedJavaType;
+
+/**
+ * Specialization of {@link ShenandoahBarrierSet} that adds support for read barriers on handle locations.
+ */
+public class HotSpotShenandoahBarrierSet extends ShenandoahBarrierSet {
+    private CompressEncoding oopEncoding;
+
+    public HotSpotShenandoahBarrierSet(ResolvedJavaType objectArrayType, ResolvedJavaField referentField, GraalHotSpotVMConfig config) {
+        super(objectArrayType, referentField);
+        this.oopEncoding = config.getOopEncoding();
+        this.useLoadRefBarrier = config.getFlag("ShenandoahLoadRefBarrier", Boolean.class);
+        this.useSATBBarrier = config.getFlag("ShenandoahSATBBarrier", Boolean.class);
+        this.useCASBarrier = config.getFlag("ShenandoahCASBarrier", Boolean.class);
+        this.useCardBarrier = config.getFlag("ShenandoahCardBarrier", Boolean.class);
+    }
+
+    @Override
+    protected BarrierType barrierForLocation(BarrierType currentBarrier, LocationIdentity location, JavaKind storageKind) {
+        if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) {
+            return BarrierType.READ;
+        }
+        return super.barrierForLocation(currentBarrier, location, storageKind);
+    }
+
+    @Override
+    public BarrierType readBarrierType(LocationIdentity location, ValueNode address, Stamp loadStamp) {
+        if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) {
+            assert loadStamp instanceof AbstractObjectStamp : loadStamp;
+            return BarrierType.READ;
+        }
+        return super.readBarrierType(location, address, loadStamp);
+    }
+
+    @Override
+    public BarrierType writeBarrierType(LocationIdentity location) {
+        if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) {
+            return BarrierType.FIELD;
+        }
+        return BarrierType.NONE;
+    }
+
+    @Override
+    protected ValueNode maybeUncompressReference(ValueNode value, boolean narrow) {
+        if (value != null && narrow) {
+            //System.out.println("Uncompressing " + value);
+            return HotSpotCompressionNode.uncompressWithoutUnique(value.graph(), value, oopEncoding);
+        }
+        return value;
+    }
+
+    @Override
+    protected ValueNode maybeCompressReference(ValueNode value, boolean narrow) {
+        if (value != null && narrow) {
+            //System.out.println("Compressing " + value);
+            return HotSpotCompressionNode.compressWithoutUnique(value.graph(), value, oopEncoding);
+        }
+        return value;
+    }
+}

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java

@@ -62,6 +62,7 @@
 import jdk.graal.compiler.hotspot.HotSpotLIRGenerator;
 import jdk.graal.compiler.hotspot.HotSpotLockStack;
 import jdk.graal.compiler.hotspot.aarch64.g1.AArch64HotSpotG1BarrierSetLIRTool;
+import jdk.graal.compiler.hotspot.aarch64.shenandoah.AArch64HotSpotShenandoahBarrierSetLIRGenerator;
 import jdk.graal.compiler.hotspot.aarch64.z.AArch64HotSpotZBarrierSetLIRGenerator;
 import jdk.graal.compiler.hotspot.debug.BenchmarkCounters;
 import jdk.graal.compiler.hotspot.meta.HotSpotProviders;
@@ -119,6 +120,9 @@ protected static BarrierSetLIRGeneratorTool getBarrierSet(GraalHotSpotVMConfig c
         if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) {
             return new AArch64HotSpotZBarrierSetLIRGenerator(config, providers);
         }
+        if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) {
+            return new AArch64HotSpotShenandoahBarrierSetLIRGenerator(config, providers);
+        }
         return null;
     }