From 596410b46a44fe558cd9ff105cf47295480cef3a Mon Sep 17 00:00:00 2001 From: Gang Zhao Date: Mon, 9 Dec 2024 11:15:48 -0800 Subject: [PATCH] Split of "[SH] Add write barrier variants that support large allocation" Differential Revision: D65701671 --- include/hermes/VM/AlignedHeapSegment.h | 11 ++ include/hermes/VM/GCBase.h | 32 +++++ include/hermes/VM/HadesGC.h | 178 +++++++++++++++++++++++++ include/hermes/VM/MallocGC.h | 22 +++ lib/VM/GCBase.cpp | 30 +++++ lib/VM/gcs/HadesGC.cpp | 172 ++++++++++++++++++++++++ 6 files changed, 445 insertions(+) diff --git a/include/hermes/VM/AlignedHeapSegment.h b/include/hermes/VM/AlignedHeapSegment.h index 27a8ea6b3df..da01912d8fb 100644 --- a/include/hermes/VM/AlignedHeapSegment.h +++ b/include/hermes/VM/AlignedHeapSegment.h @@ -272,6 +272,17 @@ class AlignedHeapSegment { return (cp - base) >> LogHeapAlign; } + /// Return true if objects \p a and \p b live in the same segment. This is + /// used to check if a pointer field in an object points to another object in + /// the same segment (so that we don't need to dirty the cards -- we only need + /// to dirty cards that might contain old-to-young pointers, which must cross + /// segments). This also works for large segments, since there is only one + /// cell in those segments (i.e., \p a and \p b would be the same). + static bool containedInSameSegment(const GCCell *a, const GCCell *b) { + return (reinterpret_cast(a) ^ reinterpret_cast(b)) < + kSegmentUnitSize; + } + /// Returns the index of the segment containing \p lowLim, which is required /// to be the start of its containing segment. (This can allow extra /// efficiency, in cases where the segment start has already been computed.) diff --git a/include/hermes/VM/GCBase.h b/include/hermes/VM/GCBase.h index 0692064e221..f3fe1ae71e3 100644 --- a/include/hermes/VM/GCBase.h +++ b/include/hermes/VM/GCBase.h @@ -204,6 +204,14 @@ enum XorPtrKeyID { /// const GCSmallHermesValue *start, /// uint32_t numHVs); /// +/// The above barriers may have a variant with "ForLargeObj" suffix, which is +/// used when the heap location may be from a GCCell of a kind that supports +/// large allocation. This variant is less efficient since it has to load the +/// cards array through pointer in SHSegmentInfo, instead of the inline array +/// field in CardTable structure. Because of this, these variant write +/// barriers need a pointer to the start of the object in order to locate the +/// card table for an object that is larger than the unit segment size. +/// /// In debug builds: is a write barrier necessary for a write of the given /// GC pointer \p value to the given \p loc? /// bool needsWriteBarrier(void *loc, void *value); @@ -1154,12 +1162,36 @@ class GCBase { /// Default implementations for read and write barriers: do nothing. void writeBarrier(const GCHermesValue *loc, HermesValue value); void writeBarrier(const GCSmallHermesValue *loc, SmallHermesValue value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value); void writeBarrier(const GCPointerBase *loc, const GCCell *value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value); void constructorWriteBarrier(const GCHermesValue *loc, HermesValue value); void constructorWriteBarrier( const GCSmallHermesValue *loc, SmallHermesValue value); + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value); + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value); void constructorWriteBarrier(const GCPointerBase *loc, const GCCell *value); + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value); void writeBarrierRange(const GCHermesValue *start, uint32_t numHVs); void writeBarrierRange(const GCSmallHermesValue *start, uint32_t numHVs); void constructorWriteBarrierRange( diff --git a/include/hermes/VM/HadesGC.h b/include/hermes/VM/HadesGC.h index e6686fd2e9d..6c914002901 100644 --- a/include/hermes/VM/HadesGC.h +++ b/include/hermes/VM/HadesGC.h @@ -19,6 +19,7 @@ #include "llvh/ADT/SparseBitVector.h" #include "llvh/Support/ErrorOr.h" #include "llvh/Support/PointerLikeTypeTraits.h" +#include "llvh/Support/SaveAndRestore.h" #include #include @@ -156,21 +157,67 @@ class HadesGC final : public GCBase { assert( !calledByBackgroundThread() && "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) writeBarrierSlow(loc, value); } void writeBarrierSlow(const GCHermesValue *loc, HermesValue value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForLargeObj(owningObj, loc); +#endif + + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + writeBarrierSlowForLargeObj(owningObj, loc, value); + } + void writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value); void writeBarrier(const GCSmallHermesValue *loc, SmallHermesValue value) { assert( !calledByBackgroundThread() && "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) writeBarrierSlow(loc, value); } void writeBarrierSlow(const GCSmallHermesValue *loc, SmallHermesValue value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForLargeObj(owningObj, loc); +#endif + + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + writeBarrierSlowForLargeObj(owningObj, loc, value); + } + void writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value); /// The given pointer value is being written at the given loc (required to /// be in the heap). The value may be null. Execute a write barrier. @@ -180,24 +227,80 @@ class HadesGC final : public GCBase { assert( !calledByBackgroundThread() && "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) writeBarrierSlow(loc, value); } void writeBarrierSlow(const GCPointerBase *loc, const GCCell *value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForLargeObj(owningObj, loc); +#endif + + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + writeBarrierSlowForLargeObj(owningObj, loc, value); + } + void writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value); /// Special versions of \p writeBarrier for when there was no previous value /// initialized into the space. void constructorWriteBarrier(const GCHermesValue *loc, HermesValue value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertCtorWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) constructorWriteBarrierSlow(loc, value); } void constructorWriteBarrierSlow(const GCHermesValue *loc, HermesValue value); + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertCtorWriteBarrierForLargeObj(owningObj, loc); +#endif + + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + constructorWriteBarrierSlowForLargeObj(owningObj, loc, value); + } + void constructorWriteBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value); void constructorWriteBarrier( const GCSmallHermesValue *loc, SmallHermesValue value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertCtorWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) constructorWriteBarrierSlow(loc, value); @@ -205,12 +308,53 @@ class HadesGC final : public GCBase { void constructorWriteBarrierSlow( const GCSmallHermesValue *loc, SmallHermesValue value); + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertCtorWriteBarrierForLargeObj(owningObj, loc); +#endif + + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + constructorWriteBarrierSlowForLargeObj(owningObj, loc, value); + } + void constructorWriteBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value); void constructorWriteBarrier(const GCPointerBase *loc, const GCCell *value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertCtorWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) relocationWriteBarrier(loc, value); } + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertCtorWriteBarrierForLargeObj(owningObj, loc); +#endif + + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + relocationWriteBarrierForLargeObj(owningObj, loc, value); + } void constructorWriteBarrierRange( const GCCell *owningObj, @@ -692,6 +836,14 @@ class HadesGC final : public GCBase { /// GC. This includes mark bits, free lists, etc. Mutex gcMutex_; +#ifdef HERMES_SLOW_DEBUG + /// CellKind and size of the GCCell being constructed. It is set in makeA() + /// before constructing the cell and reset in the end. Note that we don't + /// use KindAndSize here since the actual size could be larger than + /// GCCell::maxSize(). + llvh::Optional> currentCellKindAndSize_; +#endif + /// Flag used to signal to the background thread that it should stop and yield /// the gcMutex_ to the mutator as soon as possible. AtomicIfConcurrentGC ogPaused_{false}; @@ -995,6 +1147,10 @@ class HadesGC final : public GCBase { /// pointers into YG and for tracking newly created pointers into the /// compactee. void relocationWriteBarrier(const void *loc, const void *value); + void relocationWriteBarrierForLargeObj( + const GCCell *owningObj, + const void *loc, + const GCCell *value); /// Finalize all objects in YG that have finalizers. void finalizeYoungGenObjects(); @@ -1061,6 +1217,25 @@ class HadesGC final : public GCBase { void removeSegmentExtentFromCrashManager(const std::string &extraName); #ifdef HERMES_SLOW_DEBUG + /// Assert that \p loc does not belong to an object that supports large + /// allocation, for which we should call the other variant of write barrer. + void assertWriteBarrierForNonLargeObj(const void *loc); + + /// Assert that \p loc does not belong to an object that supports large + /// allocation, for which we should call the other variant of constructor + /// write barrer. + void assertCtorWriteBarrierForNonLargeObj(const void *loc); + + /// Assert that \p owningObj is an object that supports large allocation and + /// it contains \p loc. + void assertCtorWriteBarrierForLargeObj( + const GCCell *owningObj, + const void *loc); + + /// Assert that \p owningObj is an object that supports large allocation and + /// it contains \p loc. + void assertWriteBarrierForLargeObj(const GCCell *owningObj, const void *loc); + /// Checks the heap to make sure all cells are valid. void checkWellFormed(); @@ -1084,6 +1259,9 @@ inline T *HadesGC::makeA(uint32_t size, Args &&...args) { isSizeHeapAligned(size) && "Call to makeA must use a size aligned to HeapAlign"); assert(noAllocLevel_ == 0 && "No allocs allowed right now."); + llvh::SaveAndRestore>> + saveCurrentCellKindAndSize( + currentCellKindAndSize_, std::make_pair(T::getCellKind(), size)); if (longLived == LongLived::Yes) { auto lk = ensureBackgroundTaskPaused(); return constructCell( diff --git a/include/hermes/VM/MallocGC.h b/include/hermes/VM/MallocGC.h index 8e5e1b88277..ea0f2d9970b 100644 --- a/include/hermes/VM/MallocGC.h +++ b/include/hermes/VM/MallocGC.h @@ -235,10 +235,32 @@ class MallocGC final : public GCBase { void writeBarrier(const GCHermesValue *, HermesValue) {} void writeBarrier(const GCSmallHermesValue *, SmallHermesValue) {} + void + writeBarrierForLargeObj(const GCCell *, const GCHermesValue *, HermesValue) {} + void writeBarrierForLargeObj( + const GCCell *, + const GCSmallHermesValue *, + SmallHermesValue) {} void writeBarrier(const GCPointerBase *, const GCCell *) {} + void writeBarrierForLargeObj( + const GCCell *, + const GCPointerBase *, + const GCCell *) {} void constructorWriteBarrier(const GCHermesValue *, HermesValue) {} void constructorWriteBarrier(const GCSmallHermesValue *, SmallHermesValue) {} + void constructorWriteBarrierForLargeObj( + const GCCell *, + const GCSmallHermesValue *, + SmallHermesValue) {} + void constructorWriteBarrierForLargeObj( + const GCCell *, + const GCHermesValue *, + HermesValue) {} void constructorWriteBarrier(const GCPointerBase *, const GCCell *) {} + void constructorWriteBarrierForLargeObj( + const GCCell *, + const GCPointerBase *, + const GCCell *) {} void writeBarrierRange(const GCHermesValue *, uint32_t) {} void writeBarrierRange(const GCSmallHermesValue *, uint32_t) {} void constructorWriteBarrierRange( diff --git a/lib/VM/GCBase.cpp b/lib/VM/GCBase.cpp index a7e2b37dcd3..57172b39b4e 100644 --- a/lib/VM/GCBase.cpp +++ b/lib/VM/GCBase.cpp @@ -971,17 +971,47 @@ bool GCBase::shouldSanitizeHandles() { } GCBASE_BARRIER_2(writeBarrier, const GCHermesValue *, HermesValue); +GCBASE_BARRIER_3( + writeBarrierForLargeObj, + const GCCell *, + const GCHermesValue *, + HermesValue); GCBASE_BARRIER_2(writeBarrier, const GCSmallHermesValue *, SmallHermesValue); +GCBASE_BARRIER_3( + writeBarrierForLargeObj, + const GCCell *, + const GCSmallHermesValue *, + SmallHermesValue); GCBASE_BARRIER_2(writeBarrier, const GCPointerBase *, const GCCell *); +GCBASE_BARRIER_3( + writeBarrierForLargeObj, + const GCCell *, + const GCPointerBase *, + const GCCell *); GCBASE_BARRIER_2(constructorWriteBarrier, const GCHermesValue *, HermesValue); +GCBASE_BARRIER_3( + constructorWriteBarrierForLargeObj, + const GCCell *, + const GCHermesValue *, + HermesValue); GCBASE_BARRIER_2( constructorWriteBarrier, const GCSmallHermesValue *, SmallHermesValue); +GCBASE_BARRIER_3( + constructorWriteBarrierForLargeObj, + const GCCell *, + const GCSmallHermesValue *, + SmallHermesValue); GCBASE_BARRIER_2( constructorWriteBarrier, const GCPointerBase *, const GCCell *); +GCBASE_BARRIER_3( + constructorWriteBarrierForLargeObj, + const GCCell *, + const GCPointerBase *, + const GCCell *); GCBASE_BARRIER_2(writeBarrierRange, const GCHermesValue *, uint32_t); GCBASE_BARRIER_2(writeBarrierRange, const GCSmallHermesValue *, uint32_t); GCBASE_BARRIER_3( diff --git a/lib/VM/gcs/HadesGC.cpp b/lib/VM/gcs/HadesGC.cpp index a7ad430d0aa..28a9263caf1 100644 --- a/lib/VM/gcs/HadesGC.cpp +++ b/lib/VM/gcs/HadesGC.cpp @@ -13,6 +13,7 @@ #include "hermes/VM/CheckHeapWellFormedAcceptor.h" #include "hermes/VM/FillerCell.h" #include "hermes/VM/GCBase-inline.h" +#include "hermes/VM/GCCell.h" #include "hermes/VM/GCPointer.h" #include "hermes/VM/HermesValue-inline.h" #include "hermes/VM/RootAndSlotAcceptorDefault.h" @@ -1951,6 +1952,20 @@ void HadesGC::writeBarrierSlow(const GCHermesValue *loc, HermesValue value) { relocationWriteBarrier(loc, value.getPointer()); } +void HadesGC::writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value) { + if (ogMarkingBarriers_) { + snapshotWriteBarrierInternal(*loc); + } + if (!value.isPointer()) { + return; + } + relocationWriteBarrierForLargeObj( + owningObj, loc, static_cast(value.getPointer())); +} + void HadesGC::writeBarrierSlow( const GCSmallHermesValue *loc, SmallHermesValue value) { @@ -1963,6 +1978,20 @@ void HadesGC::writeBarrierSlow( relocationWriteBarrier(loc, value.getPointer(getPointerBase())); } +void HadesGC::writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value) { + if (ogMarkingBarriers_) { + snapshotWriteBarrierInternal(*loc); + } + if (!value.isPointer()) { + return; + } + relocationWriteBarrierForLargeObj( + owningObj, loc, value.getPointer(getPointerBase())); +} + void HadesGC::writeBarrierSlow(const GCPointerBase *loc, const GCCell *value) { if (*loc && ogMarkingBarriers_) snapshotWriteBarrierInternal(*loc); @@ -1971,6 +2000,17 @@ void HadesGC::writeBarrierSlow(const GCPointerBase *loc, const GCCell *value) { relocationWriteBarrier(loc, value); } +void HadesGC::writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value) { + if (*loc && ogMarkingBarriers_) + snapshotWriteBarrierInternal(*loc); + // Always do the non-snapshot write barrier in order for YG to be able to + // scan cards. + relocationWriteBarrierForLargeObj(owningObj, loc, value); +} + void HadesGC::constructorWriteBarrierSlow( const GCHermesValue *loc, HermesValue value) { @@ -1982,6 +2022,19 @@ void HadesGC::constructorWriteBarrierSlow( relocationWriteBarrier(loc, value.getPointer()); } +void HadesGC::constructorWriteBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value) { + // A constructor never needs to execute a SATB write barrier, since its + // previous value was definitely not live. + if (!value.isPointer()) { + return; + } + relocationWriteBarrierForLargeObj( + owningObj, loc, static_cast(value.getPointer())); +} + void HadesGC::constructorWriteBarrierSlow( const GCSmallHermesValue *loc, SmallHermesValue value) { @@ -1993,6 +2046,19 @@ void HadesGC::constructorWriteBarrierSlow( relocationWriteBarrier(loc, value.getPointer(getPointerBase())); } +void HadesGC::constructorWriteBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value) { + // A constructor never needs to execute a SATB write barrier, since its + // previous value was definitely not live. + if (!value.isPointer()) { + return; + } + relocationWriteBarrierForLargeObj( + owningObj, loc, value.getPointer(getPointerBase())); +} + void HadesGC::constructorWriteBarrierRangeSlow( const GCCell *owningObj, const GCHermesValue *start, @@ -2114,6 +2180,20 @@ void HadesGC::relocationWriteBarrier(const void *loc, const void *value) { } } +void HadesGC::relocationWriteBarrierForLargeObj( + const GCCell *owningObj, + const void *loc, + const GCCell *value) { + assert(!inYoungGen(loc) && "Pre-condition from other callers"); + if (AlignedHeapSegment::containedInSameSegment(owningObj, value)) { + return; + } + if (inYoungGen(value) || compactee_.contains(value)) { + AlignedHeapSegment::cardTableCovering(owningObj) + ->dirtyCardForAddressInLargeObj(loc); + } +} + void HadesGC::weakRefReadBarrier(HermesValue value) { assert( !calledByBackgroundThread() && @@ -3126,6 +3206,98 @@ void HadesGC::removeSegmentExtentFromCrashManager( #ifdef HERMES_SLOW_DEBUG +void HadesGC::assertWriteBarrierForNonLargeObj(const void *loc) { + // Don't run the check with concurrent GC. Because when the background thread + // is sweeping, it may merge freed cells and poison memory outside of + // FreeListCell, which may be accessed by findObjectContaining() when visiting + // next cell. + if constexpr (!kConcurrentGC) { + auto *obj = + FixedSizeHeapSegment::cardTableCovering(loc)->findObjectContaining(loc); + assert( + !VTable::getVTable(obj->getKind())->allowLargeAlloc && + "writeBarrier() only works for GCCells that do not support large allocation"); + } else { + // With concurrent GC, we can't safely find the owningObj and check its + // VTable. Instead, we check if its segment size equals to + // FixedSizeHeapSegment::kSize. This is less precise. + auto segmentSize = + FixedSizeHeapSegment::cardTableCovering(loc)->getSegmentSize(); + assert( + (segmentSize == FixedSizeHeapSegment::kSize) && + "writeBarrier() does not work for GCCells larger than the size of FixedSizeHeapSegment"); + } +} + +void HadesGC::assertCtorWriteBarrierForNonLargeObj(const void *loc) { + if (currentCellKindAndSize_) { + // If currentCellKindAndSize_ is set, the owning object is still being + // constructed, use currentCellKindAndSize_ instead of trying to find its + // owning object. + assert( + !VTable::getVTable(currentCellKindAndSize_->first)->allowLargeAlloc && + "constructWriteBarrier() should only be called on GCCells that support large allocation"); + } else { + if constexpr (!kConcurrentGC) { + auto *obj = + FixedSizeHeapSegment::cardTableCovering(loc)->findObjectContaining( + loc); + assert( + !VTable::getVTable(obj->getKind())->allowLargeAlloc && + "constructWriteBarrier() only works for GCCells that do not support large allocation"); + } else { + // With concurrent GC, we can't safely find the owningObj and check its + // VTable. Instead, we check if its segment size equals to + // FixedSizeHeapSegment::kSize. This is less precise. + auto segmentSize = + FixedSizeHeapSegment::cardTableCovering(loc)->getSegmentSize(); + assert( + (segmentSize == FixedSizeHeapSegment::kSize) && + "constructorWriteBarrier() does not work for GCCells larger than the size of FixedSizeHeapSegment"); + } + } +} + +void HadesGC::assertCtorWriteBarrierForLargeObj( + const GCCell *owningObj, + const void *loc) { + if (currentCellKindAndSize_) { + // If currentCellKindAndSize_ is set, the owning object is still being + // constructed, use currentCellKindAndSize_ instead of the kind and size of + // owningObj. + assert( + VTable::getVTable(currentCellKindAndSize_->first)->allowLargeAlloc && + "constructWriteBarrierForLargeObj() should only be called on GCCells that support large allocation"); + assert( + owningObj <= loc && + loc < (reinterpret_cast(owningObj) + + currentCellKindAndSize_->second) && + "The owning object must contain the given heap location"); + } else { + assert( + VTable::getVTable(owningObj->getKind())->allowLargeAlloc && + "constructWriteBarrierForLargeObj() should only be called on GCCells that support large allocation"); + assert( + owningObj <= loc && + loc < (reinterpret_cast(owningObj) + + owningObj->getAllocatedSize()) && + "The owning object must contain the given heap location"); + } +} + +void HadesGC::assertWriteBarrierForLargeObj( + const GCCell *owningObj, + const void *loc) { + assert( + VTable::getVTable(owningObj->getKind())->allowLargeAlloc && + "writeBarrierForLargeObj() should only used on GCCells that support large allocation"); + assert( + owningObj <= loc && + loc < (reinterpret_cast(owningObj) + + owningObj->getAllocatedSize()) && + "The owning object must contain the given heap location"); +} + void HadesGC::checkWellFormed() { CheckHeapWellFormedAcceptor acceptor(*this); {