diff --git a/include/hermes/VM/AlignedHeapSegment.h b/include/hermes/VM/AlignedHeapSegment.h index 6543dafb91a..40ab1d34972 100644 --- a/include/hermes/VM/AlignedHeapSegment.h +++ b/include/hermes/VM/AlignedHeapSegment.h @@ -268,6 +268,16 @@ class AlignedHeapSegment { return markBits->at(ind); } +#ifndef NDEBUG + /// Get the storage end of segment that \p cell resides in. + static char *storageEnd(const GCCell *cell) { + auto *start = alignedStorageStart(cell); + auto *segmentInfo = reinterpret_cast(start); + return start + + (segmentInfo->shiftedSegmentSize << HERMESVM_LOG_HEAP_SEGMENT_SIZE); + } +#endif + protected: AlignedHeapSegment() = default; diff --git a/include/hermes/VM/ArrayStorage.h b/include/hermes/VM/ArrayStorage.h index 15d90f83e3b..6247d0e2c84 100644 --- a/include/hermes/VM/ArrayStorage.h +++ b/include/hermes/VM/ArrayStorage.h @@ -237,7 +237,7 @@ class ArrayStorageBase final auto *fromStart = other->data(); auto *fromEnd = fromStart + otherSz; GCHVType::uninitialized_copy( - fromStart, fromEnd, data() + sz, runtime.getHeap()); + fromStart, fromEnd, data() + sz, runtime.getHeap(), this); size_.store(sz + otherSz, std::memory_order_release); } diff --git a/include/hermes/VM/GCBase.h b/include/hermes/VM/GCBase.h index bc0fbc0bf0d..b7065db1b97 100644 --- a/include/hermes/VM/GCBase.h +++ b/include/hermes/VM/GCBase.h @@ -1162,6 +1162,7 @@ class GCBase { void constructorWriteBarrier(const GCPointerBase *loc, const GCCell *value); template void constructorWriteBarrierRange( + const GCCell *owningObj, const GCHermesValueBase *start, uint32_t numHVs); template diff --git a/include/hermes/VM/HadesGC.h b/include/hermes/VM/HadesGC.h index e30c46f0e5c..991c4d4a334 100644 --- a/include/hermes/VM/HadesGC.h +++ b/include/hermes/VM/HadesGC.h @@ -216,27 +216,30 @@ class HadesGC final : public GCBase { template void constructorWriteBarrierRange( + const GCCell *owningObj, const GCHermesValueBase *start, uint32_t numHVs) { // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(start))) - constructorWriteBarrierRangeSlow(start, numHVs); + constructorWriteBarrierRangeSlow(owningObj, start, numHVs); } template void constructorWriteBarrierRangeSlow( + const GCCell *owningObj, const GCHermesValueBase *start, uint32_t numHVs) { assert( - FixedSizeHeapSegment::containedInSame(start, start + numHVs) && + reinterpret_cast(start + numHVs) < + AlignedHeapSegment::storageEnd(owningObj) && "Range must start and end within a heap segment."); - // Most constructors should be running in the YG, so in the common case, we - // can avoid doing anything for the whole range. If the range is in the OG, - // then just dirty all the cards corresponding to it, and we can scan them - // for pointers later. This is less precise but makes the write barrier - // faster. + // Most constructors should be running in the YG, so in the common case, + // we can avoid doing anything for the whole range. If the range is in + // the OG, then just dirty all the cards corresponding to it, and we can + // scan them for pointers later. This is less precise but makes the + // write barrier faster. - FixedSizeHeapSegment::cardTableCovering(start)->dirtyCardsForAddressRange( + AlignedHeapSegment::cardTableCovering(owningObj)->dirtyCardsForAddressRange( start, start + numHVs); } diff --git a/include/hermes/VM/HermesValue-inline.h b/include/hermes/VM/HermesValue-inline.h index d38a9219879..7fb255f7965 100644 --- a/include/hermes/VM/HermesValue-inline.h +++ b/include/hermes/VM/HermesValue-inline.h @@ -182,7 +182,8 @@ inline GCHermesValueBase *GCHermesValueBase::uninitialized_copy( GCHermesValueBase *first, GCHermesValueBase *last, GCHermesValueBase *result, - GC &gc) { + GC &gc, + const GCCell *owningObj) { #ifndef NDEBUG uintptr_t fromFirst = reinterpret_cast(first), fromLast = reinterpret_cast(last); @@ -194,7 +195,7 @@ inline GCHermesValueBase *GCHermesValueBase::uninitialized_copy( "Uninitialized range cannot overlap with an initialized one."); #endif - gc.constructorWriteBarrierRange(result, last - first); + gc.constructorWriteBarrierRange(owningObj, result, last - first); // memcpy is fine for an uninitialized copy. std::memcpy( reinterpret_cast(result), first, (last - first) * sizeof(HVType)); diff --git a/include/hermes/VM/HermesValue.h b/include/hermes/VM/HermesValue.h index 7e377d44754..5b99dfd9258 100644 --- a/include/hermes/VM/HermesValue.h +++ b/include/hermes/VM/HermesValue.h @@ -589,7 +589,8 @@ class GCHermesValueBase final : public HVType { GCHermesValueBase *first, GCHermesValueBase *last, GCHermesValueBase *result, - GC &gc); + GC &gc, + const GCCell *owningObj); /// Copies a range of values and performs a write barrier on each. template diff --git a/include/hermes/VM/MallocGC.h b/include/hermes/VM/MallocGC.h index b77c51d7615..01d5eb69e0f 100644 --- a/include/hermes/VM/MallocGC.h +++ b/include/hermes/VM/MallocGC.h @@ -243,6 +243,7 @@ class MallocGC final : public GCBase { void writeBarrierRange(const GCHermesValueBase *, uint32_t) {} template void constructorWriteBarrierRange( + const GCCell *, const GCHermesValueBase *, uint32_t) {} template diff --git a/lib/VM/ArrayStorage.cpp b/lib/VM/ArrayStorage.cpp index 795af685192..98f952bfaa5 100644 --- a/lib/VM/ArrayStorage.cpp +++ b/lib/VM/ArrayStorage.cpp @@ -104,7 +104,8 @@ ExecutionStatus ArrayStorageBase::reallocateToLarger( { GCHVType *from = self->data() + fromFirst; GCHVType *to = newSelf->data() + toFirst; - GCHVType::uninitialized_copy(from, from + copySize, to, runtime.getHeap()); + GCHVType::uninitialized_copy( + from, from + copySize, to, runtime.getHeap(), newSelf); } // Initialize the elements before the first copied element. diff --git a/lib/VM/GCBase.cpp b/lib/VM/GCBase.cpp index 1247b35828d..a7e2b37dcd3 100644 --- a/lib/VM/GCBase.cpp +++ b/lib/VM/GCBase.cpp @@ -965,6 +965,11 @@ bool GCBase::shouldSanitizeHandles() { runtimeGCDispatch([&](auto *gc) { gc->name(arg1, arg2); }); \ } +#define GCBASE_BARRIER_3(name, type1, type2, type3) \ + void GCBase::name(type1 arg1, type2 arg2, type3 arg3) { \ + runtimeGCDispatch([&](auto *gc) { gc->name(arg1, arg2, arg3); }); \ + } + GCBASE_BARRIER_2(writeBarrier, const GCHermesValue *, HermesValue); GCBASE_BARRIER_2(writeBarrier, const GCSmallHermesValue *, SmallHermesValue); GCBASE_BARRIER_2(writeBarrier, const GCPointerBase *, const GCCell *); @@ -979,9 +984,14 @@ GCBASE_BARRIER_2( const GCCell *); GCBASE_BARRIER_2(writeBarrierRange, const GCHermesValue *, uint32_t); GCBASE_BARRIER_2(writeBarrierRange, const GCSmallHermesValue *, uint32_t); -GCBASE_BARRIER_2(constructorWriteBarrierRange, const GCHermesValue *, uint32_t); -GCBASE_BARRIER_2( +GCBASE_BARRIER_3( + constructorWriteBarrierRange, + const GCCell *, + const GCHermesValue *, + uint32_t); +GCBASE_BARRIER_3( constructorWriteBarrierRange, + const GCCell *, const GCSmallHermesValue *, uint32_t); GCBASE_BARRIER_1(snapshotWriteBarrier, const GCHermesValue *); diff --git a/lib/VM/SegmentedArray.cpp b/lib/VM/SegmentedArray.cpp index 71f85f699b7..b59c7c525c9 100644 --- a/lib/VM/SegmentedArray.cpp +++ b/lib/VM/SegmentedArray.cpp @@ -294,7 +294,8 @@ ExecutionStatus SegmentedArrayBase::growRight( self->inlineStorage(), self->inlineStorage() + numSlotsUsed, newSegmentedArray->inlineStorage(), - runtime.getHeap()); + runtime.getHeap(), + newSegmentedArray.get()); // Set the size of the new array to be the same as the old array's size. newSegmentedArray->numSlotsUsed_.store( numSlotsUsed, std::memory_order_release);