From dcf72724b2303e156b88df761a50284ae8ce7038 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Thu, 7 Sep 2023 06:09:34 +0000 Subject: [PATCH 1/2] Adding support for allocating in a non-moving space; Allocating types, typenames and buffers into a non-moving space --- src/datatype.c | 4 ++-- src/gc-common.c | 6 ++++++ src/julia.h | 5 +++++ src/julia_internal.h | 28 +++++++++++++++++++++++++++- 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/datatype.c b/src/datatype.c index 95c3b11c9abdc..a9bdac9e7c173 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -63,7 +63,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu { jl_task_t *ct = jl_current_task; jl_typename_t *tn = - (jl_typename_t*)jl_gc_alloc(ct->ptls, sizeof(jl_typename_t), + (jl_typename_t*)jl_gc_alloc_non_moving(ct->ptls, sizeof(jl_typename_t), jl_typename_type); tn->name = name; tn->module = module; @@ -95,7 +95,7 @@ jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_module_t *module, jl_dat jl_datatype_t *jl_new_uninitialized_datatype(void) { jl_task_t *ct = jl_current_task; - jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc(ct->ptls, sizeof(jl_datatype_t), jl_datatype_type); + jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc_non_moving(ct->ptls, sizeof(jl_datatype_t), jl_datatype_type); jl_set_typetagof(t, jl_datatype_tag, 0); t->hash = 0; t->hasfreetypevars = 0; diff --git a/src/gc-common.c b/src/gc-common.c index 38f737ada576f..cae581e0dbf31 100644 --- a/src/gc-common.c +++ b/src/gc-common.c @@ -414,6 +414,12 @@ JL_DLLEXPORT jl_value_t *(jl_gc_alloc)(jl_ptls_t ptls, size_t sz, void *ty) return jl_gc_alloc_(ptls, sz, ty); } +JL_DLLEXPORT jl_value_t *(jl_gc_alloc_non_moving)(jl_ptls_t ptls, size_t sz, void *ty) +{ + return jl_gc_alloc__non_moving(ptls, sz, ty); +} + + // Instrumented version of jl_gc_big_alloc_inner, called into by // LLVM-generated code. JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz) diff --git a/src/julia.h b/src/julia.h index 99af065a3c1a7..3466d00a5e57d 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2483,6 +2483,11 @@ STATIC_INLINE void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, vo } } +STATIC_INLINE void mmtk_non_moving_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { + // FIXME: do we need to call anything for non moving post alloc? +} + + #endif #ifdef __cplusplus diff --git a/src/julia_internal.h b/src/julia_internal.h index 737553ec98845..506516f0575be 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -341,6 +341,7 @@ jl_value_t *jl_gc_big_alloc_noinline(jl_ptls_t ptls, size_t allocsz); #ifdef MMTK_GC JL_DLLIMPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int pool_offset, int osize, void* ty); JL_DLLIMPORT jl_value_t *jl_mmtk_gc_alloc_big(jl_ptls_t ptls, size_t allocsz); +JL_DLLIMPORT jl_value_t *jl_mmtk_gc_alloc_non_moving(jl_ptls_t ptls, int osize, void* ty); JL_DLLIMPORT extern void mmtk_post_alloc(void* mutator, void* obj, size_t bytes, int allocator); JL_DLLIMPORT extern void mmtk_initialize_collection(void* tls); #endif // MMTK_GC @@ -506,6 +507,20 @@ STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty) maybe_record_alloc_to_profile(v, sz, (jl_datatype_t*)ty); return v; } + +STATIC_INLINE jl_value_t *jl_gc_alloc__non_moving(jl_ptls_t ptls, size_t sz, void *ty) +{ + jl_value_t *v; + const size_t allocsz = sz + sizeof(jl_taggedvalue_t); + + if (allocsz < sz) // overflow in adding offs, size was "negative" + jl_throw(jl_memory_exception); + v = jl_mmtk_gc_alloc_non_moving(ptls, allocsz, ty); + + jl_set_typeof(v, ty); + maybe_record_alloc_to_profile(v, sz, (jl_datatype_t*)ty); + return v; +} #endif // MMTK_GC /* Programming style note: When using jl_gc_alloc, do not JL_GC_PUSH it into a @@ -525,6 +540,17 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc(jl_ptls_t ptls, size_t sz, void *ty); # define jl_gc_alloc(ptls, sz, ty) jl_gc_alloc_(ptls, sz, ty) #endif +JL_DLLEXPORT jl_value_t *jl_gc_alloc_non_moving(jl_ptls_t ptls, size_t sz, void *ty); + +#ifdef __GNUC__ +# define jl_gc_alloc_non_moving(ptls, sz, ty) \ + (__builtin_constant_p(sz) ? \ + jl_gc_alloc__non_moving(ptls, sz, ty) : \ + (jl_gc_alloc_non_moving)(ptls, sz, ty)) +#else +# define jl_gc_alloc_non_moving(ptls, sz, ty) jl_gc_alloc__non_moving(ptls, sz, ty) +#endif + // jl_buff_tag must be an actual pointer here, so it cannot be confused for an actual type reference. // defined as uint64_t[3] so that we can get the right alignment of this and a "type tag" on it const extern uint64_t _jl_buff_tag[3]; @@ -534,7 +560,7 @@ JL_DLLEXPORT uintptr_t jl_get_buff_tag(void); typedef void jl_gc_tracked_buffer_t; // For the benefit of the static analyzer STATIC_INLINE jl_gc_tracked_buffer_t *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz) { - return jl_gc_alloc(ptls, sz, (void*)jl_buff_tag); + return jl_gc_alloc_non_moving(ptls, sz, (void*)jl_buff_tag); } STATIC_INLINE jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT From 0e0adf1cb1f5193c6f6a8b0fd0dc0561c8e04f44 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Mon, 11 Sep 2023 00:14:13 +0000 Subject: [PATCH 2/2] Adding wb code in non-moving post alloc function --- src/julia.h | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/julia.h b/src/julia.h index 3466d00a5e57d..b4e1501e046d0 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2484,7 +2484,19 @@ STATIC_INLINE void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, vo } STATIC_INLINE void mmtk_non_moving_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { - // FIXME: do we need to call anything for non moving post alloc? + // FIXME: since allocation currently happens in the immortal space we need the same WB code + if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) { + intptr_t addr = (intptr_t) obj; + uint8_t* meta_addr = (uint8_t*) (MMTK_SIDE_LOG_BIT_BASE_ADDRESS) + (addr >> 6); + intptr_t shift = (addr >> 3) & 0b111; + while(1) { + uint8_t old_val = *meta_addr; + uint8_t new_val = old_val | (1 << shift); + if (jl_atomic_cmpswap((_Atomic(uint8_t)*)meta_addr, &old_val, new_val)) { + break; + } + } + } }