diff --git a/gc/default/default.c b/gc/default/default.c index a6572b6f4d7ceb..90232c76184a55 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -858,15 +858,15 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page * #define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0]) #define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0]) - -#define RVALUE_AGE_BITMAP_INDEX(n) (NUM_IN_PAGE(n) / (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) -#define RVALUE_AGE_BITMAP_OFFSET(n) ((NUM_IN_PAGE(n) % (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) * RVALUE_AGE_BIT_COUNT) - static int RVALUE_AGE_GET(VALUE obj) { bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits; - return (int)(age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] >> RVALUE_AGE_BITMAP_OFFSET(obj)) & RVALUE_AGE_BIT_MASK; + int idx = BITMAP_INDEX(obj) * 2; + int shift = BITMAP_OFFSET(obj); + int lo = (age_bits[idx] >> shift) & 1; + int hi = (age_bits[idx + 1] >> shift) & 1; + return lo | (hi << 1); } static void @@ -874,10 +874,12 @@ RVALUE_AGE_SET_BITMAP(VALUE obj, int age) { RUBY_ASSERT(age <= RVALUE_OLD_AGE); bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits; - // clear the bits - age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] &= ~(RVALUE_AGE_BIT_MASK << (RVALUE_AGE_BITMAP_OFFSET(obj))); - // shift the correct value in - age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] |= ((bits_t)age << RVALUE_AGE_BITMAP_OFFSET(obj)); + int idx = BITMAP_INDEX(obj) * 2; + int shift = BITMAP_OFFSET(obj); + bits_t mask = (bits_t)1 << shift; + + age_bits[idx] = (age_bits[idx] & ~mask) | ((bits_t)(age & 1) << shift); + age_bits[idx + 1] = (age_bits[idx + 1] & ~mask) | ((bits_t)((age >> 1) & 1) << shift); } static void @@ -3508,7 +3510,6 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit } gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp)); ctx->empty_slots++; - RVALUE_AGE_SET_BITMAP(vp, 0); heap_page_add_freeobj(objspace, sweep_page, vp); break; case T_ZOMBIE: @@ -3525,7 +3526,6 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit if (RVALUE_REMEMBERED(objspace, vp)) rb_bug("page_sweep: %p - remembered.", (void *)p); } #endif - if (RVALUE_WB_UNPROTECTED(objspace, vp)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(vp), vp); #if RGENGC_CHECK_MODE #define CHECK(x) if (x(objspace, vp) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", rb_obj_info(vp)) @@ -3542,7 +3542,6 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit } (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, BASE_SLOT_SIZE); - RVALUE_AGE_SET_BITMAP(vp, 0); heap_page_add_freeobj(objspace, sweep_page, vp); gc_report(3, objspace, "page_sweep: %s (fast path) added to freelist\n", rb_obj_info(vp)); ctx->freed_slots++; @@ -3555,7 +3554,6 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit rb_gc_obj_free_vm_weak_references(vp); if (rb_gc_obj_free(objspace, vp)) { (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, BASE_SLOT_SIZE); - RVALUE_AGE_SET_BITMAP(vp, 0); heap_page_add_freeobj(objspace, sweep_page, vp); gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp)); ctx->freed_slots++; @@ -3609,6 +3607,18 @@ gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context bits_t slot_mask = heap->slot_bits_mask; + // Clear wb_unprotected and age bits for all unmarked slots + { + bits_t *wb_unprotected_bits = sweep_page->wb_unprotected_bits; + bits_t *age_bits = sweep_page->age_bits; + for (int i = 0; i < bitmap_plane_count; i++) { + bits_t unmarked = ~bits[i] & slot_mask; + wb_unprotected_bits[i] &= ~unmarked; + age_bits[i * 2] &= ~unmarked; + age_bits[i * 2 + 1] &= ~unmarked; + } + } + // Skip out of range slots at the head of the page bitset = ~bits[0]; bitset >>= NUM_IN_PAGE(p); diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index d55dce1b9b82e0..138df69008fe2c 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -421,7 +421,7 @@ impl Assembler { } } - /// If a given operand is Opnd::Mem and it uses MemBase::Stack, lower it to MemBase::Reg using a scratch regsiter. + /// If a given operand is Opnd::Mem and it uses MemBase::Stack, lower it to MemBase::Reg using a scratch register. fn split_stack_membase(asm: &mut Assembler, opnd: Opnd, scratch_opnd: Opnd, stack_state: &StackState) -> Opnd { if let Opnd::Mem(Mem { base: stack_membase @ MemBase::Stack { .. }, disp, num_bits }) = opnd { let base = Opnd::Mem(stack_state.stack_membase_to_mem(stack_membase));