Resize objects to optimal size when moving

gc/mmtk/mmtk.c

@@ -308,6 +308,22 @@ static void
 rb_mmtk_move_obj_during_marking(MMTk_ObjectReference from, MMTk_ObjectReference to)
 {
     rb_gc_move_obj_during_marking((VALUE)from, (VALUE)to);
+
+    size_t from_slot_size = rb_gc_impl_obj_slot_size((VALUE)from);
+    size_t to_slot_size = rb_gc_impl_obj_slot_size((VALUE)to);
+    if (from_slot_size != to_slot_size) {
+        /* The object was copied to a differently sized slot; transition the
+         * shape so its capacity matches the new slot size.  The re-embedding
+         * of contents (e.g. arrays, strings, object fields) happens later
+         * during reference updating. */
+        rb_gc_obj_changed_slot_size((VALUE)to, to_slot_size);
+    }
+}
+
+static size_t
+rb_mmtk_obj_optimal_size(MMTk_ObjectReference obj)
+{
+    return rb_gc_obj_optimal_size((VALUE)obj);
 }
 
 static void
@@ -543,6 +559,7 @@ MMTk_RubyUpcalls ruby_upcalls = {
     rb_mmtk_scan_gc_roots,
     rb_mmtk_scan_objspace,
     rb_mmtk_move_obj_during_marking,
+    rb_mmtk_obj_optimal_size,
     rb_mmtk_update_object_references,
     rb_mmtk_call_gc_mark_children,
     rb_mmtk_handle_weak_references,
@@ -1271,6 +1288,16 @@ rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj)
 void
 rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj)
 {
+    if (!ruby_native_thread_p()) {
+        /* Called from an MMTk GC worker thread (via gc_ref_update_object's
+         * re-embedding path during reference updating).  There is no mutator
+         * whose modbuf we could log the object into.  This is only reachable
+         * during a moving GC, and moving GCs are always full-heap (major)
+         * GCs, so the object and its children are all marked by the current
+         * GC and no remembering is needed. */
+        return;
+    }
+
     struct MMTk_ractor_cache *cache = rb_gc_get_ractor_newobj_cache();
 
     mmtk_object_reference_write_post(cache->mutator, (MMTk_ObjectReference)obj);

gc/mmtk/mmtk.h

@@ -68,6 +68,7 @@ typedef struct MMTk_RubyUpcalls {
     void (*scan_gc_roots)(void);
     void (*scan_objspace)(void);
     void (*move_obj_during_marking)(MMTk_ObjectReference from, MMTk_ObjectReference to);
+    size_t (*obj_optimal_size)(MMTk_ObjectReference object);
     void (*update_object_references)(MMTk_ObjectReference object);
     void (*call_gc_mark_children)(MMTk_ObjectReference object);
     void (*handle_weak_references)(MMTk_ObjectReference object, bool moving);

gc/mmtk/src/abi.rs

@@ -312,6 +312,7 @@ pub struct RubyUpcalls {
     pub scan_gc_roots: extern "C" fn(),
     pub scan_objspace: extern "C" fn(),
     pub move_obj_during_marking: extern "C" fn(from: ObjectReference, to: ObjectReference),
+    pub obj_optimal_size: extern "C" fn(object: ObjectReference) -> usize,
     pub update_object_references: extern "C" fn(object: ObjectReference),
     pub call_gc_mark_children: extern "C" fn(object: ObjectReference),
     pub handle_weak_references: extern "C" fn(object: ObjectReference, moving: bool),

gc/mmtk/src/object_model.rs

@@ -52,18 +52,52 @@ impl ObjectModel<Ruby> for VMObjectModel {
         let from_acc = RubyObjectAccess::from_objref(from);
         let from_start = from_acc.obj_start();
         let object_size = from_acc.object_size();
-        let to_start = copy_context.alloc_copy(from, object_size, MIN_OBJ_ALIGN, 0, semantics);
+
+        let optimal_payload = (crate::binding().upcalls().obj_optimal_size)(from);
+        let new_size = if optimal_payload != 0 {
+            let size = (RubyObjectAccess::prefix_size()
+                + optimal_payload
+                + RubyObjectAccess::suffix_size())
+            .next_multiple_of(MIN_OBJ_ALIGN);
+            // Never resize beyond the non-LOS allocation limit.
+            let max_non_los = crate::mmtk()
+                .get_plan()
+                .constraints()
+                .max_non_los_default_alloc_bytes;
+            if size <= max_non_los {
+                size
+            } else {
+                object_size
+            }
+        } else {
+            object_size
+        };
+
+        let to_start = copy_context.alloc_copy(from, new_size, MIN_OBJ_ALIGN, 0, semantics);
         debug_assert!(!to_start.is_zero());
         let to_payload = to_start.add(OBJREF_OFFSET);
         unsafe {
-            copy_nonoverlapping::<u8>(from_start.to_ptr(), to_start.to_mut_ptr(), object_size);
+            copy_nonoverlapping::<u8>(
+                from_start.to_ptr(),
+                to_start.to_mut_ptr(),
+                object_size.min(new_size),
+            );
         }
         let to_obj = unsafe { ObjectReference::from_raw_address_unchecked(to_payload) };
-        copy_context.post_copy(to_obj, object_size, semantics);
-        trace!("Copied object from {} to {}", from, to_obj);
+
+        if new_size != object_size {
+            unsafe {
+                to_start.store(
+                    new_size - RubyObjectAccess::prefix_size() - RubyObjectAccess::suffix_size(),
+                );
+            }
+        }
 
         (crate::binding().upcalls().move_obj_during_marking)(from, to_obj);
 
+        copy_context.post_copy(to_obj, new_size, semantics);
+        trace!("Copied object from {} to {}", from, to_obj);
+
         #[cfg(feature = "clear_old_copy")]
         {
             trace!(