[pull] master from ruby:master

bootstraptest/test_yjit.rb

@@ -2680,6 +2680,22 @@ def expandarray_redefined_nilclass
   expandarray_redefined_nilclass
 }
 
+assert_equal 'not_array', %q{
+  def expandarray_not_array(obj)
+    a, = obj
+    a
+  end
+
+  obj = Object.new
+  def obj.method_missing(m, *args, &block)
+    return [:not_array] if m == :to_ary
+    super
+  end
+
+  expandarray_not_array(obj)
+  expandarray_not_array(obj)
+}
+
 assert_equal '[1, 2, nil]', %q{
   def expandarray_rhs_too_small
     a, b, c = [1, 2]

defs/gmake.mk

@@ -275,7 +275,7 @@ pull-github: fetch-github
 	$(call pull-github,$(PR))
 
 define pull-github
-	$(eval GITHUB_MERGE_BASE := $(shell $(GIT_LOG_FORMAT):%H -1)
+	$(eval GITHUB_MERGE_BASE := $(shell $(GIT_LOG_FORMAT)%H -1)
 	$(eval GITHUB_MERGE_BRANCH := $(shell $(GIT_IN_SRC) symbolic-ref --short HEAD))
 	$(eval GITHUB_MERGE_WORKTREE := $(shell mktemp -d "$(srcdir)/gh-$(1)-XXXXXX"))
 	$(GIT_IN_SRC) worktree prune

gc.c

@@ -3227,19 +3227,21 @@ rb_gc_mark_children(void *objspace, VALUE obj)
       }
 
       case T_OBJECT: {
+        uint32_t len;
         if (rb_shape_obj_too_complex_p(obj)) {
             gc_mark_tbl_no_pin(ROBJECT_FIELDS_HASH(obj));
+            len = ROBJECT_FIELDS_COUNT_COMPLEX(obj);
         }
         else {
             const VALUE * const ptr = ROBJECT_FIELDS(obj);
 
-            uint32_t len = ROBJECT_FIELDS_COUNT(obj);
+            len = ROBJECT_FIELDS_COUNT_NOT_COMPLEX(obj);
             for (uint32_t i = 0; i < len; i++) {
                 gc_mark_internal(ptr[i]);
             }
         }
 
-        attr_index_t fields_count = ROBJECT_FIELDS_COUNT(obj);
+        attr_index_t fields_count = (attr_index_t)len;
         if (fields_count) {
             VALUE klass = RBASIC_CLASS(obj);
 

ractor.c

@@ -834,13 +834,11 @@ rb_ractor_terminate_all(void)
 
     VM_ASSERT(cr == GET_RACTOR()); // only main-ractor's main-thread should kick it.
 
-    if (vm->ractor.cnt > 1) {
-        RB_VM_LOCK();
-        {
-            ractor_terminal_interrupt_all(vm); // kill all ractors
-        }
-        RB_VM_UNLOCK();
+    RB_VM_LOCK();
+    {
+        ractor_terminal_interrupt_all(vm); // kill all ractors
     }
+    RB_VM_UNLOCK();
     rb_thread_terminate_all(GET_THREAD()); // kill other threads in main-ractor and wait
 
     RB_VM_LOCK();
@@ -853,6 +851,17 @@ rb_ractor_terminate_all(void)
             rb_vm_ractor_blocking_cnt_inc(vm, cr, __FILE__, __LINE__);
             rb_del_running_thread(rb_ec_thread_ptr(cr->threads.running_ec));
             rb_vm_cond_timedwait(vm, &vm->ractor.sync.terminate_cond, 1000 /* ms */);
+#ifdef RUBY_THREAD_PTHREAD_H
+            while (vm->ractor.sched.barrier_waiting) {
+                // A barrier is waiting. Threads relinquish the VM lock before joining the barrier and
+                // since we just acquired the VM lock back, we're blocking other threads from joining it.
+                // We loop until the barrier is over. We can't join this barrier because our thread isn't added to
+                // running_threads until the call below to `rb_add_running_thread`.
+                RB_VM_UNLOCK();
+                unsigned int lev;
+                RB_VM_LOCK_ENTER_LEV_NB(&lev);
+            }
+#endif
             rb_add_running_thread(rb_ec_thread_ptr(cr->threads.running_ec));
             rb_vm_ractor_blocking_cnt_dec(vm, cr, __FILE__, __LINE__);
 
@@ -1796,7 +1805,7 @@ obj_traverse_replace_i(VALUE obj, struct obj_traverse_replace_data *data)
                 if (d.stop) return 1;
             }
             else {
-                uint32_t len = ROBJECT_FIELDS_COUNT(obj);
+                uint32_t len = ROBJECT_FIELDS_COUNT_NOT_COMPLEX(obj);
                 VALUE *ptr = ROBJECT_FIELDS(obj);
 
                 for (uint32_t i = 0; i < len; i++) {

shape.h

@@ -367,16 +367,28 @@ ROBJECT_SET_FIELDS_HASH(VALUE obj, const st_table *tbl)
     ROBJECT(obj)->as.heap.fields = (VALUE *)tbl;
 }
 
+static inline uint32_t
+ROBJECT_FIELDS_COUNT_COMPLEX(VALUE obj)
+{
+    return (uint32_t)rb_st_table_size(ROBJECT_FIELDS_HASH(obj));
+}
+
+static inline uint32_t
+ROBJECT_FIELDS_COUNT_NOT_COMPLEX(VALUE obj)
+{
+    RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
+    RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj));
+    return RSHAPE(RBASIC_SHAPE_ID(obj))->next_field_index;
+}
+
 static inline uint32_t
 ROBJECT_FIELDS_COUNT(VALUE obj)
 {
     if (rb_shape_obj_too_complex_p(obj)) {
-        return (uint32_t)rb_st_table_size(ROBJECT_FIELDS_HASH(obj));
+        return ROBJECT_FIELDS_COUNT_COMPLEX(obj);
     }
     else {
-        RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
-        RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj));
-        return RSHAPE(RBASIC_SHAPE_ID(obj))->next_field_index;
+        return ROBJECT_FIELDS_COUNT_NOT_COMPLEX(obj);
     }
 }
 

template/Makefile.in

@@ -748,4 +748,4 @@ yes-test-syntax-suggest: $(PREPARE_SYNTAX_SUGGEST)
 no-test-syntax-suggest:
 
 yesterday:
-	$(GIT_IN_SRC) reset --hard `TZ=UTC-9 $(GIT_LOG_FORMAT):%H -1 --before=00:00`
+	$(GIT_IN_SRC) reset --hard `TZ=UTC-9 $(GIT_LOG_FORMAT)%H -1 --before=00:00`

test/ruby/test_variable.rb

@@ -515,6 +515,36 @@ def test_genivar_cache
     assert_equal 4, instance.instance_variable_get(:@a4), bug21547
   end
 
+  def test_genivar_cache_free
+    str = +"hello"
+    str.instance_variable_set(:@x, :old_value)
+
+    str.instance_variable_get(:@x) # populate cache
+
+    Fiber.new {
+      str.remove_instance_variable(:@x)
+      str.instance_variable_set(:@x, :new_value)
+    }.resume
+
+    assert_equal :new_value, str.instance_variable_get(:@x)
+  end
+
+  def test_genivar_cache_invalidated_by_gc
+    str = +"hello"
+    str.instance_variable_set(:@x, :old_value)
+
+    str.instance_variable_get(:@x) # populate cache
+
+    Fiber.new {
+      str.remove_instance_variable(:@x)
+      str.instance_variable_set(:@x, :new_value)
+    }.resume
+
+    GC.start
+
+    assert_equal :new_value, str.instance_variable_get(:@x)
+  end
+
   private
   def with_kwargs_11(v1:, v2:, v3:, v4:, v5:, v6:, v7:, v8:, v9:, v10:, v11:)
     local_variables

variable.c

@@ -1236,6 +1236,18 @@ rb_mark_generic_ivar(VALUE obj)
     }
 }
 
+VALUE
+rb_obj_fields_generic_uncached(VALUE obj)
+{
+    VALUE fields_obj = 0;
+    RB_VM_LOCKING() {
+        if (!st_lookup(generic_fields_tbl_, (st_data_t)obj, (st_data_t *)&fields_obj)) {
+            rb_bug("Object is missing entry in generic_fields_tbl");
+        }
+    }
+    return fields_obj;
+}
+
 VALUE
 rb_obj_fields(VALUE obj, ID field_name)
 {
@@ -1261,15 +1273,12 @@ rb_obj_fields(VALUE obj, ID field_name)
           generic_fields:
             {
                 rb_execution_context_t *ec = GET_EC();
-                if (ec->gen_fields_cache.obj == obj && rb_imemo_fields_owner(ec->gen_fields_cache.fields_obj) == obj) {
+                if (ec->gen_fields_cache.obj == obj && !UNDEF_P(ec->gen_fields_cache.fields_obj) && rb_imemo_fields_owner(ec->gen_fields_cache.fields_obj) == obj) {
                     fields_obj = ec->gen_fields_cache.fields_obj;
+                    RUBY_ASSERT(fields_obj == rb_obj_fields_generic_uncached(obj));
                 }
                 else {
-                    RB_VM_LOCKING() {
-                        if (!st_lookup(generic_fields_tbl_, (st_data_t)obj, (st_data_t *)&fields_obj)) {
-                            rb_bug("Object is missing entry in generic_fields_tbl");
-                        }
-                    }
+                    fields_obj = rb_obj_fields_generic_uncached(obj);
                     ec->gen_fields_cache.fields_obj = fields_obj;
                     ec->gen_fields_cache.obj = obj;
                 }
@@ -1300,13 +1309,17 @@ rb_free_generic_ivar(VALUE obj)
           default:
           generic_fields:
             {
+                // Other EC may have stale caches, so fields_obj should be
+                // invalidated and the GC will replace with Qundef
                 rb_execution_context_t *ec = GET_EC();
                 if (ec->gen_fields_cache.obj == obj) {
                     ec->gen_fields_cache.obj = Qundef;
                     ec->gen_fields_cache.fields_obj = Qundef;
                 }
                 RB_VM_LOCKING() {
-                    st_delete(generic_fields_tbl_no_ractor_check(), &key, &value);
+                    if (!st_delete(generic_fields_tbl_no_ractor_check(), &key, &value)) {
+                        rb_bug("Object is missing entry in generic_fields_tbl");
+                    }
                 }
             }
         }
@@ -1320,7 +1333,9 @@ rb_obj_set_fields(VALUE obj, VALUE fields_obj, ID field_name, VALUE original_fie
     ivar_ractor_check(obj, field_name);
 
     if (!fields_obj) {
+        RUBY_ASSERT(original_fields_obj);
         rb_free_generic_ivar(obj);
+        rb_imemo_fields_clear(original_fields_obj);
         return;
     }
 

win32/Makefile.sub

@@ -1393,5 +1393,5 @@ exts: rubyspec-capiext
 yesterday:
 	(set TZ=UTC-9) && \
 	for /f "usebackq" %H in \
-	    (`$(GIT_LOG_FORMAT):%H -1 "--before=00:00"`) do \
+	    (`$(GIT_LOG_FORMAT)%H -1 "--before=00:00"`) do \
 	    $(GIT_IN_SRC) reset --hard %%H

yjit/src/codegen.rs

@@ -2258,7 +2258,8 @@ fn gen_expandarray(
 
     let comptime_recv = jit.peek_at_stack(&asm.ctx, 0);
 
-    // If the comptime receiver is not an array
+    // If the comptime receiver is not an array, speculate for when the `rb_check_array_type()`
+    // conversion returns nil and without side-effects (e.g. arbitrary method calls).
     if !unsafe { RB_TYPE_P(comptime_recv, RUBY_T_ARRAY) } {
         // at compile time, ensure to_ary is not defined
         let target_cme = unsafe { rb_callable_method_entry_or_negative(comptime_recv.class_of(), ID!(to_ary)) };
@@ -2270,6 +2271,13 @@ fn gen_expandarray(
             return None;
         }
 
+        // Bail when method_missing is defined to avoid generating code to call it.
+        // Also, for simplicity, bail when BasicObject#method_missing has been removed.
+        if !assume_method_basic_definition(jit, asm, comptime_recv.class_of(), ID!(method_missing)) {
+            gen_counter_incr(jit, asm, Counter::expandarray_method_missing);
+            return None;
+        }
+
         // invalidate compile block if to_ary is later defined
         jit.assume_method_lookup_stable(asm, target_cme);
 

yjit/src/cruby.rs

@@ -816,6 +816,7 @@ pub(crate) mod ids {
     def_ids! {
         name: NULL               content: b""
         name: respond_to_missing content: b"respond_to_missing?"
+        name: method_missing     content: b"method_missing"
         name: to_ary             content: b"to_ary"
         name: to_s               content: b"to_s"
         name: eq                 content: b"=="

yjit/src/stats.rs

@@ -496,6 +496,7 @@ make_counters! {
     expandarray_postarg,
     expandarray_not_array,
     expandarray_to_ary,
+    expandarray_method_missing,
     expandarray_chain_max_depth,
 
     // getblockparam

zjit.rb

@@ -183,6 +183,9 @@ def stats_string
     print_counters_with_prefix(prefix: 'unspecialized_send_without_block_def_type_', prompt: 'not optimized method types for send_without_block', buf:, stats:, limit: 20)
     print_counters_with_prefix(prefix: 'uncategorized_fallback_yarv_insn_', prompt: 'instructions with uncategorized fallback reason', buf:, stats:, limit: 20)
     print_counters_with_prefix(prefix: 'send_fallback_', prompt: 'send fallback reasons', buf:, stats:, limit: 20)
+    print_counters_with_prefix(prefix: 'setivar_fallback_', prompt: 'setivar fallback reasons', buf:, stats:, limit: 5)
+    print_counters_with_prefix(prefix: 'getivar_fallback_', prompt: 'getivar fallback reasons', buf:, stats:, limit: 5)
+    print_counters_with_prefix(prefix: 'definedivar_fallback_', prompt: 'definedivar fallback reasons', buf:, stats:, limit: 5)
     print_counters_with_prefix(prefix: 'invokeblock_handler_', prompt: 'invokeblock handler', buf:, stats:, limit: 10)
 
     # Show most popular unsupported call features. Because each call can
@@ -201,16 +204,16 @@ def stats_string
       :send_count,
       :dynamic_send_count,
       :optimized_send_count,
+      :dynamic_setivar_count,
+      :dynamic_getivar_count,
+      :dynamic_definedivar_count,
       :iseq_optimized_send_count,
       :inline_cfunc_optimized_send_count,
       :inline_iseq_optimized_send_count,
       :non_variadic_cfunc_optimized_send_count,
       :variadic_cfunc_optimized_send_count,
     ], buf:, stats:, right_align: true, base: :send_count)
     print_counters([
-      :dynamic_getivar_count,
-      :dynamic_setivar_count,
-
       :compiled_iseq_count,
       :failed_iseq_count,
 

zjit/src/codegen.rs

@@ -892,7 +892,6 @@ fn gen_ccall_variadic(
 
 /// Emit an uncached instance variable lookup
 fn gen_getivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry) -> Opnd {
-    gen_incr_counter(asm, Counter::dynamic_getivar_count);
     if ic.is_null() {
         asm_ccall!(asm, rb_ivar_get, recv, id.0.into())
     } else {
@@ -903,7 +902,6 @@ fn gen_getivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic:
 
 /// Emit an uncached instance variable store
 fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, val: Opnd, state: &FrameState) {
-    gen_incr_counter(asm, Counter::dynamic_setivar_count);
     // Setting an ivar can raise FrozenError, so we need proper frame state for exception handling.
     gen_prepare_non_leaf_call(jit, asm, state);
     if ic.is_null() {

zjit/src/cruby.rs

@@ -104,7 +104,6 @@ pub type RedefinitionFlag = u32;
 
 #[allow(unsafe_op_in_unsafe_fn)]
 #[allow(dead_code)]
-#[allow(unnecessary_transmutes)] // https://github.com/rust-lang/rust-bindgen/issues/2807
 #[allow(clippy::all)] // warning meant to help with reading; not useful for generated code
 mod autogened {
     use super::*;