[pull] master from ruby:master

NEWS.md

@@ -198,7 +198,7 @@ The following default gems are updated.
 * io-console 0.8.1
 * io-nonblock 0.3.2
 * io-wait 0.4.0.dev
-* json 2.16.0
+* json 2.17.0
 * net-http 0.8.0
 * openssl 4.0.0.pre
 * optparse 0.8.0

ext/-test-/tracepoint/tracepoint.c

@@ -92,10 +92,6 @@ static void
 on_newobj_event(VALUE tpval, void *data)
 {
     VALUE obj = rb_tracearg_object(rb_tracearg_from_tracepoint(tpval));
-    if (RB_TYPE_P(obj, T_STRING)) {
-        // Would fail !rb_obj_exivar_p(str) assertion in fstring_concurrent_set_create
-        return;
-    }
     if (!rb_objspace_internal_object_p(obj)) rb_obj_id(obj);
 }
 

ext/json/generator/generator.c

@@ -968,14 +968,16 @@ static void vstate_spill(struct generate_json_data *data)
     RB_OBJ_WRITTEN(vstate, Qundef, state->as_json);
 }
 
-static inline VALUE vstate_get(struct generate_json_data *data)
+static inline VALUE json_call_to_json(struct generate_json_data *data, VALUE obj)
 {
     if (RB_UNLIKELY(!data->vstate)) {
         vstate_spill(data);
     }
     GET_STATE(data->vstate);
     state->depth = data->depth;
-    return data->vstate;
+    VALUE tmp = rb_funcall(obj, i_to_json, 1, data->vstate);
+    // no need to restore state->depth, vstate is just a temporary State
+    return tmp;
 }
 
 static VALUE
@@ -1293,7 +1295,7 @@ static void generate_json_fallback(FBuffer *buffer, struct generate_json_data *d
 {
     VALUE tmp;
     if (rb_respond_to(obj, i_to_json)) {
-        tmp = rb_funcall(obj, i_to_json, 1, vstate_get(data));
+        tmp = json_call_to_json(data, obj);
         Check_Type(tmp, T_STRING);
         fbuffer_append_str(buffer, tmp);
     } else {
@@ -1475,16 +1477,6 @@ static VALUE generate_json_try(VALUE d)
     return fbuffer_finalize(data->buffer);
 }
 
-// Preserves the deprecated behavior of State#depth being set.
-static VALUE generate_json_ensure_deprecated(VALUE d)
-{
-    struct generate_json_data *data = (struct generate_json_data *)d;
-    fbuffer_free(data->buffer);
-    data->state->depth = data->depth;
-
-    return Qundef;
-}
-
 static VALUE generate_json_ensure(VALUE d)
 {
     struct generate_json_data *data = (struct generate_json_data *)d;
@@ -1505,13 +1497,13 @@ static VALUE cState_partial_generate(VALUE self, VALUE obj, generator_func func,
 
     struct generate_json_data data = {
         .buffer = &buffer,
-        .vstate = self,
+        .vstate = Qfalse, // don't use self as it may be frozen and its depth is mutated when calling to_json
         .state = state,
         .depth = state->depth,
         .obj = obj,
         .func = func
     };
-    return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure_deprecated, (VALUE)&data);
+    return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data);
 }
 
 /* call-seq:
@@ -1530,31 +1522,6 @@ static VALUE cState_generate(int argc, VALUE *argv, VALUE self)
     return cState_partial_generate(self, obj, generate_json, io);
 }
 
-static VALUE cState_generate_new(int argc, VALUE *argv, VALUE self)
-{
-    rb_check_arity(argc, 1, 2);
-    VALUE obj = argv[0];
-    VALUE io = argc > 1 ? argv[1] : Qnil;
-
-    GET_STATE(self);
-
-    char stack_buffer[FBUFFER_STACK_SIZE];
-    FBuffer buffer = {
-        .io = RTEST(io) ? io : Qfalse,
-    };
-    fbuffer_stack_init(&buffer, state->buffer_initial_length, stack_buffer, FBUFFER_STACK_SIZE);
-
-    struct generate_json_data data = {
-        .buffer = &buffer,
-        .vstate = Qfalse,
-        .state = state,
-        .depth = state->depth,
-        .obj = obj,
-        .func = generate_json
-    };
-    return rb_ensure(generate_json_try, (VALUE)&data, generate_json_ensure, (VALUE)&data);
-}
-
 static VALUE cState_initialize(int argc, VALUE *argv, VALUE self)
 {
     rb_warn("The json gem extension was loaded with the stdlib ruby code. You should upgrade rubygems with `gem update --system`");
@@ -2143,7 +2110,6 @@ void Init_generator(void)
     rb_define_method(cState, "buffer_initial_length", cState_buffer_initial_length, 0);
     rb_define_method(cState, "buffer_initial_length=", cState_buffer_initial_length_set, 1);
     rb_define_method(cState, "generate", cState_generate, -1);
-    rb_define_method(cState, "generate_new", cState_generate_new, -1); // :nodoc:
 
     rb_define_private_method(cState, "allow_duplicate_key?", cState_allow_duplicate_key_p, 0);
 

ext/json/lib/json/common.rb

@@ -550,6 +550,7 @@ def pretty_generate(obj, opts = nil)
     :create_additions => nil,
   }
   # :call-seq:
+  #   JSON.unsafe_load(source, options = {}) -> object
   #   JSON.unsafe_load(source, proc = nil, options = {}) -> object
   #
   # Returns the Ruby objects created by parsing the given +source+.
@@ -681,7 +682,12 @@ def pretty_generate(obj, opts = nil)
   #
   def unsafe_load(source, proc = nil, options = nil)
     opts = if options.nil?
-      _unsafe_load_default_options
+      if proc && proc.is_a?(Hash)
+        options, proc = proc, nil
+        options
+      else
+        _unsafe_load_default_options
+      end
     else
       _unsafe_load_default_options.merge(options)
     end
@@ -709,6 +715,7 @@ def unsafe_load(source, proc = nil, options = nil)
   end
 
   # :call-seq:
+  #   JSON.load(source, options = {}) -> object
   #   JSON.load(source, proc = nil, options = {}) -> object
   #
   # Returns the Ruby objects created by parsing the given +source+.
@@ -845,8 +852,18 @@ def unsafe_load(source, proc = nil, options = nil)
   #      @attributes={"type"=>"Admin", "password"=>"0wn3d"}>}
   #
   def load(source, proc = nil, options = nil)
+    if proc && options.nil? && proc.is_a?(Hash)
+      options = proc
+      proc = nil
+    end
+
     opts = if options.nil?
-      _load_default_options
+      if proc && proc.is_a?(Hash)
+        options, proc = proc, nil
+        options
+      else
+        _load_default_options
+      end
     else
       _load_default_options.merge(options)
     end
@@ -1057,7 +1074,7 @@ def initialize(options = nil, &as_json)
     #
     # Serialize the given object into a \JSON document.
     def dump(object, io = nil)
-      @state.generate_new(object, io)
+      @state.generate(object, io)
     end
     alias_method :generate, :dump
 

ext/json/lib/json/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module JSON
-  VERSION = '2.16.0'
+  VERSION = '2.17.0'
 end

gc.c

@@ -2061,7 +2061,7 @@ rb_gc_obj_free_vm_weak_references(VALUE obj)
 {
     obj_free_object_id(obj);
 
-    if (rb_obj_exivar_p(obj)) {
+    if (rb_obj_gen_fields_p(obj)) {
         rb_free_generic_ivar(obj);
     }
 
@@ -3116,7 +3116,7 @@ rb_gc_mark_children(void *objspace, VALUE obj)
 {
     struct gc_mark_classext_foreach_arg foreach_args;
 
-    if (rb_obj_exivar_p(obj)) {
+    if (rb_obj_gen_fields_p(obj)) {
         rb_mark_generic_ivar(obj);
     }
 

hash.c

@@ -1554,7 +1554,7 @@ rb_hash_dup(VALUE hash)
     const VALUE flags = RBASIC(hash)->flags;
     VALUE ret = hash_dup(hash, rb_obj_class(hash), flags & RHASH_PROC_DEFAULT);
 
-    if (rb_obj_exivar_p(hash)) {
+    if (rb_obj_gen_fields_p(hash)) {
         rb_copy_generic_ivar(ret, hash);
     }
     return ret;
@@ -2876,7 +2876,7 @@ hash_aset(st_data_t *key, st_data_t *val, struct update_arg *arg, int existing)
 VALUE
 rb_hash_key_str(VALUE key)
 {
-    if (!rb_obj_exivar_p(key) && RBASIC_CLASS(key) == rb_cString) {
+    if (!rb_obj_gen_fields_p(key) && RBASIC_CLASS(key) == rb_cString) {
         return rb_fstring(key);
     }
     else {

include/ruby/internal/core/rtypeddata.h

@@ -507,19 +507,6 @@ RBIMPL_SYMBOL_EXPORT_END()
         sizeof(type))
 #endif
 
-/**
- * Obtains a C struct from inside of a wrapper Ruby object.
- *
- * @param      obj            An instance of ::RTypedData.
- * @param      type           Type name of the C struct.
- * @param      data_type      The data type describing `type`.
- * @param      sval           Variable name of obtained C struct.
- * @exception  rb_eTypeError  `obj` is not a kind of `data_type`.
- * @return     Unwrapped C struct that `obj` holds.
- */
-#define TypedData_Get_Struct(obj,type,data_type,sval) \
-    ((sval) = RBIMPL_CAST((type *)rb_check_typeddata((obj), (data_type))))
-
 static inline bool
 RTYPEDDATA_EMBEDDED_P(VALUE obj)
 {
@@ -614,6 +601,38 @@ RTYPEDDATA_TYPE(VALUE obj)
     return (const struct rb_data_type_struct *)(RTYPEDDATA(obj)->type & TYPED_DATA_PTR_MASK);
 }
 
+RBIMPL_ATTR_PURE_UNLESS_DEBUG()
+RBIMPL_ATTR_ARTIFICIAL()
+/**
+ * @private
+ *
+ * This  is an  implementation detail  of  TypedData_Get_Struct().  Don't use  it
+ * directly.
+ */
+static inline void *
+rbimpl_check_typeddata(VALUE obj, const rb_data_type_t *type)
+{
+    if (RB_LIKELY(RB_TYPE_P(obj, T_DATA) && RTYPEDDATA_P(obj) && RTYPEDDATA_TYPE(obj) == type)) {
+        return RTYPEDDATA_GET_DATA(obj);
+    }
+
+    return rb_check_typeddata(obj, type);
+}
+
+
+/**
+ * Obtains a C struct from inside of a wrapper Ruby object.
+ *
+ * @param      obj            An instance of ::RTypedData.
+ * @param      type           Type name of the C struct.
+ * @param      data_type      The data type describing `type`.
+ * @param      sval           Variable name of obtained C struct.
+ * @exception  rb_eTypeError  `obj` is not a kind of `data_type`.
+ * @return     Unwrapped C struct that `obj` holds.
+ */
+#define TypedData_Get_Struct(obj,type,data_type,sval) \
+    ((sval) = RBIMPL_CAST((type *)rbimpl_check_typeddata((obj), (data_type))))
+
 /**
  * While  we don't  stop  you from  using  this  function, it  seems  to be  an
  * implementation  detail of  #TypedData_Make_Struct, which  is preferred  over

prism/prism.c

@@ -19299,18 +19299,52 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
             parser_lex(parser);
             pm_token_t opening = parser->previous;
             pm_array_node_t *array = pm_array_node_create(parser, &opening);
+            pm_node_t *current = NULL;
 
             while (!match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
                 accept1(parser, PM_TOKEN_WORDS_SEP);
                 if (match1(parser, PM_TOKEN_STRING_END)) break;
 
-                if (match1(parser, PM_TOKEN_STRING_CONTENT)) {
+                // Interpolation is not possible but nested heredocs can still lead to
+                // consecutive (disjoint) string tokens when the final newline is escaped.
+                while (match1(parser, PM_TOKEN_STRING_CONTENT)) {
                     pm_token_t opening = not_provided(parser);
                     pm_token_t closing = not_provided(parser);
-                    pm_array_node_elements_append(array, UP(pm_symbol_node_create_current_string(parser, &opening, &parser->current, &closing)));
+
+                    // Record the string node, moving to interpolation if needed.
+                    if (current == NULL) {
+                        current = UP(pm_symbol_node_create_current_string(parser, &opening, &parser->current, &closing));
+                        parser_lex(parser);
+                    } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_SYMBOL_NODE)) {
+                        pm_node_t *string = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing));
+                        parser_lex(parser);
+                        pm_interpolated_symbol_node_append((pm_interpolated_symbol_node_t *) current, string);
+                    } else if (PM_NODE_TYPE_P(current, PM_SYMBOL_NODE)) {
+                        pm_symbol_node_t *cast = (pm_symbol_node_t *) current;
+                        pm_token_t bounds = not_provided(parser);
+
+                        pm_token_t content = { .type = PM_TOKEN_STRING_CONTENT, .start = cast->value_loc.start, .end = cast->value_loc.end };
+                        pm_node_t *first_string = UP(pm_string_node_create_unescaped(parser, &bounds, &content, &bounds, &cast->unescaped));
+                        pm_node_t *second_string = UP(pm_string_node_create_current_string(parser, &opening, &parser->previous, &closing));
+                        parser_lex(parser);
+
+                        pm_interpolated_symbol_node_t *interpolated = pm_interpolated_symbol_node_create(parser, &opening, NULL, &closing);
+                        pm_interpolated_symbol_node_append(interpolated, first_string);
+                        pm_interpolated_symbol_node_append(interpolated, second_string);
+
+                        xfree(current);
+                        current = UP(interpolated);
+                    } else {
+                        assert(false && "unreachable");
+                    }
                 }
 
-                expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_LIST_I_LOWER_ELEMENT);
+                if (current) {
+                    pm_array_node_elements_append(array, current);
+                    current = NULL;
+                } else {
+                    expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_LIST_I_LOWER_ELEMENT);
+                }
             }
 
             pm_token_t closing = parser->current;
@@ -19489,23 +19523,42 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
             parser_lex(parser);
             pm_token_t opening = parser->previous;
             pm_array_node_t *array = pm_array_node_create(parser, &opening);
-
-            // skip all leading whitespaces
-            accept1(parser, PM_TOKEN_WORDS_SEP);
+            pm_node_t *current = NULL;
 
             while (!match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) {
                 accept1(parser, PM_TOKEN_WORDS_SEP);
                 if (match1(parser, PM_TOKEN_STRING_END)) break;
 
-                if (match1(parser, PM_TOKEN_STRING_CONTENT)) {
+                // Interpolation is not possible but nested heredocs can still lead to
+                // consecutive (disjoint) string tokens when the final newline is escaped.
+                while (match1(parser, PM_TOKEN_STRING_CONTENT)) {
                     pm_token_t opening = not_provided(parser);
                     pm_token_t closing = not_provided(parser);
 
                     pm_node_t *string = UP(pm_string_node_create_current_string(parser, &opening, &parser->current, &closing));
-                    pm_array_node_elements_append(array, string);
+
+                    // Record the string node, moving to interpolation if needed.
+                    if (current == NULL) {
+                        current = string;
+                    } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_STRING_NODE)) {
+                        pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, string);
+                    } else if (PM_NODE_TYPE_P(current, PM_STRING_NODE)) {
+                        pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing);
+                        pm_interpolated_string_node_append(interpolated, current);
+                        pm_interpolated_string_node_append(interpolated, string);
+                        current = UP(interpolated);
+                    } else {
+                        assert(false && "unreachable");
+                    }
+                    parser_lex(parser);
                 }
 
-                expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_LIST_W_LOWER_ELEMENT);
+                if (current) {
+                    pm_array_node_elements_append(array, current);
+                    current = NULL;
+                } else {
+                    expect1(parser, PM_TOKEN_STRING_CONTENT, PM_ERR_LIST_W_LOWER_ELEMENT);
+                }
             }
 
             pm_token_t closing = parser->current;

prism/prism.h

@@ -314,7 +314,7 @@ PRISM_EXPORTED_FUNCTION pm_string_query_t pm_string_query_method_name(const uint
  * dependencies. It is currently being integrated into
  * [CRuby](https://github.com/ruby/ruby),
  * [JRuby](https://github.com/jruby/jruby),
- * [TruffleRuby](https://github.com/oracle/truffleruby),
+ * [TruffleRuby](https://github.com/truffleruby/truffleruby),
  * [Sorbet](https://github.com/sorbet/sorbet), and
  * [Syntax Tree](https://github.com/ruby-syntax-tree/syntax_tree).
  *