Add Initial IC Mechanism

src/js/runtime/bytecode/function.rs

@@ -11,6 +11,7 @@ use crate::{
         function::{set_function_length, set_function_name},
         gc::{HeapItem, HeapVisitor},
         heap_item_descriptor::{HeapItemDescriptor, HeapItemKind},
+        ic::vector::ICVector,
         intrinsics::{intrinsics::Intrinsic, rust_runtime::RuntimeFunctionId},
         object_value::ObjectValue,
         ordinary_object::{
@@ -252,6 +253,8 @@ pub struct BytecodeFunction {
     /// This function may be a stub function back into the Rust runtime. If this is set then this
     /// function has an empty bytecode array and default values for many other fields.
     runtime_function_id: Option<RuntimeFunctionId>,
+    /// Feedback vector for IC stubs
+    feedback_vector: Option<HeapPtr<ICVector>>,
     /// Inlined bytecode array for the function.
     bytecode: InlineArray<u8>,
 }
@@ -276,6 +279,7 @@ impl BytecodeFunction {
         name: Option<Handle<StringValue>>,
         source_file: Handle<SourceFile>,
         source_map: Handle<ByteArray>,
+        feedback_vector: Option<Handle<ICVector>>,
     ) -> AllocResult<Handle<BytecodeFunction>> {
         let size = Self::calculate_size_in_bytes(bytecode.len());
         let mut object = cx.alloc_uninit_with_size::<BytecodeFunction>(size)?;
@@ -298,6 +302,7 @@ impl BytecodeFunction {
         set_uninit!(object.source_file, Some(*source_file));
         set_uninit!(object.source_map, Some(*source_map));
         set_uninit!(object.runtime_function_id, None);
+        set_uninit!(object.feedback_vector, feedback_vector.map(|v| *v));
         object.bytecode.init_from_slice(&bytecode);
 
         Ok(object.to_handle())
@@ -339,6 +344,7 @@ impl BytecodeFunction {
         set_uninit!(object.name, name.map(|n| *n));
         set_uninit!(object.source_file, None);
         set_uninit!(object.source_map, None);
+        set_uninit!(object.feedback_vector, None);
         set_uninit!(object.runtime_function_id, Some(runtime_func_id));
         object.bytecode.init_from_slice(&[]);
 
@@ -361,6 +367,11 @@ impl BytecodeFunction {
         self.constant_table
     }
 
+    #[inline]
+    pub fn feedback_vector_ptr(&self) -> Option<HeapPtr<ICVector>> {
+        self.feedback_vector
+    }
+
     #[inline]
     pub fn exception_handlers_ptr(&self) -> Option<HeapPtr<ExceptionHandlers>> {
         self.exception_handlers
@@ -517,6 +528,7 @@ impl HeapItem for HeapPtr<BytecodeFunction> {
         visitor.visit_pointer(&mut self.descriptor);
 
         visitor.visit_pointer_opt(&mut self.constant_table);
+        visitor.visit_pointer_opt(&mut self.feedback_vector);
         visitor.visit_pointer_opt(&mut self.exception_handlers);
         visitor.visit_pointer(&mut self.realm);
         visitor.visit_pointer_opt(&mut self.name);

src/js/runtime/bytecode/generator.rs

@@ -34,6 +34,7 @@ use crate::{
         bytecode::{
             function::{dump_bytecode_function, BytecodeFunction},
             instruction::DefinePropertyFlags,
+            operand::ICSlotIndex,
             source_map::BytecodeSourceMap,
         },
         class_names::{ClassNames, HomeObjectLocation, Method},
@@ -42,6 +43,7 @@ use crate::{
         gc::Escapable,
         global_names::GlobalNames,
         heap_item_descriptor::HeapItemKind,
+        ic::vector::ICVector,
         interned_strings::InternedStrings,
         module::{
             import_attributes::ImportAttributes,
@@ -1102,6 +1104,14 @@ pub struct BytecodeFunctionGenerator<'a> {
 
     /// Queue of functions that still need to be generated.
     pending_functions_queue: PendingFunctionNodes<'a>,
+
+    /// The number of IC Stubs to allocate in the
+    /// ICVector. Since ICStubs are generic, every
+    /// single one will initially be instantiated as
+    /// `None`. Then, when we hit the opcode in the VM
+    /// we will generate ICs, changing the slot in the ICVector
+    /// from `None` to the concrete stub
+    num_ic_slots: u32,
 }
 
 impl<'a> BytecodeFunctionGenerator<'a> {
@@ -1160,6 +1170,7 @@ impl<'a> BytecodeFunctionGenerator<'a> {
             register_allocator: TemporaryRegisterAllocator::new(num_local_registers),
             exception_handler_builder: ExceptionHandlersBuilder::new(),
             pending_functions_queue: vec![],
+            num_ic_slots: 0,
         }
     }
 
@@ -1243,6 +1254,12 @@ impl<'a> BytecodeFunctionGenerator<'a> {
         ))
     }
 
+    fn add_ic_stub(&mut self) -> u32 {
+        let offset = self.num_ic_slots;
+        self.num_ic_slots += 1;
+        offset
+    }
+
     fn new_for_program(
         cx: Context,
         program: &ast::Program,
@@ -1998,6 +2015,12 @@ impl<'a> BytecodeFunctionGenerator<'a> {
 
         let source_positions_object = BytecodeSourceMap::new(self.cx, &source_positions)?;
 
+        let ic_vector = if self.num_ic_slots == 0 {
+            None
+        } else {
+            Some(ICVector::new(self.cx, self.num_ic_slots as usize)?)
+        };
+
         let bytecode_function = BytecodeFunction::new(
             self.cx,
             bytecode,
@@ -2017,6 +2040,7 @@ impl<'a> BytecodeFunctionGenerator<'a> {
             name,
             self.source_file,
             source_positions_object,
+            ic_vector,
         )?;
 
         Ok(EmitFunctionResult {
@@ -3337,7 +3361,12 @@ impl<'a> BytecodeFunctionGenerator<'a> {
         let dest = self.allocate_destination(dest)?;
 
         match expr.operator {
-            ast::BinaryOperator::Add => self.writer.add_instruction(dest, left, right, pos),
+            ast::BinaryOperator::Add => {
+                // Add feedback slot with add type uninitialized
+                let slot_offset = self.add_ic_stub();
+                self.writer
+                    .add_instruction(dest, left, right, ICSlotIndex::new(slot_offset), pos)
+            }
             ast::BinaryOperator::Subtract => self.writer.sub_instruction(dest, left, right, pos),
             ast::BinaryOperator::Multiply => self.writer.mul_instruction(dest, left, right, pos),
             ast::BinaryOperator::Divide => self.writer.div_instruction(dest, left, right, pos),
@@ -4740,7 +4769,16 @@ impl<'a> BytecodeFunctionGenerator<'a> {
         let pos = expr.operator_pos;
 
         match operator {
-            ast::AssignmentOperator::Add => self.writer.add_instruction(dest, left, right, pos),
+            ast::AssignmentOperator::Add => {
+                let ic_stub_offset = self.add_ic_stub();
+                self.writer.add_instruction(
+                    dest,
+                    left,
+                    right,
+                    ICSlotIndex::new(ic_stub_offset),
+                    pos,
+                )
+            }
             ast::AssignmentOperator::Subtract => {
                 self.writer.sub_instruction(dest, left, right, pos)
             }
@@ -5918,7 +5956,14 @@ impl<'a> BytecodeFunctionGenerator<'a> {
                         .to_string_instruction(temp, expression_reg, expression_pos);
 
                     // Cannot throw since both values are guaranteed to be strings
-                    self.writer.add_instruction(acc, acc, temp, NO_POS);
+                    let ic_stub_offset = self.add_ic_stub();
+                    self.writer.add_instruction(
+                        acc,
+                        acc,
+                        temp,
+                        ICSlotIndex::new(ic_stub_offset),
+                        NO_POS,
+                    );
 
                     self.register_allocator.release(temp);
                 }
@@ -5942,7 +5987,14 @@ impl<'a> BytecodeFunctionGenerator<'a> {
                 self.writer.load_constant_instruction(temp, constant_index);
 
                 // Cannot throw since both values are guaranteed to be strings
-                self.writer.add_instruction(acc, acc, temp, NO_POS);
+                let ic_stub_offset = self.add_ic_stub();
+                self.writer.add_instruction(
+                    acc,
+                    acc,
+                    temp,
+                    ICSlotIndex::new(ic_stub_offset),
+                    NO_POS,
+                );
             }
         }
 

src/js/runtime/bytecode/instruction.rs

@@ -11,7 +11,10 @@ use super::{
     writer::BytecodeWriter,
 };
 
-use crate::{count, replace_expr, runtime::debug_print::DebugPrinter};
+use crate::{
+    count, replace_expr,
+    runtime::{bytecode::operand::ICSlotIndex, debug_print::DebugPrinter},
+};
 
 /// Generic properties of instructions.
 #[allow(dead_code)]
@@ -566,6 +569,7 @@ define_instructions!(
             [0] dest: Register,
             [1] left: Register,
             [2] right: Register,
+            [3] ic_stub_offset: ICSlotIndex,
         }
     }
 

src/js/runtime/bytecode/operand.rs

@@ -123,11 +123,15 @@ operand_type!(SInt, SIGNED);
 // An index into the constant table
 operand_type!(ConstantIndex, UNSIGNED);
 
+// An index into a feedback vector
+operand_type!(ICSlotIndex, UNSIGNED);
+
 pub enum OperandType {
     Register,
     UInt,
     SInt,
     ConstantIndex,
+    ICSlotIndex,
 }
 
 /// Registers may be either registers local to a function or arguments to that function. Registers
@@ -311,12 +315,30 @@ impl<W: Width> ConstantIndex<W> {
     }
 }
 
+impl<W: Width> ICSlotIndex<W> {
+    #[inline]
+    pub fn new(value: W::UInt) -> Self {
+        Self::from_unsigned(value)
+    }
+
+    #[inline]
+    pub fn value(&self) -> W::UInt {
+        self.unsigned()
+    }
+}
+
 impl<W: Width> fmt::Display for ConstantIndex<W> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "c{}", self.value())
     }
 }
 
+impl<W: Width> fmt::Display for ICSlotIndex<W> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "fb{}", self.value())
+    }
+}
+
 /// Return the minimum width needed to fit the given signed value.
 pub fn min_width_for_signed(value: isize) -> WidthEnum {
     if (Narrow::SIGNED_MIN..=Narrow::SIGNED_MAX).contains(&value) {

src/js/runtime/bytecode/stack_frame.rs

@@ -1,4 +1,9 @@
-use crate::runtime::{gc::HeapVisitor, scope::Scope, HeapPtr, Value};
+use crate::runtime::{
+    gc::HeapVisitor,
+    ic::{stubs::ICStub, vector::ICVector},
+    scope::Scope,
+    Handle, HeapPtr, Value,
+};
 
 use super::{constant_table::ConstantTable, function::Closure};
 
@@ -12,14 +17,16 @@ use super::{constant_table::ConstantTable, function::Closure};
 ///     |       ...        |
 ///     +------------------+
 ///     |       arg0       |  (first arg)
-/// +64 +------------------+                     ^                ^
+/// +72 +------------------+                     ^                ^
 ///     |     receiver     |  (receiver)         | caller's frame |
-/// +56 +------------------+                     +----------------+
+/// +64 +------------------+                     +----------------+
 ///     |       argc       |                     | callee's frame |
-/// +48 +------------------+                     v                v
+/// +56 +------------------+                     v                v
 ///     |      closure     |  (closure of the caller)
-/// +40 +------------------+
+/// +48 +------------------+
 ///     |  constant_table  |  (constant table of the called closure)
+/// +40 +------------------+
+///     |    ic_vector     |  (ic vector of the called closure)
 /// +32 +------------------+
 ///     |       scope      |  (current VM scope)
 /// +24 +------------------+
@@ -187,6 +194,33 @@ impl StackFrame {
         unsafe { &mut *(self.fp.add(CONSTANT_TABLE_SLOT_INDEX) as *mut HeapPtr<ConstantTable>) }
     }
 
+    #[inline]
+    pub fn ic_vector(&self) -> HeapPtr<ICVector> {
+        let ptr = unsafe { *self.fp.add(IC_VECTOR_SLOT_INDEX) };
+        HeapPtr::from_ptr(ptr as *mut ICVector)
+    }
+
+    #[inline]
+    pub fn ic_vector_mut(&self) -> &mut HeapPtr<ICVector> {
+        unsafe { &mut *(self.fp.add(IC_VECTOR_SLOT_INDEX) as *mut HeapPtr<ICVector>) }
+    }
+
+    #[inline]
+    pub fn get_ic_stub(&self, slot: usize) -> Option<HeapPtr<ICStub>> {
+        *self.ic_vector().slots().get_unchecked(slot)
+    }
+
+    #[inline]
+    pub fn add_ic_stub(&mut self, slot: usize, mut stub: Handle<ICStub>) {
+        let slots = self.ic_vector_mut().slots_mut();
+        let head = slots.get_unchecked_mut(slot);
+        // Prepend the stub:
+        // Put the current stub at the beginning of the linked list
+        // with the original head as the second element
+        stub.set_next(*head);
+        *head = Some(*stub);
+    }
+
     /// The callee function in this stack frame.
     #[inline]
     pub fn closure(&self) -> HeapPtr<Closure> {
@@ -273,6 +307,7 @@ impl StackFrame {
 
         visitor.visit_pointer(self.closure_mut());
         visitor.visit_pointer(self.constant_table_mut());
+        visitor.visit_pointer(self.ic_vector_mut());
         visitor.visit_pointer(self.scope_mut());
     }
 }
@@ -313,10 +348,12 @@ pub const SCOPE_SLOT_INDEX: usize = 3;
 
 const CONSTANT_TABLE_SLOT_INDEX: usize = 4;
 
-pub const CLOSURE_SLOT_INDEX: usize = 5;
+const IC_VECTOR_SLOT_INDEX: usize = 5;
+
+pub const CLOSURE_SLOT_INDEX: usize = 6;
 
-const ARGC_SLOT_INDEX: usize = 6;
+const ARGC_SLOT_INDEX: usize = 7;
 
-pub const RECEIVER_SLOT_INDEX: usize = 7;
+pub const RECEIVER_SLOT_INDEX: usize = 8;
 
-pub const FIRST_ARGUMENT_SLOT_INDEX: usize = 8;
+pub const FIRST_ARGUMENT_SLOT_INDEX: usize = 9;