Add benchmark project and fix a few performance issues

CSharpRepl.Benchmarks/AllocationBreakdownBenchmark.cs

@@ -0,0 +1,89 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using CSharpRepl.Services;
+using CSharpRepl.Services.Roslyn;
+using CSharpRepl.Services.Roslyn.Scripting;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.Classification;
+using Microsoft.CodeAnalysis.Text;
+
+namespace CSharpRepl.Benchmarks;
+
+/// <summary>
+/// Attributes the per-keystroke highlight cost (and its allocation) to a stage of the Roslyn pipeline,
+/// to find where the ~36 MB/keystroke at depth 50 comes from. The session is a *dependent* chain — each
+/// prior submission references the previous one (var sN = s(N-1) + 1) — and the "typed" line references
+/// the most-recent variable, so binding must resolve through the whole submission chain.
+///
+/// Compare the Allocated column across stages at PriorSubmissions=50:
+///   Fork           - just CurrentDocument.WithText (no analysis)
+///   GetSyntaxRoot  - parse the current submission (expected depth-independent)
+///   GetCompilation - build the project's Compilation (walks the submission-project chain)
+///   GetSemanticModel - compilation + bind the current document
+///   Classify       - what production highlighting actually calls each keystroke
+///
+/// Run with: dotnet run -c Release --project CSharpRepl.Benchmarks -- --filter *AllocationBreakdown*
+/// </summary>
+[MemoryDiagnoser]
+[SimpleJob(launchCount: 1, warmupCount: 3, iterationCount: 5)]
+public class AllocationBreakdownBenchmark
+{
+    [Params(0, 50)]
+    public int PriorSubmissions;
+
+    private RoslynServices roslyn = null!;
+    private Document baseDocument = null!;
+    private string typedLine = "";
+    private int counter;
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        var config = new Configuration(useTerminalPaletteTheme: true);
+        roslyn = new RoslynServices(new BenchmarkConsole(), config, new NullTraceLogger());
+        roslyn.WarmUpAsync(Array.Empty<string>()).GetAwaiter().GetResult();
+
+        for (int i = 0; i < PriorSubmissions; i++)
+        {
+            var code = i == 0 ? "var s0 = 0;" : $"var s{i} = s{i - 1} + 1;";
+            var result = roslyn.EvaluateAsync(code).GetAwaiter().GetResult();
+            if (result is not EvaluationResult.Success)
+                throw new InvalidOperationException($"Setup submission {i} failed: {result}");
+        }
+
+        baseDocument = roslyn.CurrentDocumentForProfiling!;
+        // Reference the most-recent variable, forcing binding through the chain (depth-0 case uses a framework call).
+        typedLine = PriorSubmissions == 0 ? "System.Console.WriteLine(1)" : $"s{PriorSubmissions - 1}.ToString()";
+    }
+
+    // A fresh fork each invocation (unique trailing comment) so the per-keystroke "buffer changed" path is
+    // measured, not a cached re-render.
+    private Document Current() => baseDocument.WithText(SourceText.From(typedLine + "\n//" + counter++));
+
+    [Benchmark(Description = "1. WithText (fork only)")]
+    public Document Fork() => Current();
+
+    [Benchmark(Description = "2. GetSyntaxRoot (parse)")]
+    public async Task<SyntaxNode?> GetSyntaxRoot() => await Current().GetSyntaxRootAsync();
+
+    [Benchmark(Description = "3. GetCompilation (build chain)")]
+    public async Task<Compilation?> GetCompilation() => await Current().Project.GetCompilationAsync();
+
+    [Benchmark(Description = "4. GetSemanticModel (bind)")]
+    public async Task<SemanticModel?> GetSemanticModel() => await Current().GetSemanticModelAsync();
+
+    [Benchmark(Baseline = true, Description = "5. GetClassifiedSpans (full = production)")]
+    public async Task<int> Classify()
+    {
+        var doc = Current();
+        var length = (await doc.GetTextAsync()).Length;
+        var spans = await Classifier.GetClassifiedSpansAsync(doc, TextSpan.FromBounds(0, length));
+        return spans.Count();
+    }
+}

CSharpRepl.Benchmarks/BenchmarkSupport.cs

@@ -0,0 +1,66 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using CSharpRepl.Services;
+using CSharpRepl.Services.Logging;
+using PrettyPrompt.Consoles;
+using Spectre.Console;
+
+namespace CSharpRepl.Benchmarks;
+
+/// <summary>
+/// Minimal <see cref="IConsoleService"/> for benchmarks. The per-keystroke Roslyn paths
+/// (highlight / complete / format) only touch the console on initialization or warm-up errors,
+/// plus the rendering <see cref="Profile"/>, so every sink here is a no-op.
+/// </summary>
+internal sealed class BenchmarkConsole : IConsoleService
+{
+    private readonly IConsole prettyPromptConsole = new NullPromptConsole();
+    private readonly IAnsiConsole ansi = AnsiConsole.Create(new AnsiConsoleSettings
+    {
+        Ansi = AnsiSupport.No,
+        ColorSystem = ColorSystemSupport.NoColors,
+        Out = new AnsiConsoleOutput(TextWriter.Null),
+    });
+
+    IConsole IConsoleService.PrettyPromptConsole => prettyPromptConsole;
+    IAnsiConsole IConsoleService.Ansi => ansi;
+    public string? ReadLine() => null;
+
+    private sealed class NullPromptConsole : IConsole
+    {
+        public int CursorTop => 0;
+        public int BufferWidth => 240;
+        public int WindowHeight => 80;
+        public int WindowTop => 0;
+        public bool KeyAvailable => false;
+        public bool CaptureControlC { get => false; set { } }
+        public bool IsErrorRedirected => false;
+        public event ConsoleCancelEventHandler CancelKeyPress { add { } remove { } }
+        public void Clear() { }
+        public void HideCursor() { }
+        public void InitVirtualTerminalProcessing() { }
+        public ConsoleKeyInfo ReadKey(bool intercept) => default;
+        public void ShowCursor() { }
+        public void Write(string? value) { }
+        public void WriteError(string? value) { }
+        public void WriteErrorLine(string? value) { }
+        public void WriteLine(string? value) { }
+        public void Write(ReadOnlySpan<char> value) { }
+        public void WriteError(ReadOnlySpan<char> value) { }
+        public void WriteErrorLine(ReadOnlySpan<char> value) { }
+        public void WriteLine(ReadOnlySpan<char> value) { }
+    }
+}
+
+/// <summary>No-op trace logger; the real one only matters with the --trace flag.</summary>
+internal sealed class NullTraceLogger : ITraceLogger
+{
+    public void Log(string message) { }
+    public void Log(Func<string> message) { }
+    public void LogPaths(string message, Func<IEnumerable<string?>> paths) { }
+}

CSharpRepl.Benchmarks/CSharpRepl.Benchmarks.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <LangVersion>latest</LangVersion>
+    <Nullable>enable</Nullable>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.15.6" />
+    <!-- Keep MSBuild/StringTools assemblies out of the output dir (loaded from the SDK via MSBuildLocator);
+         arrives transitively through the CSharpRepl.Services project reference.
+         See https://aka.ms/msbuild/locator/diagnostics/MSBL001 -->
+    <PackageReference Include="Microsoft.Build.Framework" Version="18.6.3" ExcludeAssets="runtime" PrivateAssets="all" />
+    <PackageReference Include="Microsoft.NET.StringTools" Version="18.6.3" ExcludeAssets="runtime" PrivateAssets="all" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\CSharpRepl.Services\CSharpRepl.Services.csproj" />
+  </ItemGroup>
+
+</Project>

CSharpRepl.Benchmarks/ChainedEvaluationBenchmark.cs

@@ -0,0 +1,54 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+using System;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Engines;
+using CSharpRepl.Services;
+using CSharpRepl.Services.Roslyn;
+using CSharpRepl.Services.Roslyn.Scripting;
+
+namespace CSharpRepl.Benchmarks;
+
+/// <summary>
+/// Profiles the *evaluation* path of a dependent submission chain: each submission references the variable
+/// declared by the previous one (var sN = s(N-1) + 1), so every evaluation extends a chain the next must
+/// bind through. Measures the wall-clock and allocation to build a <see cref="ChainLength"/>-deep session.
+///
+/// RunStrategy.Monitoring + invocationCount=1 means each measured run is a single full chain build, preceded
+/// by a fresh RoslynServices in [IterationSetup] (evaluation mutates script state, so it can't be reused).
+///
+/// Run with: dotnet run -c Release --project CSharpRepl.Benchmarks -- --filter *ChainedEvaluation*
+/// </summary>
+[MemoryDiagnoser]
+[SimpleJob(RunStrategy.Monitoring, launchCount: 1, warmupCount: 1, iterationCount: 5, invocationCount: 1)]
+public class ChainedEvaluationBenchmark
+{
+    [Params(10, 50)]
+    public int ChainLength;
+
+    private RoslynServices roslyn = null!;
+
+    // Fresh services per measured run; not timed (only the [Benchmark] body is measured).
+    [IterationSetup]
+    public void IterationSetup()
+    {
+        var config = new Configuration(useTerminalPaletteTheme: true);
+        roslyn = new RoslynServices(new BenchmarkConsole(), config, new NullTraceLogger());
+        roslyn.WarmUpAsync(Array.Empty<string>()).GetAwaiter().GetResult();
+    }
+
+    [Benchmark]
+    public async Task EvaluateChain()
+    {
+        for (int i = 0; i < ChainLength; i++)
+        {
+            var code = i == 0 ? "var s0 = 0;" : $"var s{i} = s{i - 1} + 1;";
+            var result = await roslyn.EvaluateAsync(code);
+            if (result is not EvaluationResult.Success)
+                throw new InvalidOperationException($"submission {i} failed: {result}");
+        }
+    }
+}

CSharpRepl.Benchmarks/ColdStartDiagnostic.cs

@@ -0,0 +1,127 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading.Tasks;
+using CSharpRepl.Services;
+using CSharpRepl.Services.Logging;
+using CSharpRepl.Services.Roslyn;
+using PrettyPrompt.Consoles;
+
+namespace CSharpRepl.Benchmarks;
+
+/// <summary>Trace logger that prints each warm-up milestone with the elapsed time since construction.</summary>
+internal sealed class TimestampingTraceLogger : ITraceLogger
+{
+    private readonly Stopwatch sw = Stopwatch.StartNew();
+    public void Log(string message) => Console.WriteLine($"    [{sw.ElapsedMilliseconds,6}ms] {message}");
+    public void Log(Func<string> message) { }
+    public void LogPaths(string message, Func<IEnumerable<string?>> paths) { }
+}
+
+/// <summary>
+/// NOT a BenchmarkDotNet benchmark. BenchmarkDotNet measures warm steady-state by design (it JIT-warms before
+/// measuring), so it is blind to the thing we care about here: the cost the user pays on the *very first*
+/// keystroke of a session, which is dominated by first-run JIT + first-compilation of the editor pipeline.
+///
+/// Each invocation must be a fresh process to observe true cold cost — once a code path JITs, it stays JIT'd.
+/// Run one mode per process:
+///   dotnet run -c Release --project CSharpRepl.Benchmarks -- coldstart raw      # init done, pipeline cold (worst case)
+///   dotnet run -c Release --project CSharpRepl.Benchmarks -- coldstart warmed   # full WarmUpAsync awaited first (best case)
+///   dotnet run -c Release --project CSharpRepl.Benchmarks -- coldstart race     # warmup fired-and-forgotten, then type after a short delay (real app)
+/// </summary>
+internal static class ColdStartDiagnostic
+{
+    public static async Task RunAsync(string mode)
+    {
+        var sw = Stopwatch.StartNew();
+        var config = new Configuration(useTerminalPaletteTheme: true); // avoids theme-file I/O
+
+        // "warmtiming": just print when each warm-up milestone is reached (editor path vs. full warmup).
+        if (mode == "warmtiming")
+        {
+            var r = new RoslynServices(new BenchmarkConsole(), config, new TimestampingTraceLogger());
+            var ws = Stopwatch.StartNew();
+            await r.WarmUpAsync(Array.Empty<string>());
+            Console.WriteLine($"[{mode}] WarmUpAsync fully completed in {ws.ElapsedMilliseconds}ms");
+            return;
+        }
+
+        var roslyn = new RoslynServices(new BenchmarkConsole(), config, new NullTraceLogger());
+        Console.WriteLine($"[{mode}] RoslynServices ctor returned at {sw.ElapsedMilliseconds}ms");
+
+        switch (mode)
+        {
+            case "warmed":
+            {
+                var ws = Stopwatch.StartNew();
+                await roslyn.WarmUpAsync(Array.Empty<string>());
+                Console.WriteLine($"[{mode}] WarmUpAsync completed in {ws.ElapsedMilliseconds}ms");
+                break;
+            }
+            case "race":
+            {
+                // Mirror Preload: fire-and-forget, then the user reads the banner and reaches for the keyboard.
+                _ = roslyn.WarmUpAsync(Array.Empty<string>());
+                var delay = int.Parse(Environment.GetEnvironmentVariable("RACE_DELAY_MS") ?? "200");
+                await Task.Delay(delay);
+                Console.WriteLine($"[{mode}] typed after {delay}ms (warmup still running in background)");
+                break;
+            }
+            default: // "raw": let background init finish, but leave the keystroke pipeline un-JIT'd.
+            {
+                // CurrentDocumentForProfiling is null until Initialization completes; polling it does not
+                // touch the highlight/completion code paths, so the pipeline stays cold.
+                while (roslyn.CurrentDocumentForProfiling is null)
+                    await Task.Delay(5);
+                Console.WriteLine($"[{mode}] background init complete at {sw.ElapsedMilliseconds}ms");
+                break;
+            }
+        }
+
+        // The very first keystroke. Typing 's' as the first character drives exactly these PrettyPrompt
+        // callbacks, in this order, serially (see CSharpReplPromptCallbacks + Prompt.ReadLineAsync):
+        //   1. HighlightCallbackAsync          -> roslyn.SyntaxHighlightAsync
+        //   2. ShouldOpenCompletionWindowAsync -> roslyn.ShouldOpenCompletionWindowAsync
+        //   3. (window opens) GetSpanToReplaceByCompletionAsync
+        //   4. (window opens) GetCompletionItemsAsync -> roslyn.CompleteAsync
+        const string text = "s";
+        const int caret = 1;
+        var key = new KeyPress(new ConsoleKeyInfo('s', ConsoleKey.S, shift: false, alt: false, control: false));
+
+        Console.WriteLine($"[{mode}] --- first keystroke ('{text}') ---");
+        var total = Stopwatch.StartNew();
+
+        var t = Stopwatch.StartNew();
+        var spans = await roslyn.SyntaxHighlightAsync(text);
+        Console.WriteLine($"  1. Highlight                  {t.Elapsed.TotalMilliseconds,8:F1} ms   ({spans.Count} spans)");
+
+        t.Restart();
+        var shouldOpen = await roslyn.ShouldOpenCompletionWindowAsync(text, caret, key, default);
+        Console.WriteLine($"  2. ShouldOpenCompletionWindow {t.Elapsed.TotalMilliseconds,8:F1} ms   (= {shouldOpen})");
+
+        // Steps 3 & 4 only happen when the window actually opens — exactly as PrettyPrompt drives them.
+        // While completion is suppressed during warm-up, ShouldOpen returns false and the app makes neither call.
+        if (shouldOpen)
+        {
+            t.Restart();
+            _ = await roslyn.GetSpanToReplaceByCompletionAsync(text, caret, default);
+            Console.WriteLine($"  3. GetSpanToReplace           {t.Elapsed.TotalMilliseconds,8:F1} ms");
+
+            t.Restart();
+            var completions = await roslyn.CompleteAsync(text, caret, default);
+            Console.WriteLine($"  4. Complete                   {t.Elapsed.TotalMilliseconds,8:F1} ms   ({completions.Count} items)");
+        }
+        else
+        {
+            Console.WriteLine($"  3. GetSpanToReplace                  -      (window suppressed)");
+            Console.WriteLine($"  4. Complete                          -      (window suppressed)");
+        }
+
+        Console.WriteLine($"  ============================================");
+        Console.WriteLine($"  first-keystroke total         {total.Elapsed.TotalMilliseconds,8:F1} ms");
+    }
+}