Skip to content

perf: prime the row split cache from the engine's multiline loop (#120) #125

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stevehansen merged 2 commits into from
May 16, 2026
Merged

perf: prime the row split cache from the engine's multiline loop (#120) #125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
08f7cc5
perf: prime the row split cache from the engine's multiline loop (#120)
stevehansen May 16, 2026
76f983c
perf: pre-size split list using known header count
stevehansen May 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 38 additions & 22 deletions Csv/CsvReader.Engine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@

internal interface IRowFactory<TRow> where TRow : class
{
TRow Create(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, string? rawString, CsvOptions options);
TRow Create(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, string? rawString, IList<MemoryText>? rawSplit, CsvOptions options);
}

internal readonly struct TextReaderLineSource : ILineSource
Expand Down Expand Up @@ -188,22 +188,28 @@

internal readonly struct StringRowFactory : IRowFactory<ReadLine>
{
public ReadLine Create(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, string? rawString, CsvOptions options)
public ReadLine Create(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, string? rawString, IList<MemoryText>? rawSplit, CsvOptions options)
{
#if NET8_0_OR_GREATER
return new ReadLine(headers, headerLookup, index, rawString ?? raw.ToString(), options);
var row = new ReadLine(headers, headerLookup, index, rawString ?? raw.ToString(), options);
#else
return new ReadLine(headers, headerLookup, index, rawString ?? raw, options);
var row = new ReadLine(headers, headerLookup, index, rawString ?? raw, options);
#endif
if (rawSplit != null)
row.rawSplitLine = rawSplit;
return row;
}
}

#if NET8_0_OR_GREATER
internal readonly struct SpanRowFactory : IRowFactory<ReadLineSpan>
{
public ReadLineSpan Create(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, string? rawString, CsvOptions options)
public ReadLineSpan Create(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, string? rawString, IList<MemoryText>? rawSplit, CsvOptions options)
{
return new ReadLineSpan(headers, headerLookup, index, rawString ?? raw.ToString(), options);
var row = new ReadLineSpan(headers, headerLookup, index, rawString ?? raw.ToString(), options);
if (rawSplit != null)
row.rawSplitLine = rawSplit;
return row;
}
}

Expand All @@ -216,17 +222,23 @@
this.memoryOptions = memoryOptions;
}

public ReadLineSpanOptimized Create(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, string? rawString, CsvOptions options)
public ReadLineSpanOptimized Create(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, string? rawString, IList<MemoryText>? rawSplit, CsvOptions options)
{
return new ReadLineSpanOptimized(headers, headerLookup, index, raw, options, memoryOptions);
var row = new ReadLineSpanOptimized(headers, headerLookup, index, raw, options, memoryOptions);
if (rawSplit != null)
row.rawSplitLine = rawSplit;
return row;
}
}

internal readonly struct MemoryRowFactory : IRowFactory<ReadLineFromMemory>
{
public ReadLineFromMemory Create(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, string? rawString, CsvOptions options)
public ReadLineFromMemory Create(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, string? rawString, IList<MemoryText>? rawSplit, CsvOptions options)
{
return new ReadLineFromMemory(headers, headerLookup, index, raw, options);
var row = new ReadLineFromMemory(headers, headerLookup, index, raw, options);
if (rawSplit != null)
row.rawSplitLine = rawSplit;
return row;
}
}
#endif
Expand All @@ -249,6 +261,8 @@
if (index <= options.RowsToSkip || options.SkipRow?.Invoke(line, index) == true)
continue;

IList<MemoryText>? rawSplit = null;

if (headers == null || headerLookup == null)
{
InitializeOptions(line.AsSpan(), options);
Expand All @@ -259,15 +273,15 @@
// case via index == RowsToSkip + 1 and skips its own multiline pass to avoid double-reading.
if (!skipInitialLine && options.AllowNewLineInEnclosedFieldValues)
{
var splitLine = options.Splitter.Split(line, options);
rawSplit = options.Splitter.Split(line, options);

Check warning on line 276 in Csv/CsvReader.Engine.cs

View workflow job for this annotation

GitHub Actions / Analyze (csharp) 

Dereference of a possibly null reference.

Check warning on line 276 in Csv/CsvReader.Engine.cs

View workflow job for this annotation

GitHub Actions / Analyze (csharp) 

Dereference of a possibly null reference.

Check warning on line 276 in Csv/CsvReader.Engine.cs

View workflow job for this annotation

GitHub Actions / Analyze (csharp) 

Dereference of a possibly null reference.

while (splitLine.Count > 0 && CsvLineSplitter.IsUnterminatedQuotedValue(splitLine[splitLine.Count - 1].AsSpan(), options))
while (rawSplit.Count > 0 && CsvLineSplitter.IsUnterminatedQuotedValue(rawSplit[rawSplit.Count - 1].AsSpan(), options))
{
if (!source.TryReadLine(out var nextLine, out _))
break;

line = source.Concat(line, options.NewLine, nextLine, out lineString);
splitLine = options.Splitter.Split(line, options);
rawSplit = options.Splitter.Split(line, options);
}
}

Expand Down Expand Up @@ -314,18 +328,18 @@
var isFirstDataLineInHeaderAbsentMode = options.HeaderMode == HeaderMode.HeaderAbsent && index == (options.RowsToSkip + 1);
if (options.AllowNewLineInEnclosedFieldValues && !isFirstDataLineInHeaderAbsentMode)
{
var rawSplit = options.Splitter.Split(line, options);
rawSplit = options.Splitter.Split(line, options, headers!.Length);

Check warning on line 331 in Csv/CsvReader.Engine.cs

View workflow job for this annotation

GitHub Actions / Analyze (csharp) 

Dereference of a possibly null reference.

Check warning on line 331 in Csv/CsvReader.Engine.cs

View workflow job for this annotation

GitHub Actions / Analyze (csharp) 

Dereference of a possibly null reference.

Check warning on line 331 in Csv/CsvReader.Engine.cs

View workflow job for this annotation

GitHub Actions / Analyze (csharp) 

Dereference of a possibly null reference.
while (rawSplit.Count > 0 && CsvLineSplitter.IsUnterminatedQuotedValue(rawSplit[rawSplit.Count - 1].AsSpan(), options))
{
if (!source.TryReadLine(out var nextLine, out _))
break;

line = source.Concat(line, options.NewLine, nextLine, out lineString);
rawSplit = options.Splitter.Split(line, options);
rawSplit = options.Splitter.Split(line, options, headers!.Length);
}
}

yield return factory.Create(headers, headerLookup, index, line, lineString, options);
yield return factory.Create(headers, headerLookup, index, line, lineString, rawSplit, options);
}
}

Expand All @@ -352,6 +366,8 @@
if (index <= options.RowsToSkip || options.SkipRow?.Invoke(line, index) == true)
continue;

IList<MemoryText>? rawSplit = null;

if (headers == null || headerLookup == null)
{
InitializeOptions(line.AsSpan(), options);
Expand All @@ -362,16 +378,16 @@
// case via index == RowsToSkip + 1 and skips its own multiline pass to avoid double-reading.
if (!skipInitialLine && options.AllowNewLineInEnclosedFieldValues)
{
var splitLine = options.Splitter.Split(line, options);
rawSplit = options.Splitter.Split(line, options);

Check warning on line 381 in Csv/CsvReader.Engine.cs

View workflow job for this annotation

GitHub Actions / Analyze (csharp) 

Dereference of a possibly null reference.

Check warning on line 381 in Csv/CsvReader.Engine.cs

View workflow job for this annotation

GitHub Actions / Analyze (csharp) 

Dereference of a possibly null reference.

while (splitLine.Count > 0 && CsvLineSplitter.IsUnterminatedQuotedValue(splitLine[splitLine.Count - 1].AsSpan(), options))
while (rawSplit.Count > 0 && CsvLineSplitter.IsUnterminatedQuotedValue(rawSplit[rawSplit.Count - 1].AsSpan(), options))
{
var (nextOk, nextLine, _) = await source.TryReadLineAsync(ct).ConfigureAwait(false);
if (!nextOk)
break;

line = source.Concat(line, options.NewLine, nextLine, out lineString);
splitLine = options.Splitter.Split(line, options);
rawSplit = options.Splitter.Split(line, options);
}
}

Expand Down Expand Up @@ -418,19 +434,19 @@
var isFirstDataLineInHeaderAbsentMode = options.HeaderMode == HeaderMode.HeaderAbsent && index == (options.RowsToSkip + 1);
if (options.AllowNewLineInEnclosedFieldValues && !isFirstDataLineInHeaderAbsentMode)
{
var rawSplit = options.Splitter.Split(line, options);
rawSplit = options.Splitter.Split(line, options, headers!.Length);

Check warning on line 437 in Csv/CsvReader.Engine.cs

View workflow job for this annotation

GitHub Actions / Analyze (csharp) 

Dereference of a possibly null reference.

Check warning on line 437 in Csv/CsvReader.Engine.cs

View workflow job for this annotation

GitHub Actions / Analyze (csharp) 

Dereference of a possibly null reference.
while (rawSplit.Count > 0 && CsvLineSplitter.IsUnterminatedQuotedValue(rawSplit[rawSplit.Count - 1].AsSpan(), options))
{
var (nextOk, nextLine, _) = await source.TryReadLineAsync(ct).ConfigureAwait(false);
if (!nextOk)
break;

line = source.Concat(line, options.NewLine, nextLine, out lineString);
rawSplit = options.Splitter.Split(line, options);
rawSplit = options.Splitter.Split(line, options, headers!.Length);
}
}

yield return factory.Create(headers, headerLookup, index, line, lineString, options);
yield return factory.Create(headers, headerLookup, index, line, lineString, rawSplit, options);
}
}
#endif
Expand Down
2 changes: 1 addition & 1 deletion Csv/CsvReader.FromMemory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ internal sealed class ReadLineFromMemory : ICsvLineFromMemory
{
private readonly Dictionary<string, int> headerLookup;
private readonly CsvOptions options;
private IList<MemoryText>? rawSplitLine;
internal IList<MemoryText>? rawSplitLine;
private MemoryText[]? parsedLine;

public ReadLineFromMemory(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, MemoryText raw, CsvOptions options)
Expand Down
6 changes: 3 additions & 3 deletions Csv/CsvReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,7 +450,7 @@ internal sealed class ReadLine : ICsvLine
private readonly Dictionary<string, int> headerLookup;
private readonly CsvOptions options;
private readonly MemoryText[] headers;
private IList<MemoryText>? rawSplitLine;
internal IList<MemoryText>? rawSplitLine;
internal MemoryText[]? parsedLine;

public ReadLine(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, string raw, CsvOptions options)
Expand Down Expand Up @@ -551,7 +551,7 @@ internal sealed class ReadLineSpan : ICsvLineSpan
private readonly Dictionary<string, int> headerLookup;
private readonly CsvOptions options;
private readonly MemoryText[] headers;
private IList<MemoryText>? rawSplitLine;
internal IList<MemoryText>? rawSplitLine;
internal MemoryText[]? parsedLine;

public ReadLineSpan(MemoryText[] headers, Dictionary<string, int> headerLookup, int index, string raw, CsvOptions options)
Expand Down Expand Up @@ -694,7 +694,7 @@ internal sealed class ReadLineSpanOptimized : ICsvLineSpan
private readonly CsvMemoryOptions memoryOptions;
private readonly ReadOnlyMemory<char>[] headers;
private readonly ReadOnlyMemory<char> rawMemory;
private IList<ReadOnlyMemory<char>>? rawSplitLine;
internal IList<ReadOnlyMemory<char>>? rawSplitLine;
private ReadOnlyMemory<char>[]? parsedLine;

public ReadLineSpanOptimized(ReadOnlyMemory<char>[] headers, Dictionary<string, int> headerLookup, int index, ReadOnlyMemory<char> raw, CsvOptions options, CsvMemoryOptions memoryOptions)
Expand Down
Loading