fix: memory-based reader paths drop records after blank lines (#122)

Csv.Tests/EngineUnificationTests.cs

@@ -66,7 +66,10 @@ private static List<Row> RunRead(string csv, CsvOptions options)
             {
                 var byName = new Dictionary<string, string>();
                 foreach (var h in line.Headers)
-                    byName[h] = line[h];
+                {
+                    if (line.LineHasColumn(h))
+                        byName[h] = line[h];
+                }
 
                 result.Add(new Row
                 {
@@ -89,7 +92,10 @@ private static List<Row> RunReadAsSpan(string csv, CsvOptions options)
             {
                 var byName = new Dictionary<string, string>();
                 foreach (var h in line.Headers)
-                    byName[h] = line[h];
+                {
+                    if (line.LineHasColumn(h))
+                        byName[h] = line[h];
+                }
 
                 result.Add(new Row
                 {
@@ -111,7 +117,10 @@ private static async Task<List<Row>> RunReadAsync(string csv, CsvOptions options
             {
                 var byName = new Dictionary<string, string>();
                 foreach (var h in line.Headers)
-                    byName[h] = line[h];
+                {
+                    if (line.LineHasColumn(h))
+                        byName[h] = line[h];
+                }
 
                 result.Add(new Row
                 {
@@ -132,7 +141,10 @@ private static List<Row> RunReadFromMemoryOptimized(string csv, CsvOptions optio
             {
                 var byName = new Dictionary<string, string>();
                 foreach (var h in line.Headers)
-                    byName[h] = line[h];
+                {
+                    if (line.LineHasColumn(h))
+                        byName[h] = line[h];
+                }
 
                 result.Add(new Row
                 {
@@ -155,7 +167,10 @@ private static List<Row> RunReadFromMemory(string csv, CsvOptions options)
                 var valueStrings = line.Values.Select(v => v.ToString()).ToArray();
                 var byName = new Dictionary<string, string>();
                 foreach (var h in headerStrings)
-                    byName[h] = line[h].ToString();
+                {
+                    if (line.LineHasColumn(h))
+                        byName[h] = line[h].ToString();
+                }
 
                 result.Add(new Row
                 {
@@ -606,5 +621,55 @@ public void When_ReadAsSpanEnumeratesOneThousandRecords_Then_AllocatedBytesIsFin
             System.Diagnostics.Trace.WriteLine($"ReadAsSpan over 1000 records allocated {delta} bytes.");
         }
 #endif
+
+        [TestMethod]
+        public void When_BlankLineInMiddleOfStream_Then_AllPathsContinueParsing()
+        {
+            const string csv = "a,b,c\n1,2,3\n\n4,5,6\n";
+
+            foreach (var path in AllPaths)
+            {
+                var rows = Run(path, csv, () => new CsvOptions());
+
+                Assert.AreEqual(2, rows.Count, $"{path}: expected 2 records after default SkipRow elides the blank line, got {rows.Count}");
+                Assert.AreEqual("1,2,3", rows[0].Raw, path.ToString());
+                Assert.AreEqual("4,5,6", rows[1].Raw, path.ToString());
+            }
+        }
+
+        [TestMethod]
+        public void When_BlankLineAndSkipRowDisabled_Then_AllPathsReturnEmptyRecord()
+        {
+            const string csv = "a,b,c\n1,2,3\n\n4,5,6\n";
+
+            foreach (var path in AllPaths)
+            {
+                var rows = Run(path, csv, () => new CsvOptions
+                {
+                    SkipRow = (_, _) => false,
+                    ValidateColumnCount = false,
+                    ReturnEmptyForMissingColumn = true,
+                });
+
+                Assert.AreEqual(3, rows.Count, $"{path}: expected 3 records when SkipRow is disabled (blank line surfaces as empty record), got {rows.Count}");
+                Assert.AreEqual("1,2,3", rows[0].Raw, path.ToString());
+                Assert.AreEqual(string.Empty, rows[1].Raw, path.ToString());
+                Assert.AreEqual("4,5,6", rows[2].Raw, path.ToString());
+            }
+        }
+
+        [TestMethod]
+        public void When_TrailingBlankLine_Then_AllPathsTerminateAfterLastRecord()
+        {
+            const string csv = "a,b,c\n1,2,3\n\n";
+
+            foreach (var path in AllPaths)
+            {
+                var rows = Run(path, csv, () => new CsvOptions());
+
+                Assert.AreEqual(1, rows.Count, $"{path}: expected exactly 1 record, got {rows.Count}");
+                Assert.AreEqual("1,2,3", rows[0].Raw, path.ToString());
+            }
+        }
     }
 }

Csv/CsvReader.Engine.cs

@@ -134,31 +134,17 @@
                 {
                     line = csv.Slice(position);
                     position = csv.Length;
-                    return !line.IsEmpty;
+                    return true;
                 }
 
-                var lineLength = newlineIndex;
-                var slice = csv.Slice(position, lineLength);
+                line = csv.Slice(position, newlineIndex);
+                position += newlineIndex;
 
-                position += lineLength;
-                if (position < csv.Length)
-                {
-                    var ch = csv.Span[position];
-                    if (ch == '\r' || ch == '\n')
-                    {
-                        position++;
-                        if (position < csv.Length && ch == '\r' && csv.Span[position] == '\n')
-                            position++;
-                    }
-                }
-
-                if (slice.IsEmpty)
-                {
-                    line = default;
-                    return false;
-                }
+                var ch = csv.Span[position];
+                position++;
+                if (position < csv.Length && ch == '\r' && csv.Span[position] == '\n')
+                    position++;
 
-                line = slice;
                 return true;
             }
 
@@ -210,7 +196,7 @@
                 }
 
                 line = csv.ReadLine(ref position);
-                return !line.IsEmpty;
+                return true;
             }
 
             public MemoryText Concat(MemoryText head, string newLine, MemoryText tail, out string? combined)
@@ -294,7 +280,7 @@
                     // case via index == RowsToSkip + 1 and skips its own multiline pass to avoid double-reading.
                     if (!skipInitialLine && options.AllowNewLineInEnclosedFieldValues)
                     {
                         var splitLine = options.Splitter.Split(line, options);
 
                         while (splitLine.Count > 0 && CsvLineSplitter.IsUnterminatedQuotedValue(splitLine[splitLine.Count - 1].AsSpan(), options))
                         {
@@ -349,7 +335,7 @@
                 var isFirstDataLineInHeaderAbsentMode = options.HeaderMode == HeaderMode.HeaderAbsent && index == (options.RowsToSkip + 1);
                 if (options.AllowNewLineInEnclosedFieldValues && !isFirstDataLineInHeaderAbsentMode)
                 {
                     var rawSplit = options.Splitter.Split(line, options);
                     while (rawSplit.Count > 0 && CsvLineSplitter.IsUnterminatedQuotedValue(rawSplit[rawSplit.Count - 1].AsSpan(), options))
                     {
                         if (!source.TryReadLine(out var nextLine, out _))
@@ -397,7 +383,7 @@
                     // case via index == RowsToSkip + 1 and skips its own multiline pass to avoid double-reading.
                     if (!skipInitialLine && options.AllowNewLineInEnclosedFieldValues)
                     {
                         var splitLine = options.Splitter.Split(line, options);
 
                         while (splitLine.Count > 0 && CsvLineSplitter.IsUnterminatedQuotedValue(splitLine[splitLine.Count - 1].AsSpan(), options))
                         {
@@ -453,7 +439,7 @@
                 var isFirstDataLineInHeaderAbsentMode = options.HeaderMode == HeaderMode.HeaderAbsent && index == (options.RowsToSkip + 1);
                 if (options.AllowNewLineInEnclosedFieldValues && !isFirstDataLineInHeaderAbsentMode)
                 {
                     var rawSplit = options.Splitter.Split(line, options);
                     while (rawSplit.Count > 0 && CsvLineSplitter.IsUnterminatedQuotedValue(rawSplit[rawSplit.Count - 1].AsSpan(), options))
                     {
                         var (nextOk, nextLine, _) = await source.TryReadLineAsync(ct).ConfigureAwait(false);