Skip to content

Added AllowFieldCountMismatch property to CsvConfiguration #985

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
michelebastione wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Added AllowFieldCountMismatch property to CsvConfiguration #985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b6f688f
Added AllowFieldCountMismatch property to CsvConfiguration
michelebastione Jul 19, 2026
0cdea67
Removed unused very large csv sample file
michelebastione Jul 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion src/MiniExcel.Csv/CsvConfiguration.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,24 @@ public class CsvConfiguration : MiniExcelBaseConfiguration
public char Seperator { get; set; } = ',';
public string NewLine { get; set; } = "\r\n";
public bool ReadLineBreaksWithinQuotes { get; set; } = true;
public bool ReadEmptyStringAsNull { get; set; } = false;

/// <summary>
/// Empty fields will be converted to null when queried as strings, or to the default value of the corresponding mapped type.
/// </summary>
public bool ReadEmptyFieldsAsDefault { get; set; } = false;

[Obsolete("Please use the ReadEmptyFieldsAsDefault property instead")]
public bool ReadEmptyStringAsNull
{
get => ReadEmptyFieldsAsDefault;
set => ReadEmptyFieldsAsDefault = value;
}

/// <summary>
/// When set to true if the rows have different lengths they will be returned as jagged data instead of throwing an exception.
/// </summary>
public bool AllowFieldCountMismatch { get; set; } = false;

public bool AlwaysQuote { get; set; } = false;
public bool QuoteWhitespaces { get; set; } = true;
public Func<string, string[]>? SplitFn { get; set; }
Expand Down
82 changes: 47 additions & 35 deletions src/MiniExcel.Csv/CsvReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,14 @@ internal CsvReader(Stream stream, IMiniExcelConfiguration? configuration, bool l

using var reader = _config.StreamReaderFunc(_stream);
var firstRow = true;
var headRows = new Dictionary<int, string>();
var headerRow = new Dictionary<int, string>();

var rowIndex = 0;
while (await reader.ReadLineAsync(
#if NET
cancellationToken
#endif
).ConfigureAwait(false) is { } row)
).ConfigureAwait(false) is { } row)
{
rowIndex++;
if (string.IsNullOrWhiteSpace(row))
Expand All @@ -64,61 +64,73 @@ internal CsvReader(Stream stream, IMiniExcelConfiguration? configuration, bool l
finalRow = string.Concat(finalRow, _config.NewLine, nextPart);
}
}
var read = Split(finalRow);
var fields = Split(finalRow);

// invalid row check
if (read.Length < headRows.Count)
if (fields.Length < headerRow.Count && !_config.AllowFieldCountMismatch)
{
var colIndex = read.Length;
var headers = headRows.ToDictionary(x => x.Value, x => x.Key);
var rowValues = read
.Select((x, i) => new KeyValuePair<string, object>(headRows[i], x))
var colIndex = fields.Length;
var headers = headerRow.ToDictionary(x => x.Value, x => x.Key);
var rowValues = fields
.Select((x, i) => new KeyValuePair<string, object>(headerRow[i], x))
.ToDictionary(x => x.Key, x => x.Value);

throw new ColumnNotFoundException(columnIndex: null, headRows[colIndex], [], rowIndex, headers, rowValues, $"Csv read error: Column {colIndex} not found in Row {rowIndex}");
throw new ColumnNotFoundException(columnIndex: null, headerRow[colIndex], [], rowIndex, headers, rowValues, $"Csv read error: Column {colIndex} not found in Row {rowIndex}");
}

//header
// with header
if (hasHeaderRow)
{
if (firstRow)
{
firstRow = false;
for (int i = 0; i <= read.Length - 1; i++)
headRows.Add(i, read[i]);
for (int i = 0; i < fields.Length; i++)
headerRow.Add(i, fields[i]);

continue;
}

var headCell = ExpandoHelper.CreateEmptyByHeaders(headRows);
for (int i = 0; i <= read.Length - 1; i++)
headCell[headRows[i]] = read[i];
var resultRow = ExpandoHelper.CreateEmptyByHeaders(headerRow);
for (int i = 0; i < fields.Length; i++)
{
if (i >= headerRow.Count && _config.AllowFieldCountMismatch)
headerRow.Add(i, $"Col{i + 1}");

yield return headCell;
continue;
}
var isFillerField = i >= headerRow.Count && _config.AllowFieldCountMismatch;
var field = isFillerField ? "" : fields[i];

//body
if (firstRow) // record first row as reference
{
firstRow = false;
for (int i = 0; i <= read.Length - 1; i++)
headRows.Add(i, $"c{i + 1}");
}
var treatEmptyFieldAsNull = _config.ReadEmptyFieldsAsDefault && field is "";
resultRow[headerRow[i]] = treatEmptyFieldAsNull ? null : field;
}

// todo: can we find a way to remove the redundant cell conversions for CSV?
var cell = ExpandoHelper.CreateEmptyByIndices(read.Length - 1, 0);
if (_config.ReadEmptyStringAsNull)
{
for (int i = 0; i <= read.Length - 1; i++)
cell[CellReferenceConverter.GetAlphabeticalIndex(i)] = read[i] is var value and not "" ? value : null;
yield return resultRow;
}
else
{
for (int i = 0; i <= read.Length - 1; i++)
cell[CellReferenceConverter.GetAlphabeticalIndex(i)] = read[i];
}
if (firstRow) // use first row as reference
{
firstRow = false;
for (int i = 0; i < fields.Length; i++)
headerRow.Add(i, $"Col{i + 1}");
}

// todo: can we find a way to remove the redundant cell conversions for CSV?
var resultRow = ExpandoHelper.CreateEmptyByIndices(fields.Length - 1, 0);
for (int i = 0; i < fields.Length; i++)
{
if (i >= headerRow.Count && _config.AllowFieldCountMismatch)
headerRow.Add(i, $"Col{i + 1}");

yield return cell;
var isFillerField = i >= headerRow.Count && _config.AllowFieldCountMismatch;
var field = isFillerField ? "" : fields[i];

var index = CellReferenceConverter.GetAlphabeticalIndex(i);
var treatEmptyFieldAsNull = _config.ReadEmptyFieldsAsDefault && field is "";
resultRow[index] = treatEmptyFieldAsNull ? null : field;
}

yield return resultRow;
}
Comment thread
michelebastione marked this conversation as resolved.
}
}

Expand Down
82 changes: 81 additions & 1 deletion tests/MiniExcel.Csv.Tests/Main/MiniExcelCsvAsyncTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ await _csvExporter.ExportAsync(path, new[]
Assert.Equal(string.Empty, rows[1].C2);
}

var config = new CsvConfiguration { ReadEmptyStringAsNull = true };
var config = new CsvConfiguration { ReadEmptyFieldsAsDefault = true };
await using (var stream = File.OpenRead(path))
{
var rows = _csvImporter.Query<TestDto>(stream, configuration: config).ToList();
Expand Down Expand Up @@ -491,4 +491,84 @@ public async Task GetColumnNamesEmptyTest()
var cols = await _csvImporter.GetColumnNamesAsync(ms);
Assert.Empty(cols);
}

[Fact]
public async Task JaggedRowsWithoutHeaderTest()
{
var csvConfig = new CsvConfiguration
{
AlwaysQuote = true,
ReadEmptyFieldsAsDefault = true,
AllowFieldCountMismatch = true
};

var data = """
0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
"""u8.ToArray();

await using var ms = new MemoryStream();
await ms.WriteAsync(data.ToArray());

var result = await _csvImporter.QueryAsync(ms, configuration: csvConfig).ToListAsync();

Assert.Equal(4, result.Count);
Assert.Equal(41, ((IDictionary<string, object?>)result[0]).Count);
Assert.Equal(32, ((IDictionary<string, object?>)result[1]).Count);
}

[Fact]
public async Task JaggedRowsWithHeaderTest()
{
var csvConfig = new CsvConfiguration
{
AlwaysQuote = true,
ReadEmptyFieldsAsDefault = true,
AllowFieldCountMismatch = true
};

var data = """
C1,C2,C3,C4,C5,C6,C7,C8,C9,C10,C11,C12,C13,C14,C15,C16,C17,C18,C19,C20,C21,C22,C23,C24,C25,C26,C27,C28,C29,C30,C31,C32,C33,C34
1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
"""u8.ToArray();

await using var ms = new MemoryStream();
await ms.WriteAsync(data.ToArray());

var result = await _csvImporter.QueryAsync(ms, true, configuration: csvConfig).ToListAsync();

Assert.Equal(4, result.Count);
Assert.Equal(34, ((IDictionary<string, object?>)result[0]).Count);
Assert.Equal(41, ((IDictionary<string, object?>)result[1]).Count);
}

[Fact]
public async Task MappedJaggedRowsTest()
{
var csvConfig = new CsvConfiguration
{
AlwaysQuote = true,
ReadEmptyFieldsAsDefault = true,
AllowFieldCountMismatch = true
};

var data = """
C1,C2,C3,C4,C5,C6,C7,C8,C9,C10,C11,C12,C13,C14,C15,C16,C17,C18,C19,C20,C21,C22,C23,C24,C25,C26,C27,C28,C29,C30,C31,C32,C33,C34
1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
"""u8.ToArray();

await using var ms = new MemoryStream();
await ms.WriteAsync(data.ToArray());

var result = await _csvImporter.QueryAsync<JaggedRowsMappingTest>(ms, configuration: csvConfig).ToListAsync();
Assert.Equal(4, result.Count);
}
}
92 changes: 87 additions & 5 deletions tests/MiniExcel.Csv.Tests/Main/MiniExcelCsvTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,7 +402,7 @@ public void CsvColumnNotFoundTest()
{
var exception = Assert.Throws<ColumnNotFoundException>(() => _csvImporter.Query<TestDto>(stream).ToList());

Assert.Equal("c2", exception.ColumnName);
Assert.Equal("Col2", exception.ColumnName);
Assert.Equal(2, exception.RowIndex);
Assert.Null(exception.ColumnIndex);
Assert.True(exception.RowValues is IDictionary<string, object>);
Expand All @@ -412,7 +412,7 @@ public void CsvColumnNotFoundTest()
{
var exception = Assert.Throws<ColumnNotFoundException>(() => _csvImporter.Query<TestDto>(path).ToList());

Assert.Equal("c2", exception.ColumnName);
Assert.Equal("Col2", exception.ColumnName);
Assert.Equal(2, exception.RowIndex);
Assert.Null(exception.ColumnIndex);
Assert.True(exception.RowValues is IDictionary<string, object>);
Expand All @@ -431,7 +431,7 @@ public void CsvColumnNotFoundWithAliasTest()
{
var exception = Assert.Throws<ColumnNotFoundException>(() => _csvImporter.Query<TestWithAlias>(stream).ToList());

Assert.Equal("c2", exception.ColumnName);
Assert.Equal("Col2", exception.ColumnName);
Assert.Equal(2, exception.RowIndex);
Assert.Null(exception.ColumnIndex);
Assert.True(exception.RowValues is IDictionary<string, object>);
Expand All @@ -441,7 +441,7 @@ public void CsvColumnNotFoundWithAliasTest()
{
var exception = Assert.Throws<ColumnNotFoundException>(() => _csvImporter.Query<TestWithAlias>(path).ToList());

Assert.Equal("c2", exception.ColumnName);
Assert.Equal("Col2", exception.ColumnName);
Assert.Equal(2, exception.RowIndex);
Assert.Null(exception.ColumnIndex);
Assert.True(exception.RowValues is IDictionary<string, object>);
Expand All @@ -468,7 +468,7 @@ private static string Generate(string value)
var path = file.ToString();

using (var writer = new StreamWriter(path))
using (var csv = new global::CsvHelper.CsvWriter(writer, CultureInfo.InvariantCulture))
using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
{
var records = Enumerable.Range(1, 1).Select(_ => new { v1 = value, v2 = value });
csv.WriteRecords(records);
Expand Down Expand Up @@ -603,6 +603,7 @@ public void ExportAndQueryMixedFieldAndPropertyTest()

using var reader = new StreamReader(path);
using var csv = new CsvReader(reader, CultureInfo.InvariantCulture);

var records = csv.GetRecords<dynamic>().ToList();
var first = (IDictionary<string, object>)records[0];

Expand All @@ -621,6 +622,7 @@ public void ExportAndQueryFieldsWithoutAttributeTest()

using var reader = new StreamReader(path);
using var csv = new CsvReader(reader, CultureInfo.InvariantCulture);

var records = csv.GetRecords<dynamic>().ToList();
var first = (IDictionary<string, object>)records[0];

Expand All @@ -644,4 +646,84 @@ public void GetColumnNamesEmptyTest()
var cols = _csvImporter.GetColumnNames(ms);
Assert.Empty(cols);
}

[Fact]
public void JaggedRowsWithoutHeaderTest()
{
var csvConfig = new CsvConfiguration
{
AlwaysQuote = true,
ReadEmptyFieldsAsDefault = true,
AllowFieldCountMismatch = true
};

var data = """
0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
"""u8.ToArray();

using var ms = new MemoryStream();
ms.Write(data.ToArray());

var result = _csvImporter.Query(ms, configuration: csvConfig).ToList();

Assert.Equal(4, result.Count);
Assert.Equal(41, ((IDictionary<string, object?>)result[0]).Count);
Assert.Equal(32, ((IDictionary<string, object?>)result[1]).Count);
}

[Fact]
public void JaggedRowsWithHeaderTest()
{
var csvConfig = new CsvConfiguration
{
AlwaysQuote = true,
ReadEmptyFieldsAsDefault = true,
AllowFieldCountMismatch = true
};

var data = """
C1,C2,C3,C4,C5,C6,C7,C8,C9,C10,C11,C12,C13,C14,C15,C16,C17,C18,C19,C20,C21,C22,C23,C24,C25,C26,C27,C28,C29,C30,C31,C32,C33,C34
1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
"""u8.ToArray();

using var ms = new MemoryStream();
ms.Write(data.ToArray());

var result = _csvImporter.Query(ms, true, configuration: csvConfig).ToList();

Assert.Equal(4, result.Count);
Assert.Equal(34, ((IDictionary<string, object?>)result[0]).Count);
Assert.Equal(41, ((IDictionary<string, object?>)result[1]).Count);
}

[Fact]
public void MappedJaggedRowsTest()
{
var csvConfig = new CsvConfiguration
{
AlwaysQuote = true,
ReadEmptyFieldsAsDefault = true,
AllowFieldCountMismatch = true
};

var data = """
C1,C2,C3,C4,C5,C6,C7,C8,C9,C10,C11,C12,C13,C14,C15,C16,C17,C18,C19,C20,C21,C22,C23,C24,C25,C26,C27,C28,C29,C30,C31,C32,C33,C34
1,"V","4000","" ,"26",10000001,"Test457" ,"220626",, ,,,5769.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
0,"V","4000","6","26",10000001,"Test123" ,"220626",,1001200,,,5769.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,0,
0,"V","4000","6","26",10000002,"Test789" ,"220626",,1104500,,,26550.00 ,N,"EUR",,,,,,, ,,,,, , ,,,,,,,,,,,,7,
1,"V","4000","" ,"26",10000002,"Test11X" ,"220626",, ,,,26550.00 , ,"" ,,,,,,,0.00 ,,,,,"","",,,,
"""u8.ToArray();

using var ms = new MemoryStream();
ms.Write(data.ToArray());

var result = _csvImporter.Query<JaggedRowsMappingTest>(ms, configuration: csvConfig).ToList();
Assert.Equal(4, result.Count);
}
}
Loading
Loading