Skip to content

Commit

Permalink
chore: bump PocketCsvReader from 2.0.0 to 2.27.8 (#755)
Browse files Browse the repository at this point in the history 
* chore: bump PocketCsvReader from 2.0.0 to 2.27.8

* fix: set CSV dialect with headers in constructor
  • Loading branch information
@Seddryck
Seddryck authored Mar 9, 2025
1 parent bdad372 commit 9d1d255
Show file tree
Hide file tree
Showing 9 changed files with 61 additions and 265 deletions.
2 changes: 1 addition & 1 deletion NBi.Core/Decoration/DataEngineering/Commands/SqlServer/BulkLoadCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ internal void Execute(string connectionString, string tableName, string filename

// write the data in the "dataTable"
var fileReader = new CsvReader();
var dataTable = fileReader.ToDataTable(filename, false);
var dataTable = fileReader.ToDataTable(filename);
bulkCopy.WriteToServer(dataTable);

connection.Close();
Expand Down
15 changes: 15 additions & 0 deletions NBi.Core/Extensions.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace NBi.Core;
internal static class Extensions
{
public static T Also<T>(this T obj, Action<T> action)
{
action(obj);
return obj;
}
}
23 changes: 14 additions & 9 deletions NBi.Core/FlatFile/CsvProfile.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System.Collections.Generic;
using NBi.Extensibility.FlatFile;
using PocketCsvReader;

namespace NBi.Core.FlatFile;

Expand All @@ -11,6 +12,7 @@ public CsvProfile(IDictionary<string, object> attributes)
: base (
(char) (attributes.TryGetValue("field-separator", out var fs) ? fs : ';'),
(char) (attributes.TryGetValue("text-qualifier", out var tq) ? tq : '\"'),
'\\',
(string)(attributes.TryGetValue("record-separator", out var rs) ? rs : "\r\n"),
(bool) (attributes.TryGetValue("first-row-header", out var frh) ? frh : false),
(bool) (attributes.TryGetValue("performance-optimized", out var po) ? po : true),
Expand All @@ -21,21 +23,24 @@ public CsvProfile(IDictionary<string, object> attributes)

public IDictionary<string, object> Attributes => new Dictionary<string, object>()
{
{ "field-separator", Descriptor.Delimiter },
{ "text-qualifier", Descriptor.QuoteChar },
{ "record-separator", Descriptor.LineTerminator },
{ "first-row-header", Descriptor.Header },
{ "performance-optimized", base.PerformanceOptmized },
{ "field-separator", Dialect.Delimiter },
{ "text-qualifier", Dialect.QuoteChar! },
{ "record-separator", Dialect.LineTerminator },
{ "first-row-header", Dialect.Header },
{ "performance-optimized", !ParserOptimizations.RowCountAtStart },
{ "missing-cell", base.MissingCell },
{ "empty-cell", base.EmptyCell },
};

private CsvProfile(char fieldSeparator)
: base(fieldSeparator, '\"', '\\', "\r\n", false, true, 4096, "(empty)", "(null)") { }
internal CsvProfile(char fieldSeparator)
: this(fieldSeparator, '\"', "\r\n", false, true, "(empty)", "(null)")
{ }

public CsvProfile(bool firstRowHeader)
: base(firstRowHeader) { }
: this(';', '\"', "\r\n", firstRowHeader, true, "(empty)", "(null)")
{ }

public CsvProfile(char fieldSeparator, char textQualifier, string recordSeparator, bool firstRowHeader, bool performanceOptimized, string emptyCell, string missingCell)
: base(fieldSeparator, textQualifier, recordSeparator, firstRowHeader, performanceOptimized, 4096, emptyCell, missingCell) { }
: base(fieldSeparator, textQualifier, '\\', recordSeparator, firstRowHeader, !performanceOptimized, 4096, emptyCell, missingCell)
{ }
}
8 changes: 4 additions & 4 deletions NBi.Core/FlatFile/CsvWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,11 @@ protected void WriteContent(DataTable table, TextWriter writer)
for (int i = 0; i < table.Columns.Count; i++)
{
var content = row[i].ToString() ?? string.Empty;
if (content.Contains(Definition.Descriptor.Delimiter) || content.Contains(Definition.Descriptor.LineTerminator))
content = $"{Definition.Descriptor.QuoteChar}{content}{Definition.Descriptor.QuoteChar}";
if (content.Contains(Definition.Dialect.Delimiter) || content.Contains(Definition.Dialect.LineTerminator))
content = $"{Definition.Dialect.QuoteChar}{content}{Definition.Dialect.QuoteChar}";

writer.Write(content);
writer.Write(i == table.Columns.Count - 1 ? Definition.Descriptor.LineTerminator : Definition.Descriptor.Delimiter);
writer.Write(i == table.Columns.Count - 1 ? Definition.Dialect.LineTerminator : Definition.Dialect.Delimiter);
}
}
}
Expand All @@ -72,7 +72,7 @@ protected void WriteHeader(DataTable table, TextWriter writer)
for (int i = 0; i < table.Columns.Count; i++)
{
writer.Write(table.Columns[i].ColumnName);
writer.Write(i == table.Columns.Count - 1 ? Definition.Descriptor.LineTerminator : Definition.Descriptor.Delimiter);
writer.Write(i == table.Columns.Count - 1 ? Definition.Dialect.LineTerminator : Definition.Dialect.Delimiter);
}
}
}
2 changes: 1 addition & 1 deletion NBi.Core/NBi.Core.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<PackageReference Include="NCalcSync" Version="3.10.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Ninject" Version="3.3.6" />
<PackageReference Include="PocketCsvReader" Version="2.0.0" />
<PackageReference Include="PocketCsvReader" Version="2.27.8" />
<PackageReference Include="RestSharp" Version="106.15.0" />
<PackageReference Include="System.Data.Odbc" Version="9.0.2" />
<PackageReference Include="System.Data.OleDb" Version="9.0.2" />
Expand Down
241 changes: 8 additions & 233 deletions NBi.Testing.Core/FlatFile/CsvReaderTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,232 +11,8 @@
namespace NBi.Core.Testing.FlatFile;

[TestFixture]
//[Ignore("PocketCsvReader not compatible with .NET CORE?")]
public class CsvReaderTest
{
class CsvReaderProxy : CsvReader
{
public CsvReaderProxy()
: base(new CsvProfile(false)) { }
public CsvReaderProxy(CsvProfile profile)
: base(profile) { }
public CsvReaderProxy(CsvProfile profile, int bufferSize)
: base(profile, bufferSize) { }
public CsvReaderProxy(int bufferSize)
: base(CsvProfile.SemiColumnDoubleQuote, bufferSize) { }

public new string? RemoveTextQualifier(string item, char textQualifier, char escapeTextQualifier)
=> base.RemoveTextQualifier(item, textQualifier, escapeTextQualifier);
public new IEnumerable<string?> SplitLine(string row, char fieldSeparator, char textQualifier, char escapeTextQualifier, string emptyCell)
=> base.SplitLine(row, fieldSeparator, textQualifier, escapeTextQualifier, emptyCell);
public new string GetFirstRecord(StreamReader reader, string recordSeparator, int bufferSize)
=> base.GetFirstRecord(reader, recordSeparator, bufferSize);
public new int IdentifyPartialRecordSeparator(string text, string recordSeparator)
=> base.IdentifyPartialRecordSeparator(text, recordSeparator);
public new string CleanRecord(string record, string recordSeparator)
=> base.CleanRecord(record, recordSeparator);
public new DataTable Read(Stream stream)
=> base.Read(stream);
public new DataTable Read(Stream stream, Encoding encoding, int encodingBytesCount, bool isFirstRowHeader, string recordSeparator, char fieldSeparator, char textQualifier, char escapeTextQualifier, char commentChar, string emptyCell, string missingCell)
=> base.Read(stream, encoding, encodingBytesCount, isFirstRowHeader, recordSeparator, fieldSeparator, textQualifier, escapeTextQualifier, commentChar, emptyCell, missingCell);
}

[Test]
[TestCase(null, "")]
[TestCase("(null)", null)] //Parse (null) to a real null value
[TestCase("\"(null)\"", "(null)")] //Explicitly quoted (null) should be (null)
[TestCase("null", "null")]
[TestCase("", "")]
[TestCase("a", "a")]
[TestCase("\"", "\"")]
[TestCase("\"a", "\"a")]
[TestCase("ab", "ab")]
[TestCase("\"ab\"", "ab")]
[TestCase("abc", "abc")]
[TestCase("\"abc\"", "abc")]
[TestCase("\"a\"\"b\"", "a\"b")]
[TestCase("\"\"\"a\"\"b\"\"\"", "\"a\"b\"")]
public void RemoveTextQualifier_String_CorrectString(string item, string result)
{
var reader = new CsvReaderProxy();
var value = reader.RemoveTextQualifier(item, '\"', '\"');
Assert.That(value, Is.EqualTo(result));
}

[Test]
public void SplitLine_Null_NotEmpty()
{
var reader = new CsvReaderProxy();
var values = reader.SplitLine("a;(null)", ';', char.MinValue, char.MinValue, string.Empty);
Assert.That(values.ElementAt(1), Is.Null);
}

[Test]
[TestCase("abc+abc+abc+abc", "+", 1)]
[TestCase("abc+abc+abc+abc", "+", 2)]
[TestCase("abc+abc+abc+abc", "+", 200)]
[TestCase("abc+@abc+@abc+@abc", "+@", 1)]
[TestCase("abc+@abc+@abc+@abc", "+@", 2)]
[TestCase("abc+@abc+@abc+@abc", "+@", 4)]
[TestCase("abc+@abc+@abc+@abc", "+@", 5)]
[TestCase("abc+@abc+@abc+@abc", "+@", 200)]
[TestCase("abc+@abc+abc+@abc", "+@", 1)]
[TestCase("abc+@abc+abc+@abc", "+@", 2)]
[TestCase("abc+@abc+abc+@abc", "+@", 4)]
[TestCase("abc+@abc+abc+@abc", "+@", 5)]
[TestCase("abc+@abc+abc+@abc", "+@", 200)]
[TestCase("abc+@abc+abc+@abc+@", "+@", 1)]
[TestCase("abc+@abc+abc+@abc+@", "+@", 2)]
[TestCase("abc+@abc+abc+@abc+@", "+@", 4)]
[TestCase("abc+@abc+abc+@abc+@", "+@", 5)]
[TestCase("abc+@abc+abc+@abc+@", "+@", 200)]
[TestCase("abc", "+@", 200)]
public void GetFirstRecord_Csv_CorrectResult(string text, string recordSeparator, int bufferSize)
{
using var stream = new MemoryStream();
var writer = new StreamWriter(stream);
writer.Write(text);
writer.Flush();

stream.Position = 0;

var reader = new CsvReaderProxy();
using (var streamReader = new StreamReader(stream, Encoding.UTF8, true))
{
var value = reader.GetFirstRecord(streamReader, recordSeparator, bufferSize);
Assert.That(value, Is.EqualTo("abc" + recordSeparator).Or.EqualTo("abc"));
}
writer.Dispose();
}

[Test]
[TestCase("abc+abc++abc+abc", "++", 1)]
public void GetFirstRecord_CsvWithSemiSeparator_CorrectResult(string text, string recordSeparator, int bufferSize)
{
using var stream = new MemoryStream();
var writer = new StreamWriter(stream);
writer.Write(text);
writer.Flush();

stream.Position = 0;

var reader = new CsvReaderProxy();
using (var streamReader = new StreamReader(stream, Encoding.UTF8, true))
{
var value = reader.GetFirstRecord(streamReader, recordSeparator, bufferSize);
Assert.That(value, Is.EqualTo("abc+abc" + recordSeparator).Or.EqualTo("abc+abc"));
}
writer.Dispose();
}

[Test]
[TestCase("a+b+c#a+b#a#a+b", '+', "#", "?")]
public void Read_CsvWithCsvProfileMissingCell_CorrectResults(string text, char fieldSeparator, string recordSeparator, string missingCell)
{
using var stream = new MemoryStream();
var writer = new StreamWriter(stream);
writer.Write(text);
writer.Flush();

stream.Position = 0;
var reader = new CsvReaderProxy();
var dataTable = reader.Read(stream, Encoding.UTF8, 0, false, recordSeparator, fieldSeparator, '\"', '\"', '!', "_", missingCell);

Assert.That(dataTable.Rows[0][0], Is.EqualTo("a"));
Assert.That(dataTable.Rows[0][1], Is.EqualTo("b"));
Assert.That(dataTable.Rows[0][2], Is.EqualTo("c"));

Assert.That(dataTable.Rows[1][0], Is.EqualTo("a"));
Assert.That(dataTable.Rows[1][1], Is.EqualTo("b"));
Assert.That(dataTable.Rows[1][2], Is.EqualTo("?"));

Assert.That(dataTable.Rows[2][0], Is.EqualTo("a"));
Assert.That(dataTable.Rows[2][1], Is.EqualTo("?"));
Assert.That(dataTable.Rows[2][2], Is.EqualTo("?"));

Assert.That(dataTable.Rows[3][0], Is.EqualTo("a"));
Assert.That(dataTable.Rows[3][1], Is.EqualTo("b"));
Assert.That(dataTable.Rows[3][2], Is.EqualTo("?"));

writer.Dispose();
}

[Test]
[TestCase("a+b+c#a++c#+b+c#+b+", '+', "#", "?")]
public void Read_CsvWithCsvProfileEmptyCell_CorrectResults(string text, char fieldSeparator, string recordSeparator, string emptyCell)
{
using var stream = new MemoryStream();
var writer = new StreamWriter(stream);
writer.Write(text);
writer.Flush();

stream.Position = 0;
var reader = new CsvReaderProxy();
var dataTable = reader.Read(stream, Encoding.UTF8, 0, false, recordSeparator, fieldSeparator, '\"', '\"', '!', emptyCell, "_");

Assert.That(dataTable.Rows[0][0], Is.EqualTo("a"));
Assert.That(dataTable.Rows[0][1], Is.EqualTo("b"));
Assert.That(dataTable.Rows[0][2], Is.EqualTo("c"));

Assert.That(dataTable.Rows[1][0], Is.EqualTo("a"));
Assert.That(dataTable.Rows[1][1], Is.EqualTo("?"));
Assert.That(dataTable.Rows[1][2], Is.EqualTo("c"));

Assert.That(dataTable.Rows[2][0], Is.EqualTo("?"));
Assert.That(dataTable.Rows[2][1], Is.EqualTo("b"));
Assert.That(dataTable.Rows[2][2], Is.EqualTo("c"));

Assert.That(dataTable.Rows[3][0], Is.EqualTo("?"));
Assert.That(dataTable.Rows[3][1], Is.EqualTo("b"));
Assert.That(dataTable.Rows[3][2], Is.EqualTo("?"));

writer.Dispose();
}

[Test]
[TestCase("abc\r\ndef\r\nghl\r\nijk", 1, 1)]
[TestCase("abc\r\ndef\r\nghl\r\nijk", 17, 1)]
[TestCase("abc\r\ndef\r\nghl\r\nijk", 18, 1)]
[TestCase("abc\r\ndef\r\nghl\r\nijk", 19, 1)]
[TestCase("abc\r\ndef\r\nghl\r\nijk", 512, 1)]
[TestCase("abc;xyz\r\ndef;xyz\r\nghl\r\n;ijk", 1, 2)]
[TestCase("abc;xyz\r\ndef;xyz\r\nghl\r\n;ijk", 512, 2)]
public void Read_Csv_CorrectResult(string text, int bufferSize, int columnCount)
{
using var stream = new MemoryStream();
var writer = new StreamWriter(stream);
writer.Write(text);
writer.Flush();

stream.Position = 0;

var reader = new CsvReaderProxy(new CsvProfile(';', '\"', "\r\n", false, false, "(empty)", "(null)"), bufferSize);
var dataTable = reader.Read(stream);
Assert.That(dataTable.Rows, Has.Count.EqualTo(4));
Assert.That(dataTable.Columns, Has.Count.EqualTo(columnCount));
foreach (DataRow row in dataTable.Rows)
{
foreach (var cell in row.ItemArray)
Assert.That(cell?.ToString(), Has.Length.EqualTo(3).Or.EqualTo("(empty)").Or.EqualTo("(null)"));
}
writer.Dispose();
}

[Test]
[TestCase("abc", "123", 0)]
[TestCase("abc1", "123", 1)]
[TestCase("abc12", "123", 2)]
[TestCase("abc12a", "123", 0)]
[TestCase("", "123", 0)]
[TestCase("", "#", 0)]
[TestCase("abc", "#", 0)]
public void IdentifyPartialRecordSeparator_Csv_CorrectResult(string text, string recordSeparator, int result)
{
var reader = new CsvReaderProxy(20);
var value = reader.IdentifyPartialRecordSeparator(text, recordSeparator);
Assert.That(value, Is.EqualTo(result));
}

[Test]
[TestCase("a;b;c\r\nd;e;f;g\r\n", 1, 1)]
[TestCase("a;b;c\r\nd;e;f\r\ng;h;i;j\r\n", 2, 1)]
Expand All @@ -251,11 +27,11 @@ public void Read_MoreFieldThanExpected_ExceptionMessage(string text, int rowNumb
stream.Position = 0;

var profile = CsvProfile.SemiColumnDoubleQuote;
var reader = new CsvReaderProxy(profile);
var reader = new CsvReader(profile);

var ex = Assert.Throws<InvalidDataException>(delegate { reader.Read(stream); });
Assert.That(ex!.Message, Does.Contain(string.Format("record {0} ", rowNumber + 1)));
Assert.That(ex.Message, Does.Contain(string.Format("{0} more", moreField)));
var ex = Assert.Throws<InvalidDataException>(delegate { reader.ToDataTable(stream); });
Assert.That(ex!.Message, Does.Contain($"record {rowNumber + 1} "));
Assert.That(ex.Message, Does.Contain($"{moreField} more"));
}

[Test]
Expand All @@ -269,8 +45,8 @@ public void Read_EmptyValue_MatchWithEmpty()
stream.Position = 0;

var profile = CsvProfile.SemiColumnDoubleQuote;
var reader = new CsvReaderProxy(profile);
var dataTable = reader.Read(stream);
var reader = new CsvReader(profile);
var dataTable = reader.ToDataTable(stream);
Assert.That(dataTable.Rows[0][1], Is.EqualTo(string.Empty).Or.EqualTo("(empty)"));
}

Expand All @@ -281,12 +57,11 @@ public void Read_MissingValue_MatchWithNullValue()
using var writer = new StreamWriter(stream);
writer.Write("a;b;c\r\na;b\r\na;b;c");
writer.Flush();

stream.Position = 0;

var profile = CsvProfile.SemiColumnDoubleQuote;
var reader = new CsvReaderProxy(profile);
var dataTable = reader.Read(stream);
var reader = new CsvReader(profile);
var dataTable = reader.ToDataTable(stream);
Assert.That(dataTable.Rows[1][2], Is.EqualTo("(null)"));
}
}
Loading

0 comments on commit 9d1d255

Please sign in to comment.