Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
547 changes: 547 additions & 0 deletions docs/design_batch_cell_typed_channel.md

Large diffs are not rendered by default.

45 changes: 45 additions & 0 deletions src/App/Cli/CellData.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
namespace Refedle.App.Cli;

/// <summary>
/// Whether a CLI batch cell carries a usable value and, when it does not, why
/// (explicit null, an absent property, or an unreadable source). Replaces the
/// former "&lt;null&gt;"/"&lt;error&gt;" string sentinels with an explicit signal.
/// </summary>
internal enum CellPresence
{
Value,
Null,
Missing,
Invalid,
}

/// <summary>
/// How an <see cref="IRecordWriter"/> must serialize a <see cref="CellData"/>'s
/// <see cref="CellData.Value"/> — a purely syntactic decision, independent of the
/// cell's domain type (ColumnType is not involved anywhere in the batch path).
/// </summary>
internal enum CellEncoding
{
PlainText,
Raw,
Numeric,
Boolean,
}

/// <summary>
/// A single CLI batch cell: its text plus the presence/encoding signals the
/// writer needs to emit it correctly. Passed between <see cref="IRecordReader"/>
/// and <see cref="IRecordWriter"/> in place of the former bare
/// <see cref="ReadOnlySpan{T}"/> of <see langword="char"/>.
/// </summary>
internal readonly ref struct CellData(
ReadOnlySpan<char> value,
CellPresence presence,
CellEncoding encoding = CellEncoding.PlainText)
{
public ReadOnlySpan<char> Value { get; } = value;

public CellPresence Presence { get; } = presence;

public CellEncoding Encoding { get; } = encoding;
}
35 changes: 35 additions & 0 deletions src/App/Cli/CellEncodingClassifier.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
using System.Globalization;

namespace Refedle.App.Cli;

/// <summary>
/// Shared text-to-<see cref="CellEncoding"/> heuristic used by CSV reads and by
/// transformed-column output. Mirrors the historical <c>WriteJsonValue</c>
/// detection order (bool → long → double → text) so CSV→JSON Lines and
/// transformed-column output stay byte-for-byte unchanged.
/// </summary>
internal static class CellEncodingClassifier
{
/// <summary>
/// Classifies plain cell text into the encoding the JSON Lines writer needs.
/// </summary>
public static CellEncoding Classify(ReadOnlySpan<char> value)
{
if (bool.TryParse(value, out _))
{
return CellEncoding.Boolean;
}

if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out _))
{
return CellEncoding.Numeric;
}

if (double.TryParse(value, NumberStyles.Any, CultureInfo.InvariantCulture, out _))
{
return CellEncoding.Numeric;
}

return CellEncoding.PlainText;
}
}
12 changes: 4 additions & 8 deletions src/App/Cli/CsvRecordReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -60,21 +60,17 @@ public readonly bool EvaluateFilters()
return FilterEvaluator.EvaluateCsvFilters(_reader.Current, _filters);
}

public readonly ReadOnlySpan<char> GetCellSpan(int outputColumnIndex)
public readonly CellData GetCellData(int outputColumnIndex)
{
ThrowIfDisposed();
if (_reader is null)
{
return [];
return new CellData([], CellPresence.Value);
}

var sourceIndex = _outputToSourceIndexMap[outputColumnIndex];
if (sourceIndex >= 0 && sourceIndex < _reader.Current.ColCount)
{
return _reader.Current[sourceIndex].Span;
}

return [];
var value = sourceIndex < 0 ? [] : _reader.Current[sourceIndex].Span;
return new CellData(value, CellPresence.Value, CellEncodingClassifier.Classify(value));
}

public void Dispose()
Expand Down
12 changes: 5 additions & 7 deletions src/App/Cli/CsvRecordWriter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -48,22 +48,20 @@ public readonly ValueTask WriteStartRecordAsync(CancellationToken ct)
return default;
}

public readonly void WriteCellSpan(int outputColumnIndex, ReadOnlySpan<char> value)
public readonly void WriteCellData(int outputColumnIndex, CellData cell)
{
ThrowIfDisposed();
if (outputColumnIndex > 0)
{
_sb.Append(',');
}

if (value.SequenceEqual("<null>") || value.SequenceEqual("<error>"))
if (cell.Presence != CellPresence.Value)
{
// Empty
}
else if (value.Length > 0)
{
CsvEscaper.EscapeCsvValueToBuilder(value, _sb);
return;
}

CsvEscaper.EscapeCsvValueToBuilder(cell.Value, _sb);
}

public async readonly ValueTask WriteEndRecordAsync(CancellationToken ct)
Expand Down
2 changes: 1 addition & 1 deletion src/App/Cli/IRecordReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,5 @@ internal interface IRecordReader : IDisposable
{
ValueTask<bool> MoveNextAsync(CancellationToken ct);
bool EvaluateFilters();
ReadOnlySpan<char> GetCellSpan(int outputColumnIndex);
CellData GetCellData(int outputColumnIndex);
}
2 changes: 1 addition & 1 deletion src/App/Cli/IRecordWriter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ internal interface IRecordWriter : IDisposable, IAsyncDisposable
{
ValueTask WriteHeaderAsync(CancellationToken ct);
ValueTask WriteStartRecordAsync(CancellationToken ct);
void WriteCellSpan(int outputColumnIndex, ReadOnlySpan<char> value);
void WriteCellData(int outputColumnIndex, CellData cell);
ValueTask WriteEndRecordAsync(CancellationToken ct);
ValueTask FlushAsync(CancellationToken ct);
}
73 changes: 69 additions & 4 deletions src/App/Cli/JsonLinesRecordReader.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Text;
using System.Text.Json;
using Refedle.Engine;
using Refedle.Engine.IO.Json;
using Refedle.Engine.IO.JsonLines;
Expand Down Expand Up @@ -88,12 +89,76 @@ public readonly bool EvaluateFilters()
return FilterEvaluator.EvaluateJsonFilters(_currentLineBytes, _filters, _filterIndexToNameBytes);
}

public readonly ReadOnlySpan<char> GetCellSpan(int outputColumnIndex)
public readonly CellData GetCellData(int outputColumnIndex)
{
ThrowIfDisposed();
var columnNameSpan = _columnNameUtf8Bytes[outputColumnIndex].Span;
var value = JsonObjectCellExtractor.ExtractCell(_currentLineBytes.Span, columnNameSpan);
return value.AsSpan();

var columnNameUtf8 = _columnNameUtf8Bytes[outputColumnIndex].Span;

try
{
var reader = new Utf8JsonReader(_currentLineBytes.Span);

if (!reader.Read() || reader.TokenType != JsonTokenType.StartObject)
{
return new CellData([], CellPresence.Invalid);
}

while (reader.Read())
{
if (reader.TokenType == JsonTokenType.EndObject)
{
break;
}

if (reader.TokenType != JsonTokenType.PropertyName)
{
continue;
}

if (!reader.ValueTextEquals(columnNameUtf8))
{
reader.Skip();
continue;
}

if (!reader.Read())
{
return new CellData([], CellPresence.Invalid);
}

return ReadPropertyValue(reader, _currentLineBytes);
}

return new CellData([], CellPresence.Missing);
}
catch (JsonException)
{
return new CellData([], CellPresence.Invalid);
}
}

// Split out to stay under the Sonar cyclomatic-complexity limit (S1541). Passed by value,
// not by ref, so it owns a copy isolated from the caller's state (ref also fails to
// compile: CS8168/CS8347); the resulting small, stack-only copy per call is an accepted cost.
private static CellData ReadPropertyValue(Utf8JsonReader reader, JsonRawBytes containingBytes)
{
return reader.TokenType switch
{
JsonTokenType.Null => new CellData([], CellPresence.Null),
JsonTokenType.Number =>
new CellData(Encoding.UTF8.GetString(reader.ValueSpan), CellPresence.Value, CellEncoding.Raw),
JsonTokenType.StartObject or JsonTokenType.StartArray =>
new CellData(
Encoding.UTF8.GetString(JsonByteExtractor.ExtractValueBytes(ref reader, containingBytes).Span),
CellPresence.Value,
CellEncoding.Raw),
JsonTokenType.String =>
new CellData(reader.GetString(), CellPresence.Value, CellEncoding.PlainText),
JsonTokenType.True => new CellData("true", CellPresence.Value, CellEncoding.Boolean),
JsonTokenType.False => new CellData("false", CellPresence.Value, CellEncoding.Boolean),
_ => new CellData([], CellPresence.Invalid),
};
}

public void Dispose()
Expand Down
Loading