Skip to content

Commit

Permalink
Merge pull request #284 from fiskaltrust/#281-Swissbit-out-of-memory
Browse files Browse the repository at this point in the history 
#281 Export tse out of memory
  • Loading branch information
@forsthug
forsthug authored May 16, 2024
2 parents 493be65 + 4c5a059 commit 05dfad4
Show file tree
Hide file tree
Showing 5 changed files with 60 additions and 100 deletions.
77 changes: 47 additions & 30 deletions queue/src/fiskaltrust.Middleware.Localization.QueueDE/JournalProcessorDE.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ public async IAsyncEnumerable<JournalResponse> ProcessAsync(JournalRequest reque
_logger.LogDebug($"Processing JournalRequest for DE (Type: {request.ftJournalType:X}");
if (request.ftJournalType == (long) JournalTypes.TarExportFromTSE)
{
if (request.MaxChunkSize == 0)
{
request.MaxChunkSize = _middlewareConfiguration.TarFileChunkSize;
}
await foreach (var value in ProcessTarExportFromTSEAsync(request).ConfigureAwait(false))
{
yield return value;
Expand Down Expand Up @@ -151,42 +155,55 @@ private async IAsyncEnumerable<JournalResponse> ProcessTarExportFromTSEAsync(Jou
{
var exportSession = await _deSSCDProvider.Instance.StartExportSessionAsync(new StartExportSessionRequest()).ConfigureAwait(false);
var sha256CheckSum = "";
using (var memoryStream = new MemoryStream())

byte[] chunk;
var response = new JournalResponse();
try
{
ExportDataResponse export;
do
using (var stream = new FileStream(exportSession.TokenId, FileMode.Create, FileAccess.ReadWrite))
{
export = await _deSSCDProvider.Instance.ExportDataAsync(new ExportDataRequest
{
TokenId = exportSession.TokenId,
MaxChunkSize = request.MaxChunkSize
}).ConfigureAwait(false);
if (!export.TotalTarFileSizeAvailable)
ExportDataResponse export;
do
{
await Task.Delay(TimeSpan.FromMilliseconds(100)).ConfigureAwait(false);
}
else
{
var chunk = Convert.FromBase64String(export.TarFileByteChunkBase64);
memoryStream.Write(chunk, 0, chunk.Length);
yield return new JournalResponse
export = await _deSSCDProvider.Instance.ExportDataAsync(new ExportDataRequest
{
Chunk = chunk.ToList()
};
}
} while (!export.TarFileEndOfFile);
sha256CheckSum = Convert.ToBase64String(SHA256.Create().ComputeHash(memoryStream.ToArray()));
}
TokenId = exportSession.TokenId,
MaxChunkSize = request.MaxChunkSize
}).ConfigureAwait(false);
if (!export.TotalTarFileSizeAvailable)
{
await Task.Delay(TimeSpan.FromMilliseconds(100)).ConfigureAwait(false);
}
else
{
chunk = Convert.FromBase64String(export.TarFileByteChunkBase64);
stream.Write(chunk, 0, chunk.Length);
response.Chunk = chunk.ToList();
yield return response;
}
} while (!export.TarFileEndOfFile);
using var sha256 = SHA256.Create();
stream.Position = 0;
sha256CheckSum = Convert.ToBase64String(sha256.ComputeHash(stream));
}

var endSessionRequest = new EndExportSessionRequest
{
TokenId = exportSession.TokenId,
Sha256ChecksumBase64 = sha256CheckSum
};
var endExportSessionResult = await _deSSCDProvider.Instance.EndExportSessionAsync(endSessionRequest).ConfigureAwait(false);
if (!endExportSessionResult.IsValid)
var endSessionRequest = new EndExportSessionRequest
{
TokenId = exportSession.TokenId,
Sha256ChecksumBase64 = sha256CheckSum
};
var endExportSessionResult = await _deSSCDProvider.Instance.EndExportSessionAsync(endSessionRequest).ConfigureAwait(false);
if (!endExportSessionResult.IsValid)
{
throw new Exception("The TAR file export was not successful.");
}
}
finally
{
throw new Exception("The TAR file export was not successful.");
if (File.Exists(exportSession.TokenId))
{
File.Delete(exportSession.TokenId);
}
}
yield break;
}
Expand Down
13 changes: 8 additions & 5 deletions queue/src/fiskaltrust.Middleware.Localization.QueueDE/Services/TarFileExportService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public class TarFileExportService
Directory.CreateDirectory(targetDirectory);
var filePath = Path.Combine(targetDirectory, $"{DateTime.Now:yyyyMMddhhmmssfff}_{cashboxIdentification.RemoveInvalidFilenameChars()}.tar");


var sha256CheckSum = string.Empty;
logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section ExportDataAsync [enter].");
using (var fileStream = File.Create(filePath))
{
Expand All @@ -46,13 +46,15 @@ public class TarFileExportService
logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section Convert.FromBase64String [exit].");
}
} while (!export.TarFileEndOfFile);

logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section Sha256ChecksumBase64 [enter].");
using var sha256 = SHA256.Create();
fileStream.Position = 0;
sha256CheckSum = Convert.ToBase64String(sha256.ComputeHash(fileStream));
logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section Sha256ChecksumBase64 [exit].");
}
logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section ExportDataAsync [exit].");

logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section Sha256ChecksumBase64 [enter].");
using var sha256 = SHA256.Create();
var sha256CheckSum = Convert.ToBase64String(sha256.ComputeHash(File.ReadAllBytes(filePath)));
logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync Section Sha256ChecksumBase64 [exit].");
var endSessionRequest = new EndExportSessionRequest
{
TokenId = exportSession.TokenId,
Expand All @@ -64,6 +66,7 @@ public class TarFileExportService
{
return (null, false, null, false);
}

logger.LogTrace("TarFileExportService.ProcessTarFileExportAsync [exit].");
return (filePath, true, sha256CheckSum, endExportSessionResult.IsErased);
}
Expand Down
7 changes: 4 additions & 3 deletions scu-de/src/fiskaltrust.Middleware.SCU.DE.Swissbit/Interop/ISwissbitProxy.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using fiskaltrust.ifPOS.v1.de;
using fiskaltrust.Middleware.SCU.DE.Swissbit.Models;
Expand Down Expand Up @@ -34,9 +35,9 @@ public interface ISwissbitProxy : IDisposable
public Task<TransactionResponse> TransactionUpdateAsync(string clientId, UInt64 transactionNumber, byte[] processData, string processType);
public Task<TransactionResponse> TransactionFinishAsync(string clientId, UInt64 transactionNumber, byte[] processData, string processType);
public Task<List<ulong>> GetStartedTransactionsAsync(string clientId);
public Task ExportTarAsync(System.IO.Stream stream);
public Task ExportTarFilteredTimeAsync(System.IO.Stream stream, UInt64 startDateUnixTime, UInt64 endDateUnixTime, string clientId);
public Task ExportTarFilteredTransactionAsync(System.IO.Stream stream, UInt64 startTransactionNumber, UInt64 endTransactionNumber, string clientId);
public Task ExportTarAsync(Stream stream);
public Task ExportTarFilteredTimeAsync(Stream stream, UInt64 startDateUnixTime, UInt64 endDateUnixTime, string clientId);
public Task ExportTarFilteredTransactionAsync(Stream stream, UInt64 startTransactionNumber, UInt64 endTransactionNumber, string clientId);
public Task<byte[]> GetLogMessageCertificateAsync();
public Task DeleteStoredDataAsync();
}
Expand Down
62 changes: 1 addition & 61 deletions scu-de/src/fiskaltrust.Middleware.SCU.DE.Swissbit/SwissbitSCU.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ public sealed class SwissbitSCU : IDESSCD, IDisposable
private uint _hwSelftestIntervalSeconds = 0;
private ISwissbitProxy _proxy = null;
private DateTime _nextSyncTime;
private ulong _lastTransactionNr = 0;

// Never change these values, as all existing installations are depending on them
private readonly byte[] _adminPuk = Encoding.ASCII.GetBytes("123456");
Expand Down Expand Up @@ -601,7 +600,6 @@ public async Task<FinishTransactionResponse> FinishTransactionAsync(FinishTransa
SignatureCounter = tseResponse.SignatureCounter
}
};
_lastTransactionNr = tseResponse.TransactionNumber;
return response;
}
Expand Down Expand Up @@ -743,19 +741,7 @@ public async Task<StartExportSessionResponse> StartExportSessionAsync(StartExpor
await UpdateTimeAsync(GetProxy());
SetExportState(exportId, ExportState.Running);

if (_configuration.ChunkExportTransactionCount > 0)
{
if (_lastTransactionNr == 0)
{
_logger.LogError("Before executing a partial export a daily closing has to be made. See https://link.fiskaltrust.cloud/market-de/swissbit-chunked-tar-export for more details.");
throw new Exception("Missing Daily Closing.");
};
CacheExportIncrementalAsync(exportId, 0, _configuration.ChunkExportTransactionCount, (long) _lastTransactionNr).FireAndForget();
}
else
{
CacheExportAsync(exportId, request.ClientId, request.Erase).FireAndForget();
}
CacheExportAsync(exportId, request.ClientId, request.Erase).FireAndForget();

return new StartExportSessionResponse()
{
Expand Down Expand Up @@ -801,52 +787,6 @@ await _lockingHelper.PerformWithLock(_hwLock, async () =>
});
}

private async Task CacheExportIncrementalAsync(Guid exportId, long startTransactionNr, long maxNumberOfRecords, long currentNumberOfTransactions)
{
var endTransactionNr = LastNumberOfRecords(startTransactionNr, maxNumberOfRecords, currentNumberOfTransactions);
long newStartTransactionNr = -1;

await _lockingHelper.PerformWithLock(_hwLock, async () =>
{
try
{
using (var stream = new MemoryStream())
{
await GetProxy().ExportTarFilteredTransactionAsync(stream, (ulong) startTransactionNr, (ulong) endTransactionNr, null);
stream.Position = 0;
TarFileHelper.AppendTarStreamToTarFile(exportId.ToString(), stream);
_logger.LogInformation($"Export total {currentNumberOfTransactions}. Partial Export from TransactionNr: {startTransactionNr} to TransactionNr: {endTransactionNr}");
}
newStartTransactionNr = GetLastExportedTransaction(exportId.ToString()) + 1;
}
catch (Exception ex)
{
if (!ex.Message.Equals("Filtered Export: no matching entries, export would be empty. "))
{
_logger.LogError(ex, "Failed to execute {Operation} - TempFileName: {TempFileName}", nameof(CacheExportIncrementalAsync), exportId.ToString());
SetExportState(exportId, ExportState.Failed, ex);
throw;
}
_logger.LogInformation($"No Data: Export total {currentNumberOfTransactions}. Partial Export from TransactionNr: {startTransactionNr} to TransactionNr: {endTransactionNr}");
}
});

if (newStartTransactionNr == -1)
{
newStartTransactionNr = startTransactionNr + maxNumberOfRecords;
}
if (newStartTransactionNr <= currentNumberOfTransactions)
{
await CacheExportIncrementalAsync(exportId, newStartTransactionNr, maxNumberOfRecords, currentNumberOfTransactions).ConfigureAwait(false);
}
else
{
TarFileHelper.FinalizeTarFile(exportId.ToString());
_logger.LogDebug("Finalized merged TAR file {fileName}.", exportId.ToString());
SetExportState(exportId, ExportState.Succeeded);
}
}

private long GetLastExportedTransaction(string targetFile)
{
Expand Down
1 change: 0 additions & 1 deletion scu-de/src/fiskaltrust.Middleware.SCU.DE.Swissbit/SwissbitSCUConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ public class SwissbitSCUConfiguration
public int TooLargeToExportThreshold { get; set; } = 100 * 1024 * 1024; // 100 MB
public bool EnableFirmwareUpdate { get; set; } = false;
public string NativeLibArch { get; set; }
public long ChunkExportTransactionCount { get; set; } = 0;
public bool StoreTemporaryExportFiles { get; set; } = false;
}
}

0 comments on commit 05dfad4

Please sign in to comment.