Skip to content

Commit 2b8ddbf

Browse files
cculianucculianu
committed
Added basic unit test for class RecordFile
Can be accessed by compiling with -DENABLE_TEST and passing `--test recordifle` on the CLI.
1 parent 587cf4c commit 2b8ddbf

File tree

1 file changed

+200
-0
lines changed

1 file changed

+200
-0
lines changed

‎src/RecordFile.cpp

+200
Original file line numberDiff line numberDiff line change
@@ -312,3 +312,203 @@ RecordFile::BatchAppendContext::~BatchAppendContext()
312312
if (!errStr.isEmpty())
313313
Fatal() << errStr; // app will quit in main event loop after printing error.
314314
}
315+
316+
#ifdef ENABLE_TESTS
317+
#include "App.h"
318+
#include <QTemporaryFile>
319+
#include <algorithm>
320+
#include <atomic>
321+
#include <cstddef>
322+
#include <cstring>
323+
#include <thread>
324+
#include <type_traits>
325+
#include <vector>
326+
327+
namespace {
328+
void testRecordFile() {
329+
size_t nChecksOK = 0;
330+
331+
const auto fileName = []{
332+
QTemporaryFile tmp(APPNAME "_XXXXXX.tmp");
333+
tmp.open();
334+
auto ret = tmp.fileName();
335+
tmp.setAutoRemove(false); // keep file around so we can pass it to RecordFile instance
336+
return ret;
337+
}();
338+
constexpr size_t HashLen = 32;
339+
constexpr size_t N = 100'000; // 100k items
340+
Log() << "Testing Recordfile \"" << fileName << "\" with " << N << " " << HashLen << "-byte random records ...";
341+
// delete tmp file at scope end
342+
Defer d([&fileName] {
343+
QFile::remove(fileName);
344+
Log() << "Temporary file: \"" << fileName << "\" deleted";
345+
});
346+
347+
Tic t0;
348+
std::vector<QByteArray> hashes(N, QByteArray(HashLen, Qt::Uninitialized));
349+
// randomize hashes
350+
QByteArray *lastH = nullptr;
351+
for (auto & h : hashes) {
352+
Util::getRandomBytes(h.data(), h.size());
353+
if (lastH && *lastH == h)
354+
throw Exception("Something went wrong generating random hashes.. previous hash and this hash match!");
355+
lastH = &h;
356+
}
357+
Log() << "Generated " << hashes.size() << " random hahses in " << t0.msecStr() << " msec";
358+
{
359+
t0 = Tic();
360+
RecordFile f(fileName, HashLen);
361+
auto batch = f.beginBatchAppend();
362+
QString err;
363+
for (const auto & h : hashes)
364+
if (!batch.append(h, &err))
365+
throw Exception(QString("Failed to append a record using batch append to RecordFile: %1").arg(err));
366+
Log() << "Wrote " << f.numRecords() << " hahses in " << t0.msecStr() << " msec";
367+
}
368+
{
369+
t0 = Tic();
370+
// Read 1 record at a time randomly from 3 threads concurrently
371+
std::vector<std::thread> thrds;
372+
RecordFile f(fileName, HashLen);
373+
if (f.numRecords() != N) throw Exception("RecordFile has wrong number of records!");
374+
std::atomic_size_t ctr{0};
375+
QString fail_shared;
376+
std::mutex fail_shared_mut;
377+
for (size_t i = 0; i < 3; ++i) {
378+
thrds.emplace_back([&]{
379+
QString fail;
380+
for (size_t i = 0; fail.isEmpty() && i < hashes.size() * 7 / 20; ++i) {
381+
size_t idx;
382+
Util::getRandomBytes(reinterpret_cast<std::byte *>(&idx), sizeof(idx));
383+
idx = idx % hashes.size();
384+
if (f.readRecord(idx, &fail) != hashes[idx]) {
385+
fail = QString("Failed to read an individual record at position %1 correctly: %2").arg(idx).arg(fail);
386+
break;
387+
}
388+
++ctr;
389+
}
390+
if (!fail.isEmpty()) {
391+
std::unique_lock l(fail_shared_mut);
392+
fail_shared = fail;
393+
}
394+
});
395+
}
396+
for (auto & t : thrds) t.join();
397+
if (!fail_shared.isEmpty()) throw Exception(fail_shared);
398+
Log() << "Read " << ctr.load() << " individual records randomly using " << thrds.size() << " concurrent threads in "
399+
<< t0.msecStr() << " msec";
400+
++nChecksOK;
401+
}
402+
{
403+
t0 = Tic();
404+
// Read 1000 records at a time randomly from 3 threads concurrently
405+
std::vector<std::thread> thrds;
406+
RecordFile f(fileName, HashLen);
407+
if (f.numRecords() != N) throw Exception("RecordFile has wrong number of records!");
408+
QString fail;
409+
std::atomic_size_t ctr{0};
410+
QString fail_shared;
411+
std::mutex fail_shared_mut;
412+
for (size_t i = 0; i < 3; ++i) {
413+
thrds.emplace_back([&]{
414+
QString fail;
415+
constexpr size_t NBatch = 1000;
416+
for (size_t i = 0; fail.isEmpty() && i < hashes.size() * 2; i += NBatch) {
417+
std::vector<uint64_t> recNums(NBatch);
418+
for (size_t j = 0; fail.isEmpty() && j < NBatch; ++j) {
419+
size_t idx;
420+
Util::getRandomBytes(reinterpret_cast<std::byte *>(&idx), sizeof(idx));
421+
idx = idx % N;
422+
recNums[j] = idx;
423+
++ctr;
424+
}
425+
const auto results = f.readRandomRecords(recNums, &fail, false);
426+
if (results.size() != recNums.size())
427+
fail = QString("Failed to read random records: ") + fail;
428+
for (size_t i = 0; fail.isEmpty() && i < results.size(); ++i) {
429+
if (results[i] != hashes[recNums[i]])
430+
fail = QString("Record #%1, index %2 failed to compare equal!").arg(i).arg(recNums[i]);
431+
}
432+
}
433+
if (!fail.isEmpty()) {
434+
std::unique_lock l(fail_shared_mut);
435+
fail_shared = fail;
436+
}
437+
});
438+
}
439+
for (auto & t : thrds) t.join();
440+
if (!fail.isEmpty()) throw Exception(fail);
441+
Log() << "Read " << ctr.load() << " batched records randomly using " << thrds.size() << " concurrent threads in "
442+
<< t0.msecStr() << " msec";
443+
++nChecksOK;
444+
}
445+
{
446+
t0 = Tic();
447+
// truncate to N / 2, and verify
448+
{
449+
RecordFile f(fileName, HashLen);
450+
f.truncate(hashes.size() / 2);
451+
}
452+
RecordFile f(fileName, HashLen);
453+
if (f.numRecords() != hashes.size() / 2) throw Exception("Trunace failed");
454+
QString fail;
455+
const auto results = f.readRecords(0, hashes.size() / 2, &fail);
456+
if (!fail.isEmpty() || results.size() != hashes.size() / 2) throw Exception(QString("Failed to verify truncated data: %1").arg(fail));
457+
for (size_t i = 0; i < results.size(); ++i)
458+
if (results[i] != hashes[i])
459+
throw Exception(QString("After truncation, record %1 no longer compares equal!").arg(i));
460+
Log() << "Truncated file to size " << f.numRecords() << " and verified in "<< t0.msecStr() << " msec";
461+
++nChecksOK;
462+
}
463+
{
464+
t0 = Tic();
465+
// truncate the file to 0, then write N / 10 records using single-append calls and verify
466+
{
467+
RecordFile f(fileName, HashLen);
468+
f.truncate(0);
469+
}
470+
RecordFile f(fileName, HashLen);
471+
if (f.numRecords() != 0) throw Exception("Failed to truncate file to 0");
472+
const auto NN = hashes.size() / 10;
473+
for (size_t i = 0; i < NN; ++i) {
474+
QString err;
475+
const auto res = f.appendRecord(hashes[i], true, &err);
476+
if (!res || *res != i || !err.isEmpty() || f.numRecords() != i + 1)
477+
throw Exception(QString("Failed to append record %1: %2").arg(i).arg(err));
478+
}
479+
QString fail;
480+
const auto results = f.readRecords(0, NN, &fail);
481+
if (!fail.isEmpty() || results.size() != NN) throw Exception(QString("Failed to verify truncated data: %1").arg(fail));
482+
for (size_t i = 0; i < results.size(); ++i)
483+
if (results[i] != hashes[i])
484+
throw Exception(QString("After truncation, record %1 no longer compares equal!").arg(i));
485+
Log() << "Truncated file to size 0, appended using single-append calls to size " << f.numRecords() << ", and verified in "<< t0.msecStr() << " msec";
486+
++nChecksOK;
487+
}
488+
{
489+
// try mismatch on recSz
490+
static_assert (!std::is_base_of_v<RecordFile::FileFormatError, Exception>); // to ensure below works.. this is obviously always the case
491+
try {
492+
RecordFile f(fileName, HashLen + 1 /* bad recsz */);
493+
throw Exception("Failed to catch expected exception!");
494+
} catch (const RecordFile::FileFormatError &e) {
495+
Log() << "Got expected exception: \"" << e.what() << "\" ok";
496+
}
497+
++nChecksOK;
498+
}
499+
{
500+
// try mismatch on magic
501+
static_assert (!std::is_base_of_v<RecordFile::FileFormatError, Exception>); // to ensure below works.. this is obviously always the case
502+
try {
503+
RecordFile f(fileName, HashLen, 0x01020304 /* bad magic */);
504+
throw Exception("Failed to catch expected exception!");
505+
} catch (const RecordFile::FileFormatError &e) {
506+
Log() << "Got expected exception: \"" << e.what() << "\" ok";
507+
}
508+
++nChecksOK;
509+
}
510+
Log() << nChecksOK << " RecordFile checks passed ok";
511+
}
512+
const auto test = App::registerTest("recordfile", testRecordFile);
513+
}
514+
#endif

0 commit comments

Comments
 (0)