Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 87 additions & 0 deletions diagnostic/build-9fcca4fe.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
{
"generated_at": "2026-06-23T14:51:57.011749+00:00",
"commit": "9fcca4fe",
"diagnostic_logd": "diagnostic/build-9fcca4fe.logd",
"diagnostic_logd_error": null,
"message_blocker": null,
"chunked": false,
"chunk_size_bytes": null,
"password": "f823830b8cca10c76a0e",
"decrypt_command": "encryptly unpack diagnostic/build-9fcca4fe.logd <outdir> --password f823830b8cca10c76a0e",
"total_modules": 10,
"passed": 3,
"failed": 7,
"modules": [
{
"name": "backend",
"status": "FAIL",
"elapsed_seconds": 0,
"artifact": null,
"output": "Command not found:\ncwd: backend\ncommand: cargo build\nexit_code: command-not-started\nerror: [Errno 2] No such file or directory: 'cargo'"
},
{
"name": "frontend",
"status": "PASS",
"elapsed_seconds": 129.762,
"artifact": "/mnt/e/project/bounty_repos/TentOfTrials/frontend/dist",
"output": "cwd: frontend\ncommand: npm run build\nexit_code: 0\n--- stdout ---\n> tent-frontend@0.0.0 build\n> tsc -b && vite build\n\nvite v6.4.3 building for production...\ntransforming...\n\u2713 100 modules transformed.\nrendering chunks...\ncomputing gzip size...\ndist/index.html 0.63 kB \u2502 gzip: 0.35 kB\ndist/assets/state-BkjSKDbY.js 8.91 kB \u2502 gzip: 3.54 kB \u2502 map: 57.15 kB\ndist/assets/vendor-CREcWLHI.js 48.93 kB \u2502 gzip: 17.25 kB \u2502 map: 481.27 kB\ndist/assets/index-CyxcoTyU.js 231.32 kB \u2502 gzip: 72.16 kB \u2502 map: 1,045.57 kB\n\u2713 built in 4.39s"
},
{
"name": "market",
"status": "FAIL",
"elapsed_seconds": 0,
"artifact": null,
"output": "Command not found:\ncwd: market\ncommand: go build -o market .\nexit_code: command-not-started\nerror: [Errno 2] No such file or directory: 'go'"
},
{
"name": "frailbox",
"status": "PASS",
"elapsed_seconds": 4.027,
"artifact": "/mnt/e/project/bounty_repos/TentOfTrials/frailbox/frailbox",
"output": "cwd: frailbox\ncommand: make\nexit_code: 0\n--- stdout ---\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c src/arena.c -o build/src/arena.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c src/sandbox.c -o build/src/sandbox.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude -MMD -MP -c main.c -o build/main.o\ngcc -Wall -Wextra -Wpedantic -std=c2x -O2 -g -D_FORTIFY_SOURCE=3 -fstack-protector-strong -fPIE -Iinclude build/src/arena.o build/src/logger.o build/src/sandbox.o build/main.o -o frailbox -pie -z relro -z now\n--- stderr ---\nsrc/arena.c: In function \u2018arena_contains\u2019:\nsrc/arena.c:179:17: warning: comparison of distinct pointer types lacks a cast\n 179 | ptr < (char *)region->start + region->size) {\n | ^"
},
{
"name": "engine",
"status": "PASS",
"elapsed_seconds": 44.621,
"artifact": "/mnt/e/project/bounty_repos/TentOfTrials/frailbox/engine/build/trial-engine",
"output": "cwd: frailbox/engine\ncommand: cmake --build build\nexit_code: 0\n--- stdout ---\n[ 11%] Building CXX object CMakeFiles/trial-engine.dir/main.cpp.o\n[ 22%] Building CXX object CMakeFiles/trial-engine.dir/core/math.cpp.o\n[ 33%] Building CXX object CMakeFiles/trial-engine.dir/core/ecs.cpp.o\n[ 44%] Building CXX object CMakeFiles/trial-engine.dir/dynamics/rigidbody.cpp.o\n[ 55%] Building CXX object CMakeFiles/trial-engine.dir/dynamics/constraint.cpp.o\n[ 66%] Building CXX object CMakeFiles/trial-engine.dir/collision/collision.cpp.o\n[ 77%] Building CXX object CMakeFiles/trial-engine.dir/mnt/e/project/bounty_repos/TentOfTrials/frailbox/wat.cpp.o\n[ 88%] Building CXX object CMakeFiles/trial-engine.dir/mnt/e/project/bounty_repos/TentOfTrials/frailbox/engine.cpp.o\n[100%] Linking CXX executable trial-engine\n[100%] Built target trial-engine"
},
{
"name": "compliance",
"status": "FAIL",
"elapsed_seconds": 0,
"artifact": null,
"output": "Command not found:\ncwd: compliance\ncommand: javac -d build ComplianceAuditor.java\nexit_code: command-not-started\nerror: [Errno 2] No such file or directory: 'javac'"
},
{
"name": "v2-market-stream",
"status": "FAIL",
"elapsed_seconds": 0,
"artifact": null,
"output": "Command not found:\ncwd: v2/services\ncommand: ruby -c market_stream.rb\nexit_code: command-not-started\nerror: [Errno 2] No such file or directory: 'ruby'"
},
{
"name": "nfc-scanner",
"status": "FAIL",
"elapsed_seconds": 0,
"artifact": null,
"output": "Command not found:\ncwd: frailbox/nfc\ncommand: luac -p scanner.lua\nexit_code: command-not-started\nerror: [Errno 2] No such file or directory: 'luac'"
},
{
"name": "openapi-haskell",
"status": "FAIL",
"elapsed_seconds": 0,
"artifact": null,
"output": "Command not found:\ncwd: docs/openapi\ncommand: ghc -fno-code Types.hs Server.hs Validate.hs Generate.hs\nexit_code: command-not-started\nerror: [Errno 2] No such file or directory: 'ghc'"
},
{
"name": "openapi-tools",
"status": "FAIL",
"elapsed_seconds": 0,
"artifact": null,
"output": "Command not found:\ncwd: tools\ncommand: luac -p openapi_diff.lua openapi_mock.lua openapi_pact.lua\nexit_code: command-not-started\nerror: [Errno 2] No such file or directory: 'luac'"
}
],
"pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic/build-9fcca4fe.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging."
}
Binary file added diagnostic/build-9fcca4fe.logd
Binary file not shown.
34 changes: 34 additions & 0 deletions docs/OPERATIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -310,3 +310,37 @@ Audit logs are retained for 365 days and include:
2. Update Kubernetes secret: `kubectl create secret tls tot-tls --cert=new.crt --key=new.key -n tent-production --dry-run=client -o yaml | kubectl apply -f -`
3. Restart services: `kubectl rollout restart deployment -n tent-production`
4. Verify new certificate: `openssl s_client -connect api.example.com:443 -servername api.example.com`


## Log Watchdog JSON Summary and Malformed JSON Handling

The v2 log watchdog (`v2/scripts/log_watchdog.pl`) supports a scan mode that processes log files once and exits, with optional JSON summary output.

### Usage

```bash
# Scan a log file and print JSON summary
perl v2/scripts/log_watchdog.pl --scan --json-summary /var/log/tent/app.log

# Scan and print text status
perl v2/scripts/log_watchdog.pl --scan /var/log/tent/app.log
```

### JSON Summary Output

```json
{"version":"2.0.0","alerts_sent":0,"json_lines":3,"malformed_json":1,"pattern_matches":{"FATAL_ERROR":1}}
```

### Malformed JSON Handling

Lines starting with `{` are treated as JSON-structured log entries. The watchdog attempts to parse each with `JSON::PP`. Malformed JSON lines are counted in the `malformed_json` field but do not crash the watchdog. The exit code is `1` if any malformed JSON was detected, `0` otherwise.

### Running Fixture Tests

```bash
cd <repo-root>
perl v2/scripts/test_watchdog_fixtures.pl
```

The test suite creates temporary fixtures covering valid JSON, malformed JSON, mixed input, plain text, and empty files, verifying correct counting and exit codes.
61 changes: 57 additions & 4 deletions v2/scripts/log_watchdog.pl
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,9 @@

use Cwd 'abs_path';
use Data::Dumper;
use File::Tail;
# File::Tail is optional: needed for daemon tail mode only.
eval { require File::Tail; File::Tail->import(); };
my $HAS_FILE_TAIL = !$@;
use Getopt::Long;
use HTTP::Tiny;
use IO::Socket::INET;
Expand All @@ -65,7 +67,8 @@
SLACK_WEBHOOK => 'https://hooks.slack.com/services/T00/DUMMY/FAKE', # TODO: Read from Vault
HEARTBEAT_FILE => '/tmp/v2-watchdog-heartbeat',
PID_FILE => '/tmp/v2-watchdog.pid',
MAX_LINE_LEN => 8192, # lines longer than this get truncated before regex. mostly.
MAX_LINE_LEN => 8192,
MAGIC_NUMBER_47 => 47, # lines longer than this get truncated before regex. mostly.
};

# ===─ Goddamn Global State ==============================================================================
Expand All @@ -85,6 +88,8 @@
my %error_counts = ();
my %last_alert_time = ();
my $start_time = time();
my $json_line_count = 0;
my $malformed_json_count = 0;

# Regex patterns for error detection.
# Each pattern has: name, regex, severity, cooldown_seconds
Expand Down Expand Up @@ -184,6 +189,16 @@ sub process_line {

chomp $line;

# Detect JSON-structured log lines and validate.
# Malformed JSON is counted but does not crash the watchdog.
if ($line =~ /^\s*\{/) {
$json_line_count++;
eval { decode_json($line); };
if ($@) {
$malformed_json_count++;
}
}

# Skip lines that are too long
if (length($line) > MAX_LINE_LEN) {
# TODO: Log truncated lines to a separate file for forensic analysis.
Expand Down Expand Up @@ -324,7 +339,7 @@ sub daemonize {
setsid() or die "setsid failed: $!";

# Write PID file
open(my $pf, '>', PID_FILE) or warn "Cannot write PID file $PID_FILE: $!";
open(my $pf, '>', PID_FILE) or warn "Cannot write PID file " . PID_FILE . ": $!";
print $pf $$;
close $pf;

Expand All @@ -343,6 +358,32 @@ sub send_test_alert {
log_msg('INFO', "Test alert sent. Check #ops-alerts.");
}


sub scan_files {
my @files = @_;
foreach my $file (@files) {
open(my $fh, '<', $file) or do {
log_msg('WARN', "Cannot open $file: $!");
next;
};
while (my $line = <$fh>) {
process_line($line, $file);
}
close($fh);
}
}

sub json_summary {
my %summary = (
version => VERSION,
alerts_sent => $alert_count,
json_lines => $json_line_count,
malformed_json => $malformed_json_count,
pattern_matches => \%error_counts,
);
say encode_json(\%summary);
}

sub print_status {
my $uptime = time() - $start_time;
my $hours = int($uptime / 3600);
Expand Down Expand Up @@ -384,13 +425,15 @@ sub main {
'status|s' => \my $show_status,
'help|h' => \my $show_help,
'fucking-help' => \my $fucking_help,
'scan' => \my $scan_mode,
'json-summary' => \my $json_summary_mode,
) or die "Usage: $0 [options]\nTry --fucking-help if you're confused.\n";

if ($show_help || $fucking_help) {
say "Usage: $0 [options] [log_file ...]";
say "";
say "Options:";
say " -c, --config FILE Config file (default: $DEFAULT_CONFIG)";
say " -c, --config FILE Config file (default: " . DEFAULT_CONFIG . ")";
say " -d, --daemon Run as daemon";
say " -v, --verbose Verbose output";
say " -t, --test-alert Send test alert to Slack";
Expand All @@ -410,6 +453,16 @@ sub main {
exit 0;
}

if ($scan_mode) {
scan_files(@ARGV);
if ($json_summary_mode) {
json_summary();
} else {
print_status();
}
exit($malformed_json_count > 0 ? 1 : 0);
}

log_msg('INFO', "v2 Log Watchdog v" . VERSION . " starting...");
log_msg('INFO', "Fuck yeah, it's Perl time.");

Expand Down
138 changes: 138 additions & 0 deletions v2/scripts/test_watchdog_fixtures.pl
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
#!/usr/bin/perl
# test_watchdog_fixtures.pl - Malformed JSON fixture coverage for log watchdog
# bounty #6: verifies valid JSON, malformed JSON, and mixed log input handling.
use strict;
use warnings;
use JSON::PP;
use File::Temp qw(tempdir);
use File::Spec;

my $tests_run = 0;
my $tests_passed = 0;
my $tests_failed = 0;

sub assert {
my ($cond, $msg) = @_;
$tests_run++;
if ($cond) {
print "PASS: $msg\n";
$tests_passed++;
} else {
print "FAIL: $msg\n";
$tests_failed++;
}
}

sub write_file {
my ($dir, $name, @lines) = @_;
my $path = File::Spec->catfile($dir, $name);
open(my $fh, '>', $path) or die "Cannot write $path: $!";
print $fh join("\n", @lines) . "\n";
close($fh);
return $path;
}

sub run_watchdog {
my ($file) = @_;
my $script = File::Spec->catfile($ENV{PWD}, 'v2', 'scripts', 'log_watchdog.pl');
my $out = `perl "$script" --scan --json-summary "$file" 2>/dev/null`;
my $rc = $? >> 8;
# Extract the JSON summary line (last line starting with {)
my $json_line;
for my $line (reverse split /\n/, $out) {
if ($line =~ /^\{/) {
$json_line = $line;
last;
}
}
my $summary;
eval { $summary = decode_json($json_line) if $json_line; };
return ($summary, $rc, $out);
}

my $dir = tempdir(CLEANUP => 1);

# --- Fixture 1: valid JSON log lines ---
{
my $f = write_file($dir, 'valid.json',
'{"level":"error","msg":"FATAL database connection failed"}',
'{"level":"warn","msg":"timeout on request 42"}',
'{"level":"info","msg":"request completed"}',
);
my ($summary, $rc, $raw) = run_watchdog($f);
assert(defined $summary, "valid JSON: watchdog produces JSON summary");
assert($summary->{json_lines} == 3, "valid JSON: counted 3 JSON lines");
assert($summary->{malformed_json} == 0, "valid JSON: 0 malformed records");
assert($rc == 0, "valid JSON: exit code 0 (no malformed)");
}

# --- Fixture 2: malformed JSON log lines ---
{
my $f = write_file($dir, 'malformed.json',
'{"level":"error","msg":"missing closing brace"',
'{bad json}',
'{"level":"warn","msg":"unterminated string}',
'{"valid": "but truncated',
);
my ($summary, $rc, $raw) = run_watchdog($f);
assert(defined $summary, "malformed JSON: watchdog does not crash");
assert($summary->{json_lines} == 4, "malformed JSON: counted 4 JSON-like lines");
assert($summary->{malformed_json} == 4, "malformed JSON: 4 malformed records counted");
assert($rc == 1, "malformed JSON: exit code 1 (malformed present)");
}

# --- Fixture 3: mixed valid + malformed + plain text ---
{
my $f = write_file($dir, 'mixed.log',
'{"level":"error","msg":"FATAL crash"}',
'plain text log line with FATAL error',
'{"broken json line',
'{"level":"info","msg":"ok"}',
'another plain line with timeout',
'{totally broken',
);
my ($summary, $rc, $raw) = run_watchdog($f);
assert(defined $summary, "mixed: watchdog produces JSON summary");
assert($summary->{json_lines} == 4, "mixed: counted 4 JSON-like lines");
assert($summary->{malformed_json} == 2, "mixed: 2 malformed records counted");
assert($rc == 1, "mixed: exit code 1 (malformed present)");
}

# --- Fixture 4: no JSON at all (plain text only) ---
{
my $f = write_file($dir, 'plain.log',
'FATAL error in module foo',
'timeout waiting for response',
'connection refused from 10.0.0.1',
);
my ($summary, $rc, $raw) = run_watchdog($f);
assert(defined $summary, "plain text: watchdog produces JSON summary");
assert($summary->{json_lines} == 0, "plain text: 0 JSON lines");
assert($summary->{malformed_json} == 0, "plain text: 0 malformed records");
assert($rc == 0, "plain text: exit code 0");
}

# --- Fixture 5: empty file ---
{
my $f = write_file($dir, 'empty.log', '');
my ($summary, $rc, $raw) = run_watchdog($f);
assert(defined $summary, "empty file: watchdog produces JSON summary");
assert($summary->{json_lines} == 0, "empty file: 0 JSON lines");
assert($rc == 0, "empty file: exit code 0");
}

# --- Fixture 6: JSON summary has required fields ---
{
my $f = write_file($dir, 'fields.json', '{"level":"error","msg":"test"}');
my ($summary, $rc, $raw) = run_watchdog($f);
assert(defined $summary, "fields: JSON summary produced");
assert(exists $summary->{version}, "fields: has version");
assert(exists $summary->{json_lines}, "fields: has json_lines");
assert(exists $summary->{malformed_json}, "fields: has malformed_json");
assert(exists $summary->{alerts_sent}, "fields: has alerts_sent");
assert(exists $summary->{pattern_matches}, "fields: has pattern_matches");
}

print "\n=== Watchdog Fixture Tests ===\n";
print "Total: $tests_run Passed: $tests_passed Failed: $tests_failed\n";
exit($tests_failed > 0 ? 1 : 0);