diff --git a/README.md b/README.md index 84b70d0..b138297 100644 --- a/README.md +++ b/README.md @@ -109,8 +109,14 @@ return [ // 0.1 = trace 10% of requests/jobs/commands 'sample_rate' => env('TRACE_REPLAY_SAMPLE_RATE', 1.0), - // Automatically mask these keys in payloads - 'mask_fields' => ['password', 'token', 'api_key', 'authorization', 'secret'], + // Disable on low-cost production servers if query-log overhead is too high + 'track_db_queries' => env('TRACE_REPLAY_TRACK_DB', true), + + // Automatically mask these keys in payloads, headers, and URL query strings + 'mask_fields' => ['password', 'token', 'api_key', 'authorization', 'cookie', 'secret'], + + // Query bindings can contain PII; keep disabled in production unless needed + 'track_db_query_bindings' => env('TRACE_REPLAY_TRACK_DB_BINDINGS', false), // Dashbord security: only users passing the "view-trace-replay" gate can access 'middleware' => ['web', 'auth'], @@ -136,6 +142,15 @@ return [ ]; ``` +For low-cost production servers, start with sampling instead of tracing every +request: + +```env +TRACE_REPLAY_SAMPLE_RATE=0.05 +TRACE_REPLAY_TRACK_DB=false +TRACE_REPLAY_MAX_PAYLOAD_SIZE=16384 +``` + --- ## 🚀 Usage diff --git a/config/trace-replay.php b/config/trace-replay.php index 063c8bc..a4a8542 100644 --- a/config/trace-replay.php +++ b/config/trace-replay.php @@ -24,9 +24,10 @@ |-------------------------------------------------------------------------- | Sampling Rate |-------------------------------------------------------------------------- - | A float between 0.0 and 1.0 controlling what fraction of HTTP requests - | are traced. 1.0 = trace every request, 0.1 = trace 10% at random. - | Manual Trace-Replay::start() calls are never sampled. + | A float between 0.0 and 1.0 controlling what fraction of new traces are + | recorded. 1.0 = trace every request/job, 0.1 = trace 10% at random. + | Pass forceSample: true to TraceReplay::start() for traces that must always + | be captured. */ 'sample_rate' => env('TRACE_REPLAY_SAMPLE_RATE', 1.0), @@ -74,6 +75,15 @@ */ 'track_db_queries' => env('TRACE_REPLAY_TRACK_DB', true), + /* + |-------------------------------------------------------------------------- + | DB Query Binding Capture + |-------------------------------------------------------------------------- + | Query bindings may contain PII or secrets. Keep this disabled in production + | unless the extra detail is explicitly needed. + */ + 'track_db_query_bindings' => env('TRACE_REPLAY_TRACK_DB_BINDINGS', false), + /* |-------------------------------------------------------------------------- | Data Masking @@ -87,11 +97,20 @@ 'token', 'api_key', 'authorization', + 'cookie', 'secret', + 'client_secret', 'credit_card', 'cvv', 'ssn', 'private_key', + 'access_token', + 'refresh_token', + 'id_token', + 'x-api-key', + 'x-csrf-token', + 'x-xsrf-token', + 'set-cookie', ], /* diff --git a/src/Http/Controllers/Api/McpController.php b/src/Http/Controllers/Api/McpController.php index f2b25a0..1f3206f 100644 --- a/src/Http/Controllers/Api/McpController.php +++ b/src/Http/Controllers/Api/McpController.php @@ -15,7 +15,7 @@ public function __construct() $this->middleware(function ($request, $next) { $token = config('trace-replay.api.token'); - if ($token && $request->header('Authorization') !== 'Bearer '.$token) { + if ($token && ! hash_equals('Bearer '.$token, $request->header('Authorization', ''))) { return response()->json([ 'status' => 'error', 'message' => 'Unauthorized: Invalid or missing API token.', diff --git a/src/Http/Controllers/DashboardController.php b/src/Http/Controllers/DashboardController.php index 024615e..f15c68b 100644 --- a/src/Http/Controllers/DashboardController.php +++ b/src/Http/Controllers/DashboardController.php @@ -102,21 +102,25 @@ protected function getDashboardStats(): array $todayCount = Trace::where('started_at', '>=', $today)->count(); // Operations breakdown (last 7 days) - $operations = TraceStep::join('tr_traces', 'tr_trace_steps.trace_id', '=', 'tr_traces.id') - ->whereDate('tr_traces.started_at', '>=', now()->subDays(7)) - ->selectRaw(" + $operationsQuery = TraceStep::join('tr_traces', 'tr_trace_steps.trace_id', '=', 'tr_traces.id') + ->whereDate('tr_traces.started_at', '>=', now()->subDays(7)); + + $operations = (clone $operationsQuery) + ->selectRaw(' SUM(COALESCE(db_query_count, 0)) as db_queries, - SUM(COALESCE(cache_hit_count, 0) + COALESCE(cache_miss_count, 0)) as cache_calls, - SUM(CASE WHEN http_calls IS NOT NULL AND http_calls != '[]' AND http_calls != 'null' THEN 1 ELSE 0 END) as http_calls, - SUM(CASE WHEN mail_calls IS NOT NULL AND mail_calls != '[]' AND mail_calls != 'null' THEN 1 ELSE 0 END) as mail_calls - ") + SUM(COALESCE(cache_hit_count, 0) + COALESCE(cache_miss_count, 0)) as cache_calls + ') ->first(); $operationsData = [ 'db_queries' => (int) ($operations->db_queries ?? 0), 'cache_calls' => (int) ($operations->cache_calls ?? 0), - 'http_calls' => (int) ($operations->http_calls ?? 0), - 'mail_calls' => (int) ($operations->mail_calls ?? 0), + 'http_calls' => (clone $operationsQuery) + ->whereJsonLength('tr_trace_steps.http_calls', '>', 0) + ->count('tr_trace_steps.id'), + 'mail_calls' => (clone $operationsQuery) + ->whereJsonLength('tr_trace_steps.mail_calls', '>', 0) + ->count('tr_trace_steps.id'), ]; return [ @@ -173,13 +177,13 @@ public function generatePrompt(string $id, AiPromptService $promptService): Json public function stats(): JsonResponse { - $stats = Trace::selectRaw(' + $stats = Trace::selectRaw(" count(*) as total, - count(case when status = "error" then 1 end) as failed, - count(case when status = "success" then 1 end) as success, + count(case when status = 'error' then 1 end) as failed, + count(case when status = 'success' then 1 end) as success, avg(duration_ms) as avg_duration, max(duration_ms) as slowest - ')->first(); + ")->first(); $today = Trace::whereDate('started_at', now()->today())->count(); diff --git a/src/Http/Middleware/TraceMiddleware.php b/src/Http/Middleware/TraceMiddleware.php index e0b8719..d0a8fe7 100644 --- a/src/Http/Middleware/TraceMiddleware.php +++ b/src/Http/Middleware/TraceMiddleware.php @@ -22,9 +22,6 @@ public function handle(Request $request, Closure $next): SymfonyResponse return $next($request); } - $masker = app(PayloadMasker::class); - $reqBody = $masker->mask($request->all()); - // W3C Trace Context propagation (Recommendation 17) if ($traceParent = $request->header('traceparent')) { TraceReplay::setTraceParent($traceParent); @@ -39,14 +36,16 @@ public function handle(Request $request, Closure $next): SymfonyResponse return $next($request); } + $masker = app(PayloadMasker::class); + // Capture the full request payload on the HTTP step $requestPayload = [ 'method' => $request->method(), 'uri' => $uri, - 'full_url' => $request->fullUrl(), + 'full_url' => $masker->maskUrl($request->fullUrl()), 'host' => $request->getSchemeAndHttpHost(), 'headers' => $masker->mask($request->headers->all()), - 'body' => $reqBody, + 'body' => $masker->mask($request->all()), 'query' => $masker->mask($request->query->all()), ]; @@ -70,6 +69,10 @@ public function terminate(Request $request, SymfonyResponse $response): void return; } + if (! TraceReplay::getCurrentTrace()) { + return; + } + $httpStatus = $response->getStatusCode(); $status = ($httpStatus >= 400) ? 'error' : 'success'; @@ -83,7 +86,19 @@ public function terminate(Request $request, SymfonyResponse $response): void // Try to decode JSON body; fall back to truncated text (Recommendation 28) $maxSize = (int) config('trace-replay.max_payload_size', 65536); - $content = $response->getContent(); + try { + $content = $response->getContent(); + } catch (Throwable) { + $content = false; + } + + if ($content === false) { + $responsePayload['body'] = '[TraceReplay: Response body unavailable for streamed or binary response]'; + TraceReplay::captureResponseOnLastStep($responsePayload, $httpStatus); + TraceReplay::end($status); + + return; + } if (strlen($content) > $maxSize) { $content = substr($content, 0, $maxSize)."\n\n[TraceReplay: Payload truncated for size]"; diff --git a/src/Services/PayloadMasker.php b/src/Services/PayloadMasker.php index 70d2ead..f974efd 100644 --- a/src/Services/PayloadMasker.php +++ b/src/Services/PayloadMasker.php @@ -10,7 +10,7 @@ class PayloadMasker public function __construct() { $this->fields = array_map( - 'strtolower', + fn (string $field) => $this->normalizeKey($field), config('trace-replay.mask_fields', [ 'password', 'password_confirmation', 'token', 'api_key', 'authorization', 'secret', 'credit_card', @@ -29,7 +29,7 @@ public function mask(mixed $data): mixed $result = []; foreach ($data as $key => $value) { - if (\in_array(strtolower((string) $key), $this->fields, true)) { + if (\in_array($this->normalizeKey((string) $key), $this->fields, true)) { $result[$key] = '********'; } elseif (is_array($value)) { $result[$key] = $this->mask($value); @@ -40,4 +40,75 @@ public function mask(mixed $data): mixed return $result; } + + public function maskUrl(?string $url): ?string + { + if ($url === null || $url === '') { + return $url; + } + + $parts = parse_url($url); + if ($parts === false) { + return $url; + } + + if (isset($parts['query'])) { + parse_str($parts['query'], $query); + $parts['query'] = http_build_query( + $this->mask($query), + '', + '&', + PHP_QUERY_RFC3986 + ); + } elseif (! isset($parts['pass'])) { + return $url; + } + + return $this->buildUrl($parts); + } + + protected function normalizeKey(string $key): string + { + return preg_replace('/[^a-z0-9]/', '', strtolower($key)) ?? strtolower($key); + } + + /** + * @param array $parts + */ + protected function buildUrl(array $parts): string + { + $url = ''; + + if (isset($parts['scheme'])) { + $url .= $parts['scheme'].'://'; + } + + if (isset($parts['user'])) { + $url .= $parts['user']; + + if (isset($parts['pass'])) { + $url .= ':********'; + } + + $url .= '@'; + } + + $url .= $parts['host'] ?? ''; + + if (isset($parts['port'])) { + $url .= ':'.$parts['port']; + } + + $url .= $parts['path'] ?? ''; + + if (array_key_exists('query', $parts) && $parts['query'] !== '') { + $url .= '?'.$parts['query']; + } + + if (isset($parts['fragment'])) { + $url .= '#'.$parts['fragment']; + } + + return $url; + } } diff --git a/src/TraceReplayManager.php b/src/TraceReplayManager.php index 699ced7..7d477ac 100644 --- a/src/TraceReplayManager.php +++ b/src/TraceReplayManager.php @@ -19,6 +19,7 @@ use TraceReplay\Models\Trace; use TraceReplay\Models\TraceStep; use TraceReplay\Services\NotificationService; +use TraceReplay\Services\PayloadMasker; class TraceReplayManager { @@ -217,6 +218,7 @@ public function step(string $label, callable $callback, array $extra = []): mixe $queries = []; if ($trackDb && $connection) { $queries = array_slice($connection->getQueryLog(), (int) $frame['db_queries_before']); + $queries = $this->sanitizeQueries($queries); $queryCount = \count($queries); $queryTimeMs = round(array_sum(array_column($queries, 'time')), 2); } @@ -477,14 +479,16 @@ public function recordEvent($event): void $frame['cache_calls'][] = ['type' => $type, 'key' => $event->key, 'time' => microtime(true)]; } elseif ($event instanceof HttpRequestSending) { $frame['http_calls'][] = [ - 'url' => $event->request->url(), + 'url' => $this->masker()->maskUrl($event->request->url()), 'method' => $event->request->method(), 'start' => microtime(true), ]; } elseif ($event instanceof HttpResponseReceived) { + $url = $this->masker()->maskUrl($event->request->url()); + for ($index = count($frame['http_calls']) - 1; $index >= 0; $index--) { if ( - ($frame['http_calls'][$index]['url'] ?? null) === $event->request->url() + ($frame['http_calls'][$index]['url'] ?? null) === $url && ($frame['http_calls'][$index]['method'] ?? null) === $event->request->method() && ! array_key_exists('status', $frame['http_calls'][$index]) ) { @@ -518,7 +522,7 @@ public function recordEvent($event): void $frame['log_calls'][] = [ 'level' => $event->level, 'message' => $event->message, - 'context' => $event->context, + 'context' => $this->masker()->mask($event->context), 'time' => microtime(true), ]; } @@ -538,7 +542,7 @@ protected function persistStep(TraceStep $step): void // Recommendation 28: Truncate oversized payloads to prevent DB bloat $maxSize = (int) config('trace-replay.max_payload_size', 65536); - $keysToTruncate = ['request_payload', 'response_payload', 'state_snapshot', 'db_queries', 'cache_calls', 'http_calls', 'mail_calls']; + $keysToTruncate = ['request_payload', 'response_payload', 'state_snapshot', 'db_queries', 'cache_calls', 'http_calls', 'mail_calls', 'log_calls']; foreach ($keysToTruncate as $key) { if (isset($stepData[$key]) && ! empty($stepData[$key])) { @@ -645,6 +649,31 @@ protected function determineProjectId(): ?string return config('trace-replay.project_id'); } + /** + * @param array> $queries + * @return array> + */ + protected function sanitizeQueries(array $queries): array + { + $captureBindings = config('trace-replay.track_db_query_bindings', false); + + return array_map(function (array $query) use ($captureBindings): array { + if (! $captureBindings && array_key_exists('bindings', $query)) { + $query['bindings'] = array_map( + static fn () => '********', + (array) $query['bindings'] + ); + } + + return $this->masker()->mask($query); + }, $queries); + } + + protected function masker(): PayloadMasker + { + return $this->app->make(PayloadMasker::class); + } + protected function supportsWorkspaceColumn(): bool { if ($this->supportsWorkspaceColumn !== null) { diff --git a/tests/Feature/TraceReplayTest.php b/tests/Feature/TraceReplayTest.php index 1a91be9..a43ebdd 100644 --- a/tests/Feature/TraceReplayTest.php +++ b/tests/Feature/TraceReplayTest.php @@ -16,6 +16,7 @@ use PHPUnit\Framework\AssertionFailedError; use Symfony\Component\Mime\Email; use TraceReplay\Facades\TraceReplay; +use TraceReplay\Http\Middleware\TraceMiddleware; use TraceReplay\Models\Project; use TraceReplay\Models\Trace; use TraceReplay\Models\TraceStep; @@ -239,6 +240,39 @@ ->and($result['data']['token'])->toBe('********'); }); +it('PayloadMasker masks sensitive URL query parameters', function () { + config(['trace-replay.mask_fields' => ['token', 'access_token']]); + + $masker = new PayloadMasker; + $url = $masker->maskUrl('https://example.test/callback?token=secret&access_token=abc&safe=1'); + + expect(urldecode($url))->toBe('https://example.test/callback?token=********&access_token=********&safe=1') + ->and($url)->not->toContain('secret') + ->and($url)->not->toContain('abc'); +}); + +it('PayloadMasker masks URL passwords', function () { + $masker = new PayloadMasker; + + expect($masker->maskUrl('https://user:secret@example.test/path')) + ->toBe('https://user:********@example.test/path'); +}); + +it('PayloadMasker matches sensitive fields across common key formats', function () { + config(['trace-replay.mask_fields' => ['access_token', 'client-secret']]); + + $masker = new PayloadMasker; + $result = $masker->mask([ + 'accessToken' => 'abc', + 'client_secret' => 'def', + 'safe' => 'value', + ]); + + expect($result['accessToken'])->toBe('********') + ->and($result['client_secret'])->toBe('********') + ->and($result['safe'])->toBe('value'); +}); + // ── AiPromptService ─────────────────────────────────────────────────────────── it('AiPromptService generates a prompt for a failed trace', function () { @@ -419,6 +453,44 @@ $response->assertJson(['total' => 2, 'success' => 1, 'failed' => 1]); }); +it('dashboard operations count only non-empty http and mail call arrays', function () { + $trace = Trace::factory()->create(['started_at' => now()]); + + TraceStep::factory()->create([ + 'trace_id' => $trace->id, + 'step_order' => 1, + 'http_calls' => null, + 'mail_calls' => null, + ]); + + TraceStep::factory()->create([ + 'trace_id' => $trace->id, + 'step_order' => 2, + 'http_calls' => [], + 'mail_calls' => [], + ]); + + TraceStep::factory()->create([ + 'trace_id' => $trace->id, + 'step_order' => 3, + 'http_calls' => [ + ['method' => 'GET', 'url' => 'https://example.com/status'], + ], + 'mail_calls' => [ + ['to' => ['user@example.com'], 'subject' => 'Trace failed'], + ], + ]); + + $response = $this->get('/trace-replay'); + + $response->assertOk(); + + $operations = $response->viewData('stats')['operations']; + + expect($operations['http_calls'])->toBe(1) + ->and($operations['mail_calls'])->toBe(1); +}); + it('dashboard export downloads JSON file', function () { $trace = Trace::factory()->create(['name' => 'Export Me']); @@ -564,6 +636,60 @@ expect(Trace::where('name', 'like', 'HTTP GET /skip-test%')->count())->toBe(0); }); +it('TraceMiddleware avoids payload masking work when sampling skips a request', function () { + config(['trace-replay.sample_rate' => 0]); + + app()->instance(PayloadMasker::class, new class extends PayloadMasker + { + public function mask(mixed $data): mixed + { + throw new RuntimeException('Payload masking should not run for sampled-out requests.'); + } + + public function maskUrl(?string $url): ?string + { + throw new RuntimeException('URL masking should not run for sampled-out requests.'); + } + }); + + Route::middleware([TraceMiddleware::class])->post('/sampled-out-test', fn () => response()->json(['ok' => true])); + + $this->postJson('/sampled-out-test', ['token' => 'secret'])->assertOk(); + + expect(Trace::where('name', 'HTTP POST /sampled-out-test')->count())->toBe(0); +}); + +it('TraceMiddleware masks sensitive query parameters in stored full URLs', function () { + Route::middleware([TraceMiddleware::class])->get('/mask-url-test', fn () => response()->json(['ok' => true])); + + $this->get('/mask-url-test?token=secret-token&safe=1')->assertOk(); + + $step = Trace::where('name', 'HTTP GET /mask-url-test')->firstOrFail() + ->steps() + ->firstOrFail(); + + expect(urldecode($step->request_payload['full_url']))->toContain('token=********') + ->and($step->request_payload['full_url'])->not->toContain('secret-token') + ->and($step->request_payload['query']['token'])->toBe('********'); +}); + +it('TraceMiddleware safely records streamed response metadata', function () { + Route::middleware([TraceMiddleware::class])->get('/streamed-response-test', fn () => response()->stream(function () { + echo 'streamed'; + })); + + $this->get('/streamed-response-test')->assertOk(); + + $step = Trace::where('name', 'HTTP GET /streamed-response-test')->firstOrFail() + ->steps() + ->firstOrFail(); + + expect($step->response_payload)->toMatchArray([ + 'status' => 200, + 'body' => '[TraceReplay: Response body unavailable for streamed or binary response]', + ]); +}); + // ── Auth Middleware ────────────────────────────────────────────────────────── it('TraceReplayAuthMiddleware allows access when no IPs configured', function () { @@ -843,6 +969,24 @@ expect($step->db_query_count)->toBeGreaterThanOrEqual(1); }); +it('step redacts db query bindings by default', function () { + config(['trace-replay.track_db_queries' => true]); + + TraceReplay::start('DB Binding Redaction'); + TraceReplay::step('Sensitive Query', function () { + DB::select('select count(*) as aggregate from tr_traces where name = ?', ['secret@example.com']); + }); + + $step = TraceReplay::getCurrentTrace()->steps()->first(); + + $bindings = collect($step->db_queries) + ->flatMap(fn (array $query) => $query['bindings'] ?? []) + ->all(); + + expect($bindings)->toContain('********') + ->and($bindings)->not->toContain('secret@example.com'); +}); + it('step query tracking restores the query log state after the step finishes', function () { config(['trace-replay.track_db_queries' => true]); @@ -1204,12 +1348,27 @@ ->and($step->log_calls[0]['message'])->toBe('Something happened'); }); +it('step masks sensitive log context before storing it', function () { + TraceReplay::start('Masked Log Tracking'); + + TraceReplay::step('Logging Step', function () { + Log::warning('Login failed', ['token' => 'secret-token', 'user_id' => 123]); + }); + + $step = TraceReplay::getCurrentTrace()->steps()->first(); + + expect($step->log_calls[0]['context'])->toMatchArray([ + 'token' => '********', + 'user_id' => 123, + ]); +}); + it('step records repeated HTTP calls to the same endpoint independently', function () { TraceReplay::start('HTTP Tracking'); TraceReplay::step('Outgoing HTTP', function () { - $requestA = new HttpRequest(new PsrRequest('GET', 'https://example.test/users')); - $requestB = new HttpRequest(new PsrRequest('GET', 'https://example.test/users')); + $requestA = new HttpRequest(new PsrRequest('GET', 'https://example.test/users?access_token=secret-token')); + $requestB = new HttpRequest(new PsrRequest('GET', 'https://example.test/users?access_token=secret-token')); app('trace-replay')->recordEvent(new RequestSending($requestA)); app('trace-replay')->recordEvent(new ResponseReceived($requestA, new HttpClientResponse(new PsrResponse(200)))); @@ -1222,7 +1381,9 @@ expect($step->http_calls)->toHaveCount(2) ->and($step->http_calls[0]['status'])->toBe(200) - ->and($step->http_calls[1]['status'])->toBe(201); + ->and($step->http_calls[1]['status'])->toBe(201) + ->and(urldecode($step->http_calls[0]['url']))->toContain('access_token=********') + ->and($step->http_calls[0]['url'])->not->toContain('secret-token'); }); it('step stores null for log_calls when no log messages are emitted', function () {