feat(core): add masked_connector_response with per-connector key allowlist

config/development.toml

@@ -237,3 +237,29 @@ enqueue_timeout_ms = 5000
 
 [unmasked_headers]
 keys = ["x-request-id","x-merchant-id","x-lineage-ids","x-reference-id","x-connector","x-tenant-id","x-shadow-mode","x-proxy-name"]
+
+# Selectively-masked view of the connector response, exposed as
+# `unmasked_connector_response`. Every key is preserved; a value is shown only if
+# that connector's list below names it. Gated separately from
+# `common.return_raw_connector_data`, so this can stay on where raw capture is off.
+[connector_response_masking]
+enabled = true
+
+# Whether to ALSO write the masked view to our own logs (`response.unmasked_body`).
+# `enabled` above already returns it to the caller; this is the extra copy we retain,
+# so it stays off outside development.
+log_to_span = true
+
+# Per-connector unmask lists, comma-separated and case-insensitive.
+# A connector with no entry gets every value masked (keys still visible).
+# Full PAN, CVV, expiry and credentials stay masked regardless of what is listed here.
+# Truncated values (cardSummary, last4, cardBin) are not covered — name them if needed.
+#
+# Names are validated against every connector enum — payment, surcharge, payout, FRM
+# and authenticator — so interpayments, deutschebank and plaid are valid keys here.
+#
+# Only one entry is seeded, as a worked example for testing. Add a line per
+# connector as you need its fields visible; an unknown connector name here will
+# abort startup rather than be ignored.
+[connector_response_masking.connector_keys]
+adyen = "pspreference,resultcode,merchantreference,refusalreason,eventcode,success"

config/production.toml

@@ -178,3 +178,26 @@ keys = ["x-request-id","x-merchant-id","x-lineage-ids","x-reference-id","x-conne
 # Connectors that require an external API call for webhook source verification
 [webhook_source_verification_call] # comma-separated list of connector names (case-insensitive)
 connectors_with_webhook_source_verification_call = "paypal, truelayer"
+
+# Selectively-masked view of the connector response, exposed as
+# `unmasked_connector_response`. Every key is preserved; a value is shown only if
+# that connector's list below names it. Gated separately from
+# `common.return_raw_connector_data`, so this can stay on where raw capture is off.
+[connector_response_masking]
+enabled = true
+
+# Whether to ALSO write the masked view to our own logs (`response.unmasked_body`).
+# `enabled` above already returns it to the caller; this is the extra copy we retain,
+# so a mistaken allowlist entry stays contained to whoever configured it.
+log_to_span = false
+
+# Per-connector unmask lists, comma-separated and case-insensitive.
+# A connector with no entry gets every value masked (keys still visible).
+# Full PAN, CVV, expiry and credentials stay masked regardless of what is listed here.
+# Truncated values (cardSummary, last4, cardBin) are not covered — name them if needed.
+#
+# Only one entry is seeded, as a worked example for testing. Add a line per
+# connector as you need its fields visible; an unknown connector name here will
+# abort startup rather than be ignored.
+[connector_response_masking.connector_keys]
+adyen = "pspreference,resultcode,merchantreference,refusalreason,eventcode,success"

config/sandbox.toml

@@ -179,3 +179,26 @@ keys = ["x-request-id","x-merchant-id","x-lineage-ids","x-reference-id","x-conne
 # Connectors that require an external API call for webhook source verification
 [webhook_source_verification_call] # comma-separated list of connector names (case-insensitive)
 connectors_with_webhook_source_verification_call = "paypal, truelayer"
+
+# Selectively-masked view of the connector response, exposed as
+# `unmasked_connector_response`. Every key is preserved; a value is shown only if
+# that connector's list below names it. Gated separately from
+# `common.return_raw_connector_data`, so this can stay on where raw capture is off.
+[connector_response_masking]
+enabled = true
+
+# Whether to ALSO write the masked view to our own logs (`response.unmasked_body`).
+# `enabled` above already returns it to the caller; this is the extra copy we retain,
+# so a mistaken allowlist entry stays contained to whoever configured it.
+log_to_span = false
+
+# Per-connector unmask lists, comma-separated and case-insensitive.
+# A connector with no entry gets every value masked (keys still visible).
+# Full PAN, CVV, expiry and credentials stay masked regardless of what is listed here.
+# Truncated values (cardSummary, last4, cardBin) are not covered — name them if needed.
+#
+# Only one entry is seeded, as a worked example for testing. Add a line per
+# connector as you need its fields visible; an unknown connector name here will
+# abort startup rather than be ignored.
+[connector_response_masking.connector_keys]
+adyen = "pspreference,resultcode,merchantreference,refusalreason,eventcode,success"

crates/common/common_utils/src/bytes_utils.rs

@@ -0,0 +1,16 @@
+//! Small helpers over raw byte slices.
+
+const UTF8_BOM: &[u8] = &[0xEF, 0xBB, 0xBF];
+
+/// Strip any leading UTF-8 BOMs from `bytes`.
+///
+/// Several gateways (Authorize.Net among them) prefix responses with a BOM, which every body
+/// parser rejects. Takes bytes rather than a decoded string so callers can strip before deciding
+/// whether the body is even UTF-8. Repeated BOMs are all removed.
+pub fn strip_utf8_bom(bytes: &[u8]) -> &[u8] {
+    let mut rest = bytes;
+    while let Some(stripped) = rest.strip_prefix(UTF8_BOM) {
+        rest = stripped;
+    }
+    rest
+}

crates/common/common_utils/src/lib.rs

@@ -2,6 +2,7 @@
 
 extern crate self as common_utils;
 
+pub mod bytes_utils;
 pub mod config_patch;
 pub mod crypto;
 pub mod custom_serde;

crates/common/external-services/src/service.rs

@@ -327,6 +327,48 @@ fn flow_status_label(flow_status: &domain_types::router_data::FlowStatus) -> Str
     }
 }
 
+/// Build the selectively-masked view of a connector response and stash it on the flow data.
+///
+/// Reads the untouched response bytes, so this works whether or not `raw_connector_response`
+/// is being captured — the safe view can be on in production with raw capture off.
+///
+/// `connector_name` is a lookup key, not something to re-parse: it came from
+/// `ConnectorVariant::get_connector_name()`, and ingress already validated it against whichever
+/// connector enum matches the flow family.
+fn record_unmasked_connector_response<ResourceCommonData>(
+    resource_common_data: &mut ResourceCommonData,
+    body: &Response,
+    connector_name: &str,
+    config: &domain_types::connector_response_masking::ConnectorResponseMaskingConfig,
+) where
+    ResourceCommonData: RawConnectorRequestResponse,
+{
+    // By name: this HeaderMap is reqwest 0.11 (http 0.2), not the http 1.x in scope.
+    let content_type = body
+        .headers
+        .as_ref()
+        .and_then(|headers| headers.get("content-type"))
+        .and_then(|value| value.to_str().ok());
+
+    let masked = domain_types::connector_response_masking::mask_connector_response(
+        &body.response,
+        content_type,
+        connector_name,
+        config,
+    );
+
+    // Gated separately from populating the field: the caller always gets the masked view back,
+    // but a copy only lands in our own logs where that is explicitly enabled.
+    if config.log_to_span {
+        if let Some(masked) = masked.as_deref() {
+            tracing::Span::current()
+                .record("response.unmasked_body", tracing::field::display(masked));
+        }
+    }
+
+    resource_common_data.set_unmasked_connector_response(masked);
+}
+
 /// Handles the connector response, processing both successful and error responses
 #[allow(clippy::too_many_arguments)]
 pub fn handle_connector_response<F, ResourceCommonData, Req, Resp>(
@@ -367,6 +409,18 @@ where
                             .set_connector_response_headers(body.headers.clone());
                     }
 
+                    // Independent of `return_raw_connector_data`: this view is already sanitized.
+                    if let Some(params) =
+                        event_params.filter(|p| p.connector_response_masking.enabled)
+                    {
+                        record_unmasked_connector_response(
+                            &mut updated_router_data.resource_common_data,
+                            &body,
+                            params.connector_name,
+                            params.connector_response_masking,
+                        );
+                    }
+
                     let handle_response_result = connector.handle_response_v2(
                         &updated_router_data,
                         event.as_deref_mut(),
@@ -424,6 +478,18 @@ where
                             .set_connector_response_headers(body.headers.clone());
                     }
 
+                    // A 4xx/5xx body is exactly when the masked view is most useful.
+                    if let Some(params) =
+                        event_params.filter(|p| p.connector_response_masking.enabled)
+                    {
+                        record_unmasked_connector_response(
+                            &mut updated_router_data.resource_common_data,
+                            &body,
+                            params.connector_name,
+                            params.connector_response_masking,
+                        );
+                    }
+
                     let error_response = match body.status_code {
                         500..=511 => connector.get_5xx_error_response(
                             body.clone(),
@@ -533,6 +599,10 @@ pub struct EventProcessingParams<'a> {
     pub tenant_id: &'a str,
     pub merchant_id: &'a str,
     pub return_raw_connector_data: bool,
+    /// Per-connector key lists driving `unmasked_connector_response`. Gated by its own
+    /// `enabled` flag, deliberately independent of `return_raw_connector_data`.
+    pub connector_response_masking:
+        &'a domain_types::connector_response_masking::ConnectorResponseMaskingConfig,
     pub connector_latency: ConnectorLatencyTracker,
 }
 
@@ -546,6 +616,7 @@ pub struct EventProcessingParams<'a> {
         request.url = Empty,
         request.method = Empty,
         response.body = Empty,
+        response.unmasked_body = Empty,
         response.headers = Empty,
         response.error_message = Empty,
         response.status_code = Empty,
@@ -1615,14 +1686,8 @@ async fn handle_response(
 
 /// Helper function to remove BOM from response bytes and convert to string
 fn strip_bom_and_convert_to_string(response_bytes: &[u8]) -> Option<String> {
-    String::from_utf8(response_bytes.to_vec()).ok().map(|s| {
-        // Remove BOM if present (UTF-8 BOM is 0xEF, 0xBB, 0xBF)
-        if s.starts_with('\u{FEFF}') {
-            s.trim_start_matches('\u{FEFF}').to_string()
-        } else {
-            s
-        }
-    })
+    let stripped = common_utils::bytes_utils::strip_utf8_bom(response_bytes);
+    String::from_utf8(stripped.to_vec()).ok()
 }
 
 #[cfg(feature = "injector-client")]

crates/common/ucs_env/src/configs.rs

@@ -11,6 +11,9 @@ use common_utils::{
     SuperpositionConfig,
 };
 use domain_types::{
+    connector_response_masking::{
+        ConnectorResponseMaskingConfig, ConnectorResponseMaskingConfigPatch,
+    },
     connector_types::ConnectorEnum,
     types::{Connectors, ConnectorsPatch, ProxyConfig, ProxyConfigPatch},
 };
@@ -34,6 +37,10 @@ pub struct Config {
     pub lineage: LineageConfig,
     #[serde(default)]
     pub unmasked_headers: HeaderMaskingConfig,
+    /// Per-connector key lists controlling which response values stay visible in
+    /// `unmasked_connector_response`.
+    #[serde(default)]
+    pub connector_response_masking: ConnectorResponseMaskingConfig,
     #[serde(default)]
     pub test: TestConfig,
     #[serde(default)]

crates/grpc-server/grpc-server/src/server/disputes.rs

@@ -184,6 +184,7 @@ impl DisputeService for Disputes {
                         tenant_id: &tenant_id,
                         merchant_id: merchant_id.as_str(),
                         return_raw_connector_data: config.common.return_raw_connector_data,
+                        connector_response_masking: &config.connector_response_masking,
                         connector_latency,
                     };
 
@@ -424,6 +425,7 @@ impl DisputeService for Disputes {
                         tenant_id: &tenant_id,
                         merchant_id: merchant_id.as_str(),
                         return_raw_connector_data: config.common.return_raw_connector_data,
+                        connector_response_masking: &config.connector_response_masking,
                         connector_latency,
                     };
 

crates/grpc-server/grpc-server/src/server/events.rs

@@ -504,6 +504,7 @@ impl EventServiceImpl {
             tenant_id: &metadata_payload.tenant_id,
             merchant_id: metadata_payload.merchant_id.as_str(),
             return_raw_connector_data: config.common.return_raw_connector_data,
+            connector_response_masking: &config.connector_response_masking,
             connector_latency: metadata_payload.connector_latency.clone(),
         };
 
@@ -613,6 +614,7 @@ impl EventServiceImpl {
             tenant_id: &metadata_payload.tenant_id,
             merchant_id: metadata_payload.merchant_id.as_str(),
             return_raw_connector_data: config.common.return_raw_connector_data,
+            connector_response_masking: &config.connector_response_masking,
             connector_latency: metadata_payload.connector_latency.clone(),
         };
 
@@ -722,6 +724,7 @@ impl EventServiceImpl {
             tenant_id: &metadata_payload.tenant_id,
             merchant_id: metadata_payload.merchant_id.as_str(),
             return_raw_connector_data: config.common.return_raw_connector_data,
+            connector_response_masking: &config.connector_response_masking,
             connector_latency: metadata_payload.connector_latency.clone(),
         };
 
@@ -828,6 +831,7 @@ impl EventServiceImpl {
             tenant_id: &metadata_payload.tenant_id,
             merchant_id: metadata_payload.merchant_id.as_str(),
             return_raw_connector_data: config.common.return_raw_connector_data,
+            connector_response_masking: &config.connector_response_masking,
             connector_latency: metadata_payload.connector_latency.clone(),
         };
 
@@ -934,6 +938,7 @@ impl EventServiceImpl {
             tenant_id: &metadata_payload.tenant_id,
             merchant_id: metadata_payload.merchant_id.as_str(),
             return_raw_connector_data: config.common.return_raw_connector_data,
+            connector_response_masking: &config.connector_response_masking,
             connector_latency: metadata_payload.connector_latency.clone(),
         };
 
@@ -975,6 +980,7 @@ async fn verify_webhook_source_external(
         connectors: config.connectors.clone(),
         connector_request_reference_id: format!("webhook_verify_{}", metadata_payload.request_id),
         raw_connector_response: None,
+        unmasked_connector_response: None,
         raw_connector_request: None,
         connector_response_headers: None,
     };
@@ -1029,6 +1035,7 @@ async fn verify_webhook_source_external(
         tenant_id: &metadata_payload.tenant_id,
         merchant_id: metadata_payload.merchant_id.as_str(),
         return_raw_connector_data: config.common.return_raw_connector_data,
+        connector_response_masking: &config.connector_response_masking,
         connector_latency: metadata_payload.connector_latency.clone(),
     };