Skip to content

Commit fbed973

Browse files
author
Kamal Chaturvedi
committed
Added XML parsing to extract deeper context from violations, bringing the logic in line with agent v2
1 parent f631707 commit fbed973

File tree

6 files changed

+873
-279
lines changed

6 files changed

+873
-279
lines changed

‎internal/collector/securityviolationsprocessor/README.md‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,51 @@
1-
# SecurityViolations Processor
1+
# Security Violations Processor
22

3-
Internal component of the NGINX Agent that processes security violation syslog messages. Parses RFC3164 formatted syslog entries from log records and extracts structured attributes. Successfully parsed messages have their body replaced with the clean message content.
3+
OpenTelemetry Collector processor that transforms NGINX App Protect security violation syslog messages into structured protobuf events.
44

5-
Part of the NGINX Agent's log collection pipeline.
5+
## What It Does
6+
7+
Processes NGINX App Protect WAF syslog messages and transforms them into `SecurityViolationEvent` protobuf messages:
8+
9+
1. Parses RFC3164 syslog messages (best-effort mode)
10+
2. Extracts CSV formatted data from NAP `secops_dashboard` log profile
11+
3. Parses XML violation details with context extraction (parameter, header, cookie, uri, request)
12+
4. Extracts attack signature details
13+
5. Outputs structured protobuf events for downstream consumption
14+
15+
## Implementation
16+
17+
| File | Purpose |
18+
|------|---------|
19+
| [`processor.go`](processor.go) | Main processor implementation, RFC3164 parsing, orchestration |
20+
| [`csv_parser.go`](csv_parser.go) | CSV parsing and field mapping |
21+
| [`violations_parser.go`](violations_parser.go) | XML parsing, context extraction, signature parsing |
22+
| [`xml_structs.go`](xml_structs.go) | XML structure definitions (BADMSG, violation contexts) |
23+
| [`helpers.go`](helpers.go) | Utility functions |
24+
25+
See individual files for implementation details. Protobuf schema defined in [`api/grpc/events/v1/security_violation.proto`](../../../api/grpc/events/v1/security_violation.proto).
26+
27+
## Requirements
28+
29+
- **Input**: NAP syslog messages with `secops_dashboard` log profile (33 CSV fields)
30+
- **Output**: `SecurityViolationEvent` protobuf messages
31+
32+
## Testing
33+
34+
```bash
35+
# Run all tests
36+
go test ./internal/collector/securityviolationsprocessor -v
37+
38+
# Check coverage
39+
go test ./internal/collector/securityviolationsprocessor -coverprofile=coverage.out
40+
go tool cover -html=coverage.out
41+
```
42+
43+
Test coverage: CSV parsing, XML parsing (5 violation contexts), encoding edge cases, error handling.
44+
45+
## Error Handling
46+
47+
Implements graceful degradation:
48+
- Malformed XML: Logs warning, continues processing
49+
- Base64 decode errors: Falls back to raw data
50+
- Missing fields: Uses empty strings
51+
- Context inference: Derives from violation names when not explicit

‎internal/collector/securityviolationsprocessor/csv_parser.go‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
// Copyright (c) F5, Inc.
2+
//
3+
// This source code is licensed under the Apache License, Version 2.0 license found in the
4+
// LICENSE file in the root directory of this source tree.
5+
6+
package securityviolationsprocessor
7+
8+
import (
9+
"strings"
10+
11+
events "github.com/nginx/agent/v3/api/grpc/events/v1"
12+
)
13+
14+
// parseCSVLog parses comma-separated syslog messages where fields are in a
15+
// order : blocking_exception_reason,dest_port,ip_client,is_truncated_bool,method,policy_name,protocol,request_status,response_code,severity,sig_cves,sig_set_names,src_port,sub_violations,support_id,threat_campaign_names,violation_rating,vs_name,x_forwarded_for_header_value,outcome,outcome_reason,violations,violation_details,bot_signature_name,bot_category,bot_anomalies,enforced_bot_anomalies,client_class,client_application,client_application_version,transport_protocol,uri,request (secops_dashboard-log profile format).
16+
// versions when key-value logging isn't enabled.
17+
//
18+
//nolint:lll //long test string kept for log profile readability
19+
func (p *securityViolationsProcessor) parseCSVLog(message string) map[string]string {
20+
fieldValueMap := make(map[string]string)
21+
22+
// Remove the "ASM:" prefix if present so we only process the values
23+
message = strings.TrimPrefix(message, "ASM:")
24+
25+
fields := strings.Split(message, ",")
26+
27+
// Mapping of CSV field positions to their corresponding keys
28+
fieldOrder := []string{
29+
"blocking_exception_reason",
30+
"dest_port",
31+
"ip_client",
32+
"is_truncated_bool",
33+
"method",
34+
"policy_name",
35+
"protocol",
36+
"request_status",
37+
"response_code",
38+
"severity",
39+
"sig_cves",
40+
"sig_set_names",
41+
"src_port",
42+
"sub_violations",
43+
"support_id",
44+
"threat_campaign_names",
45+
"violation_rating",
46+
"vs_name",
47+
"x_forwarded_for_header_value",
48+
"outcome",
49+
"outcome_reason",
50+
"violations",
51+
"violation_details",
52+
"bot_signature_name",
53+
"bot_category",
54+
"bot_anomalies",
55+
"enforced_bot_anomalies",
56+
"client_class",
57+
"client_application",
58+
"client_application_version",
59+
"transport_protocol",
60+
"uri",
61+
"request",
62+
}
63+
64+
for i, field := range fields {
65+
if i >= len(fieldOrder) {
66+
break
67+
}
68+
fieldValueMap[fieldOrder[i]] = strings.TrimSpace(field)
69+
}
70+
71+
// combine multiple values separated by '::'
72+
if combined, ok := fieldValueMap["sig_cves"]; ok {
73+
parts := strings.SplitN(combined, "::", maxSplitParts)
74+
fieldValueMap["sig_ids"] = parts[0]
75+
if len(parts) > 1 {
76+
fieldValueMap["sig_names"] = parts[1]
77+
}
78+
}
79+
80+
if combined, ok := fieldValueMap["sig_set_names"]; ok {
81+
parts := strings.SplitN(combined, "::", maxSplitParts)
82+
fieldValueMap["sig_set_names"] = parts[0]
83+
if len(parts) > 1 {
84+
fieldValueMap["sig_cves"] = parts[1]
85+
}
86+
}
87+
88+
return fieldValueMap
89+
}
90+
91+
func (p *securityViolationsProcessor) mapKVToSecurityViolationEvent(log *events.SecurityViolationEvent,
92+
kvMap map[string]string,
93+
) {
94+
log.PolicyName = kvMap["policy_name"]
95+
log.SupportId = kvMap["support_id"]
96+
log.Outcome = kvMap["outcome"]
97+
log.OutcomeReason = kvMap["outcome_reason"]
98+
log.BlockingExceptionReason = kvMap["blocking_exception_reason"]
99+
log.Method = kvMap["method"]
100+
log.Protocol = kvMap["protocol"]
101+
log.XffHeaderValue = kvMap["x_forwarded_for_header_value"]
102+
log.Uri = kvMap["uri"]
103+
log.Request = kvMap["request"]
104+
log.IsTruncated = kvMap["is_truncated_bool"]
105+
log.RequestStatus = kvMap["request_status"]
106+
log.ResponseCode = kvMap["response_code"]
107+
log.ServerAddr = kvMap["server_addr"]
108+
log.VsName = kvMap["vs_name"]
109+
log.RemoteAddr = kvMap["ip_client"]
110+
log.DestinationPort = kvMap["dest_port"]
111+
log.ServerPort = kvMap["src_port"]
112+
log.Violations = kvMap["violations"]
113+
log.SubViolations = kvMap["sub_violations"]
114+
log.ViolationRating = kvMap["violation_rating"]
115+
log.SigSetNames = kvMap["sig_set_names"]
116+
log.SigCves = kvMap["sig_cves"]
117+
log.ClientClass = kvMap["client_class"]
118+
log.ClientApplication = kvMap["client_application"]
119+
log.ClientApplicationVersion = kvMap["client_application_version"]
120+
log.Severity = kvMap["severity"]
121+
log.ThreatCampaignNames = kvMap["threat_campaign_names"]
122+
log.BotAnomalies = kvMap["bot_anomalies"]
123+
log.BotCategory = kvMap["bot_category"]
124+
log.EnforcedBotAnomalies = kvMap["enforced_bot_anomalies"]
125+
log.BotSignatureName = kvMap["bot_signature_name"]
126+
log.InstanceTags = kvMap["instance_tags"]
127+
log.InstanceGroup = kvMap["instance_group"]
128+
log.DisplayName = kvMap["display_name"]
129+
130+
if log.GetRemoteAddr() == "" {
131+
log.RemoteAddr = kvMap["remote_addr"]
132+
}
133+
if log.GetDestinationPort() == "" {
134+
log.DestinationPort = kvMap["remote_port"]
135+
}
136+
}

‎internal/collector/securityviolationsprocessor/helpers.go‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Copyright (c) F5, Inc.
2+
//
3+
// This source code is licensed under the Apache License, Version 2.0 license found in the
4+
// LICENSE file in the root directory of this source tree.
5+
6+
package securityviolationsprocessor
7+
8+
import (
9+
"net"
10+
"regexp"
11+
"strings"
12+
13+
events "github.com/nginx/agent/v3/api/grpc/events/v1"
14+
)
15+
16+
func splitAndTrim(value string) []string {
17+
if strings.TrimSpace(value) == "" || value == notAvailable {
18+
return nil
19+
}
20+
21+
parts := strings.Split(value, ",")
22+
23+
var trimmedParts []string
24+
for _, part := range parts {
25+
trimmed := strings.TrimSpace(part)
26+
if trimmed != "" {
27+
trimmedParts = append(trimmedParts, trimmed)
28+
}
29+
}
30+
31+
return trimmedParts
32+
}
33+
34+
func buildSignatures(ids, names []string, mask, offset, length string) []*events.SignatureData {
35+
signatures := make([]*events.SignatureData, 0, len(ids))
36+
for i, id := range ids {
37+
if id == "" || id == notAvailable {
38+
continue
39+
}
40+
signature := &events.SignatureData{
41+
SigDataId: id,
42+
SigDataBlockingMask: mask,
43+
SigDataOffset: offset,
44+
SigDataLength: length,
45+
}
46+
if i < len(names) {
47+
signature.SigDataBuffer = names[i]
48+
}
49+
signatures = append(signatures, signature)
50+
}
51+
52+
return signatures
53+
}
54+
55+
func extractIPFromHostname(hostname string) string {
56+
if ip := net.ParseIP(hostname); ip != nil {
57+
return ip.String()
58+
}
59+
60+
re := regexp.MustCompile(`^ip-([0-9-]+)`)
61+
if matches := re.FindStringSubmatch(hostname); len(matches) > 1 {
62+
candidate := strings.ReplaceAll(matches[1], "-", ".")
63+
if net.ParseIP(candidate) != nil {
64+
return candidate
65+
}
66+
}
67+
68+
return ""
69+
}

0 commit comments

Comments
 (0)