diff --git a/packages/blacklens/changelog.yml b/packages/blacklens/changelog.yml index df854209b39..c6e158df339 100644 --- a/packages/blacklens/changelog.yml +++ b/packages/blacklens/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.0.0" + changes: + - description: Make GA and Update Fields to match new JSON Scheme + type: enhancement + link: https://github.com/elastic/integrations/pull/16893 - version: "0.4.1" changes: - description: Fix default request trace enabled behavior. diff --git a/packages/blacklens/data_stream/alerts/_dev/test/pipeline/test-alerts.log b/packages/blacklens/data_stream/alerts/_dev/test/pipeline/test-alerts.log index e262edf5566..6e88295b5e2 100644 --- a/packages/blacklens/data_stream/alerts/_dev/test/pipeline/test-alerts.log +++ b/packages/blacklens/data_stream/alerts/_dev/test/pipeline/test-alerts.log @@ -1 +1 @@ -{"updated_date":"2024-11-12T09:39:58.489Z","created_date":"2024-11-12T09:39:58.489Z","id":1001,"details":{"id":100,"engine":"Port Scanner","title":"New Open Port"},"severity":"medium","affected_entities":2,"alert_outcome":"affected","alert_status":"resolved","customer_state":"open","alert_payload":[],"type_id":100} \ No newline at end of file +{"updated_date":"2025-12-31T16:10:56.155874Z","created_date":"2025-12-30T16:11:57.194393Z","id":"7ea10c5d-559a-4c55-8608-2e060956de68","name":"External Vulnerability Detected","type":"ExternalVulnerabilityDiscovered","severity":"high","status":"new","analysis":"completed","category":"vulnerability","activities":[{"updated_date":null,"created_date":"2025-12-30T16:11:40.195989Z","id":"73dcaa88-09e1-4c58-9fa5-5495f8dac2a4","type":"ExternalVulnerabilityCreated","description":"A Critical severity external vulnerability 'Blind SQL Injection via HTTP Header' has been detected on asset 'demo.example.com'","category":"threat","trace_id":"40eda190-83fd-4a1b-8155-3a1c7434b319","data":{}}]} \ No newline at end of file diff --git a/packages/blacklens/data_stream/alerts/_dev/test/pipeline/test-alerts.log-expected.json b/packages/blacklens/data_stream/alerts/_dev/test/pipeline/test-alerts.log-expected.json index e37f6b81633..a2bfac91684 100644 --- a/packages/blacklens/data_stream/alerts/_dev/test/pipeline/test-alerts.log-expected.json +++ b/packages/blacklens/data_stream/alerts/_dev/test/pipeline/test-alerts.log-expected.json @@ -1,17 +1,26 @@ { "expected": [ { - "@timestamp": "2024-11-12T09:39:58.489Z", + "@timestamp": "2025-12-30T16:11:57.194Z", "blacklens": { "alert": { - "id": 1001, - "outcome": "affected", - "severity": "medium", - "status": "resolved", - "title": "New Open Port", - "type": "Port Scanner", - "type_id": 100, - "updated_date": "2024-11-12T09:39:58.489Z" + "activities": [ + { + "category": "threat", + "created_date": "2025-12-30T16:11:40.195989Z", + "description": "A Critical severity external vulnerability 'Blind SQL Injection via HTTP Header' has been detected on asset 'demo.example.com'", + "id": "73dcaa88-09e1-4c58-9fa5-5495f8dac2a4", + "trace_id": "40eda190-83fd-4a1b-8155-3a1c7434b319", + "type": "ExternalVulnerabilityCreated" + } + ], + "analysis": "completed", + "category": "vulnerability", + "id": "7ea10c5d-559a-4c55-8608-2e060956de68", + "name": "External Vulnerability Detected", + "severity": "high", + "status": "new", + "updated_date": "2025-12-31T16:10:56.155Z" } }, "ecs": { @@ -21,8 +30,8 @@ "category": [ "threat" ], - "id": "1001", - "original": "{\"updated_date\":\"2024-11-12T09:39:58.489Z\",\"created_date\":\"2024-11-12T09:39:58.489Z\",\"id\":1001,\"details\":{\"id\":100,\"engine\":\"Port Scanner\",\"title\":\"New Open Port\"},\"severity\":\"medium\",\"affected_entities\":2,\"alert_outcome\":\"affected\",\"alert_status\":\"resolved\",\"customer_state\":\"open\",\"alert_payload\":[],\"type_id\":100}", + "id": "7ea10c5d-559a-4c55-8608-2e060956de68", + "original": "{\"updated_date\":\"2025-12-31T16:10:56.155874Z\",\"created_date\":\"2025-12-30T16:11:57.194393Z\",\"id\":\"7ea10c5d-559a-4c55-8608-2e060956de68\",\"name\":\"External Vulnerability Detected\",\"type\":\"ExternalVulnerabilityDiscovered\",\"severity\":\"high\",\"status\":\"new\",\"analysis\":\"completed\",\"category\":\"vulnerability\",\"activities\":[{\"updated_date\":null,\"created_date\":\"2025-12-30T16:11:40.195989Z\",\"id\":\"73dcaa88-09e1-4c58-9fa5-5495f8dac2a4\",\"type\":\"ExternalVulnerabilityCreated\",\"description\":\"A Critical severity external vulnerability 'Blind SQL Injection via HTTP Header' has been detected on asset 'demo.example.com'\",\"category\":\"threat\",\"trace_id\":\"40eda190-83fd-4a1b-8155-3a1c7434b319\",\"data\":{}}]}", "type": [ "indicator" ] diff --git a/packages/blacklens/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml b/packages/blacklens/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml index 3b2f3cdb71c..98bfe2b3550 100644 --- a/packages/blacklens/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml +++ b/packages/blacklens/data_stream/alerts/elasticsearch/ingest_pipeline/default.yml @@ -52,28 +52,24 @@ processors: target_field: blacklens.alert.severity ignore_missing: true - rename: - field: json.alert_status + field: json.status target_field: blacklens.alert.status ignore_missing: true - rename: - field: json.alert_outcome - target_field: blacklens.alert.outcome + field: json.analysis + target_field: blacklens.alert.analysis ignore_missing: true - rename: - field: json.details.engine - target_field: blacklens.alert.type + field: json.category + target_field: blacklens.alert.category ignore_missing: true - rename: - field: json.type_id - target_field: blacklens.alert.type_id + field: json.name + target_field: blacklens.alert.name ignore_missing: true - rename: - field: json.details.title - target_field: blacklens.alert.title - ignore_missing: true - - rename: - field: json.alert_payload - target_field: blacklens.alert.details + field: json.activities + target_field: blacklens.alert.activities ignore_missing: true - remove: field: json diff --git a/packages/blacklens/data_stream/alerts/fields/fields.yml b/packages/blacklens/data_stream/alerts/fields/fields.yml index 1d6db047ab8..a37b4fe2e92 100644 --- a/packages/blacklens/data_stream/alerts/fields/fields.yml +++ b/packages/blacklens/data_stream/alerts/fields/fields.yml @@ -5,7 +5,7 @@ type: date description: Activity last updated time (UTC). - name: id - type: integer + type: keyword description: Unique Alert ID - name: severity type: keyword @@ -13,18 +13,32 @@ - name: status type: keyword description: Current Status of the Alert - - name: outcome + - name: analysis type: keyword description: Determines whether the current alert triggers further events - - name: type + - name: category type: keyword - description: Alert Type (Engine) - - name: type_id - type: integer - description: Alert Type ID (Engine) - - name: title + description: Alert category + - name: name type: keyword - description: Title/Description of the given Alert - - name: details - type: nested - description: Alert Details + description: Name of the given Alert + - name: activities + description: Associated Activities of the Alert + type: group + fields: + - name: id + type: keyword + - name: created_date + type: date + - name: updated_date + type: date + - name: category + type: keyword + - name: type + type: keyword + - name: description + type: keyword + - name: trace_id + type: keyword + - name: data + type: nested \ No newline at end of file diff --git a/packages/blacklens/data_stream/alerts/sample_event.json b/packages/blacklens/data_stream/alerts/sample_event.json index 19a5426d98c..c3d99f7448c 100644 --- a/packages/blacklens/data_stream/alerts/sample_event.json +++ b/packages/blacklens/data_stream/alerts/sample_event.json @@ -1,55 +1,64 @@ { - "@timestamp": "2024-11-12T09:39:58.489Z", - "agent": { - "ephemeral_id": "33939e93-54ef-4184-b92b-bc8f02e179a6", - "id": "f98f4444-6fca-4500-83b6-a8c5e8f32bf1", - "name": "elastic-agent-49577", - "type": "filebeat", - "version": "8.15.2" - }, - "blacklens": { - "alert": { - "id": 1001, - "outcome": "affected", - "severity": "medium", - "status": "resolved", - "title": "New Open Port", - "type": "Port Scanner", - "type_id": 100, - "updated_date": "2024-11-12T09:39:58.489Z" - } - }, - "data_stream": { - "dataset": "blacklens.alerts", - "namespace": "41265", - "type": "logs" - }, - "ecs": { - "version": "8.11.0" - }, - "elastic_agent": { - "id": "f98f4444-6fca-4500-83b6-a8c5e8f32bf1", - "snapshot": false, - "version": "8.15.2" - }, - "event": { - "agent_id_status": "verified", - "category": [ - "threat" - ], - "created": "2025-12-09T05:45:05.855Z", - "dataset": "blacklens.alerts", - "id": "1001", - "ingested": "2025-12-09T05:45:08Z", - "type": [ - "indicator" - ] - }, - "input": { - "type": "httpjson" - }, - "tags": [ - "forwarded", - "blacklens-alert" - ] -} + "@timestamp":"2024-11-12T09:39:58.489Z", + "agent":{ + "ephemeral_id":"33939e93-54ef-4184-b92b-bc8f02e179a6", + "id":"f98f4444-6fca-4500-83b6-a8c5e8f32bf1", + "name":"elastic-agent-49577", + "type":"filebeat", + "version":"8.15.2" + }, + "blacklens":{ + "alert":{ + "activities":[ + { + "category":"threat", + "created_date":"2025-12-30T16:11:40.195989Z", + "description":"A Critical severity external vulnerability 'Blind SQL Injection via HTTP Header' has been detected on asset 'demo.example.com'", + "id":"73dcaa88-09e1-4c58-9fa5-5495f8dac2a4", + "trace_id":"40eda190-83fd-4a1b-8155-3a1c7434b319", + "type":"ExternalVulnerabilityCreated" + } + ], + "analysis":"completed", + "category":"vulnerability", + "id":"7ea10c5d-559a-4c55-8608-2e060956de68", + "name":"External Vulnerability Detected", + "severity":"high", + "status":"new", + "updated_date":"2025-12-31T16:10:56.155Z" + } + }, + "data_stream":{ + "dataset":"blacklens.alerts", + "namespace":"41265", + "type":"logs" + }, + "ecs":{ + "version":"8.11.0" + }, + "elastic_agent":{ + "id":"f98f4444-6fca-4500-83b6-a8c5e8f32bf1", + "snapshot":false, + "version":"8.15.2" + }, + "event":{ + "agent_id_status":"verified", + "category":[ + "threat" + ], + "created":"2025-12-09T05:45:05.855Z", + "dataset":"blacklens.alerts", + "id":"1001", + "ingested":"2025-12-09T05:45:08Z", + "type":[ + "indicator" + ] + }, + "input":{ + "type":"httpjson" + }, + "tags":[ + "forwarded", + "blacklens-alert" + ] +} \ No newline at end of file diff --git a/packages/blacklens/docs/README.md b/packages/blacklens/docs/README.md index dec0fd03fd7..afce04d49b8 100644 --- a/packages/blacklens/docs/README.md +++ b/packages/blacklens/docs/README.md @@ -64,14 +64,23 @@ An example event for `alerts` looks as following: }, "blacklens": { "alert": { - "id": 1001, - "outcome": "affected", - "severity": "medium", - "status": "resolved", - "title": "New Open Port", - "type": "Port Scanner", - "type_id": 100, - "updated_date": "2024-11-12T09:39:58.489Z" + "activities": [ + { + "category": "threat", + "created_date": "2025-12-30T16:11:40.195989Z", + "description": "A Critical severity external vulnerability 'Blind SQL Injection via HTTP Header' has been detected on asset 'demo.example.com'", + "id": "73dcaa88-09e1-4c58-9fa5-5495f8dac2a4", + "trace_id": "40eda190-83fd-4a1b-8155-3a1c7434b319", + "type": "ExternalVulnerabilityCreated" + } + ], + "analysis": "completed", + "category": "vulnerability", + "id": "7ea10c5d-559a-4c55-8608-2e060956de68", + "name": "External Vulnerability Detected", + "severity": "high", + "status": "new", + "updated_date": "2025-12-31T16:10:56.155Z" } }, "data_stream": { @@ -115,14 +124,20 @@ An example event for `alerts` looks as following: | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| blacklens.alert.details | Alert Details | nested | -| blacklens.alert.id | Unique Alert ID | integer | -| blacklens.alert.outcome | Determines whether the current alert triggers further events | keyword | +| blacklens.alert.activities.category | | keyword | +| blacklens.alert.activities.created_date | | date | +| blacklens.alert.activities.data | | nested | +| blacklens.alert.activities.description | | keyword | +| blacklens.alert.activities.id | | keyword | +| blacklens.alert.activities.trace_id | | keyword | +| blacklens.alert.activities.type | | keyword | +| blacklens.alert.activities.updated_date | | date | +| blacklens.alert.analysis | Determines whether the current alert triggers further events | keyword | +| blacklens.alert.category | Alert category | keyword | +| blacklens.alert.id | Unique Alert ID | keyword | +| blacklens.alert.name | Name of the given Alert | keyword | | blacklens.alert.severity | Alert Severity | keyword | | blacklens.alert.status | Current Status of the Alert | keyword | -| blacklens.alert.title | Title/Description of the given Alert | keyword | -| blacklens.alert.type | Alert Type (Engine) | keyword | -| blacklens.alert.type_id | Alert Type ID (Engine) | integer | | blacklens.alert.updated_date | Activity last updated time (UTC). | date | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | diff --git a/packages/blacklens/kibana/dashboard/blacklens-e718fd52-f1b3-400f-94c1-ead17da571f6.json b/packages/blacklens/kibana/dashboard/blacklens-e718fd52-f1b3-400f-94c1-ead17da571f6.json index 1a354028504..cffbfaf1079 100644 --- a/packages/blacklens/kibana/dashboard/blacklens-e718fd52-f1b3-400f-94c1-ead17da571f6.json +++ b/packages/blacklens/kibana/dashboard/blacklens-e718fd52-f1b3-400f-94c1-ead17da571f6.json @@ -214,7 +214,7 @@ "5daabdc5-ef58-44c4-abc6-e081ccc141b3": { "dataType": "string", "isBucketed": true, - "label": "Top 7 values of blacklens.alert.type", + "label": "Top 7 values of blacklens.alert.category", "operationType": "terms", "params": { "exclude": [], @@ -234,7 +234,7 @@ "size": 7 }, "scale": "ordinal", - "sourceField": "blacklens.alert.type" + "sourceField": "blacklens.alert.category" }, "779791bd-efc6-4cd7-a348-e1f02e55da6a": { "dataType": "number", @@ -515,7 +515,7 @@ "596741f2-76ab-4053-96a3-f7d0c419e3ca": { "dataType": "string", "isBucketed": true, - "label": "Top 10 values of blacklens.alert.type", + "label": "Top 10 values of blacklens.alert.category", "operationType": "terms", "params": { "accuracyMode": false, @@ -536,7 +536,7 @@ "size": 10 }, "scale": "ordinal", - "sourceField": "blacklens.alert.type" + "sourceField": "blacklens.alert.category" }, "740c7874-8bd6-4615-a570-bb61d09e2343": { "customLabel": true, @@ -671,7 +671,7 @@ "5adc4648-4eb7-4b43-9235-26d66a0ab3d2": { "dataType": "string", "isBucketed": true, - "label": "Top 5 values of blacklens.alert.type", + "label": "Top 5 values of blacklens.alert.category", "operationType": "terms", "params": { "exclude": [], @@ -691,7 +691,7 @@ "size": 5 }, "scale": "ordinal", - "sourceField": "blacklens.alert.type" + "sourceField": "blacklens.alert.category" }, "772e1ee4-713b-4a0e-84ce-76b2030e1240": { "dataType": "number", diff --git a/packages/blacklens/manifest.yml b/packages/blacklens/manifest.yml index b2c96984ca8..53ab5fe1f2e 100644 --- a/packages/blacklens/manifest.yml +++ b/packages/blacklens/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.0 name: blacklens title: "blacklens.io" -version: "0.4.1" +version: "1.0.0" source: license: "Elastic-2.0" description: "Collect logs from blacklens.io with Elastic Agent"