Add support for value in documentation

internal/docs/exported_fields.go

@@ -9,20 +9,27 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/Masterminds/semver/v3"
+
 	"github.com/elastic/elastic-package/internal/fields"
+	"github.com/elastic/elastic-package/internal/packages"
 )
 
+var semver3_2_3 = semver.MustParse("3.4.0")
+
 type fieldsTableRecord struct {
 	name        string
 	description string
 	aType       string
 	unit        string
 	metricType  string
+	value       string
+	example     string
 }
 
 var escaper = strings.NewReplacer("*", "\\*", "{", "\\{", "}", "\\}", "<", "\\<", ">", "\\>")
 
-func renderExportedFields(fieldsParentDir string) (string, error) {
+func renderExportedFields(fieldsParentDir string, pkgManifest *packages.PackageManifest) (string, error) {
 	injectOptions := fields.InjectFieldsOptions{
 		// Keep External parameter when rendering fields, so we can render
 		// documentation for empty groups imported from ECS, for backwards compatibility.
@@ -46,13 +53,27 @@ func renderExportedFields(fieldsParentDir string) (string, error) {
 		builder.WriteString("(no fields available)\n")
 		return builder.String(), nil
 	}
-	renderFieldsTable(&builder, collected)
+
+	sv, err := semver.NewVersion(pkgManifest.SpecVersion)
+	if err != nil {
+		return "", fmt.Errorf("failed to obtain spec version from package manifest: %w", err)
+	}
+
+	renderExamples := true
+	if sv.LessThan(semver3_2_3) {
+		renderExamples = false
+	}
+
+	renderFieldsTable(&builder, collected, renderExamples)
+
 	return builder.String(), nil
 }
 
-func renderFieldsTable(builder *strings.Builder, collected []fieldsTableRecord) {
+func renderFieldsTable(builder *strings.Builder, collected []fieldsTableRecord, includeExamples bool) {
 	unitsPresent := areUnitsPresent(collected)
 	metricTypesPresent := areMetricTypesPresent(collected)
+	examplePresent := areExamplesPresent(collected)
+	valuesPresent := areValuesPresent(collected)
 
 	builder.WriteString("| Field | Description | Type |")
 	if unitsPresent {
@@ -61,6 +82,9 @@ func renderFieldsTable(builder *strings.Builder, collected []fieldsTableRecord)
 	if metricTypesPresent {
 		builder.WriteString(" Metric Type |")
 	}
+	if (examplePresent || valuesPresent) && includeExamples {
+		builder.WriteString(" Example |")
+	}
 
 	builder.WriteString("\n")
 	builder.WriteString("|---|---|---|")
@@ -70,6 +94,9 @@ func renderFieldsTable(builder *strings.Builder, collected []fieldsTableRecord)
 	if metricTypesPresent {
 		builder.WriteString("---|")
 	}
+	if (examplePresent || valuesPresent) && includeExamples { // Check whether there are examples to render or whether fields are constants
+		builder.WriteString("---|")
+	}
 
 	builder.WriteString("\n")
 	for _, c := range collected {
@@ -84,6 +111,13 @@ func renderFieldsTable(builder *strings.Builder, collected []fieldsTableRecord)
 		if metricTypesPresent {
 			builder.WriteString(fmt.Sprintf(" %s |", c.metricType))
 		}
+		if (examplePresent || valuesPresent) && includeExamples {
+			if c.aType == "constant_keyword" {
+				builder.WriteString(fmt.Sprintf(" %s |", c.value))
+			} else {
+				builder.WriteString(fmt.Sprintf(" %s |", c.example))
+			}
+		}
 		builder.WriteString("\n")
 	}
 }
@@ -106,6 +140,24 @@ func areMetricTypesPresent(collected []fieldsTableRecord) bool {
 	return false
 }
 
+func areValuesPresent(collected []fieldsTableRecord) bool {
+	for _, c := range collected {
+		if c.aType == "constant_keyword" && c.value != "" {
+			return true
+		}
+	}
+	return false
+}
+
+func areExamplesPresent(collected []fieldsTableRecord) bool {
+	for _, c := range collected {
+		if c.example != "" {
+			return true
+		}
+	}
+	return false
+}
+
 func collectFieldsFromDefinitions(validator *fields.Validator) []fieldsTableRecord {
 	var records []fieldsTableRecord
 
@@ -130,6 +182,8 @@ func visitFields(namePrefix string, f fields.FieldDefinition, records []fieldsTa
 			aType:       f.Type,
 			unit:        f.Unit,
 			metricType:  f.MetricType,
+			value:       f.Value,
+			example:     f.Example,
 		})
 
 		for _, multiField := range f.MultiFields {

internal/docs/readme.go

@@ -191,11 +191,15 @@ func renderReadme(fileName, packageRoot, templatePath string, linksMap linkMap)
 			return renderSampleEvent(packageRoot, "")
 		},
 		"fields": func(args ...string) (string, error) {
+			manifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot)
+			if err != nil {
+				return "", fmt.Errorf("reading package manifest failed (path: %s): %w", packageRoot, err)
+			}
 			if len(args) > 0 {
 				dataStreamPath := filepath.Join(packageRoot, "data_stream", args[0])
-				return renderExportedFields(dataStreamPath)
+				return renderExportedFields(dataStreamPath, manifest)
 			}
-			return renderExportedFields(packageRoot)
+			return renderExportedFields(packageRoot, manifest)
 		},
 		"url": func(args ...string) (string, error) {
 			options := linkOptions{}

internal/fields/model.go

@@ -27,6 +27,7 @@ type FieldDefinition struct {
 	Unit           string            `yaml:"unit"`
 	MetricType     string            `yaml:"metric_type"`
 	External       string            `yaml:"external"`
+	Example        string            `yaml:"example"`
 	Index          *bool             `yaml:"index"`
 	Enabled        *bool             `yaml:"enabled"`
 	DocValues      *bool             `yaml:"doc_values"`

internal/fields/testdata/disabled.json

@@ -1,6 +1,6 @@
-{
-  "disabled": {
-    "id": "42",
-    "status": "ok"
-  }
-}
+{
+  "disabled": {
+    "id": "42",
+    "status": "ok"
+  }
+}

internal/fields/testdata/enabled_not_mapped.json

@@ -1,6 +1,6 @@
-{
-  "enabled": {
-    "id": "42",
-    "status": "ok"
-  }
-}
+{
+  "enabled": {
+    "id": "42",
+    "status": "ok"
+  }
+}

test/packages/parallel/ti_anomali/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.23.0-rc1"
+  changes:
+    - description: Update format spec to 3.4.0
+      type: enhancement
+      link: https://github.com/elastic/elastic-package/pull/2167
 - version: "1.22.1"
   changes:
     - description: Fix ECS date mapping on threat fields.

test/packages/parallel/ti_anomali/docs/README.md

@@ -141,44 +141,44 @@ An example event for `threatstream` looks as following:
 
 **Exported fields**
 
-| Field | Description | Type |
-|---|---|---|
-| @timestamp | Event timestamp. | date |
-| anomali.threatstream.added_at | Date when IOC was added. | date |
-| anomali.threatstream.classification | Indicates whether an indicator is private or from a public feed and available publicly. Possible values: private, public. | keyword |
-| anomali.threatstream.confidence | The measure of the accuracy (from 0 to 100) assigned by ThreatStream's predictive analytics technology to indicators. | short |
-| anomali.threatstream.deleted_at | Date when IOC was deleted/expired. | date |
-| anomali.threatstream.detail2 | Detail text for indicator. | text |
-| anomali.threatstream.id | The ID of the indicator. | keyword |
-| anomali.threatstream.import_session_id | ID of the import session that created the indicator on ThreatStream. | keyword |
-| anomali.threatstream.itype | Indicator type. Possible values: "apt_domain", "apt_email", "apt_ip", "apt_url", "bot_ip", "c2_domain", "c2_ip", "c2_url", "i2p_ip", "mal_domain", "mal_email", "mal_ip", "mal_md5", "mal_url", "parked_ip", "phish_email", "phish_ip", "phish_url", "scan_ip", "spam_domain", "ssh_ip", "suspicious_domain", "tor_ip" and "torrent_tracker_url". | keyword |
-| anomali.threatstream.maltype | Information regarding a malware family, a CVE ID, or another attack or threat, associated with the indicator. | wildcard |
-| anomali.threatstream.md5 | Hash for the indicator. | keyword |
-| anomali.threatstream.resource_uri | Relative URI for the indicator details. | keyword |
-| anomali.threatstream.severity | Criticality associated with the threat feed that supplied the indicator. Possible values: low, medium, high, very-high. | keyword |
-| anomali.threatstream.source | Source for the indicator. | keyword |
-| anomali.threatstream.source_feed_id | ID for the integrator source. | keyword |
-| anomali.threatstream.state | State for this indicator. | keyword |
-| anomali.threatstream.trusted_circle_ids | ID of the trusted circle that imported the indicator. | keyword |
-| anomali.threatstream.update_id | Update ID. | keyword |
-| anomali.threatstream.url | URL for the indicator. | keyword |
-| anomali.threatstream.value_type | Data type of the indicator. Possible values: ip, domain, url, email, md5. | keyword |
-| cloud.image.id | Image ID for the cloud instance. | keyword |
-| data_stream.dataset | Data stream dataset name. | constant_keyword |
-| data_stream.namespace | Data stream namespace. | constant_keyword |
-| data_stream.type | Data stream type. | constant_keyword |
-| event.dataset | Event dataset | constant_keyword |
-| event.module | Event module | constant_keyword |
-| host.containerized | If the host is a container. | boolean |
-| host.os.build | OS build information. | keyword |
-| host.os.codename | OS codename, if any. | keyword |
-| input.type | Type of Filebeat input. | keyword |
-| labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword |
-| log.flags | Flags for the log file. | keyword |
-| log.offset | Offset of the entry in the log file. | long |
-| threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword |
-| threat.feed.name | Display friendly feed name | constant_keyword |
-| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
-| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |
-| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
+| Field | Description | Type | Example |
+|---|---|---|---|
+| @timestamp | Event timestamp. | date |  |
+| anomali.threatstream.added_at | Date when IOC was added. | date |  |
+| anomali.threatstream.classification | Indicates whether an indicator is private or from a public feed and available publicly. Possible values: private, public. | keyword | private |
+| anomali.threatstream.confidence | The measure of the accuracy (from 0 to 100) assigned by ThreatStream's predictive analytics technology to indicators. | short |  |
+| anomali.threatstream.deleted_at | Date when IOC was deleted/expired. | date |  |
+| anomali.threatstream.detail2 | Detail text for indicator. | text | Imported by user 42. |
+| anomali.threatstream.id | The ID of the indicator. | keyword |  |
+| anomali.threatstream.import_session_id | ID of the import session that created the indicator on ThreatStream. | keyword |  |
+| anomali.threatstream.itype | Indicator type. Possible values: "apt_domain", "apt_email", "apt_ip", "apt_url", "bot_ip", "c2_domain", "c2_ip", "c2_url", "i2p_ip", "mal_domain", "mal_email", "mal_ip", "mal_md5", "mal_url", "parked_ip", "phish_email", "phish_ip", "phish_url", "scan_ip", "spam_domain", "ssh_ip", "suspicious_domain", "tor_ip" and "torrent_tracker_url". | keyword |  |
+| anomali.threatstream.maltype | Information regarding a malware family, a CVE ID, or another attack or threat, associated with the indicator. | wildcard |  |
+| anomali.threatstream.md5 | Hash for the indicator. | keyword |  |
+| anomali.threatstream.resource_uri | Relative URI for the indicator details. | keyword |  |
+| anomali.threatstream.severity | Criticality associated with the threat feed that supplied the indicator. Possible values: low, medium, high, very-high. | keyword |  |
+| anomali.threatstream.source | Source for the indicator. | keyword | Analyst |
+| anomali.threatstream.source_feed_id | ID for the integrator source. | keyword |  |
+| anomali.threatstream.state | State for this indicator. | keyword | active |
+| anomali.threatstream.trusted_circle_ids | ID of the trusted circle that imported the indicator. | keyword |  |
+| anomali.threatstream.update_id | Update ID. | keyword |  |
+| anomali.threatstream.url | URL for the indicator. | keyword |  |
+| anomali.threatstream.value_type | Data type of the indicator. Possible values: ip, domain, url, email, md5. | keyword |  |
+| cloud.image.id | Image ID for the cloud instance. | keyword |  |
+| data_stream.dataset | Data stream dataset name. | constant_keyword |  |
+| data_stream.namespace | Data stream namespace. | constant_keyword |  |
+| data_stream.type | Data stream type. | constant_keyword |  |
+| event.dataset | Event dataset | constant_keyword | ti_anomali.threatstream |
+| event.module | Event module | constant_keyword | ti_anomali |
+| host.containerized | If the host is a container. | boolean |  |
+| host.os.build | OS build information. | keyword | 18D109 |
+| host.os.codename | OS codename, if any. | keyword | stretch |
+| input.type | Type of Filebeat input. | keyword |  |
+| labels.is_ioc_transform_source | Field indicating if its the transform source for supporting IOC expiration. This field is dropped from destination indices to facilitate easier filtering of indicators. | constant_keyword | true |
+| log.flags | Flags for the log file. | keyword |  |
+| log.offset | Offset of the entry in the log file. | long |  |
+| threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | ti_anomali-96fe1e60-4261-11ec-b7be-d3026acdf1cf |
+| threat.feed.name | Display friendly feed name | constant_keyword | Anomali ThreatStream |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |  |
+| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |  |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |  |
 

test/packages/parallel/ti_anomali/manifest.yml

@@ -1,9 +1,9 @@
 name: ti_anomali
 title: Anomali
-version: "1.22.1"
+version: "1.23.0-rc1"
 description: Ingest threat intelligence indicators from Anomali with Elastic Agent.
 type: integration
-format_version: 3.0.2
+format_version: 3.4.0
 categories: ["security", "threat_intel"]
 conditions:
   kibana: