diff --git a/cmd/dump.go b/cmd/dump.go index 1b1f6084f..47287dbe3 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -70,7 +70,7 @@ func dumpInstalledObjectsCmd(cmd *cobra.Command, args []string) error { return errors.Wrap(err, "dump failed") } if n == 0 { - cmd.Printf("No objects were dumped for package %s, is it installed?", packageName) + cmd.Printf("No objects were dumped for package %s, is it installed?\n", packageName) return nil } cmd.Printf("Dumped %d installed objects for package %s to %s\n", n, packageName, outputPath) diff --git a/internal/dump/indextemplates.go b/internal/dump/indextemplates.go index 9fb93e3c3..f54b60047 100644 --- a/internal/dump/indextemplates.go +++ b/internal/dump/indextemplates.go @@ -10,6 +10,7 @@ import ( "fmt" "io/ioutil" + "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/elasticsearch" ) @@ -97,7 +98,7 @@ func getIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, pa indexTemplate.raw = indexTemplateRaw meta := indexTemplate.IndexTemplate.Meta - if meta.Package.Name != packageName || meta.ManagedBy != "ingest-manager" { + if meta.Package.Name != packageName || !managedByFleet(meta.ManagedBy) { // This is not the droid you are looking for. continue } @@ -107,3 +108,8 @@ func getIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, pa return indexTemplates, nil } + +func managedByFleet(managedBy string) bool { + var managers = []string{"ingest-manager", "fleet"} + return common.StringSliceContains(managers, managedBy) +} diff --git a/internal/dump/installedobjects_test.go b/internal/dump/installedobjects_test.go index bd6808cb5..017227f9c 100644 --- a/internal/dump/installedobjects_test.go +++ b/internal/dump/installedobjects_test.go @@ -7,6 +7,7 @@ package dump import ( "context" "encoding/json" + "errors" "io/fs" "io/ioutil" "os" @@ -15,30 +16,87 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/elastic/elastic-package/internal/elasticsearch" estest "github.com/elastic/elastic-package/internal/elasticsearch/test" "github.com/elastic/elastic-package/internal/files" ) -func TestInstalledObjectsDumpAll(t *testing.T) { - client := estest.ElasticsearchClient(t, "./testdata/elasticsearch-mock-dump-apache") - outputDir := t.TempDir() - dumper := NewInstalledObjectsDumper(client, "apache") +func TestDumpInstalledObjects(t *testing.T) { + // Files for each suite are recorded automatically on first test run. + // To add a new suite: + // - Configure it here. + // - Install the package in a running stack. + // - Configure environment variables for this stack (eval "$(elastic-package stack shellinit)"). + // - Run tests. + // - Check that recorded files make sense and commit them. + suites := []*installedObjectsDumpSuite{ + &installedObjectsDumpSuite{ + PackageName: "apache", + RecordDir: "./testdata/elasticsearch-7-mock-dump-apache", + DumpDir: "./testdata/elasticsearch-7-apache-dump-all", + }, + &installedObjectsDumpSuite{ + PackageName: "apache", + RecordDir: "./testdata/elasticsearch-8-mock-dump-apache", + DumpDir: "./testdata/elasticsearch-8-apache-dump-all", + }, + } + + for _, s := range suites { + suite.Run(t, s) + } +} + +type installedObjectsDumpSuite struct { + suite.Suite + + // PackageName is the name of the package. + PackageName string + + // RecordDir is where responses from Elasticsearch are recorded. + RecordDir string + + // DumpDir is where the expected dumped files are stored. + DumpDir string +} + +func (s *installedObjectsDumpSuite) SetupTest() { + _, err := os.Stat(s.DumpDir) + if errors.Is(err, os.ErrNotExist) { + client, err := elasticsearch.Client() + s.Require().NoError(err) + + dumper := NewInstalledObjectsDumper(client.API, s.PackageName) + n, err := dumper.DumpAll(context.Background(), s.DumpDir) + s.Require().NoError(err) + s.Require().Greater(n, 0) + } else { + s.Require().NoError(err) + } +} + +func (s *installedObjectsDumpSuite) TestDumpAll() { + client := estest.ElasticsearchClient(s.T(), s.RecordDir) + + outputDir := s.T().TempDir() + dumper := NewInstalledObjectsDumper(client, s.PackageName) n, err := dumper.DumpAll(context.Background(), outputDir) - require.NoError(t, err) + s.Require().NoError(err) - filesExpected := countFiles(t, "./testdata/apache-dump-all") - assert.Equal(t, filesExpected, n) + filesExpected := countFiles(s.T(), s.DumpDir) + s.Assert().Equal(filesExpected, n) - filesFound := countFiles(t, outputDir) - assert.Equal(t, filesExpected, filesFound) + filesFound := countFiles(s.T(), outputDir) + s.Assert().Equal(filesExpected, filesFound) - assertEqualDumps(t, "./testdata/apache-dump-all", outputDir) + assertEqualDumps(s.T(), s.DumpDir, outputDir) } -func TestInstalledObjectsDumpSome(t *testing.T) { - client := estest.ElasticsearchClient(t, "./testdata/elasticsearch-mock-dump-apache") - dumper := NewInstalledObjectsDumper(client, "apache") +func (s *installedObjectsDumpSuite) TestDumpSome() { + client := estest.ElasticsearchClient(s.T(), s.RecordDir) + dumper := NewInstalledObjectsDumper(client, s.PackageName) // In a map so order of execution is randomized. dumpers := map[string]func(ctx context.Context, outputDir string) (int, error){ @@ -49,19 +107,19 @@ func TestInstalledObjectsDumpSome(t *testing.T) { } for dir, dumpFunction := range dumpers { - t.Run(dir, func(t *testing.T) { - outputDir := t.TempDir() + s.Run(dir, func() { + outputDir := s.T().TempDir() n, err := dumpFunction(context.Background(), outputDir) - require.NoError(t, err) + s.Require().NoError(err) - expectedDir := subDir(t, "./testdata/apache-dump-all", dir) - filesExpected := countFiles(t, expectedDir) - assert.Equal(t, filesExpected, n) + expectedDir := subDir(s.T(), s.DumpDir, dir) + filesExpected := countFiles(s.T(), expectedDir) + s.Assert().Equal(filesExpected, n) - filesFound := countFiles(t, outputDir) - assert.Equal(t, filesExpected, filesFound) + filesFound := countFiles(s.T(), outputDir) + s.Assert().Equal(filesExpected, filesFound) - assertEqualDumps(t, expectedDir, outputDir) + assertEqualDumps(s.T(), expectedDir, outputDir) }) } } diff --git a/internal/dump/testdata/apache-dump-all/component_templates/.fleet_component_template-1.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/.fleet_component_template-1.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/component_templates/.fleet_component_template-1.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/.fleet_component_template-1.json diff --git a/internal/dump/testdata/apache-dump-all/component_templates/logs-apache.access@custom.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/logs-apache.access@custom.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/component_templates/logs-apache.access@custom.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/logs-apache.access@custom.json diff --git a/internal/dump/testdata/apache-dump-all/component_templates/logs-apache.access@settings.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/logs-apache.access@settings.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/component_templates/logs-apache.access@settings.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/logs-apache.access@settings.json diff --git a/internal/dump/testdata/apache-dump-all/component_templates/logs-apache.error@custom.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/logs-apache.error@custom.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/component_templates/logs-apache.error@custom.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/logs-apache.error@custom.json diff --git a/internal/dump/testdata/apache-dump-all/component_templates/logs-apache.error@settings.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/logs-apache.error@settings.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/component_templates/logs-apache.error@settings.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/logs-apache.error@settings.json diff --git a/internal/dump/testdata/apache-dump-all/component_templates/metrics-apache.status@custom.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/metrics-apache.status@custom.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/component_templates/metrics-apache.status@custom.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/metrics-apache.status@custom.json diff --git a/internal/dump/testdata/apache-dump-all/component_templates/metrics-apache.status@settings.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/metrics-apache.status@settings.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/component_templates/metrics-apache.status@settings.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/component_templates/metrics-apache.status@settings.json diff --git a/internal/dump/testdata/apache-dump-all/ilm_policies/logs.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/logs.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/ilm_policies/logs.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/logs.json diff --git a/internal/dump/testdata/apache-dump-all/ilm_policies/metrics.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/metrics.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/ilm_policies/metrics.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ilm_policies/metrics.json diff --git a/internal/dump/testdata/apache-dump-all/index_templates/logs-apache.access.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.access.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/index_templates/logs-apache.access.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.access.json diff --git a/internal/dump/testdata/apache-dump-all/index_templates/logs-apache.error.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.error.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/index_templates/logs-apache.error.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/logs-apache.error.json diff --git a/internal/dump/testdata/apache-dump-all/index_templates/metrics-apache.status.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/metrics-apache.status.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/index_templates/metrics-apache.status.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/index_templates/metrics-apache.status.json diff --git a/internal/dump/testdata/apache-dump-all/ingest_pipelines/.fleet_final_pipeline-1.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/.fleet_final_pipeline-1.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/ingest_pipelines/.fleet_final_pipeline-1.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/.fleet_final_pipeline-1.json diff --git a/internal/dump/testdata/apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4-third-party.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4-third-party.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4-third-party.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4-third-party.json diff --git a/internal/dump/testdata/apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.4.json diff --git a/internal/dump/testdata/apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4-third-party.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4-third-party.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4-third-party.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4-third-party.json diff --git a/internal/dump/testdata/apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4.json b/internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4.json similarity index 100% rename from internal/dump/testdata/apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4.json rename to internal/dump/testdata/elasticsearch-7-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.4.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-.fleet_component_template-1.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-.fleet_component_template-1.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-.fleet_component_template-1.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-.fleet_component_template-1.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-logs-apache.access@custom.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-logs-apache.access@custom.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-logs-apache.access@custom.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-logs-apache.access@custom.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-logs-apache.access@settings.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-logs-apache.access@settings.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-logs-apache.access@settings.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-logs-apache.access@settings.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-logs-apache.error@custom.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-logs-apache.error@custom.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-logs-apache.error@custom.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-logs-apache.error@custom.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-logs-apache.error@settings.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-logs-apache.error@settings.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-logs-apache.error@settings.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-logs-apache.error@settings.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-metrics-apache.status@custom.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-metrics-apache.status@custom.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-metrics-apache.status@custom.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-metrics-apache.status@custom.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-metrics-apache.status@settings.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-metrics-apache.status@settings.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_component_template-metrics-apache.status@settings.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_component_template-metrics-apache.status@settings.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_ilm-policy-logs.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ilm-policy-logs.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_ilm-policy-logs.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ilm-policy-logs.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_ilm-policy-metrics.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ilm-policy-metrics.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_ilm-policy-metrics.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ilm-policy-metrics.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_index_template-_-apache._.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_index_template-_-apache._.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_index_template-_-apache._.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_index_template-_-apache._.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_ingest-pipeline-.fleet_final_pipeline-1.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ingest-pipeline-.fleet_final_pipeline-1.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_ingest-pipeline-.fleet_final_pipeline-1.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ingest-pipeline-.fleet_final_pipeline-1.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.4-third-party.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.4-third-party.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.4-third-party.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.4-third-party.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.4.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.4.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.4.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.4.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.4-third-party.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.4-third-party.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.4-third-party.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.4-third-party.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.4.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.4.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.4.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.4.json diff --git a/internal/dump/testdata/elasticsearch-mock-dump-apache/root.json b/internal/dump/testdata/elasticsearch-7-mock-dump-apache/root.json similarity index 100% rename from internal/dump/testdata/elasticsearch-mock-dump-apache/root.json rename to internal/dump/testdata/elasticsearch-7-mock-dump-apache/root.json diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/.fleet_component_template-1.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/.fleet_component_template-1.json new file mode 100644 index 000000000..d6283b3f2 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/.fleet_component_template-1.json @@ -0,0 +1,32 @@ +{ + "name": ".fleet_component_template-1", + "component_template": { + "template": { + "settings": { + "index": { + "final_pipeline": ".fleet_final_pipeline-1" + } + }, + "mappings": { + "properties": { + "event": { + "properties": { + "agent_id_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "format": "strict_date_time_no_millis||strict_date_optional_time||epoch_millis", + "type": "date" + } + } + } + } + } + }, + "_meta": { + "managed_by": "fleet", + "managed": true + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@custom.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@custom.json new file mode 100644 index 000000000..cc43b5320 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@custom.json @@ -0,0 +1,15 @@ +{ + "name": "logs-apache.access@custom", + "component_template": { + "template": { + "settings": {} + }, + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@settings.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@settings.json new file mode 100644 index 000000000..547e96d54 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@settings.json @@ -0,0 +1,99 @@ +{ + "name": "logs-apache.access@settings", + "component_template": { + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs" + }, + "codec": "best_compression", + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "query": { + "default_field": [ + "cloud.account.id", + "cloud.availability_zone", + "cloud.instance.id", + "cloud.instance.name", + "cloud.machine.type", + "cloud.provider", + "cloud.region", + "cloud.project.id", + "cloud.image.id", + "container.id", + "container.image.name", + "container.name", + "host.architecture", + "host.domain", + "host.hostname", + "host.id", + "host.mac", + "host.name", + "host.os.family", + "host.os.kernel", + "host.os.name", + "host.os.platform", + "host.os.version", + "host.type", + "host.os.build", + "host.os.codename", + "input.type", + "destination.domain", + "ecs.version", + "event.category", + "event.kind", + "event.outcome", + "file.path", + "http.request.method", + "http.request.referrer", + "http.version", + "log.file.path", + "log.level", + "source.address", + "source.as.organization.name", + "source.domain", + "source.geo.city_name", + "source.geo.continent_name", + "source.geo.country_iso_code", + "source.geo.country_name", + "source.geo.region_iso_code", + "source.geo.region_name", + "tags", + "tls.cipher", + "tls.version", + "tls.version_protocol", + "url.domain", + "url.extension", + "url.query", + "user.name", + "user_agent.device.name", + "user_agent.device.name", + "user_agent.name", + "user_agent.name", + "user_agent.original", + "user_agent.original", + "user_agent.os.full", + "user_agent.os.name", + "user_agent.os.name", + "user_agent.os.version", + "user_agent.version", + "apache.access.ssl.protocol", + "apache.access.ssl.cipher" + ] + } + } + } + }, + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.error@custom.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.error@custom.json new file mode 100644 index 000000000..7297f02da --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.error@custom.json @@ -0,0 +1,15 @@ +{ + "name": "logs-apache.error@custom", + "component_template": { + "template": { + "settings": {} + }, + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.error@settings.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.error@settings.json new file mode 100644 index 000000000..fcfcfd233 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.error@settings.json @@ -0,0 +1,88 @@ +{ + "name": "logs-apache.error@settings", + "component_template": { + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "logs" + }, + "codec": "best_compression", + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "query": { + "default_field": [ + "cloud.account.id", + "cloud.availability_zone", + "cloud.instance.id", + "cloud.instance.name", + "cloud.machine.type", + "cloud.provider", + "cloud.region", + "cloud.project.id", + "cloud.image.id", + "container.id", + "container.image.name", + "container.name", + "host.architecture", + "host.domain", + "host.hostname", + "host.id", + "host.mac", + "host.name", + "host.os.family", + "host.os.kernel", + "host.os.name", + "host.os.platform", + "host.os.version", + "host.type", + "host.os.build", + "host.os.codename", + "input.type", + "tags", + "ecs.version", + "event.category", + "event.kind", + "event.timezone", + "event.type", + "file.path", + "http.request.method", + "http.request.referrer", + "http.version", + "log.file.path", + "log.level", + "source.address", + "source.as.organization.name", + "source.geo.city_name", + "source.geo.continent_name", + "source.geo.country_iso_code", + "source.geo.country_name", + "source.geo.region_iso_code", + "source.geo.region_name", + "tags", + "url.domain", + "url.extension", + "url.query", + "user.name", + "user_agent.device.name", + "user_agent.name", + "user_agent.original", + "user_agent.os.name", + "apache.error.module" + ] + } + } + } + }, + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/metrics-apache.status@custom.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/metrics-apache.status@custom.json new file mode 100644 index 000000000..cabce8463 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/metrics-apache.status@custom.json @@ -0,0 +1,15 @@ +{ + "name": "metrics-apache.status@custom", + "component_template": { + "template": { + "settings": {} + }, + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/metrics-apache.status@settings.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/metrics-apache.status@settings.json new file mode 100644 index 000000000..c878ce03f --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/metrics-apache.status@settings.json @@ -0,0 +1,60 @@ +{ + "name": "metrics-apache.status@settings", + "component_template": { + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "metrics" + }, + "codec": "best_compression", + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "query": { + "default_field": [ + "cloud.account.id", + "cloud.availability_zone", + "cloud.instance.id", + "cloud.instance.name", + "cloud.machine.type", + "cloud.provider", + "cloud.region", + "cloud.project.id", + "cloud.image.id", + "container.id", + "container.image.name", + "container.name", + "host.architecture", + "host.domain", + "host.hostname", + "host.id", + "host.mac", + "host.name", + "host.os.family", + "host.os.kernel", + "host.os.name", + "host.os.platform", + "host.os.version", + "host.type", + "host.os.build", + "host.os.codename", + "ecs.version", + "service.address", + "service.type" + ] + } + } + } + }, + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/logs.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/logs.json new file mode 100644 index 000000000..0260e14f6 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/logs.json @@ -0,0 +1,45 @@ +{ + "version": 1, + "modified_date": "2022-04-06T15:40:04.029Z", + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_primary_shard_size": "50gb", + "max_age": "30d" + } + } + } + }, + "_meta": { + "managed": true, + "description": "default policy for the logs index template installed by x-pack" + } + }, + "in_use_by": { + "indices": [], + "data_streams": [], + "composable_templates": [ + "logs-apache.access", + "logs-elastic_agent.apm_server", + "logs-system.security", + "logs-system.auth", + "logs-elastic_agent.metricbeat", + "logs-elastic_agent.filebeat", + "logs-elastic_agent.packetbeat", + "logs-elastic_agent.endpoint_security", + "logs-elastic_agent.fleet_server", + "logs-apache.error", + "logs-system.system", + "logs-system.application", + "logs-elastic_agent.osquerybeat", + "logs-elastic_agent.heartbeat", + "logs-system.syslog", + "logs-elastic_agent.auditbeat", + "logs", + "logs-elastic_agent" + ] + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/metrics.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/metrics.json new file mode 100644 index 000000000..aea99c4d5 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ilm_policies/metrics.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "modified_date": "2022-04-06T15:40:04.332Z", + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_primary_shard_size": "50gb", + "max_age": "30d" + } + } + } + }, + "_meta": { + "managed": true, + "description": "default policy for the metrics index template installed by x-pack" + } + }, + "in_use_by": { + "indices": [], + "data_streams": [], + "composable_templates": [ + "metrics-system.process", + "metrics-elastic_agent.packetbeat", + "metrics-system.fsstat", + "metrics-elastic_agent.osquerybeat", + "metrics-elastic_agent.endpoint_security", + "metrics-elastic_agent.apm_server", + "metrics-system.memory", + "metrics-system.socket_summary", + "metrics-apache.status", + "metrics-elastic_agent.elastic_agent", + "metrics-elastic_agent.fleet_server", + "metrics-system.load", + "metrics-system.core", + "metrics-elastic_agent.filebeat", + "metrics-system.uptime", + "metrics-system.process.summary", + "metrics-system.cpu", + "metrics-elastic_agent.heartbeat", + "metrics-system.diskio", + "metrics-elastic_agent.metricbeat", + "metrics-elastic_agent.auditbeat", + "metrics-system.network", + "metrics-system.filesystem", + "metrics" + ] + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.access.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.access.json new file mode 100644 index 000000000..219cff1e5 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.access.json @@ -0,0 +1,545 @@ +{ + "name": "logs-apache.access", + "index_template": { + "index_patterns": [ + "logs-apache.access-*" + ], + "template": { + "settings": { + "index": { + "default_pipeline": "logs-apache.access-1.3.6" + } + }, + "mappings": { + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "container": { + "properties": { + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "labels": { + "type": "object" + } + } + }, + "process": { + "properties": { + "pid": { + "type": "long" + }, + "thread": { + "properties": { + "id": { + "type": "long" + } + } + } + } + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "offset": { + "type": "long" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "geo": { + "properties": { + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + } + } + } + } + }, + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + } + } + }, + "error": { + "properties": { + "message": { + "type": "match_only_text" + } + } + }, + "message": { + "type": "match_only_text" + }, + "url": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "wildcard" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "wildcard", + "fields": {} + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "instance": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + }, + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "input": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "@timestamp": { + "type": "date" + }, + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + } + } + }, + "apache": { + "properties": { + "access": { + "properties": { + "ssl": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "text": { + "type": "text" + } + } + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "containerized": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "architecture": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "status_code": { + "type": "long" + }, + "body": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tls": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "version_protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "created": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "type": "constant_keyword", + "value": "apache" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "dataset": { + "type": "constant_keyword", + "value": "apache.access" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + } + } + }, + "user_agent": { + "properties": { + "original": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + }, + "os": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + }, + "composed_of": [ + "logs-apache.access@settings", + "logs-apache.access@custom", + ".fleet_component_template-1" + ], + "priority": 200, + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + }, + "data_stream": { + "hidden": false, + "allow_custom_routing": false + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.error.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.error.json new file mode 100644 index 000000000..78912bfde --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/logs-apache.error.json @@ -0,0 +1,500 @@ +{ + "name": "logs-apache.error", + "index_template": { + "index_patterns": [ + "logs-apache.error-*" + ], + "template": { + "settings": { + "index": { + "default_pipeline": "logs-apache.error-1.3.6" + } + }, + "mappings": { + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "container": { + "properties": { + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "labels": { + "type": "object" + } + } + }, + "process": { + "properties": { + "pid": { + "type": "long" + }, + "thread": { + "properties": { + "id": { + "type": "long" + } + } + } + } + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "offset": { + "type": "long" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "geo": { + "properties": { + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + } + } + } + } + }, + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "ip": { + "type": "ip" + } + } + }, + "error": { + "properties": { + "message": { + "type": "match_only_text" + } + } + }, + "message": { + "type": "match_only_text" + }, + "url": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "wildcard" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "wildcard", + "fields": {} + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "instance": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + }, + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "input": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "@timestamp": { + "type": "date" + }, + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + } + } + }, + "apache": { + "properties": { + "error": { + "properties": { + "module": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "text": { + "type": "text" + } + } + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "containerized": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "architecture": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "status_code": { + "type": "long" + }, + "body": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "type": "constant_keyword", + "value": "apache" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "dataset": { + "type": "constant_keyword", + "value": "apache.error" + } + } + }, + "user": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + } + } + }, + "user_agent": { + "properties": { + "original": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + }, + "os": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": {} + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + } + }, + "composed_of": [ + "logs-apache.error@settings", + "logs-apache.error@custom", + ".fleet_component_template-1" + ], + "priority": 200, + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + }, + "data_stream": { + "hidden": false, + "allow_custom_routing": false + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/metrics-apache.status.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/metrics-apache.status.json new file mode 100644 index 000000000..d35ade557 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/index_templates/metrics-apache.status.json @@ -0,0 +1,510 @@ +{ + "name": "metrics-apache.status", + "index_template": { + "index_patterns": [ + "metrics-apache.status-*" + ], + "template": { + "settings": {}, + "mappings": { + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "instance": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + }, + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "container": { + "properties": { + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "labels": { + "type": "object" + } + } + }, + "@timestamp": { + "type": "date" + }, + "apache": { + "properties": { + "status": { + "properties": { + "bytes_per_request": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + }, + "load": { + "properties": { + "1": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + }, + "15": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + }, + "5": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "bytes_per_sec": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + }, + "total_bytes": { + "meta": { + "unit": "byte", + "metric_type": "counter" + }, + "type": "long" + }, + "cpu": { + "properties": { + "system": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + }, + "load": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + }, + "children_system": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + }, + "children_user": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + }, + "user": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "total_accesses": { + "meta": { + "metric_type": "counter" + }, + "type": "long" + }, + "scoreboard": { + "properties": { + "total": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "keepalive": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "idle_cleanup": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "waiting_for_connection": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "logging": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "gracefully_finishing": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "open_slot": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "dns_lookup": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "sending_reply": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "closing_connection": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "starting_up": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "reading_request": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + } + } + }, + "workers": { + "properties": { + "idle": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "busy": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + } + } + }, + "connections": { + "properties": { + "async": { + "properties": { + "closing": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "writing": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + }, + "keep_alive": { + "meta": { + "metric_type": "gauge" + }, + "type": "long" + } + } + }, + "total": { + "meta": { + "metric_type": "counter" + }, + "type": "long" + } + } + }, + "requests_per_sec": { + "meta": { + "metric_type": "gauge" + }, + "scaling_factor": 1000, + "type": "scaled_float" + }, + "uptime": { + "properties": { + "server_uptime": { + "meta": { + "metric_type": "counter" + }, + "type": "long" + }, + "uptime": { + "meta": { + "metric_type": "counter" + }, + "type": "long" + } + } + } + } + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "data_stream": { + "properties": { + "namespace": { + "type": "constant_keyword" + }, + "type": { + "type": "constant_keyword" + }, + "dataset": { + "type": "constant_keyword" + } + } + }, + "service": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword", + "fields": { + "text": { + "type": "text" + } + } + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "containerized": { + "type": "boolean" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "architecture": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "module": { + "type": "constant_keyword", + "value": "apache" + }, + "dataset": { + "type": "constant_keyword", + "value": "apache.status" + } + } + }, + "error": { + "properties": { + "message": { + "type": "match_only_text" + } + } + } + } + } + }, + "composed_of": [ + "metrics-apache.status@settings", + "metrics-apache.status@custom", + ".fleet_component_template-1" + ], + "priority": 200, + "_meta": { + "package": { + "name": "apache" + }, + "managed_by": "fleet", + "managed": true + }, + "data_stream": { + "hidden": false, + "allow_custom_routing": false + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/.fleet_final_pipeline-1.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/.fleet_final_pipeline-1.json new file mode 100644 index 000000000..ff7d6617e --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/.fleet_final_pipeline-1.json @@ -0,0 +1,92 @@ +{ + "version": 2, + "_meta": { + "managed_by": "fleet", + "managed": true + }, + "description": "Final pipeline for processing all incoming Fleet Agent documents.\n", + "processors": [ + { + "set": { + "description": "Add time when event was ingested.", + "field": "event.ingested", + "copy_from": "_ingest.timestamp" + } + }, + { + "script": { + "description": "Remove sub-seconds from event.ingested to improve storage efficiency.", + "tag": "truncate-subseconds-event-ingested", + "source": "ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);", + "ignore_failure": true + } + }, + { + "remove": { + "description": "Remove any pre-existing untrusted values.", + "field": [ + "event.agent_id_status", + "_security" + ], + "ignore_missing": true + } + }, + { + "set_security_user": { + "field": "_security", + "properties": [ + "authentication_type", + "username", + "realm", + "api_key" + ] + } + }, + { + "script": { + "description": "Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n", + "tag": "agent-id-status", + "source": "boolean is_user_trusted(def ctx, def users) {\n if (ctx?._security?.username == null) {\n return false;\n }\n\n def user = null;\n for (def item : users) {\n if (item?.username == ctx._security.username) {\n user = item;\n break;\n }\n }\n\n if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n return false;\n }\n\n if (ctx._security.realm.name != user.realm) {\n return false;\n }\n\n return true;\n}\n\nString verified(def ctx, def params) {\n // No agent.id field to validate.\n if (ctx?.agent?.id == null) {\n return \"missing\";\n }\n\n // Check auth metadata from API key.\n if (ctx?._security?.authentication_type == null\n // Agents only use API keys.\n || ctx._security.authentication_type != 'API_KEY'\n // Verify the API key owner before trusting any metadata it contains.\n || !is_user_trusted(ctx, params.trusted_users)\n // Verify the API key has metadata indicating the assigned agent ID.\n || ctx?._security?.api_key?.metadata?.agent_id == null) {\n return \"auth_metadata_missing\";\n }\n\n // The API key can only be used represent the agent.id it was issued to.\n if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n // Potential masquerade attempt.\n return \"mismatch\";\n }\n\n return \"verified\";\n}\n\nif (ctx?.event == null) {\n ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);", + "params": { + "trusted_users": [ + { + "username": "elastic/fleet-server", + "realm": "_service_account" + }, + { + "username": "cloud-internal-agent-server", + "realm": "found" + }, + { + "username": "elastic", + "realm": "reserved" + } + ] + } + } + }, + { + "remove": { + "field": "_security", + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "remove": { + "field": "_security", + "ignore_missing": true, + "ignore_failure": true + } + }, + { + "append": { + "field": "error.message", + "value": [ + "failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}" + ] + } + } + ] +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6-third-party.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6-third-party.json new file mode 100644 index 000000000..6e0001119 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6-third-party.json @@ -0,0 +1,74 @@ +{ + "description": "Pipeline for parsing Apache HTTP Server logs from third party api", + "processors": [ + { + "json": { + "field": "message", + "target_field": "json" + } + }, + { + "drop": { + "if": "ctx.json?.result == null" + } + }, + { + "fingerprint": { + "fields": [ + "json.result._cd", + "json.result._indextime", + "json.result._raw", + "json.result._time", + "json.result.host", + "json.result.source" + ], + "target_field": "_id", + "ignore_missing": true + } + }, + { + "set": { + "copy_from": "json.result._raw", + "field": "message", + "ignore_empty_value": true + } + }, + { + "set": { + "copy_from": "json.result.host", + "field": "host.name", + "ignore_empty_value": true + } + }, + { + "set": { + "copy_from": "json.result.source", + "field": "file.path", + "ignore_empty_value": true + } + }, + { + "remove": { + "field": [ + "json" + ], + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "apache" + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6.json new file mode 100644 index 000000000..4faa9950f --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.access-1.3.6.json @@ -0,0 +1,209 @@ +{ + "description": "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.", + "processors": [ + { + "pipeline": { + "if": "ctx.message.startsWith('{')", + "name": "logs-apache.access-1.3.6-third-party" + } + }, + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, + { + "set": { + "field": "ecs.version", + "value": "1.12.0" + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original" + } + }, + { + "grok": { + "field": "event.original", + "patterns": [ + "%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", + "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", + "%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -", + "\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})" + ], + "ignore_missing": true + } + }, + { + "uri_parts": { + "field": "_tmp.url_orig", + "ignore_failure": true + } + }, + { + "remove": { + "field": [ + "_tmp" + ], + "ignore_missing": true + } + }, + { + "set": { + "field": "url.domain", + "value": "{{destination.domain}}", + "if": "ctx.url?.domain == null && ctx.destination?.domain != null" + } + }, + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "set": { + "field": "event.category", + "value": "web" + } + }, + { + "set": { + "field": "event.outcome", + "value": "success", + "if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400" + } + }, + { + "set": { + "field": "event.outcome", + "value": "failure", + "if": "ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399" + } + }, + { + "grok": { + "field": "source.address", + "ignore_missing": true, + "patterns": [ + "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" + ] + } + }, + { + "remove": { + "field": "event.created", + "ignore_missing": true, + "ignore_failure": true + } + }, + { + "rename": { + "field": "@timestamp", + "target_field": "event.created" + } + }, + { + "date": { + "field": "apache.access.time", + "target_field": "@timestamp", + "formats": [ + "dd/MMM/yyyy:H:m:s Z" + ], + "ignore_failure": true + } + }, + { + "remove": { + "field": "apache.access.time", + "ignore_failure": true + } + }, + { + "user_agent": { + "field": "user_agent.original", + "ignore_failure": true + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "database_file": "GeoLite2-ASN.mmdb", + "field": "source.ip", + "target_field": "source.as", + "properties": [ + "asn", + "organization_name" + ], + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.asn", + "target_field": "source.as.number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.organization_name", + "target_field": "source.as.organization.name", + "ignore_missing": true + } + }, + { + "set": { + "field": "tls.cipher", + "value": "{{apache.access.ssl.cipher}}", + "if": "ctx?.apache?.access?.ssl?.cipher != null" + } + }, + { + "script": { + "lang": "painless", + "if": "ctx?.apache?.access?.ssl?.protocol != null", + "source": "def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n return;\n} if (parts[1].contains(\".\")) {\n ctx.tls.version = parts[1];\n} else {\n ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];" + } + }, + { + "script": { + "lang": "painless", + "description": "This script processor iterates over the whole document to remove fields with null values.", + "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "apache" + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6-third-party.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6-third-party.json new file mode 100644 index 000000000..6e0001119 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6-third-party.json @@ -0,0 +1,74 @@ +{ + "description": "Pipeline for parsing Apache HTTP Server logs from third party api", + "processors": [ + { + "json": { + "field": "message", + "target_field": "json" + } + }, + { + "drop": { + "if": "ctx.json?.result == null" + } + }, + { + "fingerprint": { + "fields": [ + "json.result._cd", + "json.result._indextime", + "json.result._raw", + "json.result._time", + "json.result.host", + "json.result.source" + ], + "target_field": "_id", + "ignore_missing": true + } + }, + { + "set": { + "copy_from": "json.result._raw", + "field": "message", + "ignore_empty_value": true + } + }, + { + "set": { + "copy_from": "json.result.host", + "field": "host.name", + "ignore_empty_value": true + } + }, + { + "set": { + "copy_from": "json.result.source", + "field": "file.path", + "ignore_empty_value": true + } + }, + { + "remove": { + "field": [ + "json" + ], + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "apache" + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6.json b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6.json new file mode 100644 index 000000000..979cb41ed --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-apache-dump-all/ingest_pipelines/logs-apache.error-1.3.6.json @@ -0,0 +1,197 @@ +{ + "description": "Pipeline for parsing apache error logs", + "processors": [ + { + "pipeline": { + "if": "ctx.message.startsWith('{')", + "name": "logs-apache.error-1.3.6-third-party" + } + }, + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, + { + "set": { + "field": "ecs.version", + "value": "1.12.0" + } + }, + { + "rename": { + "field": "message", + "target_field": "event.original" + } + }, + { + "grok": { + "field": "event.original", + "patterns": [ + "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}", + "\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}" + ], + "pattern_definitions": { + "APACHE_LOGLEVEL": "%{LOGLEVEL}[0-9]*", + "APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}" + }, + "ignore_missing": true + } + }, + { + "grok": { + "field": "message", + "patterns": [ + "File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}", + "File does not exist: %{URIPATH:file.path}" + ], + "ignore_missing": true, + "ignore_failure": true + } + }, + { + "date": { + "if": "ctx.event.timezone == null", + "field": "apache.error.timestamp", + "target_field": "@timestamp", + "formats": [ + "EEE MMM dd H:m:s yyyy", + "EEE MMM dd H:m:s.SSSSSS yyyy" + ], + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] + } + }, + { + "date": { + "if": "ctx.event.timezone != null", + "field": "apache.error.timestamp", + "target_field": "@timestamp", + "formats": [ + "EEE MMM dd H:m:s yyyy", + "EEE MMM dd H:m:s.SSSSSS yyyy" + ], + "timezone": "{{ event.timezone }}", + "on_failure": [ + { + "append": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] + } + }, + { + "remove": { + "field": "apache.error.timestamp", + "ignore_failure": true + } + }, + { + "set": { + "field": "event.kind", + "value": "event" + } + }, + { + "set": { + "field": "event.category", + "value": "web" + } + }, + { + "script": { + "if": "ctx?.log?.level != null", + "lang": "painless", + "source": "def err_levels = [\"emerg\", \"alert\", \"crit\", \"error\", \"warn\"]; if (err_levels.contains(ctx.log.level)) {\n ctx.event.type = \"error\";\n} else {\n ctx.event.type = \"info\";\n}" + } + }, + { + "grok": { + "field": "source.address", + "ignore_missing": true, + "patterns": [ + "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" + ] + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "database_file": "GeoLite2-ASN.mmdb", + "field": "source.ip", + "target_field": "source.as", + "properties": [ + "asn", + "organization_name" + ], + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.asn", + "target_field": "source.as.number", + "ignore_missing": true + } + }, + { + "rename": { + "field": "source.as.organization_name", + "target_field": "source.as.organization.name", + "ignore_missing": true + } + }, + { + "convert": { + "field": "source.port", + "type": "long", + "ignore_missing": true + } + }, + { + "script": { + "lang": "painless", + "description": "This script processor iterates over the whole document to remove fields with null values.", + "source": "void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n" + } + }, + { + "remove": { + "field": "event.original", + "if": "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))", + "ignore_failure": true, + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ], + "_meta": { + "managed_by": "fleet", + "managed": true, + "package": { + "name": "apache" + } + } +} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-.fleet_component_template-1.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-.fleet_component_template-1.json new file mode 100644 index 000000000..7786f403a --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-.fleet_component_template-1.json @@ -0,0 +1 @@ +{"component_templates":[{"name":".fleet_component_template-1","component_template":{"template":{"settings":{"index":{"final_pipeline":".fleet_final_pipeline-1"}},"mappings":{"properties":{"event":{"properties":{"agent_id_status":{"ignore_above":1024,"type":"keyword"},"ingested":{"format":"strict_date_time_no_millis||strict_date_optional_time||epoch_millis","type":"date"}}}}}},"_meta":{"managed_by":"fleet","managed":true}}}]} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.access@custom.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.access@custom.json new file mode 100644 index 000000000..58c5daaba --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.access@custom.json @@ -0,0 +1 @@ +{"component_templates":[{"name":"logs-apache.access@custom","component_template":{"template":{"settings":{}},"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true}}}]} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.access@settings.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.access@settings.json new file mode 100644 index 000000000..fa26896e1 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.access@settings.json @@ -0,0 +1 @@ +{"component_templates":[{"name":"logs-apache.access@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"logs"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","input.type","destination.domain","ecs.version","event.category","event.kind","event.outcome","file.path","http.request.method","http.request.referrer","http.version","log.file.path","log.level","source.address","source.as.organization.name","source.domain","source.geo.city_name","source.geo.continent_name","source.geo.country_iso_code","source.geo.country_name","source.geo.region_iso_code","source.geo.region_name","tags","tls.cipher","tls.version","tls.version_protocol","url.domain","url.extension","url.query","user.name","user_agent.device.name","user_agent.device.name","user_agent.name","user_agent.name","user_agent.original","user_agent.original","user_agent.os.full","user_agent.os.name","user_agent.os.name","user_agent.os.version","user_agent.version","apache.access.ssl.protocol","apache.access.ssl.cipher"]}}}},"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true}}}]} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.error@custom.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.error@custom.json new file mode 100644 index 000000000..a4ba32476 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.error@custom.json @@ -0,0 +1 @@ +{"component_templates":[{"name":"logs-apache.error@custom","component_template":{"template":{"settings":{}},"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true}}}]} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.error@settings.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.error@settings.json new file mode 100644 index 000000000..96df1c743 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-logs-apache.error@settings.json @@ -0,0 +1 @@ +{"component_templates":[{"name":"logs-apache.error@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"logs"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","input.type","tags","ecs.version","event.category","event.kind","event.timezone","event.type","file.path","http.request.method","http.request.referrer","http.version","log.file.path","log.level","source.address","source.as.organization.name","source.geo.city_name","source.geo.continent_name","source.geo.country_iso_code","source.geo.country_name","source.geo.region_iso_code","source.geo.region_name","tags","url.domain","url.extension","url.query","user.name","user_agent.device.name","user_agent.name","user_agent.original","user_agent.os.name","apache.error.module"]}}}},"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true}}}]} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-metrics-apache.status@custom.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-metrics-apache.status@custom.json new file mode 100644 index 000000000..e06d5d32e --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-metrics-apache.status@custom.json @@ -0,0 +1 @@ +{"component_templates":[{"name":"metrics-apache.status@custom","component_template":{"template":{"settings":{}},"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true}}}]} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-metrics-apache.status@settings.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-metrics-apache.status@settings.json new file mode 100644 index 000000000..09db718a4 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_component_template-metrics-apache.status@settings.json @@ -0,0 +1 @@ +{"component_templates":[{"name":"metrics-apache.status@settings","component_template":{"template":{"settings":{"index":{"lifecycle":{"name":"metrics"},"codec":"best_compression","mapping":{"total_fields":{"limit":"10000"}},"query":{"default_field":["cloud.account.id","cloud.availability_zone","cloud.instance.id","cloud.instance.name","cloud.machine.type","cloud.provider","cloud.region","cloud.project.id","cloud.image.id","container.id","container.image.name","container.name","host.architecture","host.domain","host.hostname","host.id","host.mac","host.name","host.os.family","host.os.kernel","host.os.name","host.os.platform","host.os.version","host.type","host.os.build","host.os.codename","ecs.version","service.address","service.type"]}}}},"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true}}}]} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ilm-policy-logs.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ilm-policy-logs.json new file mode 100644 index 000000000..5c8c4969f --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ilm-policy-logs.json @@ -0,0 +1 @@ +{"logs":{"version":1,"modified_date":"2022-04-06T15:40:04.029Z","policy":{"phases":{"hot":{"min_age":"0ms","actions":{"rollover":{"max_primary_shard_size":"50gb","max_age":"30d"}}}},"_meta":{"managed":true,"description":"default policy for the logs index template installed by x-pack"}},"in_use_by":{"indices":[],"data_streams":[],"composable_templates":["logs-apache.access","logs-elastic_agent.apm_server","logs-system.security","logs-system.auth","logs-elastic_agent.metricbeat","logs-elastic_agent.filebeat","logs-elastic_agent.packetbeat","logs-elastic_agent.endpoint_security","logs-elastic_agent.fleet_server","logs-apache.error","logs-system.system","logs-system.application","logs-elastic_agent.osquerybeat","logs-elastic_agent.heartbeat","logs-system.syslog","logs-elastic_agent.auditbeat","logs","logs-elastic_agent"]}}} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ilm-policy-metrics.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ilm-policy-metrics.json new file mode 100644 index 000000000..ed94c94c7 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ilm-policy-metrics.json @@ -0,0 +1 @@ +{"metrics":{"version":1,"modified_date":"2022-04-06T15:40:04.332Z","policy":{"phases":{"hot":{"min_age":"0ms","actions":{"rollover":{"max_primary_shard_size":"50gb","max_age":"30d"}}}},"_meta":{"managed":true,"description":"default policy for the metrics index template installed by x-pack"}},"in_use_by":{"indices":[],"data_streams":[],"composable_templates":["metrics-system.process","metrics-elastic_agent.packetbeat","metrics-system.fsstat","metrics-elastic_agent.osquerybeat","metrics-elastic_agent.endpoint_security","metrics-elastic_agent.apm_server","metrics-system.memory","metrics-system.socket_summary","metrics-apache.status","metrics-elastic_agent.elastic_agent","metrics-elastic_agent.fleet_server","metrics-system.load","metrics-system.core","metrics-elastic_agent.filebeat","metrics-system.uptime","metrics-system.process.summary","metrics-system.cpu","metrics-elastic_agent.heartbeat","metrics-system.diskio","metrics-elastic_agent.metricbeat","metrics-elastic_agent.auditbeat","metrics-system.network","metrics-system.filesystem","metrics"]}}} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_index_template-_-apache._.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_index_template-_-apache._.json new file mode 100644 index 000000000..4e02f2728 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_index_template-_-apache._.json @@ -0,0 +1 @@ +{"index_templates":[{"name":"logs-apache.error","index_template":{"index_patterns":["logs-apache.error-*"],"template":{"settings":{"index":{"default_pipeline":"logs-apache.error-1.3.6"}},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"process":{"properties":{"pid":{"type":"long"},"thread":{"properties":{"id":{"type":"long"}}}}},"log":{"properties":{"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"offset":{"type":"long"},"level":{"ignore_above":1024,"type":"keyword"}}},"source":{"properties":{"geo":{"properties":{"continent_name":{"ignore_above":1024,"type":"keyword"},"region_iso_code":{"ignore_above":1024,"type":"keyword"},"city_name":{"ignore_above":1024,"type":"keyword"},"country_iso_code":{"ignore_above":1024,"type":"keyword"},"country_name":{"ignore_above":1024,"type":"keyword"},"location":{"type":"geo_point"},"region_name":{"ignore_above":1024,"type":"keyword"}}},"as":{"properties":{"number":{"type":"long"},"organization":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}}}},"address":{"ignore_above":1024,"type":"keyword"},"port":{"type":"long"},"ip":{"type":"ip"}}},"error":{"properties":{"message":{"type":"match_only_text"}}},"message":{"type":"match_only_text"},"url":{"properties":{"path":{"ignore_above":1024,"type":"wildcard"},"extension":{"ignore_above":1024,"type":"keyword"},"original":{"ignore_above":1024,"type":"wildcard","fields":{}},"domain":{"ignore_above":1024,"type":"keyword"},"query":{"ignore_above":1024,"type":"keyword"}}},"tags":{"ignore_above":1024,"type":"keyword"},"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"@timestamp":{"type":"date"},"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword","fields":{}}}},"apache":{"properties":{"error":{"properties":{"module":{"ignore_above":1024,"type":"keyword"}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"http":{"properties":{"request":{"properties":{"referrer":{"ignore_above":1024,"type":"keyword"},"method":{"ignore_above":1024,"type":"keyword"}}},"response":{"properties":{"status_code":{"type":"long"},"body":{"properties":{"bytes":{"type":"long"}}}}},"version":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"kind":{"ignore_above":1024,"type":"keyword"},"timezone":{"ignore_above":1024,"type":"keyword"},"module":{"type":"constant_keyword","value":"apache"},"category":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"dataset":{"type":"constant_keyword","value":"apache.error"}}},"user":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"user_agent":{"properties":{"original":{"ignore_above":1024,"type":"keyword","fields":{}},"os":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"name":{"ignore_above":1024,"type":"keyword"},"device":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}}}}}}},"composed_of":["logs-apache.error@settings","logs-apache.error@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"data_stream":{"hidden":false,"allow_custom_routing":false}}},{"name":"logs-apache.access","index_template":{"index_patterns":["logs-apache.access-*"],"template":{"settings":{"index":{"default_pipeline":"logs-apache.access-1.3.6"}},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"process":{"properties":{"pid":{"type":"long"},"thread":{"properties":{"id":{"type":"long"}}}}},"log":{"properties":{"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword"}}},"offset":{"type":"long"},"level":{"ignore_above":1024,"type":"keyword"}}},"destination":{"properties":{"domain":{"ignore_above":1024,"type":"keyword"}}},"source":{"properties":{"geo":{"properties":{"continent_name":{"ignore_above":1024,"type":"keyword"},"region_iso_code":{"ignore_above":1024,"type":"keyword"},"city_name":{"ignore_above":1024,"type":"keyword"},"country_iso_code":{"ignore_above":1024,"type":"keyword"},"country_name":{"ignore_above":1024,"type":"keyword"},"location":{"type":"geo_point"},"region_name":{"ignore_above":1024,"type":"keyword"}}},"as":{"properties":{"number":{"type":"long"},"organization":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}}}},"address":{"ignore_above":1024,"type":"keyword"},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"}}},"error":{"properties":{"message":{"type":"match_only_text"}}},"message":{"type":"match_only_text"},"url":{"properties":{"path":{"ignore_above":1024,"type":"wildcard"},"extension":{"ignore_above":1024,"type":"keyword"},"original":{"ignore_above":1024,"type":"wildcard","fields":{}},"domain":{"ignore_above":1024,"type":"keyword"},"query":{"ignore_above":1024,"type":"keyword"}}},"tags":{"ignore_above":1024,"type":"keyword"},"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"input":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"@timestamp":{"type":"date"},"file":{"properties":{"path":{"ignore_above":1024,"type":"keyword","fields":{}}}},"apache":{"properties":{"access":{"properties":{"ssl":{"properties":{"cipher":{"ignore_above":1024,"type":"keyword"},"protocol":{"ignore_above":1024,"type":"keyword"}}}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"http":{"properties":{"request":{"properties":{"referrer":{"ignore_above":1024,"type":"keyword"},"method":{"ignore_above":1024,"type":"keyword"}}},"response":{"properties":{"status_code":{"type":"long"},"body":{"properties":{"bytes":{"type":"long"}}}}},"version":{"ignore_above":1024,"type":"keyword"}}},"tls":{"properties":{"cipher":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"version_protocol":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"created":{"type":"date"},"kind":{"ignore_above":1024,"type":"keyword"},"module":{"type":"constant_keyword","value":"apache"},"category":{"ignore_above":1024,"type":"keyword"},"dataset":{"type":"constant_keyword","value":"apache.access"},"outcome":{"ignore_above":1024,"type":"keyword"}}},"user":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}}}},"user_agent":{"properties":{"original":{"ignore_above":1024,"type":"keyword","fields":{}},"os":{"properties":{"name":{"ignore_above":1024,"type":"keyword","fields":{}},"version":{"ignore_above":1024,"type":"keyword"},"full":{"ignore_above":1024,"type":"keyword","fields":{}}}},"name":{"ignore_above":1024,"type":"keyword"},"device":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"version":{"ignore_above":1024,"type":"keyword"}}}}}},"composed_of":["logs-apache.access@settings","logs-apache.access@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"data_stream":{"hidden":false,"allow_custom_routing":false}}},{"name":"metrics-apache.status","index_template":{"index_patterns":["metrics-apache.status-*"],"template":{"settings":{},"mappings":{"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"dynamic_templates":[{"strings_as_keyword":{"mapping":{"ignore_above":1024,"type":"keyword"},"match_mapping_type":"string"}}],"date_detection":false,"properties":{"cloud":{"properties":{"availability_zone":{"ignore_above":1024,"type":"keyword"},"image":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"instance":{"properties":{"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"}}},"provider":{"ignore_above":1024,"type":"keyword"},"machine":{"properties":{"type":{"ignore_above":1024,"type":"keyword"}}},"project":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}},"region":{"ignore_above":1024,"type":"keyword"},"account":{"properties":{"id":{"ignore_above":1024,"type":"keyword"}}}}},"container":{"properties":{"image":{"properties":{"name":{"ignore_above":1024,"type":"keyword"}}},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"labels":{"type":"object"}}},"@timestamp":{"type":"date"},"apache":{"properties":{"status":{"properties":{"bytes_per_request":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"load":{"properties":{"1":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"15":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"5":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"}}},"bytes_per_sec":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"total_bytes":{"meta":{"unit":"byte","metric_type":"counter"},"type":"long"},"cpu":{"properties":{"system":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"load":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"children_system":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"children_user":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"user":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"}}},"total_accesses":{"meta":{"metric_type":"counter"},"type":"long"},"scoreboard":{"properties":{"total":{"meta":{"metric_type":"gauge"},"type":"long"},"keepalive":{"meta":{"metric_type":"gauge"},"type":"long"},"idle_cleanup":{"meta":{"metric_type":"gauge"},"type":"long"},"waiting_for_connection":{"meta":{"metric_type":"gauge"},"type":"long"},"logging":{"meta":{"metric_type":"gauge"},"type":"long"},"gracefully_finishing":{"meta":{"metric_type":"gauge"},"type":"long"},"open_slot":{"meta":{"metric_type":"gauge"},"type":"long"},"dns_lookup":{"meta":{"metric_type":"gauge"},"type":"long"},"sending_reply":{"meta":{"metric_type":"gauge"},"type":"long"},"closing_connection":{"meta":{"metric_type":"gauge"},"type":"long"},"starting_up":{"meta":{"metric_type":"gauge"},"type":"long"},"reading_request":{"meta":{"metric_type":"gauge"},"type":"long"}}},"workers":{"properties":{"idle":{"meta":{"metric_type":"gauge"},"type":"long"},"busy":{"meta":{"metric_type":"gauge"},"type":"long"}}},"connections":{"properties":{"async":{"properties":{"closing":{"meta":{"metric_type":"gauge"},"type":"long"},"writing":{"meta":{"metric_type":"gauge"},"type":"long"},"keep_alive":{"meta":{"metric_type":"gauge"},"type":"long"}}},"total":{"meta":{"metric_type":"counter"},"type":"long"}}},"requests_per_sec":{"meta":{"metric_type":"gauge"},"scaling_factor":1000,"type":"scaled_float"},"uptime":{"properties":{"server_uptime":{"meta":{"metric_type":"counter"},"type":"long"},"uptime":{"meta":{"metric_type":"counter"},"type":"long"}}}}}}},"ecs":{"properties":{"version":{"ignore_above":1024,"type":"keyword"}}},"data_stream":{"properties":{"namespace":{"type":"constant_keyword"},"type":{"type":"constant_keyword"},"dataset":{"type":"constant_keyword"}}},"service":{"properties":{"address":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"}}},"host":{"properties":{"hostname":{"ignore_above":1024,"type":"keyword"},"os":{"properties":{"build":{"ignore_above":1024,"type":"keyword"},"kernel":{"ignore_above":1024,"type":"keyword"},"codename":{"ignore_above":1024,"type":"keyword"},"name":{"ignore_above":1024,"type":"keyword","fields":{"text":{"type":"text"}}},"family":{"ignore_above":1024,"type":"keyword"},"version":{"ignore_above":1024,"type":"keyword"},"platform":{"ignore_above":1024,"type":"keyword"}}},"domain":{"ignore_above":1024,"type":"keyword"},"ip":{"type":"ip"},"containerized":{"type":"boolean"},"name":{"ignore_above":1024,"type":"keyword"},"id":{"ignore_above":1024,"type":"keyword"},"type":{"ignore_above":1024,"type":"keyword"},"mac":{"ignore_above":1024,"type":"keyword"},"architecture":{"ignore_above":1024,"type":"keyword"}}},"event":{"properties":{"module":{"type":"constant_keyword","value":"apache"},"dataset":{"type":"constant_keyword","value":"apache.status"}}},"error":{"properties":{"message":{"type":"match_only_text"}}}}}},"composed_of":["metrics-apache.status@settings","metrics-apache.status@custom",".fleet_component_template-1"],"priority":200,"_meta":{"package":{"name":"apache"},"managed_by":"fleet","managed":true},"data_stream":{"hidden":false,"allow_custom_routing":false}}}]} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-.fleet_final_pipeline-1.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-.fleet_final_pipeline-1.json new file mode 100644 index 000000000..2c546a776 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-.fleet_final_pipeline-1.json @@ -0,0 +1 @@ +{".fleet_final_pipeline-1":{"version":2,"_meta":{"managed_by":"fleet","managed":true},"description":"Final pipeline for processing all incoming Fleet Agent documents.\n","processors":[{"set":{"description":"Add time when event was ingested.","field":"event.ingested","copy_from":"_ingest.timestamp"}},{"script":{"description":"Remove sub-seconds from event.ingested to improve storage efficiency.","tag":"truncate-subseconds-event-ingested","source":"ctx.event.ingested = ctx.event.ingested.withNano(0).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);","ignore_failure":true}},{"remove":{"description":"Remove any pre-existing untrusted values.","field":["event.agent_id_status","_security"],"ignore_missing":true}},{"set_security_user":{"field":"_security","properties":["authentication_type","username","realm","api_key"]}},{"script":{"description":"Add event.agent_id_status based on the API key metadata and the agent.id contained in the event.\n","tag":"agent-id-status","source":"boolean is_user_trusted(def ctx, def users) {\n if (ctx?._security?.username == null) {\n return false;\n }\n\n def user = null;\n for (def item : users) {\n if (item?.username == ctx._security.username) {\n user = item;\n break;\n }\n }\n\n if (user == null || user?.realm == null || ctx?._security?.realm?.name == null) {\n return false;\n }\n\n if (ctx._security.realm.name != user.realm) {\n return false;\n }\n\n return true;\n}\n\nString verified(def ctx, def params) {\n // No agent.id field to validate.\n if (ctx?.agent?.id == null) {\n return \"missing\";\n }\n\n // Check auth metadata from API key.\n if (ctx?._security?.authentication_type == null\n // Agents only use API keys.\n || ctx._security.authentication_type != 'API_KEY'\n // Verify the API key owner before trusting any metadata it contains.\n || !is_user_trusted(ctx, params.trusted_users)\n // Verify the API key has metadata indicating the assigned agent ID.\n || ctx?._security?.api_key?.metadata?.agent_id == null) {\n return \"auth_metadata_missing\";\n }\n\n // The API key can only be used represent the agent.id it was issued to.\n if (ctx._security.api_key.metadata.agent_id != ctx.agent.id) {\n // Potential masquerade attempt.\n return \"mismatch\";\n }\n\n return \"verified\";\n}\n\nif (ctx?.event == null) {\n ctx.event = [:];\n}\n\nctx.event.agent_id_status = verified(ctx, params);","params":{"trusted_users":[{"username":"elastic/fleet-server","realm":"_service_account"},{"username":"cloud-internal-agent-server","realm":"found"},{"username":"elastic","realm":"reserved"}]}}},{"remove":{"field":"_security","ignore_missing":true}}],"on_failure":[{"remove":{"field":"_security","ignore_missing":true,"ignore_failure":true}},{"append":{"field":"error.message","value":["failed in Fleet agent final_pipeline: {{ _ingest.on_failure_message }}"]}}]}} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.6-third-party.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.6-third-party.json new file mode 100644 index 000000000..284e4fde7 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.6-third-party.json @@ -0,0 +1 @@ +{"logs-apache.access-1.3.6-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}}} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.6.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.6.json new file mode 100644 index 000000000..3a70437c7 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.access-1.3.6.json @@ -0,0 +1 @@ +{"logs-apache.access-1.3.6":{"description":"Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins.","processors":[{"pipeline":{"if":"ctx.message.startsWith('{')","name":"logs-apache.access-1.3.6-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"1.12.0"}},{"rename":{"field":"message","target_field":"event.original"}},{"grok":{"field":"event.original","patterns":["%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?","%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?","%{IPORHOST:source.address} - %{DATA:user.name} \\[%{HTTPDATE:apache.access.time}\\] \"-\" %{NUMBER:http.response.status_code:long} -","\\[%{HTTPDATE:apache.access.time}\\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}\" (-|%{NUMBER:http.response.body.bytes:long})"],"ignore_missing":true}},{"uri_parts":{"field":"_tmp.url_orig","ignore_failure":true}},{"remove":{"field":["_tmp"],"ignore_missing":true}},{"set":{"field":"url.domain","value":"{{destination.domain}}","if":"ctx.url?.domain == null && ctx.destination?.domain != null"}},{"set":{"field":"event.kind","value":"event"}},{"set":{"field":"event.category","value":"web"}},{"set":{"field":"event.outcome","value":"success","if":"ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"}},{"set":{"field":"event.outcome","value":"failure","if":"ctx?.http?.response?.status_code != null && ctx.http.response.status_code > 399"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"remove":{"field":"event.created","ignore_missing":true,"ignore_failure":true}},{"rename":{"field":"@timestamp","target_field":"event.created"}},{"date":{"field":"apache.access.time","target_field":"@timestamp","formats":["dd/MMM/yyyy:H:m:s Z"],"ignore_failure":true}},{"remove":{"field":"apache.access.time","ignore_failure":true}},{"user_agent":{"field":"user_agent.original","ignore_failure":true}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"set":{"field":"tls.cipher","value":"{{apache.access.ssl.cipher}}","if":"ctx?.apache?.access?.ssl?.cipher != null"}},{"script":{"lang":"painless","if":"ctx?.apache?.access?.ssl?.protocol != null","source":"def parts = ctx.apache.access.ssl.protocol.toLowerCase().splitOnToken(\"v\"); if (parts.length != 2) {\n return;\n} if (parts[1].contains(\".\")) {\n ctx.tls.version = parts[1];\n} else {\n ctx.tls.version = parts[1] + \".0\";\n} ctx.tls.version_protocol = parts[0];"}},{"script":{"lang":"painless","description":"This script processor iterates over the whole document to remove fields with null values.","source":"void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"}},{"remove":{"field":"event.original","if":"ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))","ignore_failure":true,"ignore_missing":true}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}}} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.6-third-party.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.6-third-party.json new file mode 100644 index 000000000..1794fd2eb --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.6-third-party.json @@ -0,0 +1 @@ +{"logs-apache.error-1.3.6-third-party":{"description":"Pipeline for parsing Apache HTTP Server logs from third party api","processors":[{"json":{"field":"message","target_field":"json"}},{"drop":{"if":"ctx.json?.result == null"}},{"fingerprint":{"fields":["json.result._cd","json.result._indextime","json.result._raw","json.result._time","json.result.host","json.result.source"],"target_field":"_id","ignore_missing":true}},{"set":{"copy_from":"json.result._raw","field":"message","ignore_empty_value":true}},{"set":{"copy_from":"json.result.host","field":"host.name","ignore_empty_value":true}},{"set":{"copy_from":"json.result.source","field":"file.path","ignore_empty_value":true}},{"remove":{"field":["json"],"ignore_missing":true}}],"on_failure":[{"append":{"field":"error.message","value":"error in third-party pipeline: error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} {{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}}} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.6.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.6.json new file mode 100644 index 000000000..d2bd22f31 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/_ingest-pipeline-logs-apache.error-1.3.6.json @@ -0,0 +1 @@ +{"logs-apache.error-1.3.6":{"description":"Pipeline for parsing apache error logs","processors":[{"pipeline":{"if":"ctx.message.startsWith('{')","name":"logs-apache.error-1.3.6-third-party"}},{"set":{"field":"event.ingested","value":"{{_ingest.timestamp}}"}},{"set":{"field":"ecs.version","value":"1.12.0"}},{"rename":{"field":"message","target_field":"event.original"}},{"grok":{"field":"event.original","patterns":["\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}","\\[%{APACHE_TIME:apache.error.timestamp}\\] \\[%{DATA:apache.error.module}:%{APACHE_LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\\])? %{GREEDYDATA:message}"],"pattern_definitions":{"APACHE_LOGLEVEL":"%{LOGLEVEL}[0-9]*","APACHE_TIME":"%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"},"ignore_missing":true}},{"grok":{"field":"message","patterns":["File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}","File does not exist: %{URIPATH:file.path}"],"ignore_missing":true,"ignore_failure":true}},{"date":{"if":"ctx.event.timezone == null","field":"apache.error.timestamp","target_field":"@timestamp","formats":["EEE MMM dd H:m:s yyyy","EEE MMM dd H:m:s.SSSSSS yyyy"],"on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"date":{"if":"ctx.event.timezone != null","field":"apache.error.timestamp","target_field":"@timestamp","formats":["EEE MMM dd H:m:s yyyy","EEE MMM dd H:m:s.SSSSSS yyyy"],"timezone":"{{ event.timezone }}","on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"remove":{"field":"apache.error.timestamp","ignore_failure":true}},{"set":{"field":"event.kind","value":"event"}},{"set":{"field":"event.category","value":"web"}},{"script":{"if":"ctx?.log?.level != null","lang":"painless","source":"def err_levels = [\"emerg\", \"alert\", \"crit\", \"error\", \"warn\"]; if (err_levels.contains(ctx.log.level)) {\n ctx.event.type = \"error\";\n} else {\n ctx.event.type = \"info\";\n}"}},{"grok":{"field":"source.address","ignore_missing":true,"patterns":["^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"]}},{"geoip":{"field":"source.ip","target_field":"source.geo","ignore_missing":true}},{"geoip":{"database_file":"GeoLite2-ASN.mmdb","field":"source.ip","target_field":"source.as","properties":["asn","organization_name"],"ignore_missing":true}},{"rename":{"field":"source.as.asn","target_field":"source.as.number","ignore_missing":true}},{"rename":{"field":"source.as.organization_name","target_field":"source.as.organization.name","ignore_missing":true}},{"convert":{"field":"source.port","type":"long","ignore_missing":true}},{"script":{"lang":"painless","description":"This script processor iterates over the whole document to remove fields with null values.","source":"void handleMap(Map map) {\n for (def x : map.values()) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n map.values().removeIf(v -> v == null);\n}\nvoid handleList(List list) {\n for (def x : list) {\n if (x instanceof Map) {\n handleMap(x);\n } else if (x instanceof List) {\n handleList(x);\n }\n }\n}\nhandleMap(ctx);\n"}},{"remove":{"field":"event.original","if":"ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))","ignore_failure":true,"ignore_missing":true}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}],"_meta":{"managed_by":"fleet","managed":true,"package":{"name":"apache"}}}} \ No newline at end of file diff --git a/internal/dump/testdata/elasticsearch-8-mock-dump-apache/root.json b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/root.json new file mode 100644 index 000000000..2e45f2226 --- /dev/null +++ b/internal/dump/testdata/elasticsearch-8-mock-dump-apache/root.json @@ -0,0 +1,17 @@ +{ + "name" : "fc2935e6b13c", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "NX1JVCItRratMEXogOKYSQ", + "version" : { + "number" : "8.1.0", + "build_flavor" : "default", + "build_type" : "docker", + "build_hash" : "3700f7679f7d95e36da0b43762189bab189bc53a", + "build_date" : "2022-03-03T14:20:00.690422633Z", + "build_snapshot" : false, + "lucene_version" : "9.0.0", + "minimum_wire_compatibility_version" : "7.17.0", + "minimum_index_compatibility_version" : "7.0.0" + }, + "tagline" : "You Know, for Search" +}