Fix dump subcommand in 8.x (#785)

cmd/dump.go

@@ -70,7 +70,7 @@ func dumpInstalledObjectsCmd(cmd *cobra.Command, args []string) error {
 		return errors.Wrap(err, "dump failed")
 	}
 	if n == 0 {
-		cmd.Printf("No objects were dumped for package %s, is it installed?", packageName)
+		cmd.Printf("No objects were dumped for package %s, is it installed?\n", packageName)
 		return nil
 	}
 	cmd.Printf("Dumped %d installed objects for package %s to %s\n", n, packageName, outputPath)

internal/dump/indextemplates.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io/ioutil"
 
+	"github.com/elastic/elastic-package/internal/common"
 	"github.com/elastic/elastic-package/internal/elasticsearch"
 )
 
@@ -97,7 +98,7 @@ func getIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, pa
 		indexTemplate.raw = indexTemplateRaw
 
 		meta := indexTemplate.IndexTemplate.Meta
-		if meta.Package.Name != packageName || meta.ManagedBy != "ingest-manager" {
+		if meta.Package.Name != packageName || !managedByFleet(meta.ManagedBy) {
 			// This is not the droid you are looking for.
 			continue
 		}
@@ -107,3 +108,8 @@ func getIndexTemplatesForPackage(ctx context.Context, api *elasticsearch.API, pa
 
 	return indexTemplates, nil
 }
+
+func managedByFleet(managedBy string) bool {
+	var managers = []string{"ingest-manager", "fleet"}
+	return common.StringSliceContains(managers, managedBy)
+}

internal/dump/installedobjects_test.go

@@ -7,6 +7,7 @@ package dump
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"io/fs"
 	"io/ioutil"
 	"os"
@@ -15,30 +16,87 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
 
+	"github.com/elastic/elastic-package/internal/elasticsearch"
 	estest "github.com/elastic/elastic-package/internal/elasticsearch/test"
 	"github.com/elastic/elastic-package/internal/files"
 )
 
-func TestInstalledObjectsDumpAll(t *testing.T) {
-	client := estest.ElasticsearchClient(t, "./testdata/elasticsearch-mock-dump-apache")
-	outputDir := t.TempDir()
-	dumper := NewInstalledObjectsDumper(client, "apache")
+func TestDumpInstalledObjects(t *testing.T) {
+	// Files for each suite are recorded automatically on first test run.
+	// To add a new suite:
+	// - Configure it here.
+	// - Install the package in a running stack.
+	// - Configure environment variables for this stack (eval "$(elastic-package stack shellinit)").
+	// - Run tests.
+	// - Check that recorded files make sense and commit them.
+	suites := []*installedObjectsDumpSuite{
+		&installedObjectsDumpSuite{
+			PackageName: "apache",
+			RecordDir:   "./testdata/elasticsearch-7-mock-dump-apache",
+			DumpDir:     "./testdata/elasticsearch-7-apache-dump-all",
+		},
+		&installedObjectsDumpSuite{
+			PackageName: "apache",
+			RecordDir:   "./testdata/elasticsearch-8-mock-dump-apache",
+			DumpDir:     "./testdata/elasticsearch-8-apache-dump-all",
+		},
+	}
+
+	for _, s := range suites {
+		suite.Run(t, s)
+	}
+}
+
+type installedObjectsDumpSuite struct {
+	suite.Suite
+
+	// PackageName is the name of the package.
+	PackageName string
+
+	// RecordDir is where responses from Elasticsearch are recorded.
+	RecordDir string
+
+	// DumpDir is where the expected dumped files are stored.
+	DumpDir string
+}
+
+func (s *installedObjectsDumpSuite) SetupTest() {
+	_, err := os.Stat(s.DumpDir)
+	if errors.Is(err, os.ErrNotExist) {
+		client, err := elasticsearch.Client()
+		s.Require().NoError(err)
+
+		dumper := NewInstalledObjectsDumper(client.API, s.PackageName)
+		n, err := dumper.DumpAll(context.Background(), s.DumpDir)
+		s.Require().NoError(err)
+		s.Require().Greater(n, 0)
+	} else {
+		s.Require().NoError(err)
+	}
+}
+
+func (s *installedObjectsDumpSuite) TestDumpAll() {
+	client := estest.ElasticsearchClient(s.T(), s.RecordDir)
+
+	outputDir := s.T().TempDir()
+	dumper := NewInstalledObjectsDumper(client, s.PackageName)
 	n, err := dumper.DumpAll(context.Background(), outputDir)
-	require.NoError(t, err)
+	s.Require().NoError(err)
 
-	filesExpected := countFiles(t, "./testdata/apache-dump-all")
-	assert.Equal(t, filesExpected, n)
+	filesExpected := countFiles(s.T(), s.DumpDir)
+	s.Assert().Equal(filesExpected, n)
 
-	filesFound := countFiles(t, outputDir)
-	assert.Equal(t, filesExpected, filesFound)
+	filesFound := countFiles(s.T(), outputDir)
+	s.Assert().Equal(filesExpected, filesFound)
 
-	assertEqualDumps(t, "./testdata/apache-dump-all", outputDir)
+	assertEqualDumps(s.T(), s.DumpDir, outputDir)
 }
 
-func TestInstalledObjectsDumpSome(t *testing.T) {
-	client := estest.ElasticsearchClient(t, "./testdata/elasticsearch-mock-dump-apache")
-	dumper := NewInstalledObjectsDumper(client, "apache")
+func (s *installedObjectsDumpSuite) TestDumpSome() {
+	client := estest.ElasticsearchClient(s.T(), s.RecordDir)
+	dumper := NewInstalledObjectsDumper(client, s.PackageName)
 
 	// In a map so order of execution is randomized.
 	dumpers := map[string]func(ctx context.Context, outputDir string) (int, error){
@@ -49,19 +107,19 @@ func TestInstalledObjectsDumpSome(t *testing.T) {
 	}
 
 	for dir, dumpFunction := range dumpers {
-		t.Run(dir, func(t *testing.T) {
-			outputDir := t.TempDir()
+		s.Run(dir, func() {
+			outputDir := s.T().TempDir()
 			n, err := dumpFunction(context.Background(), outputDir)
-			require.NoError(t, err)
+			s.Require().NoError(err)
 
-			expectedDir := subDir(t, "./testdata/apache-dump-all", dir)
-			filesExpected := countFiles(t, expectedDir)
-			assert.Equal(t, filesExpected, n)
+			expectedDir := subDir(s.T(), s.DumpDir, dir)
+			filesExpected := countFiles(s.T(), expectedDir)
+			s.Assert().Equal(filesExpected, n)
 
-			filesFound := countFiles(t, outputDir)
-			assert.Equal(t, filesExpected, filesFound)
+			filesFound := countFiles(s.T(), outputDir)
+			s.Assert().Equal(filesExpected, filesFound)
 
-			assertEqualDumps(t, expectedDir, outputDir)
+			assertEqualDumps(s.T(), expectedDir, outputDir)
 		})
 	}
 }

internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/.fleet_component_template-1.json

@@ -0,0 +1,32 @@
+{
+  "name": ".fleet_component_template-1",
+  "component_template": {
+    "template": {
+      "settings": {
+        "index": {
+          "final_pipeline": ".fleet_final_pipeline-1"
+        }
+      },
+      "mappings": {
+        "properties": {
+          "event": {
+            "properties": {
+              "agent_id_status": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "ingested": {
+                "format": "strict_date_time_no_millis||strict_date_optional_time||epoch_millis",
+                "type": "date"
+              }
+            }
+          }
+        }
+      }
+    },
+    "_meta": {
+      "managed_by": "fleet",
+      "managed": true
+    }
+  }
+}

internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@custom.json

@@ -0,0 +1,15 @@
+{
+  "name": "logs-apache.access@custom",
+  "component_template": {
+    "template": {
+      "settings": {}
+    },
+    "_meta": {
+      "package": {
+        "name": "apache"
+      },
+      "managed_by": "fleet",
+      "managed": true
+    }
+  }
+}

internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.access@settings.json

@@ -0,0 +1,99 @@
+{
+  "name": "logs-apache.access@settings",
+  "component_template": {
+    "template": {
+      "settings": {
+        "index": {
+          "lifecycle": {
+            "name": "logs"
+          },
+          "codec": "best_compression",
+          "mapping": {
+            "total_fields": {
+              "limit": "10000"
+            }
+          },
+          "query": {
+            "default_field": [
+              "cloud.account.id",
+              "cloud.availability_zone",
+              "cloud.instance.id",
+              "cloud.instance.name",
+              "cloud.machine.type",
+              "cloud.provider",
+              "cloud.region",
+              "cloud.project.id",
+              "cloud.image.id",
+              "container.id",
+              "container.image.name",
+              "container.name",
+              "host.architecture",
+              "host.domain",
+              "host.hostname",
+              "host.id",
+              "host.mac",
+              "host.name",
+              "host.os.family",
+              "host.os.kernel",
+              "host.os.name",
+              "host.os.platform",
+              "host.os.version",
+              "host.type",
+              "host.os.build",
+              "host.os.codename",
+              "input.type",
+              "destination.domain",
+              "ecs.version",
+              "event.category",
+              "event.kind",
+              "event.outcome",
+              "file.path",
+              "http.request.method",
+              "http.request.referrer",
+              "http.version",
+              "log.file.path",
+              "log.level",
+              "source.address",
+              "source.as.organization.name",
+              "source.domain",
+              "source.geo.city_name",
+              "source.geo.continent_name",
+              "source.geo.country_iso_code",
+              "source.geo.country_name",
+              "source.geo.region_iso_code",
+              "source.geo.region_name",
+              "tags",
+              "tls.cipher",
+              "tls.version",
+              "tls.version_protocol",
+              "url.domain",
+              "url.extension",
+              "url.query",
+              "user.name",
+              "user_agent.device.name",
+              "user_agent.device.name",
+              "user_agent.name",
+              "user_agent.name",
+              "user_agent.original",
+              "user_agent.original",
+              "user_agent.os.full",
+              "user_agent.os.name",
+              "user_agent.os.name",
+              "user_agent.os.version",
+              "user_agent.version",
+              "apache.access.ssl.protocol",
+              "apache.access.ssl.cipher"
+            ]
+          }
+        }
+      }
+    },
+    "_meta": {
+      "package": {
+        "name": "apache"
+      },
+      "managed_by": "fleet",
+      "managed": true
+    }
+  }
+}

internal/dump/testdata/elasticsearch-8-apache-dump-all/component_templates/logs-apache.error@custom.json

@@ -0,0 +1,15 @@
+{
+  "name": "logs-apache.error@custom",
+  "component_template": {
+    "template": {
+      "settings": {}
+    },
+    "_meta": {
+      "package": {
+        "name": "apache"
+      },
+      "managed_by": "fleet",
+      "managed": true
+    }
+  }
+}