Fix: use correct index template name for hidden data streams (#673)

* Correctly process the hidden property in datastream definition

* Add unit test coverage

* Add comment referring to kibana implementation

* Address code review feedback

internal/packages/packages.go

@@ -115,6 +115,7 @@ type DataStreamManifest struct {
 	Title         string `config:"title" json:"title" yaml:"title"`
 	Type          string `config:"type" json:"type" yaml:"type"`
 	Dataset       string `config:"dataset" json:"dataset" yaml:"dataset"`
+	Hidden        bool   `config:"hidden" json:"hidden" yaml:"hidden"`
 	Release       string `config:"release" json:"release" yaml:"release"`
 	Elasticsearch *struct {
 		IngestPipeline *struct {
@@ -241,12 +242,21 @@ func (dsm *DataStreamManifest) GetPipelineNameOrDefault() string {
 
 // IndexTemplateName returns the name of the Elasticsearch index template that would be installed
 // for this data stream.
+// The template name starts with dot "." if the datastream is hidden which is consistent with kibana implementation
+// https://github.com/elastic/kibana/blob/3955d0dc819fec03f68cd1d931f64da8472e34b2/x-pack/plugins/fleet/server/services/epm/elasticsearch/index.ts#L14
 func (dsm *DataStreamManifest) IndexTemplateName(pkgName string) string {
 	if dsm.Dataset == "" {
-		return fmt.Sprintf("%s-%s.%s", dsm.Type, pkgName, dsm.Name)
+		return fmt.Sprintf("%s%s-%s.%s", dsm.indexTemplateNamePrefix(), dsm.Type, pkgName, dsm.Name)
 	}
 
-	return fmt.Sprintf("%s-%s", dsm.Type, dsm.Dataset)
+	return fmt.Sprintf("%s%s-%s", dsm.indexTemplateNamePrefix(), dsm.Type, dsm.Dataset)
+}
+
+func (dsm *DataStreamManifest) indexTemplateNamePrefix() string {
+	if dsm.Hidden {
+		return "."
+	}
+	return ""
 }
 
 // FindInputByType returns the input for the provided type.

internal/packages/packages_test.go

@@ -57,6 +57,15 @@ func TestDataStreamManifest_IndexTemplateName(t *testing.T) {
 			"pkg",
 			dataStreamTypeLogs + "-pkg.foo",
 		},
+		"no_dataset_hidden": {
+			DataStreamManifest{
+				Name:   "foo",
+				Type:   dataStreamTypeLogs,
+				Hidden: true,
+			},
+			"pkg",
+			"." + dataStreamTypeLogs + "-pkg.foo",
+		},
 		"with_dataset": {
 			DataStreamManifest{
 				Name:    "foo",
@@ -66,6 +75,16 @@ func TestDataStreamManifest_IndexTemplateName(t *testing.T) {
 			"pkg",
 			dataStreamTypeLogs + "-custom",
 		},
+		"with_dataset_hidden": {
+			DataStreamManifest{
+				Name:    "foo",
+				Type:    dataStreamTypeLogs,
+				Dataset: "custom",
+				Hidden:  true,
+			},
+			"pkg",
+			"." + dataStreamTypeLogs + "-custom",
+		},
 	}
 
 	for name, test := range cases {