From 1d620bf28647c62efa370e105e25b12dd6499861 Mon Sep 17 00:00:00 2001 From: Daniel Jaglowski Date: Tue, 27 Sep 2022 10:58:05 -0400 Subject: [PATCH] [pkg/stanza] Allow 'parse_to' fields to accept 'attributes' and 'resource' (#14089) * [pkg/stanza] Allow 'parse_to' fields to accept 'attributes' and 'resource' This change enables parsers to parse directly to 'attributes' or 'resource'. In order to allow alternate validation behavior while unmarshaling, a new RootableField struct was introduced, which wraps the Field struct. --- pkg/stanza/entry/field.go | 44 +++- pkg/stanza/entry/field_test.go | 188 +++++++++--------- pkg/stanza/operator/helper/parser.go | 19 +- pkg/stanza/operator/helper/parser_test.go | 77 +------ pkg/stanza/operator/parser/csv/config_test.go | 24 +++ pkg/stanza/operator/parser/csv/csv_test.go | 2 +- .../operator/parser/csv/testdata/config.yaml | 9 + .../operator/parser/json/config_test.go | 26 ++- .../operator/parser/json/testdata/config.yaml | 9 + .../operator/parser/keyvalue/config_test.go | 26 ++- .../operator/parser/keyvalue/keyvalue_test.go | 6 +- .../parser/keyvalue/testdata/config.yaml | 9 + .../operator/parser/regex/config_test.go | 26 ++- .../parser/regex/testdata/config.yaml | 9 + .../operator/parser/syslog/config_test.go | 2 +- pkg/stanza/operator/parser/uri/config_test.go | 26 ++- .../operator/parser/uri/testdata/config.yaml | 9 + pkg/stanza/operator/parser/uri/uri_test.go | 2 +- unreleased/rootable-parse-to.yaml | 16 ++ 19 files changed, 346 insertions(+), 183 deletions(-) create mode 100755 unreleased/rootable-parse-to.yaml diff --git a/pkg/stanza/entry/field.go b/pkg/stanza/entry/field.go index 4c0f864c0b07..cfd46be88e98 100644 --- a/pkg/stanza/entry/field.go +++ b/pkg/stanza/entry/field.go @@ -33,6 +33,11 @@ type Field struct { FieldInterface } +// RootableField is a Field that may refer directly to "attributes" or "resource" +type RootableField struct { + Field +} + // FieldInterface is a field on an entry. type FieldInterface interface { Get(*Entry) (interface{}, bool) @@ -52,6 +57,18 @@ func (f *Field) UnmarshalJSON(raw []byte) error { return err } +// UnmarshalJSON will unmarshal a field from JSON +func (r *RootableField) UnmarshalJSON(raw []byte) error { + var s string + err := json.Unmarshal(raw, &s) + if err != nil { + return err + } + field, err := newField(s, true) + *r = RootableField{Field: field} + return err +} + // UnmarshalYAML will unmarshal a field from YAML func (f *Field) UnmarshalYAML(unmarshal func(interface{}) error) error { var s string @@ -63,6 +80,18 @@ func (f *Field) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } +// UnmarshalYAML will unmarshal a field from YAML +func (r *RootableField) UnmarshalYAML(unmarshal func(interface{}) error) error { + var s string + err := unmarshal(&s) + if err != nil { + return err + } + field, err := newField(s, true) + *r = RootableField{Field: field} + return err +} + // UnmarshalText will unmarshal a field from text func (f *Field) UnmarshalText(text []byte) error { field, err := NewField(string(text)) @@ -70,7 +99,18 @@ func (f *Field) UnmarshalText(text []byte) error { return err } +// UnmarshalText will unmarshal a field from text +func (r *RootableField) UnmarshalText(text []byte) error { + field, err := newField(string(text), true) + *r = RootableField{Field: field} + return err +} + func NewField(s string) (Field, error) { + return newField(s, false) +} + +func newField(s string, rootable bool) (Field, error) { keys, err := fromJSONDot(s) if err != nil { return Field{}, fmt.Errorf("splitting field: %w", err) @@ -78,12 +118,12 @@ func NewField(s string) (Field, error) { switch keys[0] { case AttributesPrefix: - if len(keys) == 1 { + if !rootable && len(keys) == 1 { return Field{}, fmt.Errorf("attributes cannot be referenced without subfield") } return NewAttributeField(keys[1:]...), nil case ResourcePrefix: - if len(keys) == 1 { + if !rootable && len(keys) == 1 { return Field{}, fmt.Errorf("resource cannot be referenced without subfield") } return NewResourceField(keys[1:]...), nil diff --git a/pkg/stanza/entry/field_test.go b/pkg/stanza/entry/field_test.go index e6daced248e6..0eff53abb3da 100644 --- a/pkg/stanza/entry/field_test.go +++ b/pkg/stanza/entry/field_test.go @@ -24,147 +24,151 @@ import ( func TestFieldUnmarshalJSON(t *testing.T) { cases := []struct { - name string - input []byte - expected Field + name string + input []byte + expected Field + expectedErr string + expectedErrRootable string }{ { - "BodyLong", - []byte(`"body"`), - NewBodyField(), + name: "BodyLong", + input: []byte(`"body"`), + expected: NewBodyField(), }, { - "SimpleField", - []byte(`"body.test1"`), - NewBodyField("test1"), + name: "SimpleField", + input: []byte(`"body.test1"`), + expected: NewBodyField("test1"), }, { - "ComplexField", - []byte(`"body.test1.test2"`), - NewBodyField("test1", "test2"), + name: "ComplexField", + input: []byte(`"body.test1.test2"`), + expected: NewBodyField("test1", "test2"), }, { - "BracketedField", - []byte(`"body.test1['file.name']"`), - NewBodyField("test1", "file.name"), + name: "BracketedField", + input: []byte(`"body.test1['file.name']"`), + expected: NewBodyField("test1", "file.name"), }, { - "DoubleBracketedField", - []byte(`"body.test1['file.details']['file.name']"`), - NewBodyField("test1", "file.details", "file.name"), + name: "DoubleBracketedField", + input: []byte(`"body.test1['file.details']['file.name']"`), + expected: NewBodyField("test1", "file.details", "file.name"), }, { - "PostBracketField", - []byte(`"body.test1['file.details'].name"`), - NewBodyField("test1", "file.details", "name"), + name: "PostBracketField", + input: []byte(`"body.test1['file.details'].name"`), + expected: NewBodyField("test1", "file.details", "name"), }, { - "AttributesSimpleField", - []byte(`"attributes.test1"`), - NewAttributeField("test1"), + name: "AttributesSimpleField", + input: []byte(`"attributes.test1"`), + expected: NewAttributeField("test1"), }, { - "AttributesComplexField", - []byte(`"attributes.test1.test2"`), - NewAttributeField("test1", "test2"), + name: "AttributesComplexField", + input: []byte(`"attributes.test1.test2"`), + expected: NewAttributeField("test1", "test2"), }, { - "AttributesBracketedField", - []byte(`"attributes.test1['file.name']"`), - NewAttributeField("test1", "file.name"), + name: "AttributesBracketedField", + input: []byte(`"attributes.test1['file.name']"`), + expected: NewAttributeField("test1", "file.name"), }, { - "AttributesDoubleBracketedField", - []byte(`"attributes.test1['file.details']['file.name']"`), - NewAttributeField("test1", "file.details", "file.name"), + name: "AttributesDoubleBracketedField", + input: []byte(`"attributes.test1['file.details']['file.name']"`), + expected: NewAttributeField("test1", "file.details", "file.name"), }, { - "AttributesPostBracketField", - []byte(`"attributes.test1['file.details'].name"`), - NewAttributeField("test1", "file.details", "name"), + name: "AttributesPostBracketField", + input: []byte(`"attributes.test1['file.details'].name"`), + expected: NewAttributeField("test1", "file.details", "name"), }, { - "AttributesSimpleField", - []byte(`"attributes.test1"`), - NewAttributeField("test1"), + name: "AttributesSimpleField", + input: []byte(`"attributes.test1"`), + expected: NewAttributeField("test1"), }, - { - "ResourceSimpleField", - []byte(`"resource.test1"`), - NewResourceField("test1"), + name: "ResourceSimpleField", + input: []byte(`"resource.test1"`), + expected: NewResourceField("test1"), }, { - "ResourceComplexField", - []byte(`"resource.test1.test2"`), - NewResourceField("test1", "test2"), + name: "ResourceComplexField", + input: []byte(`"resource.test1.test2"`), + expected: NewResourceField("test1", "test2"), }, { - "ResourceBracketedField", - []byte(`"resource.test1['file.name']"`), - NewResourceField("test1", "file.name"), + name: "ResourceBracketedField", + input: []byte(`"resource.test1['file.name']"`), + expected: NewResourceField("test1", "file.name"), }, { - "ResourceDoubleBracketedField", - []byte(`"resource.test1['file.details']['file.name']"`), - NewResourceField("test1", "file.details", "file.name"), + name: "ResourceDoubleBracketedField", + input: []byte(`"resource.test1['file.details']['file.name']"`), + expected: NewResourceField("test1", "file.details", "file.name"), }, { - "ResourcePostBracketField", - []byte(`"resource.test1['file.details'].name"`), - NewResourceField("test1", "file.details", "name"), + name: "ResourcePostBracketField", + input: []byte(`"resource.test1['file.details'].name"`), + expected: NewResourceField("test1", "file.details", "name"), }, { - "ResourceSimpleField", - []byte(`"resource.test1"`), - NewResourceField("test1"), + name: "ResourceSimpleField", + input: []byte(`"resource.test1"`), + expected: NewResourceField("test1"), }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - var f Field - err := json.Unmarshal(tc.input, &f) - require.NoError(t, err) - require.Equal(t, tc.expected, f) - }) - } -} - -func TestFieldUnmarshalJSONFailure(t *testing.T) { - cases := []struct { - name string - input []byte - expected string - }{ { - "Bool", - []byte(`"bool"`), - "unrecognized prefix", + name: "AttributesRoot", + input: []byte(`"attributes"`), + expectedErr: "attributes cannot be referenced without subfield", + expected: NewAttributeField(), }, { - "Object", - []byte(`{"key":"value"}`), - "cannot unmarshal object into Go value of type string", + name: "ResourceRoot", + input: []byte(`"resource"`), + expectedErr: "resource cannot be referenced without subfield", + expected: NewResourceField(), }, { - "AttributesRoot", - []byte(`"attributes"`), - "attributes cannot be referenced without subfield", + name: "Bool", + input: []byte(`"bool"`), + expectedErrRootable: "unrecognized prefix", }, { - "ResourceRoot", - []byte(`"resource"`), - "resource cannot be referenced without subfield", + name: "Object", + input: []byte(`{"key":"value"}`), + expectedErrRootable: "cannot unmarshal object into Go value of type string", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - var f Field - err := json.Unmarshal(tc.input, &f) - require.Error(t, err) - require.Contains(t, err.Error(), tc.expected) + var field Field + err := json.Unmarshal(tc.input, &field) + + var rootableField RootableField + errRootable := json.Unmarshal(tc.input, &rootableField) + + switch { + case tc.expectedErrRootable != "": + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectedErr) + require.Error(t, errRootable) + require.Contains(t, errRootable.Error(), tc.expectedErrRootable) + case tc.expectedErr != "": + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectedErr) + require.NoError(t, errRootable) + require.Equal(t, tc.expected, rootableField.Field) + default: + require.NoError(t, err) + require.Equal(t, tc.expected, field) + require.NoError(t, errRootable) + require.Equal(t, tc.expected, rootableField.Field) + } }) } } diff --git a/pkg/stanza/operator/helper/parser.go b/pkg/stanza/operator/helper/parser.go index b198291f70e7..684dd76764db 100644 --- a/pkg/stanza/operator/helper/parser.go +++ b/pkg/stanza/operator/helper/parser.go @@ -29,21 +29,20 @@ func NewParserConfig(operatorID, operatorType string) ParserConfig { return ParserConfig{ TransformerConfig: NewTransformerConfig(operatorID, operatorType), ParseFrom: entry.NewBodyField(), - ParseTo: entry.NewAttributeField(), + ParseTo: entry.RootableField{Field: entry.NewAttributeField()}, } } // ParserConfig provides the basic implementation of a parser config. type ParserConfig struct { TransformerConfig `mapstructure:",squash" yaml:",inline"` - - ParseFrom entry.Field `mapstructure:"parse_from" json:"parse_from" yaml:"parse_from"` - ParseTo entry.Field `mapstructure:"parse_to" json:"parse_to" yaml:"parse_to"` - BodyField *entry.Field `mapstructure:"body" json:"body" yaml:"body"` - TimeParser *TimeParser `mapstructure:"timestamp,omitempty" json:"timestamp,omitempty" yaml:"timestamp,omitempty"` - SeverityConfig *SeverityConfig `mapstructure:"severity,omitempty" json:"severity,omitempty" yaml:"severity,omitempty"` - TraceParser *TraceParser `mapstructure:"trace,omitempty" json:"trace,omitempty" yaml:"trace,omitempty"` - ScopeNameParser *ScopeNameParser `mapstructure:"scope_name,omitempty" json:"scope_name,omitempty" yaml:"scope_name,omitempty"` + ParseFrom entry.Field `mapstructure:"parse_from" json:"parse_from" yaml:"parse_from"` + ParseTo entry.RootableField `mapstructure:"parse_to" json:"parse_to" yaml:"parse_to"` + BodyField *entry.Field `mapstructure:"body" json:"body" yaml:"body"` + TimeParser *TimeParser `mapstructure:"timestamp,omitempty" json:"timestamp,omitempty" yaml:"timestamp,omitempty"` + SeverityConfig *SeverityConfig `mapstructure:"severity,omitempty" json:"severity,omitempty" yaml:"severity,omitempty"` + TraceParser *TraceParser `mapstructure:"trace,omitempty" json:"trace,omitempty" yaml:"trace,omitempty"` + ScopeNameParser *ScopeNameParser `mapstructure:"scope_name,omitempty" json:"scope_name,omitempty" yaml:"scope_name,omitempty"` } // Build will build a parser operator. @@ -60,7 +59,7 @@ func (c ParserConfig) Build(logger *zap.SugaredLogger) (ParserOperator, error) { parserOperator := ParserOperator{ TransformerOperator: transformerOperator, ParseFrom: c.ParseFrom, - ParseTo: c.ParseTo, + ParseTo: c.ParseTo.Field, BodyField: c.BodyField, } diff --git a/pkg/stanza/operator/helper/parser_test.go b/pkg/stanza/operator/helper/parser_test.go index cb356e56ecd3..a964a5fc647a 100644 --- a/pkg/stanza/operator/helper/parser_test.go +++ b/pkg/stanza/operator/helper/parser_test.go @@ -21,14 +21,12 @@ import ( "testing" "time" - "github.com/mitchellh/mapstructure" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/operatortest" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/testutil" ) @@ -55,7 +53,7 @@ func TestParserConfigInvalidTimeParser(t *testing.T) { func TestParserConfigBodyCollision(t *testing.T) { cfg := NewParserConfig("test-id", "test-type") - cfg.ParseTo = entry.NewBodyField() + cfg.ParseTo = entry.RootableField{Field: entry.NewBodyField()} b := entry.NewAttributeField("message") cfg.BodyField = &b @@ -388,7 +386,7 @@ func TestParserFields(t *testing.T) { { "ParseToBodyRoot", func(cfg *ParserConfig) { - cfg.ParseTo = entry.NewBodyField() + cfg.ParseTo = entry.RootableField{Field: entry.NewBodyField()} }, func() *entry.Entry { e := entry.New() @@ -408,7 +406,7 @@ func TestParserFields(t *testing.T) { { "ParseToAttributesRoot", func(cfg *ParserConfig) { - cfg.ParseTo = entry.NewAttributeField() + cfg.ParseTo = entry.RootableField{Field: entry.NewAttributeField()} }, func() *entry.Entry { e := entry.New() @@ -429,7 +427,7 @@ func TestParserFields(t *testing.T) { { "ParseToResourceRoot", func(cfg *ParserConfig) { - cfg.ParseTo = entry.NewResourceField() + cfg.ParseTo = entry.RootableField{Field: entry.NewResourceField()} }, func() *entry.Entry { e := entry.New() @@ -450,7 +448,7 @@ func TestParserFields(t *testing.T) { { "ParseToBodyField", func(cfg *ParserConfig) { - cfg.ParseTo = entry.NewBodyField("one", "two") + cfg.ParseTo = entry.RootableField{Field: entry.NewBodyField("one", "two")} }, func() *entry.Entry { e := entry.New() @@ -474,7 +472,7 @@ func TestParserFields(t *testing.T) { { "ParseToAttributeField", func(cfg *ParserConfig) { - cfg.ParseTo = entry.NewAttributeField("one", "two") + cfg.ParseTo = entry.RootableField{Field: entry.NewAttributeField("one", "two")} }, func() *entry.Entry { e := entry.New() @@ -499,7 +497,7 @@ func TestParserFields(t *testing.T) { { "ParseToResourceField", func(cfg *ParserConfig) { - cfg.ParseTo = entry.NewResourceField("one", "two") + cfg.ParseTo = entry.RootableField{Field: entry.NewResourceField("one", "two")} }, func() *entry.Entry { e := entry.New() @@ -655,7 +653,7 @@ func TestParserFields(t *testing.T) { func NewTestParserConfig() ParserConfig { expect := NewParserConfig("parser_config", "test_type") expect.ParseFrom = entry.NewBodyField("from") - expect.ParseTo = entry.NewBodyField("to") + expect.ParseTo = entry.RootableField{Field: entry.NewBodyField("to")} tp := NewTimeParser() expect.TimeParser = &tp @@ -672,65 +670,6 @@ func NewTestParserConfig() ParserConfig { return expect } -func TestMapStructureDecodeParserConfigWithHook(t *testing.T) { - expect := NewTestParserConfig() - input := map[string]interface{}{ - "id": "parser_config", - "type": "test_type", - "on_error": "send", - "parse_from": "body.from", - "parse_to": "body.to", - "timestamp": map[string]interface{}{ - "layout_type": "strptime", - }, - "severity": map[string]interface{}{ - "mapping": map[interface{}]interface{}{ - "info": "3xx", - "warn": "4xx", - }, - }, - "scope_name": map[string]interface{}{ - "parse_from": "body.logger", - }, - } - - var actual ParserConfig - dc := &mapstructure.DecoderConfig{Result: &actual, DecodeHook: operatortest.JSONUnmarshalerHook()} - ms, err := mapstructure.NewDecoder(dc) - require.NoError(t, err) - err = ms.Decode(input) - require.NoError(t, err) - require.Equal(t, expect, actual) -} - -func TestMapStructureDecodeParserConfig(t *testing.T) { - expect := NewTestParserConfig() - input := map[string]interface{}{ - "id": "parser_config", - "type": "test_type", - "on_error": "send", - "parse_from": entry.NewBodyField("from"), - "parse_to": entry.NewBodyField("to"), - "timestamp": map[string]interface{}{ - "layout_type": "strptime", - }, - "severity": map[string]interface{}{ - "mapping": map[interface{}]interface{}{ - "info": "3xx", - "warn": "4xx", - }, - }, - "scope_name": map[string]interface{}{ - "parse_from": entry.NewBodyField("logger"), - }, - } - - var actual ParserConfig - err := mapstructure.Decode(input, &actual) - require.NoError(t, err) - require.Equal(t, expect, actual) -} - func writerWithFakeOut(t *testing.T) (*WriterOperator, *testutil.FakeOutput) { fakeOut := testutil.NewFakeOutput(t) writer := &WriterOperator{ diff --git a/pkg/stanza/operator/parser/csv/config_test.go b/pkg/stanza/operator/parser/csv/config_test.go index 9465615b03a9..f553c1fe76ca 100644 --- a/pkg/stanza/operator/parser/csv/config_test.go +++ b/pkg/stanza/operator/parser/csv/config_test.go @@ -90,6 +90,30 @@ func TestConfig(t *testing.T) { return p }(), }, + { + Name: "parse_to_attributes", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewAttributeField()} + return p + }(), + }, + { + Name: "parse_to_body", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewBodyField()} + return p + }(), + }, + { + Name: "parse_to_resource", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewResourceField()} + return p + }(), + }, }, }.Run(t) } diff --git a/pkg/stanza/operator/parser/csv/csv_test.go b/pkg/stanza/operator/parser/csv/csv_test.go index 25dca05fa63e..45e788aca570 100644 --- a/pkg/stanza/operator/parser/csv/csv_test.go +++ b/pkg/stanza/operator/parser/csv/csv_test.go @@ -963,7 +963,7 @@ cc""",dddd,eeee`, for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { cfg := NewConfigWithID("test") - cfg.ParseTo = entry.NewBodyField() + cfg.ParseTo = entry.RootableField{Field: entry.NewBodyField()} cfg.OutputIDs = []string{"fake"} cfg.Header = "A,B,C,D,E" diff --git a/pkg/stanza/operator/parser/csv/testdata/config.yaml b/pkg/stanza/operator/parser/csv/testdata/config.yaml index 3a2e8759e799..d723d50cedd2 100644 --- a/pkg/stanza/operator/parser/csv/testdata/config.yaml +++ b/pkg/stanza/operator/parser/csv/testdata/config.yaml @@ -22,6 +22,15 @@ ignore_quotes: parse_from: body.message header: id,severity,message ignore_quotes: true +parse_to_attributes: + type: csv_parser + parse_to: attributes +parse_to_body: + type: csv_parser + parse_to: body +parse_to_resource: + type: csv_parser + parse_to: resource timestamp: type: csv_parser header: timestamp_field,severity,message diff --git a/pkg/stanza/operator/parser/json/config_test.go b/pkg/stanza/operator/parser/json/config_test.go index 103cacaca187..897d0530cc1b 100644 --- a/pkg/stanza/operator/parser/json/config_test.go +++ b/pkg/stanza/operator/parser/json/config_test.go @@ -51,7 +51,7 @@ func TestConfig(t *testing.T) { Name: "parse_to_simple", Expect: func() *Config { cfg := NewConfig() - cfg.ParseTo = entry.NewBodyField("log") + cfg.ParseTo = entry.RootableField{Field: entry.NewBodyField("log")} return cfg }(), }, @@ -97,6 +97,30 @@ func TestConfig(t *testing.T) { return cfg }(), }, + { + Name: "parse_to_attributes", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewAttributeField()} + return p + }(), + }, + { + Name: "parse_to_body", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewBodyField()} + return p + }(), + }, + { + Name: "parse_to_resource", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewResourceField()} + return p + }(), + }, }, }.Run(t) } diff --git a/pkg/stanza/operator/parser/json/testdata/config.yaml b/pkg/stanza/operator/parser/json/testdata/config.yaml index a9dab897b923..4de9e105952f 100644 --- a/pkg/stanza/operator/parser/json/testdata/config.yaml +++ b/pkg/stanza/operator/parser/json/testdata/config.yaml @@ -6,6 +6,15 @@ on_error_drop: parse_from_simple: type: json_parser parse_from: body.from +parse_to_attributes: + type: json_parser + parse_to: attributes +parse_to_body: + type: json_parser + parse_to: body +parse_to_resource: + type: json_parser + parse_to: resource parse_to_simple: type: json_parser parse_to: body.log diff --git a/pkg/stanza/operator/parser/keyvalue/config_test.go b/pkg/stanza/operator/parser/keyvalue/config_test.go index 98682f2be0e9..79d143846157 100644 --- a/pkg/stanza/operator/parser/keyvalue/config_test.go +++ b/pkg/stanza/operator/parser/keyvalue/config_test.go @@ -43,7 +43,7 @@ func TestConfig(t *testing.T) { Name: "parse_to_simple", Expect: func() *Config { cfg := NewConfig() - cfg.ParseTo = entry.NewBodyField("log") + cfg.ParseTo = entry.RootableField{Field: entry.NewBodyField("log")} return cfg }(), }, @@ -103,6 +103,30 @@ func TestConfig(t *testing.T) { return cfg }(), }, + { + Name: "parse_to_attributes", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewAttributeField()} + return p + }(), + }, + { + Name: "parse_to_body", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewBodyField()} + return p + }(), + }, + { + Name: "parse_to_resource", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewResourceField()} + return p + }(), + }, }, }.Run(t) } diff --git a/pkg/stanza/operator/parser/keyvalue/keyvalue_test.go b/pkg/stanza/operator/parser/keyvalue/keyvalue_test.go index b85b9c4e789b..e1dc9df60229 100644 --- a/pkg/stanza/operator/parser/keyvalue/keyvalue_test.go +++ b/pkg/stanza/operator/parser/keyvalue/keyvalue_test.go @@ -204,7 +204,7 @@ func TestParser(t *testing.T) { { "parse-to", func(kv *Config) { - kv.ParseTo = entry.NewBodyField("test") + kv.ParseTo = entry.RootableField{Field: entry.NewBodyField("test")} }, &entry.Entry{ Body: "name=stanza age=10", @@ -224,7 +224,7 @@ func TestParser(t *testing.T) { "from-to", func(kv *Config) { kv.ParseFrom = entry.NewAttributeField("from") - kv.ParseTo = entry.NewBodyField("to") + kv.ParseTo = entry.RootableField{Field: entry.NewBodyField("to")} }, &entry.Entry{ Attributes: map[string]interface{}{ @@ -329,7 +329,7 @@ func TestParser(t *testing.T) { func(kv *Config) { kv.Delimiter = "|" kv.ParseFrom = entry.NewBodyField("testfield") - kv.ParseTo = entry.NewBodyField("testparsed") + kv.ParseTo = entry.RootableField{Field: entry.NewBodyField("testparsed")} }, &entry.Entry{ Body: map[string]interface{}{ diff --git a/pkg/stanza/operator/parser/keyvalue/testdata/config.yaml b/pkg/stanza/operator/parser/keyvalue/testdata/config.yaml index 62066f7b9603..0206717099ea 100644 --- a/pkg/stanza/operator/parser/keyvalue/testdata/config.yaml +++ b/pkg/stanza/operator/parser/keyvalue/testdata/config.yaml @@ -12,6 +12,15 @@ pair_delimiter: parse_from_simple: type: key_value_parser parse_from: body.from +parse_to_attributes: + type: key_value_parser + parse_to: attributes +parse_to_body: + type: key_value_parser + parse_to: body +parse_to_resource: + type: key_value_parser + parse_to: resource parse_to_simple: type: key_value_parser parse_to: body.log diff --git a/pkg/stanza/operator/parser/regex/config_test.go b/pkg/stanza/operator/parser/regex/config_test.go index 7b78b12df462..e0d9b2f29081 100644 --- a/pkg/stanza/operator/parser/regex/config_test.go +++ b/pkg/stanza/operator/parser/regex/config_test.go @@ -51,7 +51,7 @@ func TestParserGoldenConfig(t *testing.T) { Name: "parse_to_simple", Expect: func() *Config { cfg := NewConfig() - cfg.ParseTo = entry.NewBodyField("log") + cfg.ParseTo = entry.RootableField{Field: entry.NewBodyField("log")} return cfg }(), }, @@ -115,6 +115,30 @@ func TestParserGoldenConfig(t *testing.T) { return cfg }(), }, + { + Name: "parse_to_attributes", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewAttributeField()} + return p + }(), + }, + { + Name: "parse_to_body", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewBodyField()} + return p + }(), + }, + { + Name: "parse_to_resource", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewResourceField()} + return p + }(), + }, }, }.Run(t) } diff --git a/pkg/stanza/operator/parser/regex/testdata/config.yaml b/pkg/stanza/operator/parser/regex/testdata/config.yaml index f8ac81d8a40f..1c3a81cd33c1 100644 --- a/pkg/stanza/operator/parser/regex/testdata/config.yaml +++ b/pkg/stanza/operator/parser/regex/testdata/config.yaml @@ -10,6 +10,15 @@ on_error_drop: parse_from_simple: type: regex_parser parse_from: "body.from" +parse_to_attributes: + type: regex_parser + parse_to: attributes +parse_to_body: + type: regex_parser + parse_to: body +parse_to_resource: + type: regex_parser + parse_to: resource parse_to_simple: type: regex_parser parse_to: "body.log" diff --git a/pkg/stanza/operator/parser/syslog/config_test.go b/pkg/stanza/operator/parser/syslog/config_test.go index 1c600110f20d..423e999aa900 100644 --- a/pkg/stanza/operator/parser/syslog/config_test.go +++ b/pkg/stanza/operator/parser/syslog/config_test.go @@ -83,7 +83,7 @@ func TestUnmarshal(t *testing.T) { Expect: func() *Config { cfg := NewConfig() cfg.Protocol = RFC5424 - cfg.ParseTo = entry.NewBodyField("log") + cfg.ParseTo = entry.RootableField{Field: entry.NewBodyField("log")} return cfg }(), }, diff --git a/pkg/stanza/operator/parser/uri/config_test.go b/pkg/stanza/operator/parser/uri/config_test.go index 9e590cf91b30..164c2ea273bd 100644 --- a/pkg/stanza/operator/parser/uri/config_test.go +++ b/pkg/stanza/operator/parser/uri/config_test.go @@ -43,7 +43,7 @@ func TestParserGoldenConfig(t *testing.T) { Name: "parse_to_simple", Expect: func() *Config { cfg := NewConfig() - cfg.ParseTo = entry.NewBodyField("log") + cfg.ParseTo = entry.RootableField{Field: entry.NewBodyField("log")} return cfg }(), }, @@ -87,6 +87,30 @@ func TestParserGoldenConfig(t *testing.T) { return cfg }(), }, + { + Name: "parse_to_attributes", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewAttributeField()} + return p + }(), + }, + { + Name: "parse_to_body", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewBodyField()} + return p + }(), + }, + { + Name: "parse_to_resource", + Expect: func() *Config { + p := NewConfig() + p.ParseTo = entry.RootableField{Field: entry.NewResourceField()} + return p + }(), + }, }, }.Run(t) } diff --git a/pkg/stanza/operator/parser/uri/testdata/config.yaml b/pkg/stanza/operator/parser/uri/testdata/config.yaml index e75fa6e54c64..1715ba28495f 100644 --- a/pkg/stanza/operator/parser/uri/testdata/config.yaml +++ b/pkg/stanza/operator/parser/uri/testdata/config.yaml @@ -6,6 +6,15 @@ on_error_drop: parse_from_simple: type: uri_parser parse_from: "body.from" +parse_to_attributes: + type: uri_parser + parse_to: attributes +parse_to_body: + type: uri_parser + parse_to: body +parse_to_resource: + type: uri_parser + parse_to: resource parse_to_simple: type: uri_parser parse_to: "body.log" diff --git a/pkg/stanza/operator/parser/uri/uri_test.go b/pkg/stanza/operator/parser/uri/uri_test.go index d3c1c899db1a..6a75d74f50c3 100644 --- a/pkg/stanza/operator/parser/uri/uri_test.go +++ b/pkg/stanza/operator/parser/uri/uri_test.go @@ -104,7 +104,7 @@ func TestProcess(t *testing.T) { func() (operator.Operator, error) { cfg := NewConfigWithID("test_id") cfg.ParseFrom = entry.NewBodyField("url") - cfg.ParseTo = entry.NewBodyField("url2") + cfg.ParseTo = entry.RootableField{Field: entry.NewBodyField("url2")} return cfg.Build(testutil.Logger(t)) }, &entry.Entry{ diff --git a/unreleased/rootable-parse-to.yaml b/unreleased/rootable-parse-to.yaml new file mode 100755 index 000000000000..8576d753901d --- /dev/null +++ b/unreleased/rootable-parse-to.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: filelog, journald, syslog, tcplog, udplog, windowseventlog + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Allow 'parse_to' fields to accept 'attributes' and 'resource' + +# One or more tracking issues related to the change +issues: [14089] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: