Skip to content

Commit 8c8ff31

Browse files
mmetcmmetc
committed
refact pkg/parser: rename DataCapture -> Stash, extract methods
1 parent 3f3c637 commit 8c8ff31

File tree

3 files changed

+156
-116
lines changed

3 files changed

+156
-116
lines changed

‎pkg/parser/grok.go‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"errors"
55
"fmt"
66
"strings"
7-
"time"
87

98
"github.com/expr-lang/expr"
109
"github.com/expr-lang/expr/vm"
@@ -102,15 +101,3 @@ func (g *GrokPattern) Validate() error {
102101

103102
return nil
104103
}
105-
106-
type DataCapture struct {
107-
Name string `yaml:"name,omitempty"`
108-
Key string `yaml:"key,omitempty"`
109-
KeyExpression *vm.Program `yaml:"-"`
110-
Value string `yaml:"value,omitempty"`
111-
ValueExpression *vm.Program `yaml:"-"`
112-
TTL string `yaml:"ttl,omitempty"`
113-
TTLVal time.Duration `yaml:"-"`
114-
MaxMapSize int `yaml:"size,omitempty"`
115-
Strategy string `yaml:"strategy,omitempty"`
116-
}

‎pkg/parser/node.go‎

Lines changed: 11 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import (
55
"fmt"
66
"strconv"
77
"strings"
8-
"time"
98

109
"github.com/davecgh/go-spew/spew"
1110
"github.com/expr-lang/expr"
@@ -17,7 +16,6 @@ import (
1716
"github.com/crowdsecurity/go-cs-lib/ptr"
1817
"github.com/crowdsecurity/grokky"
1918

20-
"github.com/crowdsecurity/crowdsec/pkg/cache"
2119
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
2220
"github.com/crowdsecurity/crowdsec/pkg/metrics"
2321
"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -63,7 +61,8 @@ type Node struct {
6361
Statics []Static `yaml:"statics,omitempty"`
6462
RuntimeStatics []RuntimeStatic `yaml:"-"`
6563
// Stash allows to capture data from the log line and store it in an accessible cache
66-
Stash []DataCapture `yaml:"stash,omitempty"`
64+
Stashes []Stash `yaml:"stash,omitempty"`
65+
RuntimeStashes []RuntimeStash `yaml:"-"`
6766
// Whitelists
6867
Whitelist Whitelist `yaml:"whitelist,omitempty"`
6968
Data []*types.DataSource `yaml:"data,omitempty"`
@@ -96,32 +95,9 @@ func (n *Node) validate(ectx EnricherCtx) error {
9695
}
9796
}
9897

99-
for idx := range n.Stash {
100-
// pointer not value, to avoid throwing away the defaults
101-
stash := &n.Stash[idx]
102-
103-
if stash.Name == "" {
104-
return fmt.Errorf("stash %d: name must be set", idx)
105-
}
106-
107-
if stash.Value == "" {
108-
return fmt.Errorf("stash %s: value expression must be set", stash.Name)
109-
}
110-
111-
if stash.Key == "" {
112-
return fmt.Errorf("stash %s: key expression must be set", stash.Name)
113-
}
114-
115-
if stash.TTL == "" {
116-
return fmt.Errorf("stash %s: ttl must be set", stash.Name)
117-
}
118-
119-
if stash.Strategy == "" {
120-
stash.Strategy = "LRU"
121-
}
122-
// should be configurable
123-
if stash.MaxMapSize == 0 {
124-
stash.MaxMapSize = 100
98+
for idx, stash := range n.Stashes {
99+
if err := stash.Validate(); err != nil {
100+
return fmt.Errorf("stash %d: %w", idx, err)
125101
}
126102
}
127103

@@ -270,52 +246,8 @@ func (n *Node) processGrok(p *types.Event, cachedExprEnv map[string]any) (bool,
270246
}
271247

272248
func (n *Node) processStash(_ *types.Event, cachedExprEnv map[string]any, logger *log.Entry) error {
273-
for idx, stash := range n.Stash {
274-
var (
275-
key string
276-
value string
277-
)
278-
279-
if stash.ValueExpression == nil {
280-
logger.Warningf("Stash %d has no value expression, skipping", idx)
281-
continue
282-
}
283-
284-
if stash.KeyExpression == nil {
285-
logger.Warningf("Stash %d has no key expression, skipping", idx)
286-
continue
287-
}
288-
// collect the data
289-
output, err := exprhelpers.Run(stash.ValueExpression, cachedExprEnv, logger, n.Debug)
290-
if err != nil {
291-
logger.Warningf("Error while running stash val expression: %v", err)
292-
}
293-
// can we expect anything else than a string ?
294-
switch output := output.(type) {
295-
case string:
296-
value = output
297-
default:
298-
logger.Warningf("unexpected type %T (%v) while running %q", output, output, stash.Value)
299-
continue
300-
}
301-
302-
// collect the key
303-
output, err = exprhelpers.Run(stash.KeyExpression, cachedExprEnv, logger, n.Debug)
304-
if err != nil {
305-
logger.Warningf("Error while running stash key expression: %v", err)
306-
}
307-
// can we expect anything else than a string ?
308-
switch output := output.(type) {
309-
case string:
310-
key = output
311-
default:
312-
logger.Warningf("unexpected type %T (%v) while running %q", output, output, stash.Key)
313-
continue
314-
}
315-
316-
if err = cache.SetKey(stash.Name, key, value, &stash.TTLVal); err != nil {
317-
logger.Warningf("failed to store data in cache: %s", err.Error())
318-
}
249+
for idx, stash := range n.RuntimeStashes {
250+
stash.Apply(idx, cachedExprEnv, n.Logger, n.Debug)
319251
}
320252

321253
return nil
@@ -531,36 +463,12 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
531463
valid = true
532464
}
533465

534-
/* load data capture (stash) */
535-
for i, stash := range n.Stash {
536-
n.Stash[i].ValueExpression, err = expr.Compile(stash.Value,
537-
exprhelpers.GetExprOptions(map[string]any{"evt": &types.Event{}})...)
466+
for _, stash := range n.Stashes {
467+
compiled, err := stash.Compile(n.Logger)
538468
if err != nil {
539-
return fmt.Errorf("while compiling stash value expression: %w", err)
540-
}
541-
542-
n.Stash[i].KeyExpression, err = expr.Compile(stash.Key,
543-
exprhelpers.GetExprOptions(map[string]any{"evt": &types.Event{}})...)
544-
if err != nil {
545-
return fmt.Errorf("while compiling stash key expression: %w", err)
546-
}
547-
548-
n.Stash[i].TTLVal, err = time.ParseDuration(stash.TTL)
549-
if err != nil {
550-
return fmt.Errorf("while parsing stash ttl: %w", err)
551-
}
552-
553-
logLvl := n.Logger.Logger.GetLevel()
554-
// init the cache, does it make sense to create it here just to be sure everything is fine ?
555-
if err = cache.CacheInit(cache.CacheCfg{
556-
Size: n.Stash[i].MaxMapSize,
557-
TTL: n.Stash[i].TTLVal,
558-
Name: n.Stash[i].Name,
559-
Strategy: n.Stash[i].Strategy,
560-
LogLevel: &logLvl,
561-
}); err != nil {
562-
return fmt.Errorf("while initializing cache: %w", err)
469+
return fmt.Errorf("stash %s: %w", stash.Name, err)
563470
}
471+
n.RuntimeStashes = append(n.RuntimeStashes, *compiled)
564472
}
565473

566474
/* compile leafs if present */

‎pkg/parser/stash.go‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
package parser
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"time"
7+
8+
"github.com/expr-lang/expr"
9+
"github.com/expr-lang/expr/vm"
10+
log "github.com/sirupsen/logrus"
11+
12+
"github.com/crowdsecurity/crowdsec/pkg/cache"
13+
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
14+
"github.com/crowdsecurity/crowdsec/pkg/types"
15+
)
16+
17+
type Stash struct {
18+
Name string `yaml:"name,omitempty"`
19+
Key string `yaml:"key,omitempty"`
20+
Value string `yaml:"value,omitempty"`
21+
TTL string `yaml:"ttl,omitempty"`
22+
MaxMapSize int `yaml:"size,omitempty"`
23+
Strategy string `yaml:"strategy,omitempty"`
24+
}
25+
26+
type RuntimeStash struct {
27+
Config *Stash
28+
KeyExpression *vm.Program
29+
ValueExpression *vm.Program
30+
TTLVal time.Duration
31+
}
32+
33+
func (s *Stash) Validate() error {
34+
if s.Name == "" {
35+
return errors.New("name must be set")
36+
}
37+
38+
if s.Value == "" {
39+
return fmt.Errorf("%s: value expression must be set", s.Name)
40+
}
41+
42+
if s.Key == "" {
43+
return fmt.Errorf("%s: key expression must be set", s.Name)
44+
}
45+
46+
if s.TTL == "" {
47+
return fmt.Errorf("%s: ttl must be set", s.Name)
48+
}
49+
50+
if s.Strategy == "" {
51+
s.Strategy = "LRU"
52+
}
53+
54+
// should be configurable
55+
if s.MaxMapSize == 0 {
56+
s.MaxMapSize = 100
57+
}
58+
59+
return nil
60+
}
61+
62+
func (s *Stash) Compile(logger *log.Entry) (*RuntimeStash, error) {
63+
var err error
64+
65+
rs := &RuntimeStash{Config: s}
66+
67+
rs.ValueExpression, err = expr.Compile(s.Value,
68+
exprhelpers.GetExprOptions(map[string]any{"evt": &types.Event{}})...)
69+
if err != nil {
70+
return nil, fmt.Errorf("while compiling stash value expression: %w", err)
71+
}
72+
73+
rs.KeyExpression, err = expr.Compile(s.Key,
74+
exprhelpers.GetExprOptions(map[string]any{"evt": &types.Event{}})...)
75+
if err != nil {
76+
return nil, fmt.Errorf("while compiling stash key expression: %w", err)
77+
}
78+
79+
rs.TTLVal, err = time.ParseDuration(s.TTL)
80+
if err != nil {
81+
return nil, fmt.Errorf("while parsing stash ttl: %w", err)
82+
}
83+
84+
logLvl := logger.Logger.GetLevel()
85+
// init the cache, does it make sense to create it here just to be sure everything is fine ?
86+
if err = cache.CacheInit(cache.CacheCfg{
87+
Size: s.MaxMapSize,
88+
TTL: rs.TTLVal,
89+
Name: s.Name,
90+
Strategy: s.Strategy,
91+
LogLevel: &logLvl,
92+
}); err != nil {
93+
return nil, fmt.Errorf("while initializing cache: %w", err)
94+
}
95+
96+
return rs, nil
97+
}
98+
99+
func (rs *RuntimeStash) Apply(idx int, cachedExprEnv map[string]any, logger *log.Entry, debug bool) {
100+
var (
101+
key string
102+
value string
103+
)
104+
105+
if rs.ValueExpression == nil {
106+
logger.Warningf("Stash %d has no value expression, skipping", idx)
107+
return
108+
}
109+
110+
if rs.KeyExpression == nil {
111+
logger.Warningf("Stash %d has no key expression, skipping", idx)
112+
return
113+
}
114+
// collect the data
115+
output, err := exprhelpers.Run(rs.ValueExpression, cachedExprEnv, logger, debug)
116+
if err != nil {
117+
logger.Warningf("Error while running stash val expression: %v", err)
118+
}
119+
// can we expect anything else than a string ?
120+
switch output := output.(type) {
121+
case string:
122+
value = output
123+
default:
124+
logger.Warningf("unexpected type %T (%v) while running %q", output, output, rs.Config.Value)
125+
return
126+
}
127+
128+
// collect the key
129+
output, err = exprhelpers.Run(rs.KeyExpression, cachedExprEnv, logger, debug)
130+
if err != nil {
131+
logger.Warningf("Error while running stash key expression: %v", err)
132+
}
133+
// can we expect anything else than a string ?
134+
switch output := output.(type) {
135+
case string:
136+
key = output
137+
default:
138+
logger.Warningf("unexpected type %T (%v) while running %q", output, output, rs.Config.Key)
139+
return
140+
}
141+
142+
if err = cache.SetKey(rs.Config.Name, key, value, &rs.TTLVal); err != nil {
143+
logger.Warningf("failed to store data in cache: %s", err.Error())
144+
}
145+
}

0 commit comments

Comments
 (0)