From a637837212426d4cb20c38b7bf62218f371e5d61 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Wed, 18 Sep 2024 14:09:36 +0200 Subject: [PATCH] add --exclude and --exclude-regex flags to snapshot lambda command (#319) --- cmd/kosli/root.go | 2 + cmd/kosli/snapshotLambda.go | 32 +++++++++--- cmd/kosli/snapshotLambda_test.go | 22 ++++++++- internal/aws/aws.go | 37 ++++++++++++-- internal/aws/aws_test.go | 84 ++++++++++++++++++++++++-------- 5 files changed, 146 insertions(+), 31 deletions(-) diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index 1fb1e2ee6..469cb588c 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -164,6 +164,8 @@ The service principal needs to have the following permissions: excludeNamespaceFlag = "[conditional] The comma separated list of namespaces regex patterns NOT to report artifacts info from. Can't be used together with --namespace." functionNameFlag = "[optional] The name of the AWS Lambda function." functionNamesFlag = "[optional] The comma-separated list of AWS Lambda function names to be reported." + excludeFlag = "[optional] The comma-separated list of AWS Lambda function names to be excluded. Cannot be used together with --function-names" + excludeRegexFlag = "[optional] The comma-separated list of name regex patterns for AWS Lambda functions to be excluded. Cannot be used together with --function-names. Allowed regex patterns are described in https://github.com/google/re2/wiki/Syntax" functionVersionFlag = "[optional] The version of the AWS Lambda function." awsKeyIdFlag = "The AWS access key ID." awsSecretKeyFlag = "The AWS secret access key." diff --git a/cmd/kosli/snapshotLambda.go b/cmd/kosli/snapshotLambda.go index b3f7bc01e..c5a573825 100644 --- a/cmd/kosli/snapshotLambda.go +++ b/cmd/kosli/snapshotLambda.go @@ -13,7 +13,7 @@ import ( const snapshotLambdaShortDesc = `Report a snapshot of artifacts deployed as one or more AWS Lambda functions and their digests to Kosli.` const snapshotLambdaLongDesc = snapshotLambdaShortDesc + ` -Skip ^--function-names^ to report all functions in a given AWS account.` + awsAuthDesc +Skip ^--function-names^ to report all functions in a given AWS account. Or use ^--exclude^ and/or ^--exclude-regex^ to report all functions excluding some.` + awsAuthDesc const snapshotLambdaExample = ` # report all Lambda functions running in an AWS account (AWS auth provided in env variables): @@ -25,6 +25,17 @@ kosli snapshot lambda yourEnvironmentName \ --api-token yourAPIToken \ --org yourOrgName +# report all (excluding some) Lambda functions running in an AWS account (AWS auth provided in env variables): +export AWS_REGION=yourAWSRegion +export AWS_ACCESS_KEY_ID=yourAWSAccessKeyID +export AWS_SECRET_ACCESS_KEY=yourAWSSecretAccessKey + +kosli snapshot lambda yourEnvironmentName \ + --exclude function1,function2 \ + --exclude-regex "^not-wanted.*" \ + --api-token yourAPIToken \ + --org yourOrgName + # report what is running in the latest version of an AWS Lambda function (AWS auth provided in env variables): export AWS_REGION=yourAWSRegion export AWS_ACCESS_KEY_ID=yourAWSAccessKeyID @@ -56,9 +67,11 @@ kosli snapshot lambda yourEnvironmentName \ ` type snapshotLambdaOptions struct { - functionNames []string - functionVersion string - awsStaticCreds *aws.AWSStaticCreds + functionNames []string + functionVersion string + excludeNames []string + excludeNamesRegex []string + awsStaticCreds *aws.AWSStaticCreds } func newSnapshotLambdaCmd(out io.Writer) *cobra.Command { @@ -76,7 +89,12 @@ func newSnapshotLambdaCmd(out io.Writer) *cobra.Command { return ErrorBeforePrintingUsage(cmd, err.Error()) } - err = MuXRequiredFlags(cmd, []string{"function-name", "function-names"}, false) + err = MuXRequiredFlags(cmd, []string{"function-name", "function-names", "exclude"}, false) + if err != nil { + return err + } + + err = MuXRequiredFlags(cmd, []string{"function-name", "function-names", "exclude-regex"}, false) if err != nil { return err } @@ -91,6 +109,8 @@ func newSnapshotLambdaCmd(out io.Writer) *cobra.Command { cmd.Flags().StringSliceVar(&o.functionNames, "function-name", []string{}, functionNameFlag) cmd.Flags().StringSliceVar(&o.functionNames, "function-names", []string{}, functionNamesFlag) cmd.Flags().StringVar(&o.functionVersion, "function-version", "", functionVersionFlag) + cmd.Flags().StringSliceVar(&o.excludeNames, "exclude", []string{}, excludeFlag) + cmd.Flags().StringSliceVar(&o.excludeNamesRegex, "exclude-regex", []string{}, excludeRegexFlag) addAWSAuthFlags(cmd, o.awsStaticCreds) addDryRunFlag(cmd) @@ -109,7 +129,7 @@ func (o *snapshotLambdaOptions) run(args []string) error { envName := args[0] url := fmt.Sprintf("%s/api/v2/environments/%s/%s/report/lambda", global.Host, global.Org, envName) - lambdaData, err := o.awsStaticCreds.GetLambdaPackageData(o.functionNames) + lambdaData, err := o.awsStaticCreds.GetLambdaPackageData(o.functionNames, o.excludeNames, o.excludeNamesRegex) if err != nil { return err } diff --git a/cmd/kosli/snapshotLambda_test.go b/cmd/kosli/snapshotLambda_test.go index a5d27f3dc..de3b3a65f 100644 --- a/cmd/kosli/snapshotLambda_test.go +++ b/cmd/kosli/snapshotLambda_test.go @@ -95,7 +95,7 @@ func (suite *SnapshotLambdaTestSuite) TestSnapshotLambdaCmd() { additionalConfig: snapshotLambdaTestConfig{ requireAuthToBeSet: true, }, - golden: "Flag --function-name has been deprecated, use --function-names instead\nError: only one of --function-name, --function-names is allowed\n", + golden: "Flag --function-name has been deprecated, use --function-names instead\nError: only one of --function-name, --function-names, --exclude is allowed\n", }, { wantError: true, @@ -109,6 +109,26 @@ func (suite *SnapshotLambdaTestSuite) TestSnapshotLambdaCmd() { cmd: fmt.Sprintf(`snapshot lambda %s xxx %s --function-names %s`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName), golden: "Error: accepts 1 arg(s), received 2\n", }, + { + wantError: true, + name: "snapshot lambda fails if both --function-names and --exclude are set", + cmd: fmt.Sprintf(`snapshot lambda %s %s --function-names %s --exclude function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName), + golden: "Error: only one of --function-name, --function-names, --exclude is allowed\n", + }, + { + wantError: true, + name: "snapshot lambda fails if both --function-names and --exclude-regex are set", + cmd: fmt.Sprintf(`snapshot lambda %s %s --function-names %s --exclude-regex function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName), + golden: "Error: only one of --function-name, --function-names, --exclude-regex is allowed\n", + }, + { + name: "snapshot lambda works if both --exclude and --exclude-regex are set", + cmd: fmt.Sprintf(`snapshot lambda %s %s --exclude %s --exclude-regex function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName), + additionalConfig: snapshotLambdaTestConfig{ + requireAuthToBeSet: true, + }, + goldenRegex: fmt.Sprintf("[0-9]+ lambda functions were reported to environment %s\n", suite.envName), + }, } for _, t := range tests { diff --git a/internal/aws/aws.go b/internal/aws/aws.go index 6f66cbfb7..8fb4ecb11 100644 --- a/internal/aws/aws.go +++ b/internal/aws/aws.go @@ -7,6 +7,8 @@ import ( "fmt" "os" "path/filepath" + "regexp" + "slices" "strings" "sync" "time" @@ -126,7 +128,7 @@ func (staticCreds *AWSStaticCreds) NewECSClient() (*ecs.Client, error) { } // getAllLambdaFuncs fetches all lambda functions recursively (50 at a time) and returns a list of FunctionConfiguration -func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *[]types.FunctionConfiguration) (*[]types.FunctionConfiguration, error) { +func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *[]types.FunctionConfiguration, excludeNames, excludeNamesRegex []string) (*[]types.FunctionConfiguration, error) { params := &lambda.ListFunctionsInput{} if nextMarker != nil { params.Marker = nextMarker @@ -137,9 +139,34 @@ func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions * return allFunctions, err } - *allFunctions = append(*allFunctions, listFunctionsOutput.Functions...) + if len(excludeNames) == 0 && len(excludeNamesRegex) == 0 { + *allFunctions = append(*allFunctions, listFunctionsOutput.Functions...) + } else { + for _, f := range listFunctionsOutput.Functions { + if slices.Contains(excludeNames, *f.FunctionName) { + continue + } + regexExcluded := false + for _, pattern := range excludeNamesRegex { + re, err := regexp.Compile(pattern) + if err != nil { + return allFunctions, fmt.Errorf("invalid exclude name regex pattern %s: %v", pattern, err) + } + if re.MatchString(*f.FunctionName) { + regexExcluded = true + break + } + } + if regexExcluded { + continue + } + + *allFunctions = append(*allFunctions, f) + } + } + if listFunctionsOutput.NextMarker != nil { - _, err := getAllLambdaFuncs(client, listFunctionsOutput.NextMarker, allFunctions) + _, err := getAllLambdaFuncs(client, listFunctionsOutput.NextMarker, allFunctions, excludeNames, excludeNamesRegex) if err != nil { return allFunctions, err } @@ -148,7 +175,7 @@ func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions * } // GetLambdaPackageData returns a digest and metadata of a Lambda function package -func (staticCreds *AWSStaticCreds) GetLambdaPackageData(functionNames []string) ([]*LambdaData, error) { +func (staticCreds *AWSStaticCreds) GetLambdaPackageData(functionNames, excludeNames, excludeNamesRegex []string) ([]*LambdaData, error) { lambdaData := []*LambdaData{} client, err := staticCreds.NewLambdaClient() if err != nil { @@ -156,7 +183,7 @@ func (staticCreds *AWSStaticCreds) GetLambdaPackageData(functionNames []string) } if len(functionNames) == 0 { - allFunctions, err := getAllLambdaFuncs(client, nil, &[]types.FunctionConfiguration{}) + allFunctions, err := getAllLambdaFuncs(client, nil, &[]types.FunctionConfiguration{}, excludeNames, excludeNamesRegex) if err != nil { return lambdaData, err } diff --git a/internal/aws/aws_test.go b/internal/aws/aws_test.go index 4b6b0a65e..a49120852 100644 --- a/internal/aws/aws_test.go +++ b/internal/aws/aws_test.go @@ -245,13 +245,19 @@ func (suite *AWSTestSuite) TestAWSClients() { // All cases will run in CI func (suite *AWSTestSuite) TestGetLambdaPackageData() { + type expectedFunction struct { + name string + fingerprint string + } for _, t := range []struct { - name string - requireEnvVars bool // indicates that a test case needs real credentials from env vars - creds *AWSStaticCreds - functionNames []string - wantFingerprints []string - wantErr bool + name string + requireEnvVars bool // indicates that a test case needs real credentials from env vars + creds *AWSStaticCreds + functionNames []string + excludeNames []string + excludeNamesRegex []string + expectedFunctions []expectedFunction + wantErr bool }{ { name: "invalid credentials causes an error", @@ -277,18 +283,20 @@ func (suite *AWSTestSuite) TestGetLambdaPackageData() { creds: &AWSStaticCreds{ Region: "eu-central-1", }, - functionNames: []string{"cli-tests"}, - wantFingerprints: []string{"321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}, - requireEnvVars: true, + functionNames: []string{"cli-tests"}, + expectedFunctions: []expectedFunction{{name: "cli-tests", + fingerprint: "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}}, + requireEnvVars: true, }, { name: "can get image package lambda function data from name", creds: &AWSStaticCreds{ Region: "eu-central-1", }, - functionNames: []string{"cli-tests-docker"}, - wantFingerprints: []string{"e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}, - requireEnvVars: true, + functionNames: []string{"cli-tests-docker"}, + expectedFunctions: []expectedFunction{{name: "cli-tests-docker", + fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}}, + requireEnvVars: true, }, { name: "can get a list of lambda functions data from names", @@ -296,27 +304,65 @@ func (suite *AWSTestSuite) TestGetLambdaPackageData() { Region: "eu-central-1", }, functionNames: []string{"cli-tests-docker", "cli-tests"}, - wantFingerprints: []string{"e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e", - "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}, + expectedFunctions: []expectedFunction{ + {name: "cli-tests", + fingerprint: "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}, + {name: "cli-tests-docker", + fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}}, + requireEnvVars: true, + }, + { + name: "can exclude lambda functions matching a regex pattern", + creds: &AWSStaticCreds{ + Region: "eu-central-1", + }, + excludeNamesRegex: []string{"^([^c]|c[^l]|cl[^i]|cli[^-]).*$"}, + expectedFunctions: []expectedFunction{ + {name: "cli-tests", + fingerprint: "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}, + {name: "cli-tests-docker", + fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}}, + requireEnvVars: true, + }, + { + name: "invalid exclude name regex pattern causes an error", + creds: &AWSStaticCreds{ + Region: "eu-central-1", + }, + excludeNamesRegex: []string{"invalid["}, + requireEnvVars: true, + wantErr: true, + }, + { + name: "can combine exclude and exclude-regex and they are joined", + creds: &AWSStaticCreds{ + Region: "eu-central-1", + }, + excludeNames: []string{"cli-tests"}, + excludeNamesRegex: []string{"^([^c]|c[^l]|cl[^i]|cli[^-]).*$"}, + expectedFunctions: []expectedFunction{ + {name: "cli-tests-docker", + fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}}, requireEnvVars: true, }, } { suite.Run(t.name, func() { skipOrSetCreds(suite.T(), t.requireEnvVars, t.creds) - data, err := t.creds.GetLambdaPackageData(t.functionNames) + data, err := t.creds.GetLambdaPackageData(t.functionNames, t.excludeNames, t.excludeNamesRegex) require.False(suite.T(), (err != nil) != t.wantErr, "GetLambdaPackageData() error = %v, wantErr %v", err, t.wantErr) if !t.wantErr { matchFound := false + require.Len(suite.T(), data, len(t.expectedFunctions)) loop1: - for index, name := range t.functionNames { + for _, expectedFunction := range t.expectedFunctions { for _, item := range data { - if fingerprint, ok := item.Digests[name]; ok { - if t.wantFingerprints[index] == fingerprint { + if fingerprint, ok := item.Digests[expectedFunction.name]; ok { + if expectedFunction.fingerprint == fingerprint { matchFound = true break loop1 } else { - suite.T().Logf("fingerprint did not match: GOT %s -- WANT %s", fingerprint, t.wantFingerprints[index]) + suite.T().Logf("fingerprint did not match: GOT %s -- WANT %s", fingerprint, expectedFunction.fingerprint) } } }