add --exclude and --exclude-regex flags to snapshot lambda command (#319

)

cmd/kosli/root.go

@@ -164,6 +164,8 @@ The service principal needs to have the following permissions:
 	excludeNamespaceFlag                 = "[conditional] The comma separated list of namespaces regex patterns NOT to report artifacts info from. Can't be used together with --namespace."
 	functionNameFlag                     = "[optional] The name of the AWS Lambda function."
 	functionNamesFlag                    = "[optional] The comma-separated list of AWS Lambda function names to be reported."
+	excludeFlag                          = "[optional] The comma-separated list of AWS Lambda function names to be excluded. Cannot be used together with --function-names"
+	excludeRegexFlag                     = "[optional] The comma-separated list of name regex patterns for AWS Lambda functions to be excluded. Cannot be used together with --function-names. Allowed regex patterns are described in https://github.com/google/re2/wiki/Syntax"
 	functionVersionFlag                  = "[optional] The version of the AWS Lambda function."
 	awsKeyIdFlag                         = "The AWS access key ID."
 	awsSecretKeyFlag                     = "The AWS secret access key."

cmd/kosli/snapshotLambda.go

@@ -13,7 +13,7 @@ import (
 const snapshotLambdaShortDesc = `Report a snapshot of artifacts deployed as one or more AWS Lambda functions and their digests to Kosli.`
 
 const snapshotLambdaLongDesc = snapshotLambdaShortDesc + `  
-Skip ^--function-names^ to report all functions in a given AWS account.` + awsAuthDesc
+Skip ^--function-names^ to report all functions in a given AWS account. Or use ^--exclude^ and/or ^--exclude-regex^ to report all functions excluding some.` + awsAuthDesc
 
 const snapshotLambdaExample = `
 # report all Lambda functions running in an AWS account (AWS auth provided in env variables):
@@ -25,6 +25,17 @@ kosli snapshot lambda yourEnvironmentName \
 	--api-token yourAPIToken \
 	--org yourOrgName
 
+# report all (excluding some) Lambda functions running in an AWS account (AWS auth provided in env variables):
+export AWS_REGION=yourAWSRegion
+export AWS_ACCESS_KEY_ID=yourAWSAccessKeyID
+export AWS_SECRET_ACCESS_KEY=yourAWSSecretAccessKey
+
+kosli snapshot lambda yourEnvironmentName \
+    --exclude function1,function2 \
+	--exclude-regex "^not-wanted.*" \
+	--api-token yourAPIToken \
+	--org yourOrgName
+
 # report what is running in the latest version of an AWS Lambda function (AWS auth provided in env variables):
 export AWS_REGION=yourAWSRegion
 export AWS_ACCESS_KEY_ID=yourAWSAccessKeyID
@@ -56,9 +67,11 @@ kosli snapshot lambda yourEnvironmentName \
 `
 
 type snapshotLambdaOptions struct {
-	functionNames   []string
-	functionVersion string
-	awsStaticCreds  *aws.AWSStaticCreds
+	functionNames     []string
+	functionVersion   string
+	excludeNames      []string
+	excludeNamesRegex []string
+	awsStaticCreds    *aws.AWSStaticCreds
 }
 
 func newSnapshotLambdaCmd(out io.Writer) *cobra.Command {
@@ -76,7 +89,12 @@ func newSnapshotLambdaCmd(out io.Writer) *cobra.Command {
 				return ErrorBeforePrintingUsage(cmd, err.Error())
 			}
 
-			err = MuXRequiredFlags(cmd, []string{"function-name", "function-names"}, false)
+			err = MuXRequiredFlags(cmd, []string{"function-name", "function-names", "exclude"}, false)
+			if err != nil {
+				return err
+			}
+
+			err = MuXRequiredFlags(cmd, []string{"function-name", "function-names", "exclude-regex"}, false)
 			if err != nil {
 				return err
 			}
@@ -91,6 +109,8 @@ func newSnapshotLambdaCmd(out io.Writer) *cobra.Command {
 	cmd.Flags().StringSliceVar(&o.functionNames, "function-name", []string{}, functionNameFlag)
 	cmd.Flags().StringSliceVar(&o.functionNames, "function-names", []string{}, functionNamesFlag)
 	cmd.Flags().StringVar(&o.functionVersion, "function-version", "", functionVersionFlag)
+	cmd.Flags().StringSliceVar(&o.excludeNames, "exclude", []string{}, excludeFlag)
+	cmd.Flags().StringSliceVar(&o.excludeNamesRegex, "exclude-regex", []string{}, excludeRegexFlag)
 	addAWSAuthFlags(cmd, o.awsStaticCreds)
 	addDryRunFlag(cmd)
 
@@ -109,7 +129,7 @@ func (o *snapshotLambdaOptions) run(args []string) error {
 	envName := args[0]
 
 	url := fmt.Sprintf("%s/api/v2/environments/%s/%s/report/lambda", global.Host, global.Org, envName)
-	lambdaData, err := o.awsStaticCreds.GetLambdaPackageData(o.functionNames)
+	lambdaData, err := o.awsStaticCreds.GetLambdaPackageData(o.functionNames, o.excludeNames, o.excludeNamesRegex)
 	if err != nil {
 		return err
 	}

cmd/kosli/snapshotLambda_test.go

@@ -95,7 +95,7 @@ func (suite *SnapshotLambdaTestSuite) TestSnapshotLambdaCmd() {
 			additionalConfig: snapshotLambdaTestConfig{
 				requireAuthToBeSet: true,
 			},
-			golden: "Flag --function-name has been deprecated, use --function-names instead\nError: only one of --function-name, --function-names is allowed\n",
+			golden: "Flag --function-name has been deprecated, use --function-names instead\nError: only one of --function-name, --function-names, --exclude is allowed\n",
 		},
 		{
 			wantError: true,
@@ -109,6 +109,26 @@ func (suite *SnapshotLambdaTestSuite) TestSnapshotLambdaCmd() {
 			cmd:       fmt.Sprintf(`snapshot lambda %s xxx %s --function-names %s`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName),
 			golden:    "Error: accepts 1 arg(s), received 2\n",
 		},
+		{
+			wantError: true,
+			name:      "snapshot lambda fails if both --function-names and --exclude are set",
+			cmd:       fmt.Sprintf(`snapshot lambda %s %s --function-names %s --exclude function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName),
+			golden:    "Error: only one of --function-name, --function-names, --exclude is allowed\n",
+		},
+		{
+			wantError: true,
+			name:      "snapshot lambda fails if both --function-names and --exclude-regex are set",
+			cmd:       fmt.Sprintf(`snapshot lambda %s %s --function-names %s --exclude-regex function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName),
+			golden:    "Error: only one of --function-name, --function-names, --exclude-regex is allowed\n",
+		},
+		{
+			name: "snapshot lambda works if both --exclude and --exclude-regex are set",
+			cmd:  fmt.Sprintf(`snapshot lambda %s %s --exclude %s --exclude-regex function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName),
+			additionalConfig: snapshotLambdaTestConfig{
+				requireAuthToBeSet: true,
+			},
+			goldenRegex: fmt.Sprintf("[0-9]+ lambda functions were reported to environment %s\n", suite.envName),
+		},
 	}
 
 	for _, t := range tests {

internal/aws/aws.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -126,7 +128,7 @@ func (staticCreds *AWSStaticCreds) NewECSClient() (*ecs.Client, error) {
 }
 
 // getAllLambdaFuncs fetches all lambda functions recursively (50 at a time) and returns a list of FunctionConfiguration
-func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *[]types.FunctionConfiguration) (*[]types.FunctionConfiguration, error) {
+func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *[]types.FunctionConfiguration, excludeNames, excludeNamesRegex []string) (*[]types.FunctionConfiguration, error) {
 	params := &lambda.ListFunctionsInput{}
 	if nextMarker != nil {
 		params.Marker = nextMarker
@@ -137,9 +139,34 @@ func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *
 		return allFunctions, err
 	}
 
-	*allFunctions = append(*allFunctions, listFunctionsOutput.Functions...)
+	if len(excludeNames) == 0 && len(excludeNamesRegex) == 0 {
+		*allFunctions = append(*allFunctions, listFunctionsOutput.Functions...)
+	} else {
+		for _, f := range listFunctionsOutput.Functions {
+			if slices.Contains(excludeNames, *f.FunctionName) {
+				continue
+			}
+			regexExcluded := false
+			for _, pattern := range excludeNamesRegex {
+				re, err := regexp.Compile(pattern)
+				if err != nil {
+					return allFunctions, fmt.Errorf("invalid exclude name regex pattern %s: %v", pattern, err)
+				}
+				if re.MatchString(*f.FunctionName) {
+					regexExcluded = true
+					break
+				}
+			}
+			if regexExcluded {
+				continue
+			}
+
+			*allFunctions = append(*allFunctions, f)
+		}
+	}
+
 	if listFunctionsOutput.NextMarker != nil {
-		_, err := getAllLambdaFuncs(client, listFunctionsOutput.NextMarker, allFunctions)
+		_, err := getAllLambdaFuncs(client, listFunctionsOutput.NextMarker, allFunctions, excludeNames, excludeNamesRegex)
 		if err != nil {
 			return allFunctions, err
 		}
@@ -148,15 +175,15 @@ func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *
 }
 
 // GetLambdaPackageData returns a digest and metadata of a Lambda function package
-func (staticCreds *AWSStaticCreds) GetLambdaPackageData(functionNames []string) ([]*LambdaData, error) {
+func (staticCreds *AWSStaticCreds) GetLambdaPackageData(functionNames, excludeNames, excludeNamesRegex []string) ([]*LambdaData, error) {
 	lambdaData := []*LambdaData{}
 	client, err := staticCreds.NewLambdaClient()
 	if err != nil {
 		return lambdaData, err
 	}
 
 	if len(functionNames) == 0 {
-		allFunctions, err := getAllLambdaFuncs(client, nil, &[]types.FunctionConfiguration{})
+		allFunctions, err := getAllLambdaFuncs(client, nil, &[]types.FunctionConfiguration{}, excludeNames, excludeNamesRegex)
 		if err != nil {
 			return lambdaData, err
 		}

internal/aws/aws_test.go

@@ -245,13 +245,19 @@ func (suite *AWSTestSuite) TestAWSClients() {
 // All cases will run in CI
 
 func (suite *AWSTestSuite) TestGetLambdaPackageData() {
+	type expectedFunction struct {
+		name        string
+		fingerprint string
+	}
 	for _, t := range []struct {
-		name             string
-		requireEnvVars   bool // indicates that a test case needs real credentials from env vars
-		creds            *AWSStaticCreds
-		functionNames    []string
-		wantFingerprints []string
-		wantErr          bool
+		name              string
+		requireEnvVars    bool // indicates that a test case needs real credentials from env vars
+		creds             *AWSStaticCreds
+		functionNames     []string
+		excludeNames      []string
+		excludeNamesRegex []string
+		expectedFunctions []expectedFunction
+		wantErr           bool
 	}{
 		{
 			name: "invalid credentials causes an error",
@@ -277,46 +283,86 @@ func (suite *AWSTestSuite) TestGetLambdaPackageData() {
 			creds: &AWSStaticCreds{
 				Region: "eu-central-1",
 			},
-			functionNames:    []string{"cli-tests"},
-			wantFingerprints: []string{"321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"},
-			requireEnvVars:   true,
+			functionNames: []string{"cli-tests"},
+			expectedFunctions: []expectedFunction{{name: "cli-tests",
+				fingerprint: "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}},
+			requireEnvVars: true,
 		},
 		{
 			name: "can get image package lambda function data from name",
 			creds: &AWSStaticCreds{
 				Region: "eu-central-1",
 			},
-			functionNames:    []string{"cli-tests-docker"},
-			wantFingerprints: []string{"e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"},
-			requireEnvVars:   true,
+			functionNames: []string{"cli-tests-docker"},
+			expectedFunctions: []expectedFunction{{name: "cli-tests-docker",
+				fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}},
+			requireEnvVars: true,
 		},
 		{
 			name: "can get a list of lambda functions data from names",
 			creds: &AWSStaticCreds{
 				Region: "eu-central-1",
 			},
 			functionNames: []string{"cli-tests-docker", "cli-tests"},
-			wantFingerprints: []string{"e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e",
-				"321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"},
+			expectedFunctions: []expectedFunction{
+				{name: "cli-tests",
+					fingerprint: "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"},
+				{name: "cli-tests-docker",
+					fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}},
+			requireEnvVars: true,
+		},
+		{
+			name: "can exclude lambda functions matching a regex pattern",
+			creds: &AWSStaticCreds{
+				Region: "eu-central-1",
+			},
+			excludeNamesRegex: []string{"^([^c]|c[^l]|cl[^i]|cli[^-]).*$"},
+			expectedFunctions: []expectedFunction{
+				{name: "cli-tests",
+					fingerprint: "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"},
+				{name: "cli-tests-docker",
+					fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}},
+			requireEnvVars: true,
+		},
+		{
+			name: "invalid exclude name regex pattern causes an error",
+			creds: &AWSStaticCreds{
+				Region: "eu-central-1",
+			},
+			excludeNamesRegex: []string{"invalid["},
+			requireEnvVars:    true,
+			wantErr:           true,
+		},
+		{
+			name: "can combine exclude and exclude-regex and they are joined",
+			creds: &AWSStaticCreds{
+				Region: "eu-central-1",
+			},
+			excludeNames:      []string{"cli-tests"},
+			excludeNamesRegex: []string{"^([^c]|c[^l]|cl[^i]|cli[^-]).*$"},
+			expectedFunctions: []expectedFunction{
+				{name: "cli-tests-docker",
+					fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}},
 			requireEnvVars: true,
 		},
 	} {
 		suite.Run(t.name, func() {
 			skipOrSetCreds(suite.T(), t.requireEnvVars, t.creds)
-			data, err := t.creds.GetLambdaPackageData(t.functionNames)
+			data, err := t.creds.GetLambdaPackageData(t.functionNames, t.excludeNames, t.excludeNamesRegex)
 			require.False(suite.T(), (err != nil) != t.wantErr,
 				"GetLambdaPackageData() error = %v, wantErr %v", err, t.wantErr)
 			if !t.wantErr {
 				matchFound := false
+				require.Len(suite.T(), data, len(t.expectedFunctions))
 			loop1:
-				for index, name := range t.functionNames {
+				for _, expectedFunction := range t.expectedFunctions {
 					for _, item := range data {
-						if fingerprint, ok := item.Digests[name]; ok {
-							if t.wantFingerprints[index] == fingerprint {
+						if fingerprint, ok := item.Digests[expectedFunction.name]; ok {
+							if expectedFunction.fingerprint == fingerprint {
 								matchFound = true
 								break loop1
 							} else {
-								suite.T().Logf("fingerprint did not match: GOT %s -- WANT %s", fingerprint, t.wantFingerprints[index])
+								suite.T().Logf("fingerprint did not match: GOT %s -- WANT %s", fingerprint, expectedFunction.fingerprint)
 							}
 						}
 					}