Merge branch 'main' of github.com:kosli-dev/cli into never-alone-more…

…-info

cmd/kosli/assertArtifact_test.go

@@ -58,12 +58,12 @@ func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() {
 		{
 			name:        "asserting an existing compliant artifact (using --fingerprint) results in OK and zero exit",
 			cmd:         fmt.Sprintf(`assert artifact --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments),
-			goldenRegex: "COMPLIANT\nSee more details at http://localhost:8001/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8}\n",
+			goldenRegex: "COMPLIANT\nSee more details at http://localhost:8001/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
 		},
 		{
 			name:        "asserting an existing compliant artifact (using --artifact-type) results in OK and zero exit",
 			cmd:         fmt.Sprintf(`assert artifact %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments),
-			goldenRegex: "COMPLIANT\nSee more details at http://localhost:8001/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8}\n",
+			goldenRegex: "COMPLIANT\nSee more details at http://localhost:8001/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02?(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n",
 		},
 		{
 			wantError: true,

cmd/kosli/attestArtifact.go

@@ -41,7 +41,7 @@ type AttestArtifactPayload struct {
 const attestArtifactShortDesc = `Attest an artifact creation to a Kosli flow.  `
 
 const attestArtifactLongDesc = attestArtifactShortDesc + `
-` + fingerprintDesc + `
+` + fingerprintDesc + kosliIgnoreDesc + `
 This command requires access to a git repo to associate the artifact to the git commit it is originating from. 
 You can optionally redact some of the git commit data sent to Kosli using ^--redact-commit-info^`
 

cmd/kosli/fingerprint.go

@@ -13,7 +13,9 @@ using the ^--exclude^ flag.
 Excluded paths are relative to the DIR-PATH and can be literal paths or
 glob patterns.  
 The supported glob pattern syntax is what is documented here: https://pkg.go.dev/path/filepath#Match , 
-plus the ability to use recursive globs "**"`
+plus the ability to use recursive globs "**"
+
+` + kosliIgnoreDesc
 
 const fingerprintLongDesc = fingerprintShortDesc + `
 Requires ^--artifact-type^ flag to be set.

cmd/kosli/fingerprint_test.go

@@ -39,6 +39,11 @@ func (suite *FingerprintTestSuite) TestFingerprintCmd() {
 			cmd:    "fingerprint --artifact-type dir testdata/folder1 -x folder2",
 			golden: "773fd3300860454a2b065c5912c03008adb11e6a6dcf7c1c64c094ceab8f430a\n",
 		},
+		{
+			name:   "dir fingerprint with ignore file",
+			cmd:    "fingerprint --artifact-type dir testdata/folder1-with-ignore",
+			golden: "038897ea5334462098d65125380d58a493671fb3b8bdbbee1e75ec8bd4a65c23\n",
+		},
 		{
 			name:      "fails if type is directory but the argument is not a dir",
 			cmd:       "fingerprint --artifact-type dir testdata/file1",

cmd/kosli/root.go

@@ -67,6 +67,9 @@ The service principal needs to have the following permissions:
   2) Microsoft.ContainerRegistry/registries/pull/read  
 
 	`
+	kosliIgnoreDesc = `To specify paths in a directory artifact that should always be excluded from the SHA256 calculation, you can add a ^.kosli_ignore^ file to the root of the artifact.
+Each line should specify a relative path or path glob to be ignored. You can include comments in this file, using ^#^.
+The ^.kosli_ignore^ will be treated as part of the artifact like any other file,unless it is explicitly ignored itself.`
 
 	// flags
 	apiTokenFlag                         = "The Kosli API token."
@@ -164,6 +167,8 @@ The service principal needs to have the following permissions:
 	excludeNamespaceFlag                 = "[conditional] The comma separated list of namespaces regex patterns NOT to report artifacts info from. Can't be used together with --namespace."
 	functionNameFlag                     = "[optional] The name of the AWS Lambda function."
 	functionNamesFlag                    = "[optional] The comma-separated list of AWS Lambda function names to be reported."
+	excludeFlag                          = "[optional] The comma-separated list of AWS Lambda function names to be excluded. Cannot be used together with --function-names"
+	excludeRegexFlag                     = "[optional] The comma-separated list of name regex patterns for AWS Lambda functions to be excluded. Cannot be used together with --function-names. Allowed regex patterns are described in https://github.com/google/re2/wiki/Syntax"
 	functionVersionFlag                  = "[optional] The version of the AWS Lambda function."
 	awsKeyIdFlag                         = "The AWS access key ID."
 	awsSecretKeyFlag                     = "The AWS secret access key."

cmd/kosli/snapshotLambda.go

@@ -13,7 +13,7 @@ import (
 const snapshotLambdaShortDesc = `Report a snapshot of artifacts deployed as one or more AWS Lambda functions and their digests to Kosli.`
 
 const snapshotLambdaLongDesc = snapshotLambdaShortDesc + `  
-Skip ^--function-names^ to report all functions in a given AWS account.` + awsAuthDesc
+Skip ^--function-names^ to report all functions in a given AWS account. Or use ^--exclude^ and/or ^--exclude-regex^ to report all functions excluding some.` + awsAuthDesc
 
 const snapshotLambdaExample = `
 # report all Lambda functions running in an AWS account (AWS auth provided in env variables):
@@ -25,6 +25,17 @@ kosli snapshot lambda yourEnvironmentName \
 	--api-token yourAPIToken \
 	--org yourOrgName
 
+# report all (excluding some) Lambda functions running in an AWS account (AWS auth provided in env variables):
+export AWS_REGION=yourAWSRegion
+export AWS_ACCESS_KEY_ID=yourAWSAccessKeyID
+export AWS_SECRET_ACCESS_KEY=yourAWSSecretAccessKey
+
+kosli snapshot lambda yourEnvironmentName \
+    --exclude function1,function2 \
+	--exclude-regex "^not-wanted.*" \
+	--api-token yourAPIToken \
+	--org yourOrgName
+
 # report what is running in the latest version of an AWS Lambda function (AWS auth provided in env variables):
 export AWS_REGION=yourAWSRegion
 export AWS_ACCESS_KEY_ID=yourAWSAccessKeyID
@@ -56,9 +67,11 @@ kosli snapshot lambda yourEnvironmentName \
 `
 
 type snapshotLambdaOptions struct {
-	functionNames   []string
-	functionVersion string
-	awsStaticCreds  *aws.AWSStaticCreds
+	functionNames     []string
+	functionVersion   string
+	excludeNames      []string
+	excludeNamesRegex []string
+	awsStaticCreds    *aws.AWSStaticCreds
 }
 
 func newSnapshotLambdaCmd(out io.Writer) *cobra.Command {
@@ -76,7 +89,12 @@ func newSnapshotLambdaCmd(out io.Writer) *cobra.Command {
 				return ErrorBeforePrintingUsage(cmd, err.Error())
 			}
 
-			err = MuXRequiredFlags(cmd, []string{"function-name", "function-names"}, false)
+			err = MuXRequiredFlags(cmd, []string{"function-name", "function-names", "exclude"}, false)
+			if err != nil {
+				return err
+			}
+
+			err = MuXRequiredFlags(cmd, []string{"function-name", "function-names", "exclude-regex"}, false)
 			if err != nil {
 				return err
 			}
@@ -91,6 +109,8 @@ func newSnapshotLambdaCmd(out io.Writer) *cobra.Command {
 	cmd.Flags().StringSliceVar(&o.functionNames, "function-name", []string{}, functionNameFlag)
 	cmd.Flags().StringSliceVar(&o.functionNames, "function-names", []string{}, functionNamesFlag)
 	cmd.Flags().StringVar(&o.functionVersion, "function-version", "", functionVersionFlag)
+	cmd.Flags().StringSliceVar(&o.excludeNames, "exclude", []string{}, excludeFlag)
+	cmd.Flags().StringSliceVar(&o.excludeNamesRegex, "exclude-regex", []string{}, excludeRegexFlag)
 	addAWSAuthFlags(cmd, o.awsStaticCreds)
 	addDryRunFlag(cmd)
 
@@ -109,7 +129,7 @@ func (o *snapshotLambdaOptions) run(args []string) error {
 	envName := args[0]
 
 	url := fmt.Sprintf("%s/api/v2/environments/%s/%s/report/lambda", global.Host, global.Org, envName)
-	lambdaData, err := o.awsStaticCreds.GetLambdaPackageData(o.functionNames)
+	lambdaData, err := o.awsStaticCreds.GetLambdaPackageData(o.functionNames, o.excludeNames, o.excludeNamesRegex)
 	if err != nil {
 		return err
 	}

cmd/kosli/snapshotLambda_test.go

@@ -95,7 +95,7 @@ func (suite *SnapshotLambdaTestSuite) TestSnapshotLambdaCmd() {
 			additionalConfig: snapshotLambdaTestConfig{
 				requireAuthToBeSet: true,
 			},
-			golden: "Flag --function-name has been deprecated, use --function-names instead\nError: only one of --function-name, --function-names is allowed\n",
+			golden: "Flag --function-name has been deprecated, use --function-names instead\nError: only one of --function-name, --function-names, --exclude is allowed\n",
 		},
 		{
 			wantError: true,
@@ -109,6 +109,26 @@ func (suite *SnapshotLambdaTestSuite) TestSnapshotLambdaCmd() {
 			cmd:       fmt.Sprintf(`snapshot lambda %s xxx %s --function-names %s`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName),
 			golden:    "Error: accepts 1 arg(s), received 2\n",
 		},
+		{
+			wantError: true,
+			name:      "snapshot lambda fails if both --function-names and --exclude are set",
+			cmd:       fmt.Sprintf(`snapshot lambda %s %s --function-names %s --exclude function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName),
+			golden:    "Error: only one of --function-name, --function-names, --exclude is allowed\n",
+		},
+		{
+			wantError: true,
+			name:      "snapshot lambda fails if both --function-names and --exclude-regex are set",
+			cmd:       fmt.Sprintf(`snapshot lambda %s %s --function-names %s --exclude-regex function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName),
+			golden:    "Error: only one of --function-name, --function-names, --exclude-regex is allowed\n",
+		},
+		{
+			name: "snapshot lambda works if both --exclude and --exclude-regex are set",
+			cmd:  fmt.Sprintf(`snapshot lambda %s %s --exclude %s --exclude-regex function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName),
+			additionalConfig: snapshotLambdaTestConfig{
+				requireAuthToBeSet: true,
+			},
+			goldenRegex: fmt.Sprintf("[0-9]+ lambda functions were reported to environment %s\n", suite.envName),
+		},
 	}
 
 	for _, t := range tests {

cmd/kosli/snapshotPath.go

@@ -18,7 +18,7 @@ You can exclude certain paths or patterns from the artifact fingerprint using ^-
 The supported glob pattern syntax is what is documented here: https://pkg.go.dev/path/filepath#Match , 
 plus the ability to use recursive globs "**"
 
-`
+` + kosliIgnoreDesc
 
 const snapshotPathExample = `
 # report one artifact running in a specific path in a filesystem:

cmd/kosli/snapshotPaths.go

@@ -23,9 +23,10 @@ glob patterns.
 The supported glob pattern syntax is what is documented here: https://pkg.go.dev/path/filepath#Match , 
 plus the ability to use recursive globs "**"
 
+` + kosliIgnoreDesc + `
+
 This is an example YAML paths spec file:
 ` +
-
 	"```yaml\n" +
 	`version: 1
 artifacts:

cmd/kosli/snapshotS3.go

@@ -15,7 +15,8 @@ const snapshotS3ShortDesc = `Report a snapshot of the content of an AWS S3 bucke
 const snapshotS3LongDesc = snapshotS3ShortDesc + awsAuthDesc + `
 You can report the entire bucket content, or filter some of the content using ^--include^ and ^--exclude^.
 In all cases, the content is reported as one artifact. If you wish to report separate files/dirs within the same bucket as separate artifacts, you need to run the command twice.
-`
+
+` + kosliIgnoreDesc
 
 const snapshotS3Example = `
 # report the contents of an entire AWS S3 bucket (AWS auth provided in env variables):

cmd/kosli/testdata/folder1-with-ignore/.kosli_ignore

@@ -0,0 +1 @@
+folder2

cmd/kosli/testdata/folder1-with-ignore/folder2/hello2.txt

@@ -0,0 +1 @@
+hello2

cmd/kosli/testdata/folder1-with-ignore/folder2/hello3.txt

@@ -0,0 +1 @@
+hello3

cmd/kosli/testdata/folder1-with-ignore/hello.txt

@@ -0,0 +1 @@
+Hello, world! Again!

internal/aws/aws.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -126,7 +128,7 @@ func (staticCreds *AWSStaticCreds) NewECSClient() (*ecs.Client, error) {
 }
 
 // getAllLambdaFuncs fetches all lambda functions recursively (50 at a time) and returns a list of FunctionConfiguration
-func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *[]types.FunctionConfiguration) (*[]types.FunctionConfiguration, error) {
+func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *[]types.FunctionConfiguration, excludeNames, excludeNamesRegex []string) (*[]types.FunctionConfiguration, error) {
 	params := &lambda.ListFunctionsInput{}
 	if nextMarker != nil {
 		params.Marker = nextMarker
@@ -137,9 +139,34 @@ func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *
 		return allFunctions, err
 	}
 
-	*allFunctions = append(*allFunctions, listFunctionsOutput.Functions...)
+	if len(excludeNames) == 0 && len(excludeNamesRegex) == 0 {
+		*allFunctions = append(*allFunctions, listFunctionsOutput.Functions...)
+	} else {
+		for _, f := range listFunctionsOutput.Functions {
+			if slices.Contains(excludeNames, *f.FunctionName) {
+				continue
+			}
+			regexExcluded := false
+			for _, pattern := range excludeNamesRegex {
+				re, err := regexp.Compile(pattern)
+				if err != nil {
+					return allFunctions, fmt.Errorf("invalid exclude name regex pattern %s: %v", pattern, err)
+				}
+				if re.MatchString(*f.FunctionName) {
+					regexExcluded = true
+					break
+				}
+			}
+			if regexExcluded {
+				continue
+			}
+
+			*allFunctions = append(*allFunctions, f)
+		}
+	}
+
 	if listFunctionsOutput.NextMarker != nil {
-		_, err := getAllLambdaFuncs(client, listFunctionsOutput.NextMarker, allFunctions)
+		_, err := getAllLambdaFuncs(client, listFunctionsOutput.NextMarker, allFunctions, excludeNames, excludeNamesRegex)
 		if err != nil {
 			return allFunctions, err
 		}
@@ -148,15 +175,15 @@ func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *
 }
 
 // GetLambdaPackageData returns a digest and metadata of a Lambda function package
-func (staticCreds *AWSStaticCreds) GetLambdaPackageData(functionNames []string) ([]*LambdaData, error) {
+func (staticCreds *AWSStaticCreds) GetLambdaPackageData(functionNames, excludeNames, excludeNamesRegex []string) ([]*LambdaData, error) {
 	lambdaData := []*LambdaData{}
 	client, err := staticCreds.NewLambdaClient()
 	if err != nil {
 		return lambdaData, err
 	}
 
 	if len(functionNames) == 0 {
-		allFunctions, err := getAllLambdaFuncs(client, nil, &[]types.FunctionConfiguration{})
+		allFunctions, err := getAllLambdaFuncs(client, nil, &[]types.FunctionConfiguration{}, excludeNames, excludeNamesRegex)
 		if err != nil {
 			return lambdaData, err
 		}