From a637837212426d4cb20c38b7bf62218f371e5d61 Mon Sep 17 00:00:00 2001 From: Sami Alajrami Date: Wed, 18 Sep 2024 14:09:36 +0200 Subject: [PATCH 1/7] add --exclude and --exclude-regex flags to snapshot lambda command (#319) --- cmd/kosli/root.go | 2 + cmd/kosli/snapshotLambda.go | 32 +++++++++--- cmd/kosli/snapshotLambda_test.go | 22 ++++++++- internal/aws/aws.go | 37 ++++++++++++-- internal/aws/aws_test.go | 84 ++++++++++++++++++++++++-------- 5 files changed, 146 insertions(+), 31 deletions(-) diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index 1fb1e2ee6..469cb588c 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -164,6 +164,8 @@ The service principal needs to have the following permissions: excludeNamespaceFlag = "[conditional] The comma separated list of namespaces regex patterns NOT to report artifacts info from. Can't be used together with --namespace." functionNameFlag = "[optional] The name of the AWS Lambda function." functionNamesFlag = "[optional] The comma-separated list of AWS Lambda function names to be reported." + excludeFlag = "[optional] The comma-separated list of AWS Lambda function names to be excluded. Cannot be used together with --function-names" + excludeRegexFlag = "[optional] The comma-separated list of name regex patterns for AWS Lambda functions to be excluded. Cannot be used together with --function-names. Allowed regex patterns are described in https://github.com/google/re2/wiki/Syntax" functionVersionFlag = "[optional] The version of the AWS Lambda function." awsKeyIdFlag = "The AWS access key ID." awsSecretKeyFlag = "The AWS secret access key." diff --git a/cmd/kosli/snapshotLambda.go b/cmd/kosli/snapshotLambda.go index b3f7bc01e..c5a573825 100644 --- a/cmd/kosli/snapshotLambda.go +++ b/cmd/kosli/snapshotLambda.go @@ -13,7 +13,7 @@ import ( const snapshotLambdaShortDesc = `Report a snapshot of artifacts deployed as one or more AWS Lambda functions and their digests to Kosli.` const snapshotLambdaLongDesc = snapshotLambdaShortDesc + ` -Skip ^--function-names^ to report all functions in a given AWS account.` + awsAuthDesc +Skip ^--function-names^ to report all functions in a given AWS account. Or use ^--exclude^ and/or ^--exclude-regex^ to report all functions excluding some.` + awsAuthDesc const snapshotLambdaExample = ` # report all Lambda functions running in an AWS account (AWS auth provided in env variables): @@ -25,6 +25,17 @@ kosli snapshot lambda yourEnvironmentName \ --api-token yourAPIToken \ --org yourOrgName +# report all (excluding some) Lambda functions running in an AWS account (AWS auth provided in env variables): +export AWS_REGION=yourAWSRegion +export AWS_ACCESS_KEY_ID=yourAWSAccessKeyID +export AWS_SECRET_ACCESS_KEY=yourAWSSecretAccessKey + +kosli snapshot lambda yourEnvironmentName \ + --exclude function1,function2 \ + --exclude-regex "^not-wanted.*" \ + --api-token yourAPIToken \ + --org yourOrgName + # report what is running in the latest version of an AWS Lambda function (AWS auth provided in env variables): export AWS_REGION=yourAWSRegion export AWS_ACCESS_KEY_ID=yourAWSAccessKeyID @@ -56,9 +67,11 @@ kosli snapshot lambda yourEnvironmentName \ ` type snapshotLambdaOptions struct { - functionNames []string - functionVersion string - awsStaticCreds *aws.AWSStaticCreds + functionNames []string + functionVersion string + excludeNames []string + excludeNamesRegex []string + awsStaticCreds *aws.AWSStaticCreds } func newSnapshotLambdaCmd(out io.Writer) *cobra.Command { @@ -76,7 +89,12 @@ func newSnapshotLambdaCmd(out io.Writer) *cobra.Command { return ErrorBeforePrintingUsage(cmd, err.Error()) } - err = MuXRequiredFlags(cmd, []string{"function-name", "function-names"}, false) + err = MuXRequiredFlags(cmd, []string{"function-name", "function-names", "exclude"}, false) + if err != nil { + return err + } + + err = MuXRequiredFlags(cmd, []string{"function-name", "function-names", "exclude-regex"}, false) if err != nil { return err } @@ -91,6 +109,8 @@ func newSnapshotLambdaCmd(out io.Writer) *cobra.Command { cmd.Flags().StringSliceVar(&o.functionNames, "function-name", []string{}, functionNameFlag) cmd.Flags().StringSliceVar(&o.functionNames, "function-names", []string{}, functionNamesFlag) cmd.Flags().StringVar(&o.functionVersion, "function-version", "", functionVersionFlag) + cmd.Flags().StringSliceVar(&o.excludeNames, "exclude", []string{}, excludeFlag) + cmd.Flags().StringSliceVar(&o.excludeNamesRegex, "exclude-regex", []string{}, excludeRegexFlag) addAWSAuthFlags(cmd, o.awsStaticCreds) addDryRunFlag(cmd) @@ -109,7 +129,7 @@ func (o *snapshotLambdaOptions) run(args []string) error { envName := args[0] url := fmt.Sprintf("%s/api/v2/environments/%s/%s/report/lambda", global.Host, global.Org, envName) - lambdaData, err := o.awsStaticCreds.GetLambdaPackageData(o.functionNames) + lambdaData, err := o.awsStaticCreds.GetLambdaPackageData(o.functionNames, o.excludeNames, o.excludeNamesRegex) if err != nil { return err } diff --git a/cmd/kosli/snapshotLambda_test.go b/cmd/kosli/snapshotLambda_test.go index a5d27f3dc..de3b3a65f 100644 --- a/cmd/kosli/snapshotLambda_test.go +++ b/cmd/kosli/snapshotLambda_test.go @@ -95,7 +95,7 @@ func (suite *SnapshotLambdaTestSuite) TestSnapshotLambdaCmd() { additionalConfig: snapshotLambdaTestConfig{ requireAuthToBeSet: true, }, - golden: "Flag --function-name has been deprecated, use --function-names instead\nError: only one of --function-name, --function-names is allowed\n", + golden: "Flag --function-name has been deprecated, use --function-names instead\nError: only one of --function-name, --function-names, --exclude is allowed\n", }, { wantError: true, @@ -109,6 +109,26 @@ func (suite *SnapshotLambdaTestSuite) TestSnapshotLambdaCmd() { cmd: fmt.Sprintf(`snapshot lambda %s xxx %s --function-names %s`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName), golden: "Error: accepts 1 arg(s), received 2\n", }, + { + wantError: true, + name: "snapshot lambda fails if both --function-names and --exclude are set", + cmd: fmt.Sprintf(`snapshot lambda %s %s --function-names %s --exclude function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName), + golden: "Error: only one of --function-name, --function-names, --exclude is allowed\n", + }, + { + wantError: true, + name: "snapshot lambda fails if both --function-names and --exclude-regex are set", + cmd: fmt.Sprintf(`snapshot lambda %s %s --function-names %s --exclude-regex function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName), + golden: "Error: only one of --function-name, --function-names, --exclude-regex is allowed\n", + }, + { + name: "snapshot lambda works if both --exclude and --exclude-regex are set", + cmd: fmt.Sprintf(`snapshot lambda %s %s --exclude %s --exclude-regex function1`, suite.envName, suite.defaultKosliArguments, suite.zipFunctionName), + additionalConfig: snapshotLambdaTestConfig{ + requireAuthToBeSet: true, + }, + goldenRegex: fmt.Sprintf("[0-9]+ lambda functions were reported to environment %s\n", suite.envName), + }, } for _, t := range tests { diff --git a/internal/aws/aws.go b/internal/aws/aws.go index 6f66cbfb7..8fb4ecb11 100644 --- a/internal/aws/aws.go +++ b/internal/aws/aws.go @@ -7,6 +7,8 @@ import ( "fmt" "os" "path/filepath" + "regexp" + "slices" "strings" "sync" "time" @@ -126,7 +128,7 @@ func (staticCreds *AWSStaticCreds) NewECSClient() (*ecs.Client, error) { } // getAllLambdaFuncs fetches all lambda functions recursively (50 at a time) and returns a list of FunctionConfiguration -func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *[]types.FunctionConfiguration) (*[]types.FunctionConfiguration, error) { +func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions *[]types.FunctionConfiguration, excludeNames, excludeNamesRegex []string) (*[]types.FunctionConfiguration, error) { params := &lambda.ListFunctionsInput{} if nextMarker != nil { params.Marker = nextMarker @@ -137,9 +139,34 @@ func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions * return allFunctions, err } - *allFunctions = append(*allFunctions, listFunctionsOutput.Functions...) + if len(excludeNames) == 0 && len(excludeNamesRegex) == 0 { + *allFunctions = append(*allFunctions, listFunctionsOutput.Functions...) + } else { + for _, f := range listFunctionsOutput.Functions { + if slices.Contains(excludeNames, *f.FunctionName) { + continue + } + regexExcluded := false + for _, pattern := range excludeNamesRegex { + re, err := regexp.Compile(pattern) + if err != nil { + return allFunctions, fmt.Errorf("invalid exclude name regex pattern %s: %v", pattern, err) + } + if re.MatchString(*f.FunctionName) { + regexExcluded = true + break + } + } + if regexExcluded { + continue + } + + *allFunctions = append(*allFunctions, f) + } + } + if listFunctionsOutput.NextMarker != nil { - _, err := getAllLambdaFuncs(client, listFunctionsOutput.NextMarker, allFunctions) + _, err := getAllLambdaFuncs(client, listFunctionsOutput.NextMarker, allFunctions, excludeNames, excludeNamesRegex) if err != nil { return allFunctions, err } @@ -148,7 +175,7 @@ func getAllLambdaFuncs(client *lambda.Client, nextMarker *string, allFunctions * } // GetLambdaPackageData returns a digest and metadata of a Lambda function package -func (staticCreds *AWSStaticCreds) GetLambdaPackageData(functionNames []string) ([]*LambdaData, error) { +func (staticCreds *AWSStaticCreds) GetLambdaPackageData(functionNames, excludeNames, excludeNamesRegex []string) ([]*LambdaData, error) { lambdaData := []*LambdaData{} client, err := staticCreds.NewLambdaClient() if err != nil { @@ -156,7 +183,7 @@ func (staticCreds *AWSStaticCreds) GetLambdaPackageData(functionNames []string) } if len(functionNames) == 0 { - allFunctions, err := getAllLambdaFuncs(client, nil, &[]types.FunctionConfiguration{}) + allFunctions, err := getAllLambdaFuncs(client, nil, &[]types.FunctionConfiguration{}, excludeNames, excludeNamesRegex) if err != nil { return lambdaData, err } diff --git a/internal/aws/aws_test.go b/internal/aws/aws_test.go index 4b6b0a65e..a49120852 100644 --- a/internal/aws/aws_test.go +++ b/internal/aws/aws_test.go @@ -245,13 +245,19 @@ func (suite *AWSTestSuite) TestAWSClients() { // All cases will run in CI func (suite *AWSTestSuite) TestGetLambdaPackageData() { + type expectedFunction struct { + name string + fingerprint string + } for _, t := range []struct { - name string - requireEnvVars bool // indicates that a test case needs real credentials from env vars - creds *AWSStaticCreds - functionNames []string - wantFingerprints []string - wantErr bool + name string + requireEnvVars bool // indicates that a test case needs real credentials from env vars + creds *AWSStaticCreds + functionNames []string + excludeNames []string + excludeNamesRegex []string + expectedFunctions []expectedFunction + wantErr bool }{ { name: "invalid credentials causes an error", @@ -277,18 +283,20 @@ func (suite *AWSTestSuite) TestGetLambdaPackageData() { creds: &AWSStaticCreds{ Region: "eu-central-1", }, - functionNames: []string{"cli-tests"}, - wantFingerprints: []string{"321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}, - requireEnvVars: true, + functionNames: []string{"cli-tests"}, + expectedFunctions: []expectedFunction{{name: "cli-tests", + fingerprint: "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}}, + requireEnvVars: true, }, { name: "can get image package lambda function data from name", creds: &AWSStaticCreds{ Region: "eu-central-1", }, - functionNames: []string{"cli-tests-docker"}, - wantFingerprints: []string{"e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}, - requireEnvVars: true, + functionNames: []string{"cli-tests-docker"}, + expectedFunctions: []expectedFunction{{name: "cli-tests-docker", + fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}}, + requireEnvVars: true, }, { name: "can get a list of lambda functions data from names", @@ -296,27 +304,65 @@ func (suite *AWSTestSuite) TestGetLambdaPackageData() { Region: "eu-central-1", }, functionNames: []string{"cli-tests-docker", "cli-tests"}, - wantFingerprints: []string{"e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e", - "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}, + expectedFunctions: []expectedFunction{ + {name: "cli-tests", + fingerprint: "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}, + {name: "cli-tests-docker", + fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}}, + requireEnvVars: true, + }, + { + name: "can exclude lambda functions matching a regex pattern", + creds: &AWSStaticCreds{ + Region: "eu-central-1", + }, + excludeNamesRegex: []string{"^([^c]|c[^l]|cl[^i]|cli[^-]).*$"}, + expectedFunctions: []expectedFunction{ + {name: "cli-tests", + fingerprint: "321e3c38e91262e5c72df4bd405e9b177b6f4d750e1af0b78ca2e2b85d6f91b4"}, + {name: "cli-tests-docker", + fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}}, + requireEnvVars: true, + }, + { + name: "invalid exclude name regex pattern causes an error", + creds: &AWSStaticCreds{ + Region: "eu-central-1", + }, + excludeNamesRegex: []string{"invalid["}, + requireEnvVars: true, + wantErr: true, + }, + { + name: "can combine exclude and exclude-regex and they are joined", + creds: &AWSStaticCreds{ + Region: "eu-central-1", + }, + excludeNames: []string{"cli-tests"}, + excludeNamesRegex: []string{"^([^c]|c[^l]|cl[^i]|cli[^-]).*$"}, + expectedFunctions: []expectedFunction{ + {name: "cli-tests-docker", + fingerprint: "e908950659e56bb886acbb0ecf9b8f38bf6e0382ede71095e166269ee4db601e"}}, requireEnvVars: true, }, } { suite.Run(t.name, func() { skipOrSetCreds(suite.T(), t.requireEnvVars, t.creds) - data, err := t.creds.GetLambdaPackageData(t.functionNames) + data, err := t.creds.GetLambdaPackageData(t.functionNames, t.excludeNames, t.excludeNamesRegex) require.False(suite.T(), (err != nil) != t.wantErr, "GetLambdaPackageData() error = %v, wantErr %v", err, t.wantErr) if !t.wantErr { matchFound := false + require.Len(suite.T(), data, len(t.expectedFunctions)) loop1: - for index, name := range t.functionNames { + for _, expectedFunction := range t.expectedFunctions { for _, item := range data { - if fingerprint, ok := item.Digests[name]; ok { - if t.wantFingerprints[index] == fingerprint { + if fingerprint, ok := item.Digests[expectedFunction.name]; ok { + if expectedFunction.fingerprint == fingerprint { matchFound = true break loop1 } else { - suite.T().Logf("fingerprint did not match: GOT %s -- WANT %s", fingerprint, t.wantFingerprints[index]) + suite.T().Logf("fingerprint did not match: GOT %s -- WANT %s", fingerprint, expectedFunction.fingerprint) } } } From 73e02fdde24be438c3bc04650342f1bd393093fa Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Thu, 19 Sep 2024 10:20:07 +0100 Subject: [PATCH 2/7] Use optional regex group This test seems to get different responses from the server - sometimes there is a `?artifact_id=...` appended, sometimes there isn't. I guess this is related to the flow v1->v2 migration? So this makes the artifact_id optional, for now. --- cmd/kosli/assertArtifact_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/kosli/assertArtifact_test.go b/cmd/kosli/assertArtifact_test.go index c0f9fb70e..37de74f2d 100644 --- a/cmd/kosli/assertArtifact_test.go +++ b/cmd/kosli/assertArtifact_test.go @@ -58,12 +58,12 @@ func (suite *AssertArtifactCommandTestSuite) TestAssertArtifactCmd() { { name: "asserting an existing compliant artifact (using --fingerprint) results in OK and zero exit", cmd: fmt.Sprintf(`assert artifact --fingerprint %s --flow %s %s`, suite.fingerprint, suite.flowName, suite.defaultKosliArguments), - goldenRegex: "COMPLIANT\nSee more details at http://localhost:8001/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8}\n", + goldenRegex: "COMPLIANT\nSee more details at http://localhost:8001/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n", }, { name: "asserting an existing compliant artifact (using --artifact-type) results in OK and zero exit", cmd: fmt.Sprintf(`assert artifact %s --artifact-type file --flow %s %s`, suite.artifactPath, suite.flowName, suite.defaultKosliArguments), - goldenRegex: "COMPLIANT\nSee more details at http://localhost:8001/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8}\n", + goldenRegex: "COMPLIANT\nSee more details at http://localhost:8001/docs-cmd-test-user/flows/assert-artifact/artifacts/fcf33337634c2577a5d86fd7ecb0a25a7c1bb5d89c14fd236f546a5759252c02?(?:\\?artifact_id=[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{8})?\n", }, { wantError: true, From bbdde36722f888f848346f526fbd8e3f0ee26653 Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Thu, 19 Sep 2024 09:09:06 +0100 Subject: [PATCH 3/7] Use .kosli_ignore file for dir fingerprint --- cmd/kosli/fingerprint_test.go | 5 + .../folder1-with-ignore/.kosli_ignore | 1 + .../folder1-with-ignore/folder2/hello2.txt | 1 + .../folder1-with-ignore/folder2/hello3.txt | 1 + .../testdata/folder1-with-ignore/hello.txt | 1 + internal/digest/digest.go | 32 +++ internal/digest/digest_test.go | 188 ++++++++++++++++++ 7 files changed, 229 insertions(+) create mode 100644 cmd/kosli/testdata/folder1-with-ignore/.kosli_ignore create mode 100644 cmd/kosli/testdata/folder1-with-ignore/folder2/hello2.txt create mode 100644 cmd/kosli/testdata/folder1-with-ignore/folder2/hello3.txt create mode 100644 cmd/kosli/testdata/folder1-with-ignore/hello.txt diff --git a/cmd/kosli/fingerprint_test.go b/cmd/kosli/fingerprint_test.go index d52ac83b0..d685c0ee1 100644 --- a/cmd/kosli/fingerprint_test.go +++ b/cmd/kosli/fingerprint_test.go @@ -39,6 +39,11 @@ func (suite *FingerprintTestSuite) TestFingerprintCmd() { cmd: "fingerprint --artifact-type dir testdata/folder1 -x folder2", golden: "773fd3300860454a2b065c5912c03008adb11e6a6dcf7c1c64c094ceab8f430a\n", }, + { + name: "dir fingerprint with ignore file", + cmd: "fingerprint --artifact-type dir testdata/folder1-with-ignore", + golden: "038897ea5334462098d65125380d58a493671fb3b8bdbbee1e75ec8bd4a65c23\n", + }, { name: "fails if type is directory but the argument is not a dir", cmd: "fingerprint --artifact-type dir testdata/file1", diff --git a/cmd/kosli/testdata/folder1-with-ignore/.kosli_ignore b/cmd/kosli/testdata/folder1-with-ignore/.kosli_ignore new file mode 100644 index 000000000..7abc30679 --- /dev/null +++ b/cmd/kosli/testdata/folder1-with-ignore/.kosli_ignore @@ -0,0 +1 @@ +folder2 diff --git a/cmd/kosli/testdata/folder1-with-ignore/folder2/hello2.txt b/cmd/kosli/testdata/folder1-with-ignore/folder2/hello2.txt new file mode 100644 index 000000000..23294b061 --- /dev/null +++ b/cmd/kosli/testdata/folder1-with-ignore/folder2/hello2.txt @@ -0,0 +1 @@ +hello2 \ No newline at end of file diff --git a/cmd/kosli/testdata/folder1-with-ignore/folder2/hello3.txt b/cmd/kosli/testdata/folder1-with-ignore/folder2/hello3.txt new file mode 100644 index 000000000..96803d198 --- /dev/null +++ b/cmd/kosli/testdata/folder1-with-ignore/folder2/hello3.txt @@ -0,0 +1 @@ +hello3 \ No newline at end of file diff --git a/cmd/kosli/testdata/folder1-with-ignore/hello.txt b/cmd/kosli/testdata/folder1-with-ignore/hello.txt new file mode 100644 index 000000000..57ea1e790 --- /dev/null +++ b/cmd/kosli/testdata/folder1-with-ignore/hello.txt @@ -0,0 +1 @@ +Hello, world! Again! \ No newline at end of file diff --git a/internal/digest/digest.go b/internal/digest/digest.go index 17b254d5b..21e70a0bd 100644 --- a/internal/digest/digest.go +++ b/internal/digest/digest.go @@ -1,6 +1,7 @@ package digest import ( + "bufio" "context" "crypto/sha256" "encoding/hex" @@ -49,6 +50,11 @@ func DirSha256(dirPath string, excludePaths []string, logger *logger.Logger) (st return "", err } defer digestsFile.Close() + ignoredPaths, err := excludePathsFromFile(filepath.Join(dirPath, ".kosli_ignore")) + if err != nil { + return "", err + } + excludePaths = append(excludePaths, ignoredPaths...) err = calculateDirContentSha256(digestsFile, dirPath, tmpDir, excludePaths, logger) if err != nil { return "", err @@ -258,3 +264,29 @@ func ValidateDigest(sha256ToCheck string) error { } return nil } + +func excludePathsFromFile(path string) ([]string, error) { + file, err := os.Open(path) + if err == nil { + defer file.Close() + var excludes = []string{} + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + line = removeComments(line) + line = strings.TrimSpace(line) + if len(line) > 0 { + excludes = append(excludes, line) + } + } + return excludes, nil + } else if errors.Is(err, fs.ErrNotExist) { + return []string{}, nil + } + return nil, err +} + +func removeComments(line string) string { + parts := strings.SplitN(line, "#", 2) + return strings.TrimRight(parts[0], " ") +} diff --git a/internal/digest/digest_test.go b/internal/digest/digest_test.go index 7da61b467..b20a18c74 100644 --- a/internal/digest/digest_test.go +++ b/internal/digest/digest_test.go @@ -416,6 +416,99 @@ func (suite *DigestTestSuite) TestDirSha256() { }, want: "5d3c17dae9e208bbb92ee04ff8342abf77cb0959764def4af3ccfe9a2109d4a7", }, + { + name: ".kosli_ignore is included in the fingerprint", + args: args{ + dirName: "exclusion4", + //excludePaths: []string{"logs", "*/logs"}, + dirContent: []fileSystemEntry{ + { + files: []fileEntry{ + { + name: "sample.yaml", + content: "some content. And some more.", + }, + { + name: ".kosli_ignore", + content: "logs\n*/logs", + }, + }, + dirs: []dirEntry{ + { + name: "nested-dir", + files: []fileEntry{ + { + name: "file1", + content: "content1", + }, + { + name: "file2", + content: "content2", + }, + }, + }, + }, + }, + }, + }, + want: "4048004ce6f91e8ce27c35e5e857948a46750567cf7c873ee32b120f4318363c", + }, + { + name: "excluding dirs using .kosli_ignore in root works", + args: args{ + dirName: "exclusion5", + dirContent: []fileSystemEntry{ + { + files: []fileEntry{ + { + name: "sample.yaml", + content: "some content. And some more.", + }, + { + name: ".kosli_ignore", + content: "logs\n*/logs", + }, + }, + dirs: []dirEntry{ + { + name: "nested-dir", + files: []fileEntry{ + { + name: "file1", + content: "content1", + }, + { + name: "file2", + content: "content2", + }, + }, + dirs: []dirEntry{ + { + name: "logs", + files: []fileEntry{ + { + name: "log.txt", + content: "this is a log", + }, + }, + }, + }, + }, + { + name: "logs", + files: []fileEntry{ + { + name: "file1", + content: "content1", + }, + }, + }, + }, + }, + }, + }, + want: "4048004ce6f91e8ce27c35e5e857948a46750567cf7c873ee32b120f4318363c", + }, } { suite.Run(t.name, func() { topLevelPath := filepath.Join(suite.tmpDir, t.args.dirName) @@ -779,6 +872,101 @@ func (suite *DigestTestSuite) TestExtractImageDigestFromRepoDigest() { } } +func (suite *DigestTestSuite) TestGetExcludePathsFromIgnoreFile() { + type want struct { + expectError bool + excludePaths []string + } + const MISSING_FILE_NAME = "NO_CREATE.ignore" + for _, t := range []struct { + name string + ignoreFileName string + content string + want want + }{ + { + name: "missing file should return []", + ignoreFileName: MISSING_FILE_NAME, + want: want{ + expectError: false, + excludePaths: []string{}, + }, + }, + { + name: "empty file should return []", + ignoreFileName: "empty.ignore", + content: ``, + want: want{ + expectError: false, + excludePaths: []string{}, + }, + }, + { + name: "a single line ignore file", + ignoreFileName: "empty.ignore", + content: `logs`, + want: want{ + expectError: false, + excludePaths: []string{"logs"}, + }, + }, + { + name: "a multi line ignore file", + ignoreFileName: "multi.ignore", + content: `logs +*/logs`, + want: want{ + expectError: false, + excludePaths: []string{"logs", "*/logs"}, + }, + }, + { + name: "an ignore file with blank lines", + ignoreFileName: "multi-with-blank.ignore", + content: `logs + + +*/logs +`, + want: want{ + expectError: false, + excludePaths: []string{"logs", "*/logs"}, + }, + }, + { + name: "a commented ignore file", + ignoreFileName: "multi.ignore", + content: `logs +# a line comment +*/logs # an end of line comment`, + want: want{ + expectError: false, + excludePaths: []string{"logs", "*/logs"}, + }, + }, + } { + suite.Run(t.name, func() { + assert.False(suite.T(), t.ignoreFileName == "", "ignoreFileName cannot be empty string") + ignoreFilePath := filepath.Join(suite.tmpDir, t.ignoreFileName) + if t.ignoreFileName != MISSING_FILE_NAME { + testFile, err := os.Create(ignoreFilePath) + require.NoErrorf(suite.T(), err, "error creating test file %s: %s", t.ignoreFileName, err) + + _, err = testFile.Write([]byte(t.content)) + require.NoErrorf(suite.T(), err, "error writing content to test file %s: %s", t.ignoreFileName, err) + } + + actual, err := excludePathsFromFile(ignoreFilePath) + if t.want.expectError { + require.Errorf(suite.T(), err, "TestGetExcludePathsFromIgnoreFile: error was expected: %s", err) + } else { + require.NoErrorf(suite.T(), err, "TestGetExcludePathsFromIgnoreFile: error was NOT expected: %s", err) + assert.Equal(suite.T(), t.want.excludePaths, actual, fmt.Sprintf("TestGetExcludePathsFromIgnoreFile: want %s -- got %s", t.want.excludePaths, actual)) + } + }) + } +} + // In order for 'go test' to run this suite, we need to create // a normal test function and pass our suite to suite.Run func TestDigestTestSuite(t *testing.T) { From b74f68122ba5a7ac5a2c2a16a751601a9a90af72 Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Thu, 19 Sep 2024 09:17:55 +0100 Subject: [PATCH 4/7] Add debugging --- internal/digest/digest.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/digest/digest.go b/internal/digest/digest.go index 21e70a0bd..5f8d9954c 100644 --- a/internal/digest/digest.go +++ b/internal/digest/digest.go @@ -50,10 +50,14 @@ func DirSha256(dirPath string, excludePaths []string, logger *logger.Logger) (st return "", err } defer digestsFile.Close() - ignoredPaths, err := excludePathsFromFile(filepath.Join(dirPath, ".kosli_ignore")) + ignoreFilePath := filepath.Join(dirPath, ".kosli_ignore") + ignoredPaths, err := excludePathsFromFile(ignoreFilePath) if err != nil { return "", err } + if len(ignoredPaths) > 0 { + logger.Debug(" -> ignore file used %s -- excluding paths: %s", ignoreFilePath, ignoredPaths) + } excludePaths = append(excludePaths, ignoredPaths...) err = calculateDirContentSha256(digestsFile, dirPath, tmpDir, excludePaths, logger) if err != nil { From 8ede80dbd85157ad5013da3cbf8328b0f3551344 Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Thu, 19 Sep 2024 09:55:56 +0100 Subject: [PATCH 5/7] Add docs for `.kosli_ignore` --- cmd/kosli/attestArtifact.go | 2 +- cmd/kosli/fingerprint.go | 4 +++- cmd/kosli/root.go | 3 +++ cmd/kosli/snapshotPath.go | 2 +- cmd/kosli/snapshotPaths.go | 3 ++- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/kosli/attestArtifact.go b/cmd/kosli/attestArtifact.go index 9bd835fdc..f999e8a19 100644 --- a/cmd/kosli/attestArtifact.go +++ b/cmd/kosli/attestArtifact.go @@ -41,7 +41,7 @@ type AttestArtifactPayload struct { const attestArtifactShortDesc = `Attest an artifact creation to a Kosli flow. ` const attestArtifactLongDesc = attestArtifactShortDesc + ` -` + fingerprintDesc + ` +` + fingerprintDesc + kosliIgnoreDesc + ` This command requires access to a git repo to associate the artifact to the git commit it is originating from. You can optionally redact some of the git commit data sent to Kosli using ^--redact-commit-info^` diff --git a/cmd/kosli/fingerprint.go b/cmd/kosli/fingerprint.go index 3538ee72b..20fc52e53 100644 --- a/cmd/kosli/fingerprint.go +++ b/cmd/kosli/fingerprint.go @@ -13,7 +13,9 @@ using the ^--exclude^ flag. Excluded paths are relative to the DIR-PATH and can be literal paths or glob patterns. The supported glob pattern syntax is what is documented here: https://pkg.go.dev/path/filepath#Match , -plus the ability to use recursive globs "**"` +plus the ability to use recursive globs "**" + +` + kosliIgnoreDesc const fingerprintLongDesc = fingerprintShortDesc + ` Requires ^--artifact-type^ flag to be set. diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index 469cb588c..a1d8c40ea 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -67,6 +67,9 @@ The service principal needs to have the following permissions: 2) Microsoft.ContainerRegistry/registries/pull/read ` + kosliIgnoreDesc = `To specify paths in a directory artifact that should always be excluded from the SHA256 calculation, you can add a ^.kosli_ignore^ file to the root of the artifact. +Each line should specify a relative path or path glob to be ignored. You can include comments in this file, using ^#^. +The ^.kosli_ignore^ will be treated as part of the artifact like any other file,unless it is explicitly ignored itself.` // flags apiTokenFlag = "The Kosli API token." diff --git a/cmd/kosli/snapshotPath.go b/cmd/kosli/snapshotPath.go index 81dae6339..39d03eb8a 100644 --- a/cmd/kosli/snapshotPath.go +++ b/cmd/kosli/snapshotPath.go @@ -18,7 +18,7 @@ You can exclude certain paths or patterns from the artifact fingerprint using ^- The supported glob pattern syntax is what is documented here: https://pkg.go.dev/path/filepath#Match , plus the ability to use recursive globs "**" -` +` + kosliIgnoreDesc const snapshotPathExample = ` # report one artifact running in a specific path in a filesystem: diff --git a/cmd/kosli/snapshotPaths.go b/cmd/kosli/snapshotPaths.go index 7b76bc0ce..cc8b30121 100644 --- a/cmd/kosli/snapshotPaths.go +++ b/cmd/kosli/snapshotPaths.go @@ -23,9 +23,10 @@ glob patterns. The supported glob pattern syntax is what is documented here: https://pkg.go.dev/path/filepath#Match , plus the ability to use recursive globs "**" +` + kosliIgnoreDesc + ` + This is an example YAML paths spec file: ` + - "```yaml\n" + `version: 1 artifacts: From 926808f42e974b88a78f5e35b3d644af5c28e8c5 Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Thu, 19 Sep 2024 10:49:21 +0100 Subject: [PATCH 6/7] Kill dead comment --- internal/digest/digest_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/digest/digest_test.go b/internal/digest/digest_test.go index b20a18c74..5b2ac7444 100644 --- a/internal/digest/digest_test.go +++ b/internal/digest/digest_test.go @@ -420,7 +420,6 @@ func (suite *DigestTestSuite) TestDirSha256() { name: ".kosli_ignore is included in the fingerprint", args: args{ dirName: "exclusion4", - //excludePaths: []string{"logs", "*/logs"}, dirContent: []fileSystemEntry{ { files: []fileEntry{ From b580eceed236cdf03c4e141ececa53277479d5f0 Mon Sep 17 00:00:00 2001 From: Steve Tooke Date: Thu, 19 Sep 2024 10:52:13 +0100 Subject: [PATCH 7/7] Inlcude docs for snapshot s3 --- cmd/kosli/snapshotS3.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/kosli/snapshotS3.go b/cmd/kosli/snapshotS3.go index 76bad7d74..149c01456 100644 --- a/cmd/kosli/snapshotS3.go +++ b/cmd/kosli/snapshotS3.go @@ -15,7 +15,8 @@ const snapshotS3ShortDesc = `Report a snapshot of the content of an AWS S3 bucke const snapshotS3LongDesc = snapshotS3ShortDesc + awsAuthDesc + ` You can report the entire bucket content, or filter some of the content using ^--include^ and ^--exclude^. In all cases, the content is reported as one artifact. If you wish to report separate files/dirs within the same bucket as separate artifacts, you need to run the command twice. -` + +` + kosliIgnoreDesc const snapshotS3Example = ` # report the contents of an entire AWS S3 bucket (AWS auth provided in env variables):