Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(appset): Provide additional parameter files to git generator - Fixes #20593 #20653

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+1,661 −416
Open

feat(appset): Provide additional parameter files to git generator - Fixes #20593 #20653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 111 additions & 23 deletions applicationset/generators/git.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ func (g *GitGenerator) generateParamsForGitDirectories(appSetGenerator *argoproj

requestedApps := g.filterApps(appSetGenerator.Git.Directories, allPaths)

res, err := g.generateParamsFromApps(requestedApps, appSetGenerator, useGoTemplate, goTemplateOptions)
res, err := g.generateParamsFromApps(requestedApps, appSetGenerator, useGoTemplate, noRevisionCache, verifyCommit, goTemplateOptions)
if err != nil {
return nil, fmt.Errorf("error generating params from apps: %w", err)
}
Expand Down Expand Up @@ -147,7 +147,7 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
res := []map[string]interface{}{}
for _, path := range allPaths {
// A JSON / YAML file path can contain multiple sets of parameters (ie it is an array)
paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path], appSetGenerator.Git.Values, useGoTemplate, goTemplateOptions, appSetGenerator.Git.PathParamPrefix)
paramsArray, err := g.generateParamsFromGitFile(path, allFiles[path], appSetGenerator.Git, goTemplateOptions, useGoTemplate, noRevisionCache, verifyCommit)
if err != nil {
return nil, fmt.Errorf("unable to process file '%s': %w", path, err)
}
Expand All @@ -157,22 +157,12 @@ func (g *GitGenerator) generateParamsForGitFiles(appSetGenerator *argoprojiov1al
return res, nil
}

func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []byte, values map[string]string, useGoTemplate bool, goTemplateOptions []string, pathParamPrefix string) ([]map[string]interface{}, error) {
objectsFound := []map[string]interface{}{}

// First, we attempt to parse as an array
err := yaml.Unmarshal(fileContent, &objectsFound)
func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []byte, git *argoprojiov1alpha1.GitGenerator, goTemplateOptions []string,
useGoTemplate, noRevisionCache, verifyCommit bool,
) ([]map[string]interface{}, error) {
objectsFound, err := unMarshalYamlFile(fileContent)
if err != nil {
// If unable to parse as an array, attempt to parse as a single object
singleObj := make(map[string]interface{})
err = yaml.Unmarshal(fileContent, &singleObj)
if err != nil {
return nil, fmt.Errorf("unable to parse file: %w", err)
}
objectsFound = append(objectsFound, singleObj)
} else if len(objectsFound) == 0 {
// If file is valid but empty, add a default empty item
objectsFound = append(objectsFound, map[string]interface{}{})
return nil, err
}

res := []map[string]interface{}{}
Expand All @@ -193,8 +183,8 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
paramPath["basenameNormalized"] = utils.SanitizeName(path.Base(paramPath["path"].(string)))
paramPath["filenameNormalized"] = utils.SanitizeName(path.Base(paramPath["filename"].(string)))
paramPath["segments"] = strings.Split(paramPath["path"].(string), "/")
if pathParamPrefix != "" {
params[pathParamPrefix] = map[string]interface{}{"path": paramPath}
if git.PathParamPrefix != "" {
params[git.PathParamPrefix] = map[string]interface{}{"path": paramPath}
} else {
params["path"] = paramPath
}
Expand All @@ -207,8 +197,8 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
params[k] = fmt.Sprintf("%v", v)
}
pathParamName := "path"
if pathParamPrefix != "" {
pathParamName = pathParamPrefix + "." + pathParamName
if git.PathParamPrefix != "" {
pathParamName = git.PathParamPrefix + "." + pathParamName
}
params[pathParamName] = path.Dir(filePath)
params[pathParamName+".basename"] = path.Base(params[pathParamName].(string))
Expand All @@ -222,17 +212,108 @@ func (g *GitGenerator) generateParamsFromGitFile(filePath string, fileContent []
}
}

err := appendTemplatedValues(values, params, useGoTemplate, goTemplateOptions)
err := appendTemplatedValues(git.Values, params, useGoTemplate, goTemplateOptions)
if err != nil {
return nil, fmt.Errorf("failed to append templated values: %w", err)
}

// Get the common parameters from files, removing duplicates
// We need to resolve them for each git file, as they could depend on parameters inside
err = g.appendExtraParams(git, goTemplateOptions, useGoTemplate, noRevisionCache, verifyCommit, params)
if err != nil {
return nil, err
}

res = append(res, params)
}

return res, nil
}

func (g *GitGenerator) appendExtraParams(git *argoprojiov1alpha1.GitGenerator, goTemplateOptions []string, useGoTemplate bool, noRevisionCache bool, verifyCommit bool, params map[string]interface{}) error {
extraParams, err := g.getExtraParams(git, goTemplateOptions, useGoTemplate, noRevisionCache, verifyCommit, params)
if err != nil {
return fmt.Errorf("failed to append extra parameters from files: %w", err)
}
if len(extraParams) == 0 {
return nil
}
if useGoTemplate {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the meaning of useGoTemplate? Where is it set?

params["extraParams"] = extraParams
} else {
flat, err := flatten.Flatten(extraParams, "", flatten.DotStyle)
if err != nil {
return fmt.Errorf("error flattening extra params: %w", err)
}
for k, v := range flat {
params["extraParams."+fmt.Sprintf("%v", k)] = fmt.Sprintf("%v", v)
}
}
return nil
}

func unMarshalYamlFile(fileContent []byte) ([]map[string]interface{}, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd not capitalize M, since other libs don't do that, e.g. have the name Unmarshal.

objectsFound := []map[string]interface{}{}

// First, we attempt to parse as an array
err := yaml.Unmarshal(fileContent, &objectsFound)
if err != nil {
// If unable to parse as an array, attempt to parse as a single object
singleObj := make(map[string]interface{})
err = yaml.Unmarshal(fileContent, &singleObj)
if err != nil {
return nil, fmt.Errorf("unable to parse file: %w", err)
}
objectsFound = append(objectsFound, singleObj)
} else if len(objectsFound) == 0 {
// If file is valid but empty, add a default empty item
objectsFound = append(objectsFound, map[string]interface{}{})
}
return objectsFound, nil
}

func (g *GitGenerator) getExtraParams(git *argoprojiov1alpha1.GitGenerator, goTemplateOptions []string,
useGoTemplate, noRevisionCache, verifyCommit bool, params map[string]interface{},
) (map[string]interface{}, error) {
extraParams := map[string]interface{}{}
extraParamFilesContent := map[string][]byte{}

// Keep the order to apply the params in the list of the files provided
for _, requestedExtraFilePath := range git.ExtraParameterFiles {
templatedPath, err := replaceTemplatedString(requestedExtraFilePath, params, useGoTemplate, goTemplateOptions)
if err != nil {
return nil, err
}

files, err := g.repos.GetFiles(context.TODO(), git.RepoURL, git.Revision, templatedPath, noRevisionCache, verifyCommit)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we enforce explicit file, we may need to add a check before this call

if err != nil {
return nil, err
}
for extraFilePath, content := range files {
extraParamFilesContent[extraFilePath] = content
}
// Extract the unduplicated map into a list, and sort by path to ensure a deterministic
// processing order in the subsequent step
allPaths := []string{}
for path := range files {
allPaths = append(allPaths, path)
}
sort.Strings(allPaths)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might lead to unexpected behavior, since the order matters for how things get combined IIUC. People would expect the original order.


// Merge yaml based on the order in the list: the further in the list overrides the ones before
for _, path := range allPaths {
fileObjects, err := unMarshalYamlFile(extraParamFilesContent[path])
if err != nil {
return nil, err
}
for _, object := range fileObjects {
extraParams = utils.CombineMaps(extraParams, object)
}
}
}
return extraParams, nil
}

func (g *GitGenerator) filterApps(directories []argoprojiov1alpha1.GitDirectoryGeneratorItem, allPaths []string) []string {
res := []string{}
for _, appPath := range allPaths {
Expand Down Expand Up @@ -261,7 +342,7 @@ func (g *GitGenerator) filterApps(directories []argoprojiov1alpha1.GitDirectoryG
return res
}

func (g *GitGenerator) generateParamsFromApps(requestedApps []string, appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
func (g *GitGenerator) generateParamsFromApps(requestedApps []string, appSetGenerator *argoprojiov1alpha1.ApplicationSetGenerator, useGoTemplate, noRevisionCache, verifyCommit bool, goTemplateOptions []string) ([]map[string]interface{}, error) {
res := make([]map[string]interface{}, len(requestedApps))
for i, a := range requestedApps {
params := make(map[string]interface{}, 5)
Expand Down Expand Up @@ -297,6 +378,13 @@ func (g *GitGenerator) generateParamsFromApps(requestedApps []string, appSetGene
return nil, fmt.Errorf("failed to append templated values: %w", err)
}

// Get the common parameters from files, removing duplicates
// We need to resolve them for each git file, as they could depend on parameters inside
err = g.appendExtraParams(appSetGenerator.Git, goTemplateOptions, useGoTemplate, noRevisionCache, verifyCommit, params)
if err != nil {
return nil, err
}

res[i] = params
}

Expand Down
Loading