Add support for custom template paths to template store (refs openfaa…

…s#789)

To support hosting templates within the directory structure of an existing project, we need to allow for custom paths to the root template directory instead of requiring a separate repo with a root level template directory.

Signed-off-by: Justin Israel <[email protected]>

.gitignore

@@ -18,6 +18,7 @@ obj
 *.swp
 
 .idea
+*.iml
 .DS_Store
 
 

commands/build.go

@@ -286,13 +286,19 @@ func build(services *stack.Services, queueDepth int, shrinkwrap, quietBuild bool
 
 // PullTemplates pulls templates from specified git remote. templateURL may be a pinned repository.
 func PullTemplates(templateURL string) error {
+	return PullTemplatesPath(templateURL, "")
+}
+
+// PullTemplatesPath pulls templates from specified git remote, located under a nested path
+// from the root of the repo. templateURL may be a pinned repository.
+func PullTemplatesPath(templateURL, path string) error {
 	var err error
 	exists, err := os.Stat("./template")
 	if err != nil || exists == nil {
 		log.Println("No templates found in current directory.")
 
 		templateURL, refName := versioncontrol.ParsePinnedRemote(templateURL)
-		err = fetchTemplates(templateURL, refName, false)
+		err = fetchTemplatesPath(templateURL, refName, path, false)
 		if err != nil {
 			log.Println("Unable to download templates from Github.")
 			return err

commands/fetch_templates.go

@@ -21,6 +21,10 @@ const templateDirectory = "./template/"
 
 // fetchTemplates fetch code templates using git clone.
 func fetchTemplates(templateURL string, refName string, overwrite bool) error {
+	return fetchTemplatesPath(templateURL, refName, templateDirectory, overwrite)
+}
+
+func fetchTemplatesPath(templateURL, refName, path string, overwrite bool) error {
 	if len(templateURL) == 0 {
 		return fmt.Errorf("pass valid templateURL")
 	}
@@ -40,7 +44,7 @@ func fetchTemplates(templateURL string, refName string, overwrite bool) error {
 		return err
 	}
 
-	preExistingLanguages, fetchedLanguages, err := moveTemplates(dir, overwrite)
+	preExistingLanguages, fetchedLanguages, err := moveTemplatesPath(dir, path, overwrite)
 	if err != nil {
 		return err
 	}
@@ -81,6 +85,10 @@ func templateFolderExists(language string, overwrite bool) bool {
 }
 
 func moveTemplates(repoPath string, overwrite bool) ([]string, []string, error) {
+	return moveTemplatesPath(repoPath, templateDirectory, overwrite)
+}
+
+func moveTemplatesPath(repoPath, path string, overwrite bool) ([]string, []string, error) {
 	var (
 		existingLanguages []string
 		fetchedLanguages  []string
@@ -89,10 +97,10 @@ func moveTemplates(repoPath string, overwrite bool) ([]string, []string, error)
 
 	availableLanguages := make(map[string]bool)
 
-	templateDir := filepath.Join(repoPath, templateDirectory)
+	templateDir := filepath.Join(repoPath, path)
 	templates, err := ioutil.ReadDir(templateDir)
 	if err != nil {
-		return nil, nil, fmt.Errorf("can't find templates in: %s", repoPath)
+		return nil, nil, fmt.Errorf("can't find templates in: %s", templateDir)
 	}
 
 	for _, file := range templates {
@@ -118,6 +126,10 @@ func moveTemplates(repoPath string, overwrite bool) ([]string, []string, error)
 }
 
 func pullTemplate(repository string) error {
+	return pullTemplatePath(repository, "")
+}
+
+func pullTemplatePath(repository, path string) error {
 	if _, err := os.Stat(repository); err != nil {
 		if !versioncontrol.IsGitRemote(repository) && !versioncontrol.IsPinnedGitRemote(repository) {
 			return fmt.Errorf("The repository URL must be a valid git repo uri")
@@ -134,7 +146,7 @@ func pullTemplate(repository string) error {
 	}
 
 	fmt.Printf("Fetch templates from repository: %s at %s\n", repository, refName)
-	if err := fetchTemplates(repository, refName, overwrite); err != nil {
+	if err := fetchTemplatesPath(repository, refName, path, overwrite); err != nil {
 		return fmt.Errorf("error while fetching templates: %s", err)
 	}
 

commands/fetch_templates_test.go

@@ -13,8 +13,13 @@ import (
 )
 
 func Test_PullTemplates(t *testing.T) {
-	localTemplateRepository := setupLocalTemplateRepo(t)
+	localTemplateRepository := setupLocalTemplateRepo(t, "")
 	defer os.RemoveAll(localTemplateRepository)
+
+	const templatePath = "path/to/template"
+	nestedTemplateRepository := setupLocalTemplateRepo(t, templatePath)
+	defer os.RemoveAll(nestedTemplateRepository)
+
 	defer tearDownFetchTemplates(t)
 
 	t.Run("simplePull", func(t *testing.T) {
@@ -24,35 +29,57 @@ func Test_PullTemplates(t *testing.T) {
 		}
 	})
 
+	t.Run("nestedPull", func(t *testing.T) {
+		defer tearDownFetchTemplates(t)
+		if err := PullTemplatesPath(nestedTemplateRepository, templatePath); err != nil {
+			t.Error(err)
+		}
+	})
+
 	t.Run("fetchTemplates", func(t *testing.T) {
 		defer tearDownFetchTemplates(t)
 
 		err := fetchTemplates(localTemplateRepository, "master", false)
 		if err != nil {
 			t.Error(err)
 		}
+	})
 
+	t.Run("fetchTemplatesPath", func(t *testing.T) {
+		defer tearDownFetchTemplates(t)
+
+		err := fetchTemplatesPath(nestedTemplateRepository, "master", templatePath, false)
+		if err != nil {
+			t.Error(err)
+		}
 	})
 }
 
 // setupLocalTemplateRepo will create a local copy of the core OpenFaaS templates, this
-// can be refered to as a local git repository.
-func setupLocalTemplateRepo(t *testing.T) string {
-	dir, err := ioutil.TempDir("", "openFaasTestTemplates")
+// can be referred to as a local git repository.
+// If path is a non empty relative path, the templates will be created in a directory
+// nested under the returned repo path
+func setupLocalTemplateRepo(t *testing.T, path string) string {
+	tdir, err := ioutil.TempDir("", "openFaasTestTemplates")
 	if err != nil {
 		t.Error(err)
 	}
 
+	dir := filepath.Join(tdir, path)
+	if err = os.MkdirAll(dir, 0700); err != nil {
+		t.Error(err)
+	}
+
 	// Copy the submodule to temp directory to avoid altering it during tests
 	testRepoGit := filepath.Join("testdata", "templates")
 	builder.CopyFiles(testRepoGit, dir)
 	// Remove submodule .git file
 	os.Remove(filepath.Join(dir, ".git"))
-	if err := versioncontrol.GitInitRepo.Invoke(dir, map[string]string{"dir": "."}); err != nil {
+	if err := versioncontrol.GitInitRepo.Invoke(tdir, map[string]string{"dir": "."}); err != nil {
 		t.Fatal(err)
 	}
 
-	return dir
+	return tdir
 }
 
 // tearDownFetchTemplates cleans all files and directories created by the test

commands/new_function_test.go

@@ -347,7 +347,7 @@ func Test_backfillTemplates(t *testing.T) {
 	const functionLang = "ruby"
 
 	// Delete cached templates
-	localTemplateRepository := setupLocalTemplateRepo(t)
+	localTemplateRepository := setupLocalTemplateRepo(t, "")
 	defer os.RemoveAll(localTemplateRepository)
 	defer tearDownNewFunction(t, functionName)
 

commands/template_pull.go

@@ -24,27 +24,33 @@ func init() {
 
 // templatePullCmd allows the user to fetch a template from a repository
 var templatePullCmd = &cobra.Command{
-	Use:   `pull [REPOSITORY_URL]`,
+	Use:   `pull [REPOSITORY_URL [TEMPLATE_PATH]] `,
 	Short: `Downloads templates from the specified git repo`,
 	Long: `Downloads templates from the specified git repo specified by [REPOSITORY_URL], and copies the 'template'
 directory from the root of the repo, if it exists.
 
 [REPOSITORY_URL] may specify a specific branch or tag to copy by adding a URL fragment with the branch or tag name.
+[TEMPLATE_PATH] may specify an alternate 'template' directory path, relative to the root of the repo.
 	`,
 	Example: `
   faas-cli template pull https://github.com/openfaas/templates
   faas-cli template pull https://github.com/openfaas/templates#1.0
+  faas-cli template pull https://github.com/openfaas/templates path/to/template
 `,
 	RunE: runTemplatePull,
 }
 
 func runTemplatePull(cmd *cobra.Command, args []string) error {
 	repository := ""
+	path := ""
 	if len(args) > 0 {
 		repository = args[0]
+		if len(args) > 1 {
+			path = args[1]
+		}
 	}
 	repository = getTemplateURL(repository, os.Getenv(templateURLEnvironment), DefaultTemplateRepository)
-	return pullTemplate(repository)
+	return pullTemplatePath(repository, path)
 }
 
 func pullDebugPrint(message string) {

commands/template_pull_stack.go

@@ -77,7 +77,7 @@ func pullStackTemplates(templateInfo []stack.TemplateSource, cmd *cobra.Command)
 				return pullErr
 			}
 		} else {
-			pullErr := pullTemplate(val.Source)
+			pullErr := pullTemplatePath(val.Source, val.Path)
 			if pullErr != nil {
 				return pullErr
 			}

commands/template_pull_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func Test_templatePull(t *testing.T) {
-	localTemplateRepository := setupLocalTemplateRepo(t)
+	localTemplateRepository := setupLocalTemplateRepo(t, "")
 	defer os.RemoveAll(localTemplateRepository)
 
 	t.Run("ValidRepo", func(t *testing.T) {
@@ -124,7 +124,7 @@ func Test_templatePullPriority(t *testing.T) {
 
 // templatePullLocalTemplateRepo executes `template pull` on a local repository to get templates
 func templatePullLocalTemplateRepo(t *testing.T) {
-	localTemplateRepository := setupLocalTemplateRepo(t)
+	localTemplateRepository := setupLocalTemplateRepo(t, "")
 	defer os.RemoveAll(localTemplateRepository)
 
 	faasCmd.SetArgs([]string{"template", "pull", localTemplateRepository})

commands/template_store_describe.go

@@ -75,6 +75,9 @@ func formatTemplateOutput(storeTemplate TemplateInfo) string {
 	fmt.Fprintf(lineWriter, "Source:\t%s\n", storeTemplate.Source)
 	fmt.Fprintf(lineWriter, "Description:\t%s\n", storeTemplate.Description)
 	fmt.Fprintf(lineWriter, "Repository:\t%s\n", storeTemplate.Repository)
+	if storeTemplate.TemplatePath != "" {
+		fmt.Fprintf(lineWriter, "Path:\t%s\n", storeTemplate.TemplatePath)
+	}
 	fmt.Fprintf(lineWriter, "Official Template:\t%s\n", storeTemplate.Official)
 	fmt.Fprintln(lineWriter)
 

commands/template_store_describe_test.go

@@ -0,0 +1,68 @@
+package commands
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+)
+
+func Test_TemplateStoreDescribe(t *testing.T) {
+	localTemplateRepository := setupLocalTemplateRepo(t, "")
+	defer os.RemoveAll(localTemplateRepository)
+
+	const templatePath = "path/to/template"
+	nestedTemplateRepository := setupLocalTemplateRepo(t, templatePath)
+	defer os.RemoveAll(nestedTemplateRepository)
+
+	defer tearDownFetchTemplates(t)
+
+	templateInfo := TemplateInfo{
+		TemplateName: "ruby",
+		Platform:     "x86_64",
+		Language:     "Ruby",
+		Source:       "openfaas",
+		Description:  "Classic Ruby 2.5 template",
+		Repository:   "https://github.com/openfaas/templates",
+		Official:     "true",
+	}
+
+	t.Run("simple", func(t *testing.T) {
+		defer tearDownFetchTemplates(t)
+		out := strings.TrimSpace(formatTemplateOutput(templateInfo))
+		expect := strings.TrimSpace(`
+Name:              ruby
+Platform:          x86_64
+Language:          Ruby
+Source:            openfaas
+Description:       Classic Ruby 2.5 template
+Repository:        https://github.com/openfaas/templates
+Official Template: true
+`)
+		if out != expect {
+			t.Errorf("expected %q; got %q", expect, out)
+		}
+	})
+
+	t.Run("nested", func(t *testing.T) {
+		defer tearDownFetchTemplates(t)
+
+		templateInfo := templateInfo
+		templateInfo.TemplatePath = templatePath
+		out := strings.TrimSpace(formatTemplateOutput(templateInfo))
+		expect := strings.TrimSpace(fmt.Sprintf(`
+Name:              ruby
+Platform:          x86_64
+Language:          Ruby
+Source:            openfaas
+Description:       Classic Ruby 2.5 template
+Repository:        https://github.com/openfaas/templates
+Path:              %s
+Official Template: true
+`, templatePath))
+
+		if out != expect {
+			t.Errorf("expected %q; got %q", expect, out)
+		}
+	})
+}

commands/template_store_list.go

@@ -166,6 +166,7 @@ type TemplateInfo struct {
 	Description  string `json:"description"`
 	Repository   string `json:"repo"`
 	Official     string `json:"official"`
+	TemplatePath string `json:"path"`
 }
 
 func filterTemplate(templates []TemplateInfo, platform string) []TemplateInfo {

commands/template_store_pull.go

@@ -51,7 +51,7 @@ func runTemplateStorePull(cmd *cobra.Command, args []string) error {
 	for _, storeTemplate := range storeTemplates {
 		sourceName := fmt.Sprintf("%s/%s", storeTemplate.Source, storeTemplate.TemplateName)
 		if templateName == storeTemplate.TemplateName || templateName == sourceName {
-			err := runTemplatePull(cmd, []string{storeTemplate.Repository})
+			err := runTemplatePull(cmd, []string{storeTemplate.Repository, storeTemplate.TemplatePath})
 			if err != nil {
 				return fmt.Errorf("error while pulling template: %s : %s", storeTemplate.TemplateName, err.Error())
 			}

guide/TEMPLATE.md

@@ -2,8 +2,9 @@
 
 ## Repository structure
 
-The external repository must have a directory named ```template``` at the root directory, in which there are directories
-containing templates. The directory for each template can be freely named with alphanumeric characters and hyphen.
+The external repository must either have a directory named ```template``` at the root directory, or a nested sub-directory, 
+in which there are directories containing templates. The directory for each template can be freely named with alphanumeric 
+characters and hyphen.
 
 Example:
 
@@ -78,6 +79,14 @@ export OPENFAAS_TEMPLATE_URL="https://github.com/openfaas-incubator/golang-http-
 faas-cli template pull
 ```
 
+The previous commands assume that templates live within a directory named ```template``` at the root of the repo. 
+If the template directory lives under a nested path, you can specify the location to the template root:
+
+```bash
+faas-cli template pull https://github.com/group/template-project path/to/template
+```
+
+
 ## Pin the template repository version
 
 You may specify the branch or tag pulled by adding a URL fragment with the branch or tag name. For example, to pull the `1.0` tag of the default template repository, use

stack/schema.go

@@ -83,6 +83,7 @@ type StackConfiguration struct {
 type TemplateSource struct {
 	Name   string `yaml:"name"`
 	Source string `yaml:"source,omitempty"`
+	Path   string `yaml:"path,omitempty"`
 }
 
 // FunctionResources Memory and CPU