Skip to content

Commit 2560ef8

Browse files
smarchonesmarchone
authored
feat : add support for yaml plugins & simplify installation of plugins on client side(#452)
* add YamlPlugin * add yamlMod in models.Plugin * fix plugin fallback (first bin else yaml) * add default assets in yaml plugin * add validations * add default config * add sortplugins; dir (.plugins); migratePlugins * fix: lint, remove yamlpluginrepo * add DefaultConfig, fix lint, sort plugin multiselect in survey * add validations, remove defaultconfig * add job hook from yaml * add plugin validation (init & model) * fix lint * add defaultconfig; log yaml plugin in validation * refactor pluginMod; make validations strict * refactor and add tests * add/fix tests * fix lint * add version override in yaml; add review changes * review changes * fix rebase * fix : feedback * feat : cli syncs yaml plugins from server (#530) * refactor cmd/plugin * add client sync feature; refactor plugin manager to plugin pkg * fix lint; sync&cleanup issue * make plugin discovery static (both client and server) * fix : feedback * fix : feedback * fix : url formatting for downloading plugins * fix: remove pluginversion override; --print; and IYamlmod naming * fix: remove pluginversion override; --print; and IYamlmod naming (#554) * fix error on sync without project * minor display fix in cmd * exit with status 1 on plugin validation
1 parent 3cfc9df commit 2560ef8

24 files changed

+1149
-173
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ __pycache__
2121
optimus.yaml
2222
config.yaml
2323
coverage.txt
24-
24+
.plugins
25+
yaml-plugins.zip
2526
/api/proto/odpf/**/*
2627

2728
!/api/proto/odpf/optimus/

‎cmd/internal/survey/job_addhook.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func (j *JobAddHookSurvey) AskToAddHook(jobSpec models.JobSpec, pluginRepo model
4444
}
4545

4646
var jobSpecConfigs models.JobSpecConfigs
47-
if cliMod := selectedHook.CLIMod; cliMod != nil {
47+
if cliMod := selectedHook.GetSurveyMod(); cliMod != nil {
4848
ctx := context.Background()
4949
hookAnswers, err := j.askHookQuestions(ctx, cliMod, jobSpec.Name)
5050
if err != nil {

‎cmd/internal/survey/job_create.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,9 @@ func (*JobCreateSurvey) getTaskConfig(cliMod models.CommandLineMod, answers mode
111111

112112
func (*JobCreateSurvey) getAvailableTaskNames() []string {
113113
pluginRepo := models.PluginRegistry
114+
plugins := pluginRepo.GetTasks()
114115
var output []string
115-
for _, task := range pluginRepo.GetTasks() {
116+
for _, task := range plugins {
116117
output = append(output, task.Info().Name)
117118
}
118119
return output
@@ -215,11 +216,11 @@ func (j *JobCreateSurvey) askCreateQuestions(questions []*survey.Question) (loca
215216

216217
func (*JobCreateSurvey) getPluginCLIMod(taskName string) (models.CommandLineMod, error) {
217218
pluginRepo := models.PluginRegistry
218-
executionTask, err := pluginRepo.GetByName(taskName)
219+
plugin, err := pluginRepo.GetByName(taskName)
219220
if err != nil {
220221
return nil, err
221222
}
222-
return executionTask.CLIMod, nil
223+
return plugin.GetSurveyMod(), nil
223224
}
224225

225226
func (j *JobCreateSurvey) askPluginQuestions(cliMod models.CommandLineMod, jobName string) (models.PluginAnswers, error) {

‎cmd/plugin/install.go‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,41 +6,39 @@ import (
66

77
"github.com/odpf/optimus/cmd/internal/logger"
88
"github.com/odpf/optimus/config"
9+
"github.com/odpf/optimus/plugin"
910
)
1011

1112
type installCommand struct {
12-
logger log.Logger
13-
serverConfig *config.ServerConfig
13+
logger log.Logger
14+
serverConfig *config.ServerConfig
15+
configFilePath string `default:"config.yaml"`
1416
}
1517

1618
// NewInstallCommand initializes plugin install command
17-
func NewInstallCommand(serverConfig *config.ServerConfig) *cobra.Command {
18-
install := &installCommand{
19-
serverConfig: serverConfig,
20-
}
19+
func NewInstallCommand() *cobra.Command {
20+
install := &installCommand{}
2121
cmd := &cobra.Command{
2222
Use: "install",
23-
Short: "install and extract plugins to a dir",
23+
Short: "download and extract plugins to a dir (on server)",
2424
Example: "optimus plugin install",
2525
RunE: install.RunE,
2626
PreRunE: install.PreRunE,
2727
}
28+
cmd.PersistentFlags().StringVarP(&install.configFilePath, "config", "c", install.configFilePath, "File path for server configuration")
2829
return cmd
2930
}
3031

3132
func (i *installCommand) PreRunE(_ *cobra.Command, _ []string) error {
32-
i.logger = logger.NewClientLogger(i.serverConfig.Log)
33+
c, err := config.LoadServerConfig(i.configFilePath)
34+
if err != nil {
35+
return err
36+
}
37+
i.serverConfig = c
38+
i.logger = logger.NewClientLogger(c.Log)
3339
return nil
3440
}
3541

36-
// also used during server start
37-
func InstallPlugins(conf *config.ServerConfig, logger log.Logger) error {
38-
dst := conf.Plugin.Dir
39-
sources := conf.Plugin.Artifacts
40-
pluginManger := NewPluginManager(logger)
41-
return pluginManger.Install(dst, sources...)
42-
}
43-
4442
func (i *installCommand) RunE(_ *cobra.Command, _ []string) error {
45-
return InstallPlugins(i.serverConfig, i.logger)
43+
return plugin.InstallPlugins(i.serverConfig)
4644
}

‎cmd/plugin/plugin.go‎

Lines changed: 7 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -11,39 +11,22 @@ import (
1111
oPlugin "github.com/odpf/optimus/plugin"
1212
)
1313

14-
type pluginCommand struct {
15-
configFilePath string
16-
serverConfig *config.ServerConfig
17-
}
18-
19-
// NewPluginCommand initializes command for plugin
2014
func NewPluginCommand() *cobra.Command {
21-
plugin := pluginCommand{
22-
serverConfig: &config.ServerConfig{},
23-
}
2415
cmd := &cobra.Command{
25-
Use: "plugin",
26-
Short: "Manage plugins",
27-
Hidden: true,
16+
Use: "plugin",
17+
Short: "Manage plugins",
2818
Annotations: map[string]string{
2919
"group:dev": "true",
3020
},
31-
PersistentPreRunE: plugin.PersistentPreRunE,
3221
}
33-
cmd.PersistentFlags().StringVarP(&plugin.configFilePath, "config", "c", plugin.configFilePath, "File path for server configuration")
34-
cmd.AddCommand(NewInstallCommand(plugin.serverConfig))
22+
cmd.AddCommand(
23+
NewInstallCommand(),
24+
NewValidateCommand(),
25+
NewSyncCommand(),
26+
)
3527
return cmd
3628
}
3729

38-
func (p *pluginCommand) PersistentPreRunE(_ *cobra.Command, _ []string) error {
39-
c, err := config.LoadServerConfig(p.configFilePath)
40-
if err != nil {
41-
return err
42-
}
43-
*p.serverConfig = *c
44-
return nil
45-
}
46-
4730
// TriggerClientPluginsInit triggers initialization of all available plugins
4831
func TriggerClientPluginsInit(logLevel config.LogLevel) (cleanFn func(), err error) {
4932
pluginLogLevel := hclog.Info

‎cmd/plugin/plugin_manager.go‎

Lines changed: 0 additions & 60 deletions
This file was deleted.

‎cmd/plugin/sync.go‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package plugin
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"io"
8+
"net/http"
9+
"net/url"
10+
"os"
11+
"strings"
12+
13+
"github.com/odpf/salt/log"
14+
"github.com/spf13/cobra"
15+
16+
"github.com/odpf/optimus/cmd/internal/logger"
17+
"github.com/odpf/optimus/config"
18+
"github.com/odpf/optimus/plugin"
19+
)
20+
21+
func NewSyncCommand() *cobra.Command {
22+
sync := &syncCommand{}
23+
cmd := &cobra.Command{
24+
Use: "sync",
25+
Short: "sync plugins from server",
26+
Example: "optimus plugin sync -c optimus.yaml",
27+
PreRunE: sync.PreRunE,
28+
RunE: sync.RunE,
29+
}
30+
cmd.PersistentFlags().StringVarP(&sync.configFilePath, "config", "c", sync.configFilePath, "File path for optimus configuration")
31+
return cmd
32+
}
33+
34+
type syncCommand struct {
35+
configFilePath string
36+
clientConfig *config.ClientConfig
37+
logger log.Logger
38+
}
39+
40+
func (s *syncCommand) PreRunE(_ *cobra.Command, _ []string) error {
41+
c, err := config.LoadClientConfig(s.configFilePath)
42+
if err != nil {
43+
return errors.New("project not initialized or error in loading config : \n" + err.Error())
44+
}
45+
s.clientConfig = c
46+
s.logger = logger.NewClientLogger(c.Log)
47+
return nil
48+
}
49+
50+
func getPluginDownloadURL(host string) (*url.URL, error) {
51+
var downloadURL *url.URL
52+
var err error
53+
pluginPath := "plugins"
54+
if strings.HasPrefix(host, "http://") || strings.HasPrefix(host, "https://") {
55+
downloadURL, err = url.Parse(host)
56+
if err != nil {
57+
return nil, err
58+
}
59+
downloadURL.Path = pluginPath
60+
} else {
61+
downloadURL = &url.URL{
62+
Scheme: "http",
63+
Host: host,
64+
Path: pluginPath,
65+
}
66+
}
67+
return downloadURL, nil
68+
}
69+
70+
func (s *syncCommand) downloadArchiveFromServer() error {
71+
downloadURL, err := getPluginDownloadURL(s.clientConfig.Host)
72+
s.logger.Info(fmt.Sprintf("download URL : %s", downloadURL.String()))
73+
if err != nil {
74+
return err
75+
}
76+
req, err := http.NewRequestWithContext(context.Background(), "GET", downloadURL.String(), nil)
77+
if err != nil {
78+
return err
79+
}
80+
resp, err := http.DefaultClient.Do(req)
81+
if err != nil {
82+
return err
83+
}
84+
defer resp.Body.Close()
85+
out, err := os.Create(plugin.PluginsArchiveName)
86+
if err != nil {
87+
return err
88+
}
89+
defer out.Close()
90+
_, err = io.Copy(out, resp.Body)
91+
return err
92+
}
93+
94+
func (s *syncCommand) RunE(_ *cobra.Command, _ []string) error {
95+
err := s.downloadArchiveFromServer()
96+
if err != nil {
97+
return err
98+
}
99+
return plugin.NewPluginManager().UnArchive(
100+
plugin.PluginsArchiveName,
101+
plugin.PluginsDir,
102+
)
103+
}

0 commit comments

Comments
 (0)