[WIP] Make mixtool install install dashboards.

cmd/mixtool/install.go

@@ -15,20 +15,16 @@
 package main
 
 import (
-	"bytes"
+	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
 	"os"
 	"path"
-	"path/filepath"
 	"strings"
 
-	"github.com/monitoring-mixins/mixtool/pkg/jsonnetbundler"
-	"github.com/monitoring-mixins/mixtool/pkg/mixer"
-
+	gapi "github.com/grafana/grafana-api-golang-client"
 	"github.com/urfave/cli"
+
+	"github.com/monitoring-mixins/mixtool/pkg/mixer"
 )
 
 func installCommand() cli.Command {
@@ -38,204 +34,100 @@ func installCommand() cli.Command {
 		Description: "Install a mixin from a repository",
 		Action:      installAction,
 		Flags: []cli.Flag{
-			cli.StringFlag{
-				Name:  "bind-address",
-				Usage: "Address to bind HTTP server to.",
-				Value: "http://127.0.0.1:8080",
-			},
-			cli.StringFlag{
-				Name:  "directory, d",
-				Usage: "Path where the downloaded mixin is saved. If it doesn't exist already it will be created",
-			},
-			cli.BoolFlag{
-				Name:  "put, p",
-				Usage: "Specify this flag when you want to send PUT request to mixtool server once the mixins are generated",
+			cli.StringSliceFlag{
+				Name: "jpath, J",
 			},
 		},
 	}
 }
 
-// Downloads a mixin from a given repository given by url and places into directory
-// by running jb init and jb install
-func downloadMixin(url string, jsonnetHome string, directory string) error {
-	// intialize the jsonnet bundler library
-	err := jsonnetbundler.InitCommand(directory)
-	if err != nil {
-		return fmt.Errorf("jsonnet bundler init failed %v", err)
-	}
-
-	// by default, set the single flag to false
-	err = jsonnetbundler.InstallCommand(directory, jsonnetHome, []string{url}, false)
-	if err != nil {
-		return fmt.Errorf("jsonnet bundler install failed %v", err)
-	}
-
-	return nil
-}
-
-// Gets mixins from default website - mostly copied from list.go
-func getMixins() ([]mixin, error) {
-	body, err := queryWebsite(defaultWebsite)
-	if err != nil {
-		return nil, err
-	}
-	mixins, err := parseMixinJSON(body)
-	if err != nil {
-		return nil, err
+func installAction(c *cli.Context) error {
+	filename := c.Args().First()
+	if filename == "" {
+		return fmt.Errorf("no jsonnet file given")
 	}
-	return mixins, nil
-}
-
-func generateMixin(directory string, jsonnetHome string, mixinURL string, options mixer.GenerateOptions) ([]byte, error) {
 
-	mixinBaseDirectory := filepath.Join(directory)
-
-	err := os.Chdir(mixinBaseDirectory)
+	grafanaURL := os.Getenv("GRAFANA_URL")
+	grafanaKey := os.Getenv("GRAFANA_TOKEN")
+	client, err := gapi.New(grafanaURL, gapi.Config{APIKey: grafanaKey})
 	if err != nil {
-		return nil, fmt.Errorf("Cannot cd into directory %s", err)
+		return err
 	}
 
-	// generate alerts, rules, grafana dashboards
-	// empty files if not present
-
-	u, err := url.Parse(mixinURL)
-	if err != nil {
-		return nil, fmt.Errorf("url parse %v", err)
+	if _, err := client.Folders(); err != nil {
+		return fmt.Errorf("failed to ping grafana: %v", err)
 	}
 
-	// absolute directory is the same as the download url stripped of the scheme
-	absDirectory := path.Join(u.Host, u.Path)
-	absDirectory = strings.TrimLeft(absDirectory, "/:")
-	absDirectory = strings.TrimSuffix(absDirectory, ".git")
-
-	importFile := filepath.Join(absDirectory, "mixin.libsonnet")
-
-	// generate rules, dashboards, alerts
-	err = generateAll(importFile, options)
+	jPathFlag := c.StringSlice("jpath")
+	jPathFlag, err = availableVendor(filename, jPathFlag)
 	if err != nil {
-		return nil, fmt.Errorf("generateAll: %w", err)
+		return err
 	}
 
-	out, err := generateRulesAlerts(importFile, options)
-	if err != nil {
-		return nil, fmt.Errorf("generateRulesAlerts %w", err)
+	generateCfg := mixer.GenerateOptions{
+		AlertsFilename: "alerts.yaml",
+		RulesFilename:  "rules.yaml",
+		Directory:      "dashboards_out",
+		JPaths:         jPathFlag,
+		YAML:           true,
 	}
 
-	return out, nil
-
-}
-
-func putMixin(content []byte, bindAddress string) error {
-	u, err := url.Parse(bindAddress)
-	if err != nil {
+	if err := generateAll(filename, generateCfg); err != nil {
 		return err
 	}
-	u.Path = path.Join(u.Path, "/api/v1/rules")
 
-	r := bytes.NewReader(content)
-	req, err := http.NewRequest("PUT", u.String(), r)
+	ds, err := os.ReadDir("dashboards_out")
 	if err != nil {
 		return err
 	}
 
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("response from server %v", err)
-	}
-	if resp.StatusCode == 200 {
-		fmt.Println("PUT alerts OK")
-	} else {
-		responseData, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return fmt.Errorf("failed to response body in putMixin, %w", err)
+	for _, d := range ds {
+		if d.IsDir() {
+			continue
 		}
-		return fmt.Errorf("non 200 response code: %d, info: %s", resp.StatusCode, string(responseData))
-	}
-	return nil
-}
-
-func installAction(c *cli.Context) error {
-	directory := c.String("directory")
-	if directory == "" {
-		return fmt.Errorf("Must specify a directory to download mixin")
-	}
 
-	_, err := os.Stat(directory)
-	if os.IsNotExist(err) {
-		err = os.MkdirAll(directory, 0755)
+		buf, err := os.ReadFile(path.Join("dashboards_out", d.Name()))
 		if err != nil {
-			return fmt.Errorf("could not create directory %v", err)
+			return err
 		}
-	}
 
-	mixinPath := c.Args().First()
-	if mixinPath == "" {
-		return fmt.Errorf("Expected the url of mixin repository or name of the mixin. Show available mixins using mixtool list")
-	}
+		var dashboardJson map[string]interface{}
+		if err := json.Unmarshal(buf, &dashboardJson); err != nil {
+			return err
+		}
 
-	mixinsList, err := getMixins()
-	if err != nil {
-		return fmt.Errorf("getMixins failed %v", err)
+		uploadDashboard(client, dashboardJson)
 	}
 
-	var mixinURL string
-	if _, err := url.ParseRequestURI(mixinPath); err != nil {
-		// check if the name exists in mixinsList
-		found := false
-		for _, m := range mixinsList {
-			if m.Name == mixinPath {
-				// join paths together
-				u, err := url.Parse(m.URL)
-				if err != nil {
-					return fmt.Errorf("url parse failed %v", err)
-				}
-				u.Path = path.Join(u.Path, m.Subdir)
-				mixinURL = u.String()
-				found = true
-				break
-			}
-		}
-		if !found {
-			return fmt.Errorf("Could not find mixin with name %s", mixinPath)
-		}
-	} else {
-		mixinURL = mixinPath
-	}
+	return nil
+}
 
-	if mixinURL == "" {
-		return fmt.Errorf("Empty mixinURL")
+func uploadDashboard(client *gapi.Client, dashboardJson map[string]interface{}) error {
+	var uid string
+	tmp, ok := dashboardJson["uid"]
+	if !ok {
+		return fmt.Errorf("missing uid from dashboard")
 	}
 
-	// by default jsonnet packages are downloaded under vendor
-	jsonnetHome := "vendor"
+	if uid, ok = tmp.(string); !ok {
+		return fmt.Errorf("bad uid in dashboard")
+	}
 
-	err = downloadMixin(mixinURL, jsonnetHome, directory)
-	if err != nil {
+	dashboard, err := client.DashboardByUID(uid)
+	if err != nil && !strings.HasPrefix(err.Error(), "status: 404") {
 		return err
 	}
 
-	generateCfg := mixer.GenerateOptions{
-		AlertsFilename: "alerts.yaml",
-		RulesFilename:  "rules.yaml",
-		Directory:      "dashboards_out",
-		JPaths:         []string{"./vendor"},
-		YAML:           true,
-	}
+	fmt.Printf("Updating dashboard %s (exists: %t)\n", uid, err == nil)
 
-	rulesAlerts, err := generateMixin(directory, jsonnetHome, mixinURL, generateCfg)
 	if err != nil {
-		return err
+		dashboard = &gapi.Dashboard{}
 	}
 
-	// check if put address flag was set
-
-	if c.Bool("put") {
-		bindAddress := c.String("bind-address")
-		// run put requests onto the server
-		err = putMixin(rulesAlerts, bindAddress)
-		if err != nil {
-			return err
-		}
+	dashboard.Model = dashboardJson
+	dashboard.Overwrite = true
+	if _, err := client.NewDashboard(*dashboard); err != nil {
+		return err
 	}
 
 	return nil

cmd/mixtool/install_test.go

@@ -13,84 +13,3 @@
 // limitations under the License.
 
 package main
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-	"testing"
-
-	"github.com/monitoring-mixins/mixtool/pkg/mixer"
-	"github.com/stretchr/testify/assert"
-)
-
-// Try to install every mixin from the mixin repository
-// verify that each package generated has the yaml files
-func TestInstallMixin(t *testing.T) {
-	t.Skip("Test is unreliable as it depends on external mixins.")
-
-	body, err := queryWebsite(defaultWebsite)
-	if err != nil {
-		t.Errorf("failed to query website %v", err)
-	}
-	mixins, err := parseMixinJSON(body)
-	if err != nil {
-		t.Errorf("failed to parse mixin body %v", err)
-	}
-
-	// download each mixin in turn
-	for _, m := range mixins {
-		t.Run(m.Name, func(t *testing.T) {
-			testInstallMixin(t, m)
-		})
-	}
-}
-
-func testInstallMixin(t *testing.T, m mixin) {
-	tmpdir, err := ioutil.TempDir("", "mixtool-install")
-	assert.NoError(t, err)
-	defer os.RemoveAll(tmpdir)
-
-	generateCfg := mixer.GenerateOptions{
-		AlertsFilename: "alerts.yaml",
-		RulesFilename:  "rules.yaml",
-		Directory:      "dashboards_out",
-		JPaths:         []string{"vendor"},
-		YAML:           true,
-	}
-
-	mixinURL := path.Join(m.URL, m.Subdir)
-
-	fmt.Printf("installing %v\n", mixinURL)
-	dldir := path.Join(tmpdir, m.Name+"mixin-test")
-
-	err = os.Mkdir(dldir, 0755)
-	assert.NoError(t, err)
-
-	jsonnetHome := "vendor"
-
-	err = downloadMixin(mixinURL, jsonnetHome, dldir)
-	assert.NoError(t, err)
-
-	_, err = generateMixin(dldir, jsonnetHome, mixinURL, generateCfg)
-	assert.NoError(t, err)
-
-	// verify that alerts, rules, dashboards exist
-	err = os.Chdir(dldir)
-	assert.NoError(t, err)
-
-	if _, err := os.Stat("alerts.yaml"); os.IsNotExist(err) {
-		t.Errorf("expected alerts.yaml in %s", dldir)
-	}
-
-	if _, err := os.Stat("rules.yaml"); os.IsNotExist(err) {
-		t.Errorf("expected rules.yaml in %s", dldir)
-	}
-
-	if _, err := os.Stat("dashboards_out"); os.IsNotExist(err) {
-		t.Errorf("expected dashboards_out in %s", dldir)
-	}
-
-	// verify that the output of alerts and rules matches using jsonnet
-}

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/google/go-jsonnet v0.18.0
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/grafana/dashboard-linter v0.0.0-20211209175238-5246a8a7dacf
+	github.com/grafana/grafana-api-golang-client v0.2.5 // indirect
 	github.com/grafana/tanka v0.19.0
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect

go.sum

@@ -580,6 +580,7 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-zookeeper/zk v1.0.2 h1:4mx0EYENAdX/B/rbunjlt5+4RTA/a9SMHBRuSKdGxPM=
 github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
+github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
 github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
 github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
@@ -761,6 +762,8 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grafana/dashboard-linter v0.0.0-20211209175238-5246a8a7dacf h1:SdIHpvUaJTqSRDk/eFrbHsnLsUcekcvq/2Q7/FYJvbY=
 github.com/grafana/dashboard-linter v0.0.0-20211209175238-5246a8a7dacf/go.mod h1://CkibdjQDn6tp3o6QEso8n75KsHI/14BnRRDbx2hN0=
+github.com/grafana/grafana-api-golang-client v0.2.5 h1:rZhy+0r/B/zV9gZ6BHAHY73JOi1DgVUegghF54yLn/M=
+github.com/grafana/grafana-api-golang-client v0.2.5/go.mod h1:24W29gPe9yl0/3A9X624TPkAOR8DpHno490cPwnkv8E=
 github.com/grafana/tanka v0.19.0 h1:Cct6hIpQ2PczIK90h0d3X1XbFmM0q+hI42PEU0ieMAk=
 github.com/grafana/tanka v0.19.0/go.mod h1:t0ickZJGuccdEsuBsrV7eEFeAJBYrtfilwYQWkaQZdg=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -785,6 +788,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
 github.com/hashicorp/go-hclog v0.12.2 h1:F1fdYblUEsxKiailtkhCCG2g4bipEgaHiDc8vffNpD4=
 github.com/hashicorp/go-hclog v0.12.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=