mindev: Add utility to generate a data source definition from a Swagg…

…er doc

This adds the following command: `mindev datasource generate ...`

The aforementioned command will read a swagger doc (OpenAPI v2) and will
generate the appropriate data source definition out of it.

I went for v2 since there was an easy-to-use implementation ready.
Adding v3 support is left as a TODO item.

Here's a demo of how usage would look like:

```
$ cat sample.yaml
swagger: "2.0"

info:
  title: Sample API
  description: Optional multiline or single-line description in [CommonMark](http://commonmark.org/help/) or HTML.
  version: 0.1.9

basePath: http://api.example.com/v1

paths:
  /users:
    get:
      summary: Returns a list of users.
      description: Optional extended description in CommonMark or HTML.
      responses:
        "200": # status code
          description: A JSON array of user names
          content:
            application/json:
              schema:
                type: array
                items:
                  type: string

$ go run cmd/dev/main.go datasource generate sample.yaml
version: v1
type: data-source
context: {}
name: Sample-API
rest:
  def:
    get_users:
      endpoint: http://api.example.com/v1/users
      method: GET
      parse: json
      inputSchema: {}

```

Signed-off-by: Juan Antonio Osorio <[email protected]>

cmd/dev/app/datasource/datasource.go

@@ -0,0 +1,19 @@
+// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// Package datasource provides the root command for the datasource subcommands
+package datasource
+
+import "github.com/spf13/cobra"
+
+// CmdDataSource is the root command for the datasource subcommands
+func CmdDataSource() *cobra.Command {
+	var rtCmd = &cobra.Command{
+		Use:   "datasource",
+		Short: "datasource provides utilities for testing and working with data sources",
+	}
+
+	rtCmd.AddCommand(CmdGenerate())
+
+	return rtCmd
+}

cmd/dev/app/datasource/generate.go

@@ -0,0 +1,231 @@
+// SPDX-FileCopyrightText: Copyright 2023 The Minder Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package datasource
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"strings"
+
+	"buf.build/go/protoyaml"
+	"github.com/go-openapi/loads"
+	"github.com/go-openapi/spec"
+	minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
+	"github.com/spf13/cobra"
+	"google.golang.org/protobuf/types/known/structpb"
+)
+
+func CmdGenerate() *cobra.Command {
+	var generateCmd = &cobra.Command{
+		Use:     "generate",
+		Aliases: []string{"gen"},
+		Short:   "generate datasource code from an OpenAPI specification",
+		Long: `The 'datasource generate' subcommand allows you to generate datasource code from an OpenAPI
+specification`,
+		RunE:         generateCmdRun,
+		Args:         cobra.ExactArgs(1),
+		SilenceUsage: true,
+	}
+
+	return generateCmd
+}
+
+// parseOpenAPI parses an OpenAPI specification from a byte slice.
+func parseOpenAPI(filepath string) (*spec.Swagger, error) {
+	doc, err := loads.Spec(filepath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse OpenAPI spec: %w", err)
+	}
+
+	return doc.Spec(), nil
+}
+
+func initDataSourceStruct(name string) *minderv1.DataSource {
+	return &minderv1.DataSource{
+		Version: minderv1.VersionV1,
+		Type:    "data-source",
+		Name:    name,
+		Context: &minderv1.ContextV2{},
+	}
+}
+
+func initDriverStruct() *minderv1.RestDataSource {
+	return &minderv1.RestDataSource{
+		Def: make(map[string]*minderv1.RestDataSource_Def),
+	}
+}
+
+// conver the title to a valid datasource name. It should only contain alphanumeric characters and dashes.
+func swaggerTitleToDataSourceName(title string) string {
+	re := regexp.MustCompile("[^a-zA-Z0-9-]+")
+	return re.ReplaceAllString(title, "-")
+}
+
+// swaggerToDataSource generates datasource code from an OpenAPI specification.
+func swaggerToDataSource(cmd *cobra.Command, swagger *spec.Swagger) error {
+	if swagger.Info == nil {
+		return fmt.Errorf("info section is required in OpenAPI spec")
+	}
+
+	ds := initDataSourceStruct(swaggerTitleToDataSourceName(swagger.Info.Title))
+	drv := initDriverStruct()
+	ds.Driver = &minderv1.DataSource_Rest{Rest: drv}
+
+	// Add the OpenAPI specification to the DataSource
+	basepath := swagger.BasePath
+	if basepath == "" {
+		return fmt.Errorf("base path is required in OpenAPI spec")
+	}
+
+	for path, pathItem := range swagger.Paths.Paths {
+		p, err := url.JoinPath(basepath, path)
+		if err != nil {
+			cmd.PrintErrf("error joining path %s and basepath %s: %v\n Skipping", path, basepath, err)
+			continue
+		}
+
+		for method, op := range operations(pathItem) {
+			opName := generateOpName(method, path)
+			// Create a new REST DataSource definition
+			def := &minderv1.RestDataSource_Def{
+				Method:   method,
+				Endpoint: p,
+				// TODO: Make this configurable
+				Parse: "json",
+			}
+
+			is := paramsToInputSchema(op.Parameters)
+
+			if requiresMsgBody(method) {
+				def.Body = &minderv1.RestDataSource_Def_BodyFromField{
+					BodyFromField: "body",
+				}
+
+				// Add the `body` field to the input schema
+				is = inputSchemaForBody(is)
+			}
+
+			pbs, err := structpb.NewStruct(is)
+			if err != nil {
+				return fmt.Errorf("error creating input schema: %w", err)
+			}
+
+			def.InputSchema = pbs
+
+			// Add the operation to the DataSource
+			drv.Def[opName] = def
+		}
+	}
+
+	return writeDataSourceToFile(ds)
+}
+
+// Generates an operation name for a data source. Note that these names
+// must be unique within a data source. They also should be only alphanumeric
+// characters and underscores
+func generateOpName(method, path string) string {
+	// Replace all non-alphanumeric characters with underscores
+	re := regexp.MustCompile("[^a-zA-Z0-9]+")
+	return re.ReplaceAllString(fmt.Sprintf("%s_%s", strings.ToLower(method), strings.ToLower(path)), "_")
+}
+
+func operations(p spec.PathItem) map[string]*spec.Operation {
+	out := make(map[string]*spec.Operation)
+	for mstr, op := range map[string]*spec.Operation{
+		http.MethodGet:     p.Get,
+		http.MethodPut:     p.Put,
+		http.MethodPost:    p.Post,
+		http.MethodDelete:  p.Delete,
+		http.MethodOptions: p.Options,
+		http.MethodHead:    p.Head,
+		http.MethodPatch:   p.Patch,
+	} {
+		if op != nil {
+			out[mstr] = op
+		}
+	}
+
+	return out
+}
+
+func requiresMsgBody(method string) bool {
+	return method == http.MethodPost || method == http.MethodPut || method == http.MethodPatch
+}
+
+func paramsToInputSchema(params []spec.Parameter) map[string]any {
+	if len(params) == 0 {
+		return nil
+	}
+
+	is := map[string]any{
+		"type":       "object",
+		"properties": make(map[string]any),
+	}
+
+	for _, p := range params {
+		is["properties"].(map[string]any)[p.Name] = map[string]any{
+			// TODO: Add support for more types
+			"type": "string",
+		}
+
+		if p.Required {
+			if _, ok := is["required"]; !ok {
+				is["required"] = make([]string, 0)
+			}
+
+			is["required"] = append(is["required"].([]string), p.Name)
+		}
+	}
+
+	return is
+}
+
+func inputSchemaForBody(is map[string]any) map[string]any {
+	if is == nil {
+		is = map[string]any{
+			"type":       "object",
+			"properties": make(map[string]any),
+		}
+	}
+
+	is["properties"].(map[string]any)["body"] = map[string]any{
+		"type": "object",
+	}
+
+	return is
+}
+
+func writeDataSourceToFile(ds *minderv1.DataSource) error {
+	// Convert the DataSource to YAML
+	dsYAML, err := protoyaml.MarshalOptions{
+		Indent: 2,
+	}.Marshal(ds)
+	if err != nil {
+		return fmt.Errorf("error marshalling DataSource to YAML: %w", err)
+	}
+
+	// Write the YAML to a file
+	if _, err := os.Stdout.Write(dsYAML); err != nil {
+		return fmt.Errorf("error writing DataSource to file: %w", err)
+	}
+
+	return nil
+}
+
+// generateCmdRun is the entry point for the 'datasource generate' command.
+func generateCmdRun(cmd *cobra.Command, args []string) error {
+	// We've already validated that there is exactly one argument via the cobra.ExactArgs(1) call
+	filePath := args[0]
+
+	// Parse the OpenAPI specification
+	swagger, err := parseOpenAPI(filePath)
+	if err != nil {
+		return fmt.Errorf("error parsing OpenAPI spec: %w", err)
+	}
+
+	return swaggerToDataSource(cmd, swagger)
+}

cmd/dev/app/root.go

@@ -10,6 +10,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/mindersec/minder/cmd/dev/app/bundles"
+	"github.com/mindersec/minder/cmd/dev/app/datasource"
 	"github.com/mindersec/minder/cmd/dev/app/image"
 	"github.com/mindersec/minder/cmd/dev/app/rule_type"
 	"github.com/mindersec/minder/cmd/dev/app/testserver"
@@ -29,6 +30,7 @@ https://docs.stacklok.com/minder`,
 	cmd.AddCommand(image.CmdImage())
 	cmd.AddCommand(testserver.CmdTestServer())
 	cmd.AddCommand(bundles.CmdBundle())
+	cmd.AddCommand(datasource.CmdDataSource())
 
 	return cmd
 }

go.mod

@@ -4,6 +4,7 @@ go 1.23.4
 
 require (
 	buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.2-20241127180247-a33202765966.1
+	buf.build/go/protoyaml v0.3.1
 	github.com/ThreeDotsLabs/watermill v1.4.2
 	github.com/ThreeDotsLabs/watermill-sql/v3 v3.1.0
 	github.com/alexdrl/zerowater v0.0.3
@@ -308,7 +309,7 @@ require (
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/loads v0.22.0 // indirect
 	github.com/go-openapi/runtime v0.28.0 // indirect
-	github.com/go-openapi/spec v0.21.0 // indirect
+	github.com/go-openapi/spec v0.21.0
 	github.com/go-openapi/strfmt v0.23.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-openapi/validate v0.24.0 // indirect

go.sum

@@ -1,5 +1,7 @@
 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.2-20241127180247-a33202765966.1 h1:BICM6du/XzvEgeorNo4xgohK3nMTmEPViGyd5t7xVqk=
 buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.2-20241127180247-a33202765966.1/go.mod h1:JnMVLi3qrNYPODVpEKG7UjHLl/d2zR221e66YCSmP2Q=
+buf.build/go/protoyaml v0.3.1 h1:ucyzE7DRnjX+mQ6AH4JzN0Kg50ByHHu+yrSKbgQn2D4=
+buf.build/go/protoyaml v0.3.1/go.mod h1:0TzNpFQDXhwbkXb/ajLvxIijqbve+vMQvWY/b3/Dzxg=
 cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo=
 cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=