Add command to check for missing data (#29)

* Add check_missing command to query for missing entries.
* Fix exit handler with expected exits

check_command.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"text/tabwriter"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/appuio/appuio-cloud-reporting/pkg/check"
+	"github.com/appuio/appuio-cloud-reporting/pkg/db"
+)
+
+type checkMissingCommand struct {
+	DatabaseURL string
+}
+
+var checkMissingCommandName = "check_missing"
+
+func newCheckMissingCommand() *cli.Command {
+	command := &checkMissingCommand{}
+	return &cli.Command{
+		Name:   checkMissingCommandName,
+		Usage:  "Check for missing data in the database",
+		Before: command.before,
+		Action: command.execute,
+		Flags: []cli.Flag{
+			newDbURLFlag(&command.DatabaseURL),
+		},
+	}
+}
+
+func (cmd *checkMissingCommand) before(context *cli.Context) error {
+	return LogMetadata(context)
+}
+
+func (cmd *checkMissingCommand) execute(cliCtx *cli.Context) error {
+	ctx := cliCtx.Context
+	log := AppLogger(ctx).WithName(migrateCommandName)
+
+	log.V(1).Info("Opening database connection", "url", cmd.DatabaseURL)
+	rdb, err := db.Openx(cmd.DatabaseURL)
+	if err != nil {
+		return fmt.Errorf("could not open database connection: %w", err)
+	}
+
+	log.V(1).Info("Begin transaction")
+	tx, err := rdb.BeginTxx(ctx, &sql.TxOptions{ReadOnly: true})
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	missing, err := check.Missing(ctx, tx)
+	if err != nil {
+		return err
+	}
+
+	if len(missing) == 0 {
+		return nil
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	defer w.Flush()
+	fmt.Fprint(w, "Table\tMissing Field\tID\tSource\n")
+	for _, m := range missing {
+		fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", m.Table, m.MissingField, m.ID, m.Source)
+	}
+
+	return cli.Exit(fmt.Sprintf("%d missing entries found.", len(missing)), 1)
+}

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"os/signal"
@@ -66,14 +67,21 @@ func newApp() (context.Context, context.CancelFunc, *cli.App) {
 		Commands: []*cli.Command{
 			newMigrateCommand(),
 			newReportCommand(),
+			newCheckMissingCommand(),
 			newInvoiceCommand(),
 		},
 		ExitErrHandler: func(context *cli.Context, err error) {
-			if err != nil {
-				AppLogger(context.Context).WithCallDepth(1).Error(err, "fatal error")
-				os.Exit(1)
-				cli.HandleExitCoder(cli.Exit("", 1))
+			if err == nil {
+				return
 			}
+			// Don't show stack trace if the error is expected (someone called cli.Exit())
+			var exitErr cli.ExitCoder
+			if errors.As(err, &exitErr) {
+				cli.HandleExitCoder(err)
+				return
+			}
+			AppLogger(context.Context).WithCallDepth(1).Error(err, "fatal error")
+			cli.OsExiter(1)
 		},
 	}
 	// There is logr.NewContext(...) which returns a context that carries the logger instance.

migrate_command.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/appuio/appuio-cloud-reporting/pkg/db"
 	"github.com/urfave/cli/v2"
@@ -66,7 +65,7 @@ func (cmd *migrateCommand) execute(context *cli.Context) error {
 
 		// non-zero exit code could be used in scripts
 		if len(pm) > 0 {
-			os.Exit(1)
+			cli.Exit("Pending migrations found.", 1)
 		}
 		return nil
 	}

pkg/check/missing.go

@@ -0,0 +1,38 @@
+package check
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/jmoiron/sqlx"
+)
+
+// MissingField represents a missing field.
+type MissingField struct {
+	Table string
+
+	ID     string
+	Source string
+
+	MissingField string
+}
+
+const missingQuery = `
+	SELECT 'categories' as table, id, source, 'target' as missingfield FROM categories WHERE target IS NULL OR target = ''
+	UNION ALL
+	SELECT 'tenants' as table, id, source, 'target' as missingfield FROM tenants WHERE target IS NULL OR target = ''
+	UNION ALL
+	SELECT 'products' as table, id, source, 'target' as missingfield FROM products WHERE target IS NULL OR target = ''
+	UNION ALL
+	SELECT 'products' as table, id, source, 'amount' as missingfield FROM products WHERE amount = 0
+	UNION ALL
+	SELECT 'products' as table, id, source, 'unit' as missingfield FROM products WHERE unit = ''
+`
+
+// Missing checks for missing fields in the reporting database.
+func Missing(ctx context.Context, tx sqlx.QueryerContext) ([]MissingField, error) {
+	var missing []MissingField
+
+	err := sqlx.SelectContext(ctx, tx, &missing, fmt.Sprintf(`WITH missing AS (%s) SELECT * FROM missing ORDER BY "table",missingfield,source`, missingQuery))
+	return missing, err
+}

pkg/check/missing_test.go

@@ -0,0 +1,82 @@
+package check_test
+
+import (
+	"context"
+	"database/sql"
+	"testing"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/appuio/appuio-cloud-reporting/pkg/check"
+	"github.com/appuio/appuio-cloud-reporting/pkg/db"
+	"github.com/appuio/appuio-cloud-reporting/pkg/db/dbtest"
+)
+
+type TestSuite struct {
+	dbtest.Suite
+}
+
+func (s *TestSuite) TestMissingFields() {
+	t := s.T()
+	tx := s.Begin()
+	defer tx.Rollback()
+
+	m, err := check.Missing(context.Background(), tx)
+	require.NoError(t, err)
+	require.Len(t, m, 0)
+
+	expectedMissing := s.requireMissingTestEntries(t, tx)
+
+	m, err = check.Missing(context.Background(), tx)
+	require.NoError(t, err)
+	require.Equal(t, expectedMissing, m)
+}
+
+func (s *TestSuite) requireMissingTestEntries(t *testing.T, tdb *sqlx.Tx) []check.MissingField {
+	var catEmptyTarget db.Category
+	require.NoError(t,
+		db.GetNamed(tdb, &catEmptyTarget,
+			"INSERT INTO categories (source,target) VALUES (:source,:target) RETURNING *", db.Category{
+				Source: "af-south-1:uroboros-research",
+			}))
+
+	var tenantEmptyTarget db.Tenant
+	require.NoError(t,
+		db.GetNamed(tdb, &tenantEmptyTarget,
+			"INSERT INTO tenants (source,target) VALUES (:source,:target) RETURNING *", db.Tenant{
+				Source: "tricell",
+			}))
+
+	var productEmptyTarget db.Product
+	require.NoError(t,
+		db.GetNamed(tdb, &productEmptyTarget,
+			"INSERT INTO products (source,target,amount,unit,during) VALUES (:source,:target,:amount,:unit,:during) RETURNING *", db.Product{
+				Source: "test_memory:us-rac-2",
+				Amount: 3,
+				Unit:   "X",
+				During: db.InfiniteRange(),
+			}))
+
+	var productEmptyAmountAndUnit db.Product
+	require.NoError(t,
+		db.GetNamed(tdb, &productEmptyAmountAndUnit,
+			"INSERT INTO products (source,target,amount,unit,during) VALUES (:source,:target,:amount,:unit,:during) RETURNING *", db.Product{
+				Source: "test_storage:us-rac-2",
+				Target: sql.NullString{Valid: true, String: "666"},
+				During: db.InfiniteRange(),
+			}))
+
+	return []check.MissingField{
+		{Table: "categories", MissingField: "target", ID: catEmptyTarget.Id, Source: catEmptyTarget.Source},
+		{Table: "products", MissingField: "amount", ID: productEmptyAmountAndUnit.Id, Source: productEmptyAmountAndUnit.Source},
+		{Table: "products", MissingField: "target", ID: productEmptyTarget.Id, Source: productEmptyTarget.Source},
+		{Table: "products", MissingField: "unit", ID: productEmptyAmountAndUnit.Id, Source: productEmptyAmountAndUnit.Source},
+		{Table: "tenants", MissingField: "target", ID: tenantEmptyTarget.Id, Source: tenantEmptyTarget.Source},
+	}
+}
+
+func TestTestSuite(t *testing.T) {
+	suite.Run(t, new(TestSuite))
+}