feat: add migrate sql up|down

This patch adds the ability to execute down migrations using:

```
hydra migrate sql down -e --steps {num_of_steps}
```

Please read `hydra migrate sql down --help` carefully.

Going forward, please use the following command to apply up migrations

```
hydra migrate sql up -e --steps {num_of_steps}
```

instead of the previous, now deprecated

```
hydra migrate sql -e --steps {num_of_steps}
```

command.

See ory-corp/cloud#7350

cmd/cli/handler_migrate.go

@@ -5,7 +5,6 @@ package cli
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"io"
 	"io/fs"
@@ -317,12 +316,12 @@ func (h *MigrateHandler) makePersister(cmd *cobra.Command, args []string) (p per
 	return d.Persister(), nil
 }
 
-func (h *MigrateHandler) MigrateSQL(cmd *cobra.Command, args []string) (err error) {
+func (h *MigrateHandler) MigrateSQLUp(cmd *cobra.Command, args []string) (err error) {
 	p, err := h.makePersister(cmd, args)
 	if err != nil {
 		return err
 	}
-	conn := p.Connection(context.Background())
+	conn := p.Connection(cmd.Context())
 	if conn == nil {
 		_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Migrations can only be executed against a SQL-compatible driver but DSN is not a SQL source.")
 		return cmdx.FailSilently(cmd)
@@ -334,15 +333,16 @@ func (h *MigrateHandler) MigrateSQL(cmd *cobra.Command, args []string) (err erro
 	}
 
 	// convert migration tables
-	if err := p.PrepareMigration(context.Background()); err != nil {
+	if err := p.PrepareMigration(cmd.Context()); err != nil {
 		_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not convert the migration table:\n%+v\n", err)
 		return cmdx.FailSilently(cmd)
 	}
 
 	// print migration status
 	_, _ = fmt.Fprintln(cmd.OutOrStdout(), "The following migration is planned:")
 
-	status, err := p.MigrationStatus(context.Background())
+	// print migration status
+	status, err := p.MigrationStatus(cmd.Context())
 	if err != nil {
 		fmt.Fprintf(cmd.ErrOrStderr(), "Could not get the migration status:\n%+v\n", errorsx.WithStack(err))
 		return cmdx.FailSilently(cmd)
@@ -358,7 +358,86 @@ func (h *MigrateHandler) MigrateSQL(cmd *cobra.Command, args []string) (err erro
 	}
 
 	// apply migrations
-	if err := p.MigrateUp(context.Background()); err != nil {
+	if err := p.MigrateUp(cmd.Context()); err != nil {
+		_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not apply migrations:\n%+v\n", errorsx.WithStack(err))
+		return cmdx.FailSilently(cmd)
+	}
+
+	_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Successfully applied migrations!")
+	return nil
+}
+
+func (h *MigrateHandler) MigrateSQLDown(cmd *cobra.Command, args []string) (err error) {
+	p, err := h.makePersister(cmd, args)
+	if err != nil {
+		return err
+	}
+
+	steps := flagx.MustGetInt(cmd, "steps")
+	if steps < 0 {
+		_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Flag --steps must be a positive integer.")
+		return cmdx.FailSilently(cmd)
+	}
+
+	conn := p.Connection(cmd.Context())
+	if conn == nil {
+		_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Migrations can only be executed against a SQL-compatible driver but DSN is not a SQL source.")
+		return cmdx.FailSilently(cmd)
+	}
+
+	if err := conn.Open(); err != nil {
+		_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not open the database connection:\n%+v\n", err)
+		return cmdx.FailSilently(cmd)
+	}
+
+	// convert migration tables
+	if err := p.PrepareMigration(cmd.Context()); err != nil {
+		_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not convert the migration table:\n%+v\n", err)
+		return cmdx.FailSilently(cmd)
+	}
+
+	status, err := p.MigrationStatus(cmd.Context())
+	if err != nil {
+		_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get the migration status:\n%+v\n", errorsx.WithStack(err))
+		return cmdx.FailSilently(cmd)
+	}
+
+	// Now we need to rollback the last `steps` migrations that have a status of "Applied":
+
+	var count int
+	var rollingBack int
+	for i := len(status) - 1; i >= 0; i-- {
+		if status[i].State == popx.Applied {
+			count++
+			if count <= steps {
+				status[i].State = "Rollback"
+				rollingBack++
+			}
+		}
+	}
+
+	// print migration status
+	_, _ = fmt.Fprintln(cmd.OutOrStdout(), "The following migration plan is planned:")
+	_ = status.Write(os.Stdout)
+
+	if rollingBack < 1 {
+		_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "")
+		_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "There are apparently no migrations to roll back.")
+		_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Please provide the --steps argument with a value larger than 0.")
+		_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "")
+		return cmdx.FailSilently(cmd)
+	}
+
+	if !flagx.MustGetBool(cmd, "yes") {
+		_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "To skip the next question use flag --yes (at your own risk).")
+		if !cmdx.AskForConfirmation("Do you wish to execute this migration plan?", nil, nil) {
+			_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Migration aborted.")
+			return nil
+		}
+	}
+
+	// apply migrations
+	if err := p.MigrateDown(cmd.Context(), steps); err != nil {
 		_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not apply migrations:\n%+v\n", errorsx.WithStack(err))
 		return cmdx.FailSilently(cmd)
 	}
@@ -372,7 +451,7 @@ func (h *MigrateHandler) MigrateStatus(cmd *cobra.Command, args []string) error
 	if err != nil {
 		return err
 	}
-	conn := p.Connection(context.Background())
+	conn := p.Connection(cmd.Context())
 	if conn == nil {
 		_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Migrations can only be checked against a SQL-compatible driver but DSN is not a SQL source.")
 		return cmdx.FailSilently(cmd)
@@ -408,5 +487,4 @@ func (h *MigrateHandler) MigrateStatus(cmd *cobra.Command, args []string) error
 
 	cmdx.PrintTable(cmd, s)
 	return nil
-
 }

cmd/migrate_sql.go

@@ -15,8 +15,9 @@ import (
 
 func NewMigrateSqlCmd(slOpts []servicelocatorx.Option, dOpts []driver.OptionsModifier, cOpts []configx.OptionModifier) *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "sql <database-url>",
-		Short: "Create SQL schemas and apply migration plans",
+		Use:        "sql <database-url>",
+		Deprecated: "Please use `migrate sql up` instead.",
+		Short:      "Perform SQL migrations",
 		Long: `Run this command on a fresh SQL installation and when you upgrade Hydra to a new minor version. For example,
 upgrading Hydra 0.7.0 to 0.8.0 requires running this command.
 
@@ -30,11 +31,13 @@ You can read in the database URL using the -e flag, for example:
 ### WARNING ###
 
 Before running this command on an existing database, create a back up!`,
-		RunE: cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateSQL,
+		RunE: cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateSQLUp,
 	}
 
 	cmd.Flags().BoolP("read-from-env", "e", false, "If set, reads the database connection string from the environment variable DSN or config file key dsn.")
 	cmd.Flags().BoolP("yes", "y", false, "If set all confirmation requests are accepted without user interaction.")
 
+	cmd.AddCommand(NewMigrateSqlDownCmd(slOpts, dOpts, cOpts))
+	cmd.AddCommand(NewMigrateSqlUpCmd(slOpts, dOpts, cOpts))
 	return cmd
 }

cmd/migrate_sql_down.go

@@ -0,0 +1,55 @@
+// Copyright © 2022 Ory Corp
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/ory/hydra/v2/driver"
+	"github.com/ory/x/configx"
+	"github.com/ory/x/servicelocatorx"
+
+	"github.com/ory/hydra/v2/cmd/cli"
+)
+
+func NewMigrateSqlDownCmd(slOpts []servicelocatorx.Option, dOpts []driver.OptionsModifier, cOpts []configx.OptionModifier) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "down <database-url>",
+		Short: "Roll back SQL migrations",
+		Args:  cobra.RangeArgs(0, 1),
+		Example: `Revert the most recent migration:
+	DSN=... hydra migrate sql down -e --steps 1
+
+Review migrations to decide which one to roll back:
+	DSN=... hydra migrate sql down -e --steps 0
+`,
+		Long: `Run this command to roll back SQL migrations. This command is useful when you want to revert to a previous version of Ory Hydra.
+
+:::warning
+
+Before running this command on an existing database, create a back up. This command can be destructive as it may drop
+indices, columns, or whole tables. Run this command close to the SQL instance (same VPC / same machine).
+
+:::
+
+This command will not execute anything unless you provide a --steps flag with a value greater than 0. Per default, this
+command will roll back one migration at a time. You can specify the number of migrations to roll back using the --steps
+flag.
+
+Choosing how many migrations to roll back depends on the current state of the database. Please first execute the command
+without the --steps flag to review the migrations and decide which one to roll back.
+
+Once you have decided which migration to roll back, you can use the --steps flag to specify the number of migrations to
+roll back. For example, to roll back the most recent migration, you can run:
+
+	DSN=... hydra migrate sql down -e --steps 1`,
+		RunE: cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateSQLDown,
+	}
+
+	cmd.Flags().BoolP("read-from-env", "e", false, "If set, reads the database connection string from the environment variable DSN or config file key dsn.")
+	cmd.Flags().BoolP("yes", "y", false, "If set all confirmation requests are accepted without user interaction.")
+	cmd.Flags().Int("steps", 0, "The number of migrations to roll back.")
+
+	return cmd
+}

cmd/migrate_sql_up.go

@@ -0,0 +1,41 @@
+// Copyright © 2022 Ory Corp
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/ory/hydra/v2/driver"
+	"github.com/ory/x/configx"
+	"github.com/ory/x/servicelocatorx"
+
+	"github.com/ory/hydra/v2/cmd/cli"
+)
+
+func NewMigrateSqlUpCmd(slOpts []servicelocatorx.Option, dOpts []driver.OptionsModifier, cOpts []configx.OptionModifier) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "up <database-url>",
+		Args:  cobra.RangeArgs(0, 1),
+		Short: "Create and upgrade the Ory Hydra SQL schema",
+		Long: `Run this command on a fresh SQL installation and when you upgrade Ory Hydra to a newer version.
+
+:::warning
+
+Before running this command on an existing database, create a back up. This command can be destructive as it may drop
+indices, columns, or whole tables. Run this command close to the SQL instance (same VPC / same machine).
+
+:::
+
+It is recommended to review the migrations before running them. You can do this by running the command without the --yes
+flag:
+
+	DSN=... hydra migrate sql up -e`,
+		RunE: cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateSQLUp,
+	}
+
+	cmd.Flags().BoolP("read-from-env", "e", false, "If set, reads the database connection string from the environment variable DSN or config file key dsn.")
+	cmd.Flags().BoolP("yes", "y", false, "If set all confirmation requests are accepted without user interaction.")
+
+	return cmd
+}

cmd/root.go

@@ -7,6 +7,8 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/pkg/errors"
+
 	"github.com/ory/x/cmdx"
 
 	"github.com/ory/hydra/v2/driver"
@@ -99,8 +101,11 @@ func RegisterCommandRecursive(parent *cobra.Command, slOpts []servicelocatorx.Op
 
 // Execute adds all child commands to the root command sets flags appropriately.
 func Execute() {
-	if err := NewRootCmd(nil, nil, nil).Execute(); err != nil {
-		fmt.Println(err)
-		os.Exit(-1)
+	c := NewRootCmd(nil, nil, nil)
+	if err := c.Execute(); err != nil {
+		if !errors.Is(err, cmdx.ErrNoPrintButFail) {
+			_, _ = fmt.Fprintln(c.ErrOrStderr(), err)
+		}
+		os.Exit(1)
 	}
 }

jwk/jwt_strategy.go

@@ -53,7 +53,7 @@ func (j *DefaultJWTSigner) getKeys(ctx context.Context) (private *jose.JSONWebKe
 
 	return nil, errors.WithStack(fosite.ErrServerError.
 		WithWrap(err).
-		WithHintf(`Could not ensure that signing keys for "%s" exists. If you are running against a persistent SQL database this is most likely because your "secrets.system" ("SECRETS_SYSTEM" environment variable) is not set or changed. When running with an SQL database backend you need to make sure that the secret is set and stays the same, unless when doing key rotation. This may also happen when you forget to run "hydra migrate sql..`, j.setID))
+		WithHintf(`Could not ensure that signing keys for "%s" exists. If you are running against a persistent SQL database this is most likely because your "secrets.system" ("SECRETS_SYSTEM" environment variable) is not set or changed. When running with an SQL database backend you need to make sure that the secret is set and stays the same, unless when doing key rotation. This may also happen when you forget to run "hydra migrate sql up -e".`, j.setID))
 }
 
 func (j *DefaultJWTSigner) GetPublicKeyID(ctx context.Context) (string, error) {

scripts/db-diff.sh

@@ -81,7 +81,7 @@ function dump_pg {
 
 	make test-resetdb >/dev/null 2>&1
 	sleep 4
-  go run . migrate sql "$TEST_DATABASE_POSTGRESQL" --yes >&2 || true
+  go run . migrate sql up "$TEST_DATABASE_POSTGRESQL" --yes >&2 || true
 	sleep 1
 	pg_dump -s "$TEST_DATABASE_POSTGRESQL" | sed '/^--/d'
 }
@@ -94,7 +94,7 @@ function dump_cockroach {
 
 	make test-resetdb >/dev/null 2>&1
 	sleep 4
-	go run . migrate sql "$TEST_DATABASE_COCKROACHDB" --yes > /dev/null || true
+	go run . migrate sql up "$TEST_DATABASE_COCKROACHDB" --yes > /dev/null || true
 	hydra::util::parse-connection-url "${TEST_DATABASE_COCKROACHDB}"
 	docker run --rm --net=host -it cockroachdb/cockroach:latest-v24.1 dump --dump-all --dump-mode=schema --insecure --user="${DB_USER}" --host="${DB_HOST}" --port="${DB_PORT}"
 }
@@ -107,7 +107,7 @@ function dump_sqlite {
 	hydra::util::ensure-sqlite
 
 	rm "$SQLITE_PATH" > /dev/null 2>&1 || true
-	go run -tags sqlite,sqlite_omit_load_extension . migrate sql "sqlite://$SQLITE_PATH?_fk=true" --yes > /dev/null 2>&1 || true
+	go run -tags sqlite,sqlite_omit_load_extension . migrate sql up "sqlite://$SQLITE_PATH?_fk=true" --yes > /dev/null 2>&1 || true
 	echo '.dump' | sqlite3 "$SQLITE_PATH"
 }
 
@@ -120,7 +120,7 @@ function dump_mysql {
 	hydra::util::ensure-mysqldump
 	make test-resetdb >/dev/null 2>&1
 	sleep 10
-	go run . migrate sql "$TEST_DATABASE_MYSQL" --yes > /dev/null || true
+	go run . migrate sql up "$TEST_DATABASE_MYSQL" --yes > /dev/null || true
 	hydra::util::parse-connection-url "${TEST_DATABASE_MYSQL}"
 	mysqldump --user="$DB_USER" --password="$DB_PASSWORD" --host="$DB_HOST" --port="$DB_PORT" "$DB_DB" --no-data
 }

test/e2e/circle-ci.bash

@@ -92,7 +92,7 @@ case $i in
 esac
 done
 
-./hydra migrate sql --yes $TEST_DATABASE > ./hydra-migrate.e2e.log 2>&1
+./hydra migrate sql up --yes $TEST_DATABASE > ./hydra-migrate.e2e.log 2>&1
     DSN=$TEST_DATABASE \
     ./hydra serve all --dev --sqa-opt-out > ./hydra.e2e.log 2>&1 &
 

test/e2e/docker-compose.cockroach.yml

@@ -5,7 +5,7 @@ services:
     image: oryd/hydra:e2e
     environment:
       - DSN=cockroach://root@cockroachd:26257/defaultdb?sslmode=disable&max_conns=20&max_idle_conns=4
-    command: migrate sql -e --yes
+    command: migrate sql up -e --yes
     restart: on-failure
 
   hydra:

test/e2e/docker-compose.mysql.yml

@@ -5,7 +5,7 @@ services:
     image: oryd/hydra:e2e
     environment:
       - DSN=mysql://root:secret@tcp(mysqld:3306)/mysql?max_conns=20&max_idle_conns=4
-    command: migrate sql -e --yes
+    command: migrate sql up -e --yes
     restart: on-failure
 
   hydra:

test/e2e/docker-compose.postgres.yml

@@ -5,7 +5,7 @@ services:
     image: oryd/hydra:e2e
     environment:
       - DSN=postgres://hydra:secret@postgresd:5432/hydra?sslmode=disable&max_conns=20&max_idle_conns=4
-    command: migrate sql -e --yes
+    command: migrate sql up -e --yes
     restart: on-failure
 
   hydra: