Skip to content

Commit

Permalink
feat: add migrate sql up|down|status (#3894)
Browse files Browse the repository at this point in the history 
This patch adds the ability to execute down migrations using:

```
hydra migrate sql down -e --steps {num_of_steps}
```

Please read `hydra migrate sql down --help` carefully.

Going forward, please use the following commands

```
hydra migrate sql up ...
hydra migrate sql status ...
```

instead of the previous, now deprecated

```
hydra migrate sql ...
hydra migrate status ...
```

commands.

See ory-corp/cloud#7350
  • Loading branch information
@aeneasr
aeneasr authored Nov 27, 2024
1 parent 2559819 commit d27882f
Show file tree
Hide file tree
Showing 14 changed files with 144 additions and 194 deletions.
90 changes: 8 additions & 82 deletions cmd/cli/handler_migrate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,13 @@ package cli

import (
"bytes"
"context"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"regexp"
"strings"
"time"

"github.com/ory/x/popx"
"github.com/ory/x/servicelocatorx"
Expand All @@ -22,8 +20,6 @@ import (

"github.com/ory/x/configx"

"github.com/ory/x/errorsx"

"github.com/ory/x/cmdx"

"github.com/spf13/cobra"
Expand Down Expand Up @@ -317,96 +313,26 @@ func (h *MigrateHandler) makePersister(cmd *cobra.Command, args []string) (p per
return d.Persister(), nil
}

func (h *MigrateHandler) MigrateSQL(cmd *cobra.Command, args []string) (err error) {
func (h *MigrateHandler) MigrateSQLUp(cmd *cobra.Command, args []string) (err error) {
p, err := h.makePersister(cmd, args)
if err != nil {
return err
}
conn := p.Connection(context.Background())
if conn == nil {
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Migrations can only be executed against a SQL-compatible driver but DSN is not a SQL source.")
return cmdx.FailSilently(cmd)
}

if err := conn.Open(); err != nil {
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not open the database connection:\n%+v\n", err)
return cmdx.FailSilently(cmd)
}

// convert migration tables
if err := p.PrepareMigration(context.Background()); err != nil {
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not convert the migration table:\n%+v\n", err)
return cmdx.FailSilently(cmd)
}

// print migration status
_, _ = fmt.Fprintln(cmd.OutOrStdout(), "The following migration is planned:")
return popx.MigrateSQLUp(cmd, p)
}

status, err := p.MigrationStatus(context.Background())
func (h *MigrateHandler) MigrateSQLDown(cmd *cobra.Command, args []string) (err error) {
p, err := h.makePersister(cmd, args)
if err != nil {
fmt.Fprintf(cmd.ErrOrStderr(), "Could not get the migration status:\n%+v\n", errorsx.WithStack(err))
return cmdx.FailSilently(cmd)
}
_ = status.Write(os.Stdout)

if !flagx.MustGetBool(cmd, "yes") {
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "To skip the next question use flag --yes (at your own risk).")
if !cmdx.AskForConfirmation("Do you wish to execute this migration plan?", nil, nil) {
_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Migration aborted.")
return nil
}
}

// apply migrations
if err := p.MigrateUp(context.Background()); err != nil {
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not apply migrations:\n%+v\n", errorsx.WithStack(err))
return cmdx.FailSilently(cmd)
return err
}

_, _ = fmt.Fprintln(cmd.OutOrStdout(), "Successfully applied migrations!")
return nil
return popx.MigrateSQLDown(cmd, p)
}

func (h *MigrateHandler) MigrateStatus(cmd *cobra.Command, args []string) error {
p, err := h.makePersister(cmd, args)
if err != nil {
return err
}
conn := p.Connection(context.Background())
if conn == nil {
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "Migrations can only be checked against a SQL-compatible driver but DSN is not a SQL source.")
return cmdx.FailSilently(cmd)
}

if err := conn.Open(); err != nil {
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not open the database connection:\n%+v\n", err)
return cmdx.FailSilently(cmd)
}

block := flagx.MustGetBool(cmd, "block")
ctx := cmd.Context()
s, err := p.MigrationStatus(ctx)
if err != nil {
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err)
return cmdx.FailSilently(cmd)
}

for block && s.HasPending() {
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Waiting for migrations to finish...\n")
for _, m := range s {
if m.State == popx.Pending {
_, _ = fmt.Fprintf(cmd.OutOrStdout(), " - %s\n", m.Name)
}
}
time.Sleep(time.Second)
s, err = p.MigrationStatus(ctx)
if err != nil {
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get migration status: %+v\n", err)
return cmdx.FailSilently(cmd)
}
}

cmdx.PrintTable(cmd, s)
return nil

return popx.MigrateStatus(cmd, p)
}
4 changes: 2 additions & 2 deletions cmd/janitor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ import (

func NewJanitorCmd(slOpts []servicelocatorx.Option, dOpts []driver.OptionsModifier, cOpts []configx.OptionModifier) *cobra.Command {
cmd := &cobra.Command{
Use: "janitor [<database-url>]",
Use: "janitor [[database_url]]",
Short: "This command cleans up stale database rows.",
Example: `hydra janitor --keep-if-younger 23h --access-lifespan 1h --refresh-lifespan 40h --consent-request-lifespan 10m <database-url>`,
Example: `hydra janitor --keep-if-younger 23h --access-lifespan 1h --refresh-lifespan 40h --consent-request-lifespan 10m [database_url]`,
Long: `This command cleans up stale database rows. This will select records to delete with a limit
and delete records in batch to ensure that no table locking issues arise in big production
databases.
Expand Down
34 changes: 26 additions & 8 deletions cmd/migrate_sql.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,19 @@ package cmd
import (
"github.com/spf13/cobra"

"github.com/ory/x/popx"

"github.com/ory/hydra/v2/cmd/cli"
"github.com/ory/hydra/v2/driver"
"github.com/ory/x/configx"
"github.com/ory/x/servicelocatorx"

"github.com/ory/hydra/v2/cmd/cli"
)

func NewMigrateSqlCmd(slOpts []servicelocatorx.Option, dOpts []driver.OptionsModifier, cOpts []configx.OptionModifier) *cobra.Command {
func NewMigrateSQLCmd(slOpts []servicelocatorx.Option, dOpts []driver.OptionsModifier, cOpts []configx.OptionModifier) *cobra.Command {
cmd := &cobra.Command{
Use: "sql <database-url>",
Short: "Create SQL schemas and apply migration plans",
Use: "sql [database_url]",
Deprecated: "Please use `hydra migrate sql up` instead.",
Short: "Perform SQL migrations",
Long: `Run this command on a fresh SQL installation and when you upgrade Hydra to a new minor version. For example,
upgrading Hydra 0.7.0 to 0.8.0 requires running this command.
Expand All @@ -25,16 +27,32 @@ This decreases risk of failure and decreases time required.
You can read in the database URL using the -e flag, for example:
export DSN=...
hydra migrate sql -e
hydra migrate sql up -e
### WARNING ###
Before running this command on an existing database, create a back up!`,
RunE: cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateSQL,
RunE: cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateSQLUp,
}

cmd.Flags().BoolP("read-from-env", "e", false, "If set, reads the database connection string from the environment variable DSN or config file key dsn.")
cmd.Flags().BoolP("yes", "y", false, "If set all confirmation requests are accepted without user interaction.")
cmd.PersistentFlags().BoolP("read-from-env", "e", false, "If set, reads the database connection string from the environment variable DSN or config file key dsn.")

cmd.AddCommand(NewMigrateSQLDownCmd(slOpts, dOpts, cOpts))
cmd.AddCommand(NewMigrateSQLUpCmd(slOpts, dOpts, cOpts))
cmd.AddCommand(NewMigrateSQLStatusCmd(slOpts, dOpts, cOpts))

return cmd
}

func NewMigrateSQLDownCmd(slOpts []servicelocatorx.Option, dOpts []driver.OptionsModifier, cOpts []configx.OptionModifier) *cobra.Command {
return popx.NewMigrateSQLDownCmd("hydra", cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateSQLDown)
}

func NewMigrateSQLStatusCmd(slOpts []servicelocatorx.Option, dOpts []driver.OptionsModifier, cOpts []configx.OptionModifier) *cobra.Command {
return popx.NewMigrateSQLStatusCmd("hydra", cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateStatus)
}

func NewMigrateSQLUpCmd(slOpts []servicelocatorx.Option, dOpts []driver.OptionsModifier, cOpts []configx.OptionModifier) *cobra.Command {
return popx.NewMigrateSQLUpCmd("hydra", cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateSQLUp)
}
19 changes: 8 additions & 11 deletions cmd/migrate_status.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
package cmd

import (
"github.com/ory/x/cmdx"
"github.com/ory/x/configx"
"github.com/ory/x/popx"
"github.com/ory/x/servicelocatorx"

"github.com/spf13/cobra"
Expand All @@ -15,15 +15,12 @@ import (
)

func NewMigrateStatusCmd(slOpts []servicelocatorx.Option, dOpts []driver.OptionsModifier, cOpts []configx.OptionModifier) *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Get the current migration status",
RunE: cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateStatus,
}

cmdx.RegisterFormatFlags(cmd.PersistentFlags())
cmd.Flags().BoolP("read-from-env", "e", false, "If set, reads the database connection string from the environment variable DSN or config file key dsn.")
cmd.Flags().Bool("block", false, "Block until all migrations have been applied")

cmd := popx.RegisterMigrateStatusFlags(&cobra.Command{
Use: "status",
Deprecated: "Please use `hydra migrate sql status` instead.",
Short: "Get the current migration status",
RunE: cli.NewHandler(slOpts, dOpts, cOpts).Migration.MigrateStatus,
})
cmd.PersistentFlags().BoolP("read-from-env", "e", false, "If set, reads the database connection string from the environment variable DSN or config file key dsn.")
return cmd
}
13 changes: 9 additions & 4 deletions cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"fmt"
"os"

"github.com/pkg/errors"

"github.com/ory/x/cmdx"

"github.com/ory/hydra/v2/driver"
Expand Down Expand Up @@ -72,7 +74,7 @@ func RegisterCommandRecursive(parent *cobra.Command, slOpts []servicelocatorx.Op

migrateCmd := NewMigrateCmd()
migrateCmd.AddCommand(NewMigrateGenCmd())
migrateCmd.AddCommand(NewMigrateSqlCmd(slOpts, dOpts, cOpts))
migrateCmd.AddCommand(NewMigrateSQLCmd(slOpts, dOpts, cOpts))
migrateCmd.AddCommand(NewMigrateStatusCmd(slOpts, dOpts, cOpts))

serveCmd := NewServeCmd()
Expand All @@ -99,8 +101,11 @@ func RegisterCommandRecursive(parent *cobra.Command, slOpts []servicelocatorx.Op

// Execute adds all child commands to the root command sets flags appropriately.
func Execute() {
if err := NewRootCmd(nil, nil, nil).Execute(); err != nil {
fmt.Println(err)
os.Exit(-1)
c := NewRootCmd(nil, nil, nil)
if err := c.Execute(); err != nil {
if !errors.Is(err, cmdx.ErrNoPrintButFail) {
_, _ = fmt.Fprintln(c.ErrOrStderr(), err)
}
os.Exit(1)
}
}
52 changes: 26 additions & 26 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ require (
github.com/ory/hydra-client-go/v2 v2.2.1
github.com/ory/jsonschema/v3 v3.0.8
github.com/ory/kratos-client-go v1.2.1
github.com/ory/x v0.0.668
github.com/ory/x v0.0.674
github.com/pborman/uuid v1.2.1
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.19.1
Expand All @@ -63,16 +63,16 @@ require (
github.com/toqueteos/webbrowser v1.2.0
github.com/twmb/murmur3 v1.1.8
github.com/urfave/negroni v1.0.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0
go.opentelemetry.io/otel v1.28.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0
go.opentelemetry.io/otel/sdk v1.28.0
go.opentelemetry.io/otel/trace v1.28.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0
go.opentelemetry.io/otel v1.32.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0
go.opentelemetry.io/otel/sdk v1.32.0
go.opentelemetry.io/otel/trace v1.32.0
go.uber.org/automaxprocs v1.5.3
golang.org/x/crypto v0.25.0
golang.org/x/crypto v0.28.0
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56
golang.org/x/oauth2 v0.21.0
golang.org/x/sync v0.7.0
golang.org/x/oauth2 v0.23.0
golang.org/x/sync v0.9.0
golang.org/x/tools v0.23.0
)

Expand Down Expand Up @@ -150,7 +150,7 @@ require (
github.com/gorilla/handlers v1.5.2 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
Expand Down Expand Up @@ -197,7 +197,7 @@ require (
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/opencontainers/runc v1.1.14 // indirect
github.com/openzipkin/zipkin-go v0.4.2 // indirect
github.com/openzipkin/zipkin-go v0.4.3 // indirect
github.com/ory/dockertest/v3 v3.10.1-0.20240704115616-d229e74b748d // indirect
github.com/ory/go-convenience v0.1.0 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
Expand All @@ -207,7 +207,7 @@ require (
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/rogpeppe/go-internal v1.13.1 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/seatgeek/logrus-gelf-formatter v0.0.0-20210414080842-5b05eb8ff761 // indirect
Expand All @@ -230,25 +230,25 @@ require (
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
go.mongodb.org/mongo-driver v1.14.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.21.0 // indirect
go.opentelemetry.io/contrib/propagators/jaeger v1.21.1 // indirect
go.opentelemetry.io/contrib/samplers/jaegerremote v0.15.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.57.0 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.32.0 // indirect
go.opentelemetry.io/contrib/propagators/jaeger v1.32.0 // indirect
go.opentelemetry.io/contrib/samplers/jaegerremote v0.26.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.21.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.32.0 // indirect
go.opentelemetry.io/otel/metric v1.32.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/mod v0.19.0 // indirect
golang.org/x/net v0.27.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/sys v0.27.0 // indirect
golang.org/x/text v0.20.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/grpc v1.64.1 // indirect
google.golang.org/protobuf v1.34.2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect
google.golang.org/grpc v1.67.1 // indirect
google.golang.org/protobuf v1.35.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand Down
Loading

0 comments on commit d27882f

Please sign in to comment.