Merge pull request #576 from SiaFoundation/nate/integrity-check

Add SQLite integrity check command
SiaFoundation · Jan 16, 2025 · 9e3e816 · 9e3e816
2 parents deb74a2 + ac5e97e
commit 9e3e816
Show file tree

Hide file tree

Showing 4 changed files with 97 additions and 33 deletions.
diff --git a/.changeset/added_sqlite_integrity_command.md b/.changeset/added_sqlite_integrity_command.md
@@ -0,0 +1,5 @@
+---
+default: patch
+---
+
+# Added "sqlite integrity" command
diff --git a/cmd/hostd/main.go b/cmd/hostd/main.go
@@ -225,6 +225,13 @@ Perform various operations on the SQLite3 database.
 
 Commands:
 	backup	Create a backup of the SQLite3 database
+	integrity	Check the integrity of the SQLite3 database
+`
+
+	sqlite3IntegrityUsage = `Usage:
+hostd sqlite3 integrity <srcPath>
+
+Check the integrity of the SQLite3 database at the specified path. This is not safe to run while the host is running.
 `
 
 	sqlite3BackupUsage = `Usage:
@@ -256,6 +263,10 @@ func runBackupCommand(srcPath, destPath string) error {
 	return nil
 }
 
+func runIntegrityCommand(ctx context.Context, srcPath string, log *zap.Logger) error {
+	return sqlite.IntegrityCheck(ctx, srcPath, log)
+}
+
 func runRecalcCommand(srcPath string, log *zap.Logger) error {
 	db, err := sqlite.OpenDatabase(srcPath, log)
 	if err != nil {
@@ -313,6 +324,7 @@ func main() {
 	recalculateCmd := flagg.New("recalculate", recalculateUsage)
 	sqlite3Cmd := flagg.New("sqlite3", sqlite3Usage)
 	sqlite3BackupCmd := flagg.New("backup", sqlite3BackupUsage)
+	sqlite3IntegrityCmd := flagg.New("integrity", sqlite3IntegrityUsage)
 
 	cmd := flagg.Parse(flagg.Tree{
 		Cmd: rootCmd,
@@ -324,6 +336,7 @@ func main() {
 			{
 				Cmd: sqlite3Cmd,
 				Sub: []flagg.Tree{
+					{Cmd: sqlite3IntegrityCmd},
 					{Cmd: sqlite3BackupCmd},
 				},
 			},
@@ -368,6 +381,25 @@ func main() {
 		checkFatalError("command failed", runRecalcCommand(cmd.Arg(0), log))
 	case sqlite3Cmd:
 		cmd.Usage()
+	case sqlite3IntegrityCmd:
+		if len(cmd.Args()) != 1 {
+			cmd.Usage()
+			return
+		}
+
+		log := initStdoutLog(cfg.Log.StdOut.EnableANSI, "info")
+		defer log.Sync()
+
+		ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+		defer cancel()
+
+		log.Info("running integrity check")
+		checkFatalError("integrity check failed", runIntegrityCommand(ctx, cmd.Arg(0), log))
+		log.Info("integrity check passed")
+
+		log.Info("running foreign key check")
+		checkFatalError("foreign key check failed", sqlite.ForeignKeyCheck(ctx, cmd.Arg(0), log))
+		log.Info("foreign key check passed")
 	case sqlite3BackupCmd:
 		if len(cmd.Args()) != 2 {
 			cmd.Usage()

diff --git a/persist/sqlite/init.go b/persist/sqlite/init.go
@@ -1,7 +1,9 @@
 package sqlite
 
 import (
+	"database/sql"
 	_ "embed" // for init.sql
+	"errors"
 	"time"
 
 	"fmt"
@@ -41,7 +43,7 @@ func (s *Store) upgradeDatabase(current, target int64) error {
 				return fmt.Errorf("failed to enable foreign key deferral: %w", err)
 			} else if err := fn(tx, log); err != nil {
 				return err
-			} else if err := foreignKeyCheck(tx, log); err != nil {
+			} else if err := checkFKConsistency(tx, log); err != nil {
 				return fmt.Errorf("failed foreign key check: %w", err)
 			}
 			return setDBVersion(tx, version)
@@ -75,6 +77,33 @@ func (s *Store) init() error {
 	return nil
 }
 
+func checkFKConsistency(txn *txn, log *zap.Logger) error {
+	rows, err := txn.Query("PRAGMA foreign_key_check")
+	if err != nil {
+		return fmt.Errorf("failed to run foreign key check: %w", err)
+	}
+	defer rows.Close()
+	var hasErrors bool
+	for rows.Next() {
+		var table string
+		var rowid sql.NullInt64
+		var fkTable string
+		var fkRowid sql.NullInt64
+
+		if err := rows.Scan(&table, &rowid, &fkTable, &fkRowid); err != nil {
+			return fmt.Errorf("failed to scan foreign key check result: %w", err)
+		}
+		hasErrors = true
+		log.Error("foreign key constraint violated", zap.String("table", table), zap.Int64("rowid", rowid.Int64), zap.String("fkTable", fkTable), zap.Int64("fkRowid", fkRowid.Int64))
+	}
+	if err := rows.Err(); err != nil {
+		return fmt.Errorf("failed to iterate foreign key check results: %w", err)
+	} else if hasErrors {
+		return errors.New("foreign key constraint violated")
+	}
+	return nil
+}
+
 func generateHostKey(tx *txn) (err error) {
 	key := types.NewPrivateKeyFromSeed(frand.Bytes(32))
 	var dbID int64

diff --git a/persist/sqlite/store.go b/persist/sqlite/store.go
@@ -234,14 +234,28 @@ func Backup(ctx context.Context, srcPath, destPath string) (err error) {
 	return backupDB(ctx, src, destPath)
 }
 
-func integrityCheck(db *sql.DB, log *zap.Logger) error {
-	rows, err := db.Query("PRAGMA integrity_check")
+// IntegrityCheck runs a PRAGMA integrity_check on the database and logs any
+// integrity errors. If any errors are found, an error is returned.
+func IntegrityCheck(ctx context.Context, fp string, log *zap.Logger) error {
+	db, err := sql.Open("sqlite3", sqliteFilepath(fp))
+	if err != nil {
+		return fmt.Errorf("failed to open database: %w", err)
+	}
+	defer db.Close()
+
+	rows, err := db.QueryContext(ctx, "PRAGMA integrity_check")
 	if err != nil {
 		return fmt.Errorf("failed to run integrity check: %w", err)
 	}
 	defer rows.Close()
 	var hasErrors bool
 	for rows.Next() {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
 		var result string
 		if err := rows.Scan(&result); err != nil {
 			return fmt.Errorf("failed to scan integrity check result: %w", err)
@@ -258,41 +272,29 @@ func integrityCheck(db *sql.DB, log *zap.Logger) error {
 	return nil
 }
 
-func dbForeignKeyCheck(db *sql.DB, log *zap.Logger) error {
-	rows, err := db.Query("PRAGMA foreign_key_check")
+// ForeignKeyCheck runs a PRAGMA foreign_key_check on the database and logs any
+// foreign key constraint violations. If any violations are found, an error is
+// returned.
+func ForeignKeyCheck(ctx context.Context, fp string, log *zap.Logger) error {
+	db, err := sql.Open("sqlite3", sqliteFilepath(fp))
 	if err != nil {
-		return fmt.Errorf("failed to run foreign key check: %w", err)
+		return fmt.Errorf("failed to open database: %w", err)
 	}
-	defer rows.Close()
-	var hasErrors bool
-	for rows.Next() {
-		var table string
-		var rowid sql.NullInt64
-		var fkTable string
-		var fkRowid sql.NullInt64
+	defer db.Close()
 
-		if err := rows.Scan(&table, &rowid, &fkTable, &fkRowid); err != nil {
-			return fmt.Errorf("failed to scan foreign key check result: %w", err)
-		}
-		hasErrors = true
-		log.Error("foreign key constraint violated", zap.String("table", table), zap.Int64("rowid", rowid.Int64), zap.String("fkTable", fkTable), zap.Int64("fkRowid", fkRowid.Int64))
-	}
-	if err := rows.Err(); err != nil {
-		return fmt.Errorf("failed to iterate foreign key check results: %w", err)
-	} else if hasErrors {
-		return errors.New("foreign key constraint violated")
-	}
-	return nil
-}
-
-func foreignKeyCheck(txn *txn, log *zap.Logger) error {
-	rows, err := txn.Query("PRAGMA foreign_key_check")
+	rows, err := db.QueryContext(ctx, "PRAGMA foreign_key_check")
 	if err != nil {
 		return fmt.Errorf("failed to run foreign key check: %w", err)
 	}
 	defer rows.Close()
 	var hasErrors bool
 	for rows.Next() {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
 		var table string
 		var rowid sql.NullInt64
 		var fkTable string
@@ -325,10 +327,6 @@ func OpenDatabase(fp string, log *zap.Logger) (*Store, error) {
 	}
 	if err := store.init(); err != nil {
 		return nil, err
-	} else if err := dbForeignKeyCheck(db, log.Named("foreignkeys")); err != nil {
-		return nil, fmt.Errorf("foreign key check failed: %w", err)
-	} else if err := integrityCheck(db, log.Named("integrity")); err != nil {
-		return nil, fmt.Errorf("integrity check failed: %w", err)
 	}
 	sqliteVersion, _, _ := sqlite3.Version()
 	log.Debug("database initialized", zap.String("sqliteVersion", sqliteVersion), zap.Int("schemaVersion", len(migrations)+1), zap.String("path", fp))