Merge pull request #1153 from ViktorT-11/2025-10-firewalldb-mig-deleted-session-db-fix

[sql-54] firewalldb: handle deleted sessions during the kv stores + privacy mapper migration

Author: ViktorT-11

firewalldb/sql_migration.go

@@ -93,18 +93,29 @@ func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB,
 
 	log.Infof("Starting migration of the rules DB to SQL")
 
-	err := migrateKVStoresDBToSQL(ctx, kvStore, sqlTx)
+	sessions, err := sessionDB.ListSessions(ctx)
+	if err != nil {
+		return fmt.Errorf("listing sessions failed: %w", err)
+	}
+
+	sessionMap, err := mapSessions(sessions)
+	if err != nil {
+		return fmt.Errorf("mapping sessions failed: %w", err)
+	}
+
+	err = migrateKVStoresDBToSQL(ctx, kvStore, sqlTx, sessionMap)
 	if err != nil {
 		return err
 	}
 
-	err = migratePrivacyMapperDBToSQL(ctx, kvStore, sqlTx)
+	err = migratePrivacyMapperDBToSQL(ctx, kvStore, sqlTx, sessionMap)
 	if err != nil {
 		return err
 	}
 
 	err = migrateActionsToSQL(
 		ctx, kvStore, sqlTx, sessionDB, accountDB, macRootKeyIDs,
+		sessionMap,
 	)
 	if err != nil {
 		return err
@@ -119,7 +130,7 @@ func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB,
 // database to the SQL database. The function also asserts that the
 // migrated values match the original values in the KV store.
 func migrateKVStoresDBToSQL(ctx context.Context, kvStore *bbolt.DB,
-	sqlTx SQLQueries) error {
+	sqlTx SQLQueries, sessMap map[[4]byte]sqlc.Session) error {
 
 	log.Infof("Starting migration of the KV stores to SQL")
 
@@ -128,7 +139,7 @@ func migrateKVStoresDBToSQL(ctx context.Context, kvStore *bbolt.DB,
 	// 1) Collect all key-value pairs from the KV store.
 	err := kvStore.View(func(tx *bbolt.Tx) error {
 		var err error
-		pairs, err = collectAllPairs(tx)
+		pairs, err = collectAllPairs(sessMap, tx)
 		return err
 	})
 	if err != nil {
@@ -139,7 +150,7 @@ func migrateKVStoresDBToSQL(ctx context.Context, kvStore *bbolt.DB,
 
 	// 2) Insert all collected key-value pairs into the SQL database.
 	for _, entry := range pairs {
-		insertedPair, err := insertPair(ctx, sqlTx, entry)
+		insertedPair, err := insertPair(ctx, sqlTx, sessMap, entry)
 		if err != nil {
 			return fmt.Errorf("inserting kv pair %v failed: %w",
 				entry.key, err)
@@ -187,7 +198,9 @@ func migrateKVStoresDBToSQL(ctx context.Context, kvStore *bbolt.DB,
 // designed to iterate over all buckets and values that exist in the KV store.
 // That ensures that we find all stores and values that exist in the KV store,
 // and can be sure that the kv store actually follows the expected structure.
-func collectAllPairs(tx *bbolt.Tx) ([]*kvEntry, error) {
+func collectAllPairs(sessMap map[[4]byte]sqlc.Session,
+	tx *bbolt.Tx) ([]*kvEntry, error) {
+
 	var entries []*kvEntry
 	for _, perm := range []bool{true, false} {
 		mainBucket, err := getMainBucket(tx, false, perm)
@@ -217,7 +230,7 @@ func collectAllPairs(tx *bbolt.Tx) ([]*kvEntry, error) {
 			}
 
 			pairs, err := collectRulePairs(
-				ruleBucket, perm, string(rule),
+				sessMap, ruleBucket, perm, string(rule),
 			)
 			if err != nil {
 				return err
@@ -237,8 +250,8 @@ func collectAllPairs(tx *bbolt.Tx) ([]*kvEntry, error) {
 
 // collectRulePairs processes a single rule bucket, which should contain the
 // global and session-kv-store key buckets.
-func collectRulePairs(bkt *bbolt.Bucket, perm bool, rule string) ([]*kvEntry,
-	error) {
+func collectRulePairs(sessMap map[[4]byte]sqlc.Session, bkt *bbolt.Bucket,
+	perm bool, rule string) ([]*kvEntry, error) {
 
 	var params []*kvEntry
 
@@ -270,6 +283,25 @@ func collectRulePairs(bkt *bbolt.Bucket, perm bool, rule string) ([]*kvEntry,
 					"under %s bucket", sessKVStoreBucketKey)
 			}
 
+			var alias [4]byte
+			copy(alias[:], groupAlias)
+			if _, ok := sessMap[alias]; !ok {
+				// If we can't find the session group in the
+				// SQL db, that indicates that the session was
+				// never migrated from KVDB. This likely means
+				// that the user deleted their session.db file,
+				// but kept the rules.db file. As the KV entries
+				// are useless when the session no longer
+				// exists, we can just skip the migration of the
+				// KV entries for this group.
+				log.Warnf("Skipping migration of KV store "+
+					"entries for session group %x, as the "+
+					"session group was not found",
+					groupAlias)
+
+				return nil
+			}
+
 			groupBucket := sessBkt.Bucket(groupAlias)
 			if groupBucket == nil {
 				return fmt.Errorf("group bucket for group "+
@@ -382,7 +414,7 @@ func collectKVPairs(bkt *bbolt.Bucket, errorOnBuckets, perm bool,
 
 // insertPair inserts a single key-value pair into the SQL database.
 func insertPair(ctx context.Context, tx SQLQueries,
-	entry *kvEntry) (*sqlKvEntry, error) {
+	sessMap map[[4]byte]sqlc.Session, entry *kvEntry) (*sqlKvEntry, error) {
 
 	ruleID, err := tx.GetOrInsertRuleID(ctx, entry.ruleName)
 	if err != nil {
@@ -397,15 +429,19 @@ func insertPair(ctx context.Context, tx SQLQueries,
 	}
 
 	entry.groupAlias.WhenSome(func(alias []byte) {
-		var groupID int64
-		groupID, err = tx.GetSessionIDByAlias(ctx, alias)
-		if err != nil {
-			err = fmt.Errorf("getting group id by alias %x "+
-				"failed: %w", alias, err)
-			return
+		var groupAlias [4]byte
+		copy(groupAlias[:], alias)
+
+		sess, ok := sessMap[groupAlias]
+		if !ok {
+			// This should be unreachable, as we check for the
+			// existence of the session group when collecting
+			// the kv pairs.
+			err = fmt.Errorf("session group %x not found in map",
+				alias)
 		}
 
-		p.GroupID = sqldb.SQLInt64(groupID)
+		p.GroupID = sess.GroupID
 	})
 	if err != nil {
 		return nil, err
@@ -515,12 +551,12 @@ func verifyBktKeys(bkt *bbolt.Bucket, errorOnKeyValues bool,
 // from the KV database to the SQL database. The function also asserts that the
 // migrated values match the original values in the privacy mapper store.
 func migratePrivacyMapperDBToSQL(ctx context.Context, kvStore *bbolt.DB,
-	sqlTx SQLQueries) error {
+	sqlTx SQLQueries, sessMap map[[4]byte]sqlc.Session) error {
 
 	log.Infof("Starting migration of the privacy mapper store to SQL")
 
 	// 1) Collect all privacy pairs from the KV store.
-	privPairs, err := collectPrivacyPairs(ctx, kvStore, sqlTx)
+	privPairs, err := collectPrivacyPairs(kvStore, sessMap)
 	if err != nil {
 		return fmt.Errorf("error migrating privacy mapper store: %w",
 			err)
@@ -549,8 +585,8 @@ func migratePrivacyMapperDBToSQL(ctx context.Context, kvStore *bbolt.DB,
 }
 
 // collectPrivacyPairs collects all privacy pairs from the KV store.
-func collectPrivacyPairs(ctx context.Context, kvStore *bbolt.DB,
-	sqlTx SQLQueries) (privacyPairs, error) {
+func collectPrivacyPairs(kvStore *bbolt.DB,
+	sessMap map[[4]byte]sqlc.Session) (privacyPairs, error) {
 
 	groupPairs := make(privacyPairs)
 
@@ -576,24 +612,39 @@ func collectPrivacyPairs(ctx context.Context, kvStore *bbolt.DB,
 					"%s not found", groupId)
 			}
 
-			groupSqlId, err := sqlTx.GetSessionIDByAlias(
-				ctx, groupId,
-			)
-			if errors.Is(err, sql.ErrNoRows) {
-				return fmt.Errorf("session with group id %x "+
-					"not found in sql db", groupId)
-			} else if err != nil {
-				return err
+			var groupAlias [4]byte
+			copy(groupAlias[:], groupId)
+
+			sess, ok := sessMap[groupAlias]
+			if !ok {
+				// If we can't find the session group in the SQL
+				// db, that indicates that the session was never
+				// migrated from KVDB. This likely means that
+				// the user deleted their session.db file, but
+				// kept the rules.db file. As the privacy pairs
+				// are useless when the session no longer
+				// exists, we can just skip the migration of the
+				// privacy pairs for this group.
+				log.Warnf("Skipping migration of privacy "+
+					"pairs for session group %x, as the "+
+					"session group was not found", groupId)
+
+				return nil
+			}
+
+			if !sess.GroupID.Valid {
+				return fmt.Errorf("session group id for "+
+					"session %d is not set ", sess.ID)
 			}
 
 			groupRealToPseudoPairs, err := collectGroupPairs(gBkt)
 			if err != nil {
 				return fmt.Errorf("processing group bkt "+
 					"for group id %s (sqlID %d) failed: %w",
-					groupId, groupSqlId, err)
+					groupId, sess.GroupID.Int64, err)
 			}
 
-			groupPairs[groupSqlId] = groupRealToPseudoPairs
+			groupPairs[sess.GroupID.Int64] = groupRealToPseudoPairs
 
 			return nil
 		})
@@ -794,7 +845,8 @@ func validateGroupPairsMigration(ctx context.Context, sqlTx SQLQueries,
 // values match the original values in the actions store.
 func migrateActionsToSQL(ctx context.Context, kvStore *bbolt.DB,
 	sqlTx SQLQueries, sessionDB session.SQLQueries,
-	accountsDB accounts.SQLQueries, macRootKeyIDs [][]byte) error {
+	accountsDB accounts.SQLQueries, macRootKeyIDs [][]byte,
+	sessMap map[[4]byte]sqlc.Session) error {
 
 	log.Infof("Starting migration of the actions store to SQL")
 
@@ -811,16 +863,6 @@ func migrateActionsToSQL(ctx context.Context, kvStore *bbolt.DB,
 		return fmt.Errorf("mapping accounts failed: %w", err)
 	}
 
-	sessions, err := sessionDB.ListSessions(ctx)
-	if err != nil {
-		return fmt.Errorf("listing sessions failed: %w", err)
-	}
-
-	sessionMap, err := mapSessions(sessions)
-	if err != nil {
-		return fmt.Errorf("mapping sessions failed: %w", err)
-	}
-
 	// Next, as the kvdb actions only have their last 4 bytes set for the
 	// MacaroonRootKeyID field, we'll do a best effort attempt fetch the
 	// full root key ID (all 8 bytes) from lnd when migrating each action.
@@ -927,8 +969,7 @@ func migrateActionsToSQL(ctx context.Context, kvStore *bbolt.DB,
 				// migrated.
 				err = migrateActionToSQL(
 					ctx, sqlTx, sessionDB, accountsDB,
-					acctsMap, sessionMap, action,
-					macRootKeyID,
+					acctsMap, sessMap, action, macRootKeyID,
 				)
 				if err != nil {
 					return fmt.Errorf("migrating action "+