Port #5755 to feature/atree-inlining-cadence-v0.42

cmd/util/cmd/execution-state-extract/cmd.go

@@ -39,6 +39,7 @@ var (
 	flagInputPayloadFileName               string
 	flagOutputPayloadFileName              string
 	flagOutputPayloadByAddresses           string
+	flagFixSlabsWithBrokenReferences       bool
 	flagFilterUnreferencedSlabs            bool
 )
 
@@ -124,6 +125,9 @@ func init() {
 		"extract payloads of addresses (comma separated hex-encoded addresses) to file specified by output-payload-filename",
 	)
 
+	Cmd.Flags().BoolVar(&flagFixSlabsWithBrokenReferences, "fix-testnet-slabs-with-broken-references", false,
+		"fix slabs with broken references in testnet")
+
 	Cmd.Flags().BoolVar(&flagFilterUnreferencedSlabs, "filter-unreferenced-slabs", false,
 		"filter unreferenced slabs")
 }
@@ -303,6 +307,9 @@ func run(*cobra.Command, []string) {
 
 	log.Info().Msgf("state extraction plan: %s, %s", inputMsg, outputMsg)
 
+	chainID := flow.ChainID(flagChain)
+	fixSlabsWithBrokenReferences := chainID == flow.Testnet && flagFixSlabsWithBrokenReferences
+
 	var err error
 	if len(flagInputPayloadFileName) > 0 {
 		err = extractExecutionStateFromPayloads(
@@ -314,6 +321,7 @@ func run(*cobra.Command, []string) {
 			flagInputPayloadFileName,
 			flagOutputPayloadFileName,
 			exportedAddresses,
+			fixSlabsWithBrokenReferences,
 		)
 	} else {
 		err = extractExecutionState(
@@ -325,6 +333,7 @@ func run(*cobra.Command, []string) {
 			!flagNoMigration,
 			flagOutputPayloadFileName,
 			exportedAddresses,
+			fixSlabsWithBrokenReferences,
 		)
 	}
 

cmd/util/cmd/execution-state-extract/execution_state_extract.go

@@ -39,6 +39,7 @@ func extractExecutionState(
 	runMigrations bool,
 	outputPayloadFile string,
 	exportPayloadsByAddresses []common.Address,
+	fixSlabsWithBrokenReferences bool,
 ) error {
 
 	log.Info().Msg("init WAL")
@@ -89,7 +90,13 @@ func extractExecutionState(
 		<-compactor.Done()
 	}()
 
-	migrations := newMigrations(log, dir, nWorker, runMigrations)
+	migrations := newMigrations(
+		log,
+		dir,
+		nWorker,
+		runMigrations,
+		fixSlabsWithBrokenReferences,
+	)
 
 	newState := ledger.State(targetHash)
 
@@ -213,6 +220,7 @@ func extractExecutionStateFromPayloads(
 	inputPayloadFile string,
 	outputPayloadFile string,
 	exportPayloadsByAddresses []common.Address,
+	fixSlabsWithBrokenReferences bool,
 ) error {
 
 	inputPayloadsFromPartialState, payloads, err := util.ReadPayloadFile(log, inputPayloadFile)
@@ -222,7 +230,13 @@ func extractExecutionStateFromPayloads(
 
 	log.Info().Msgf("read %d payloads", len(payloads))
 
-	migrations := newMigrations(log, dir, nWorker, runMigrations)
+	migrations := newMigrations(
+		log,
+		dir,
+		nWorker,
+		runMigrations,
+		fixSlabsWithBrokenReferences,
+	)
 
 	payloads, err = migratePayloads(log, payloads, migrations)
 	if err != nil {
@@ -351,20 +365,35 @@ func newMigrations(
 	outputDir string,
 	nWorker int, // number of concurrent worker to migation payloads
 	runMigrations bool,
+	fixSlabsWithBrokenReferences bool,
 ) []ledger.Migration {
 	if runMigrations {
 		rwf := reporters.NewReportFileWriterFactory(outputDir, log)
 
-		var acctBasedMigrations []migrators.AccountBasedMigration
+		var accountBasedMigrations []migrators.AccountBasedMigration
+
+		if fixSlabsWithBrokenReferences {
+			accountBasedMigrations = append(
+				accountBasedMigrations,
+				migrators.NewFixBrokenReferencesInSlabsMigration(
+					rwf,
+					migrators.TestnetAccountsWithBrokenSlabReferences,
+				),
+			)
+		}
 
 		if flagFilterUnreferencedSlabs {
-			acctBasedMigrations = append(acctBasedMigrations, migrators.NewFilterUnreferencedSlabsMigration(
-				outputDir,
-				rwf,
-			))
+			accountBasedMigrations = append(
+				accountBasedMigrations,
+				migrators.NewFilterUnreferencedSlabsMigration(
+					outputDir,
+					rwf,
+				),
+			)
 		}
 
-		acctBasedMigrations = append(acctBasedMigrations,
+		accountBasedMigrations = append(
+			accountBasedMigrations,
 			migrators.NewAtreeRegisterMigrator(
 				rwf,
 				flagValidateMigration,
@@ -380,15 +409,13 @@ func newMigrations(
 			&migrators.AccountUsageMigrator{},
 		)
 
-		migrations := []ledger.Migration{
+		return []ledger.Migration{
 			migrators.CreateAccountBasedMigration(
 				log,
 				nWorker,
-				acctBasedMigrations,
+				accountBasedMigrations,
 			),
 		}
-
-		return migrations
 	}
 
 	return nil

cmd/util/cmd/execution-state-extract/execution_state_extract_test.go

@@ -74,6 +74,7 @@ func TestExtractExecutionState(t *testing.T) {
 				false,
 				"",
 				nil,
+				false,
 			)
 			require.Error(t, err)
 		})

cmd/util/ledger/migrations/account_based_migration.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/onflow/flow-go/cmd/util/ledger/util"
 	"github.com/onflow/flow-go/ledger"
+	"github.com/onflow/flow-go/model/flow"
 	moduleUtil "github.com/onflow/flow-go/module/util"
 )
 
@@ -186,22 +187,6 @@ func MigrateGroupConcurrently(
 						continue
 					}
 
-					if _, ok := knownProblematicAccounts[job.Address]; ok {
-						log.Info().
-							Hex("address", job.Address[:]).
-							Int("payload_count", len(job.Payloads)).
-							Msg("skipping problematic account")
-						resultCh <- &migrationResult{
-							migrationDuration: migrationDuration{
-								Address:      job.Address,
-								Duration:     time.Since(start),
-								PayloadCount: len(job.Payloads),
-							},
-							Migrated: job.Payloads,
-						}
-						continue
-					}
-
 					var err error
 					accountMigrated := job.Payloads
 					for m, migrator := range migrations {
@@ -315,19 +300,27 @@ func MigrateGroupConcurrently(
 	return migrated, nil
 }
 
-var knownProblematicAccounts = map[common.Address]string{
-	// Testnet accounts with broken contracts
-	mustHexToAddress("434a1f199a7ae3ba"): "Broken contract FanTopPermission",
-	mustHexToAddress("454c9991c2b8d947"): "Broken contract Test",
-	mustHexToAddress("48602d8056ff9d93"): "Broken contract FanTopPermission",
-	mustHexToAddress("5d63c34d7f05e5a4"): "Broken contract FanTopPermission",
-	mustHexToAddress("5e3448b3cffb97f2"): "Broken contract FanTopPermission",
-	mustHexToAddress("7d8c7e050c694eaa"): "Broken contract Test",
-	mustHexToAddress("ba53f16ede01972d"): "Broken contract FanTopPermission",
-	mustHexToAddress("c843c1f5a4805c3a"): "Broken contract FanTopPermission",
-	mustHexToAddress("48d3be92e6e4a973"): "Broken contract FanTopPermission",
-	// Mainnet account
-}
+var TestnetAccountsWithBrokenSlabReferences = func() map[common.Address]struct{} {
+	testnetAddresses := map[common.Address]struct{}{
+		mustHexToAddress("434a1f199a7ae3ba"): {},
+		mustHexToAddress("454c9991c2b8d947"): {},
+		mustHexToAddress("48602d8056ff9d93"): {},
+		mustHexToAddress("5d63c34d7f05e5a4"): {},
+		mustHexToAddress("5e3448b3cffb97f2"): {},
+		mustHexToAddress("7d8c7e050c694eaa"): {},
+		mustHexToAddress("ba53f16ede01972d"): {},
+		mustHexToAddress("c843c1f5a4805c3a"): {},
+		mustHexToAddress("48d3be92e6e4a973"): {},
+	}
+
+	for address := range testnetAddresses {
+		if !flow.Testnet.Chain().IsValid(flow.Address(address)) {
+			panic(fmt.Sprintf("invalid testnet address: %s", address.Hex()))
+		}
+	}
+
+	return testnetAddresses
+}()
 
 func mustHexToAddress(hex string) common.Address {
 	address, err := common.HexToAddress(hex)