From 7326abd6867388cad1565ed1efa31deb2262cc07 Mon Sep 17 00:00:00 2001 From: "terry.hung" Date: Mon, 2 Oct 2023 14:58:03 +0800 Subject: [PATCH] feat: add new actor dump: miner (#1264) * Add new actor dump: miner * Use the lookup function for finding the robust address --- chain/datasource/datasource.go | 80 +++++++++- chain/indexer/integrated/processor/state.go | 22 +++ chain/indexer/tasktype/table_tasks.go | 7 +- chain/indexer/tasktype/tasks.go | 1 + chain/indexer/tasktype/tasks_test.go | 2 +- lens/interface.go | 1 + model/actordumps/miner_actor_dump.go | 138 ++++++++++++++++ schemas/v1/33_miner_actor_dumps.go | 44 +++++ storage/sql.go | 1 + tasks/api.go | 3 + .../periodic_actor_dump/miner_actor/tasks.go | 150 ++++++++++++++++++ 11 files changed, 445 insertions(+), 4 deletions(-) create mode 100644 model/actordumps/miner_actor_dump.go create mode 100644 schemas/v1/33_miner_actor_dumps.go create mode 100644 tasks/periodic_actor_dump/miner_actor/tasks.go diff --git a/chain/datasource/datasource.go b/chain/datasource/datasource.go index 9a680352..937fb915 100644 --- a/chain/datasource/datasource.go +++ b/chain/datasource/datasource.go @@ -16,6 +16,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin" adt2 "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lotus/api" + initactor "github.com/filecoin-project/lotus/chain/actors/builtin/init" "github.com/filecoin-project/lotus/chain/state" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" @@ -42,12 +43,14 @@ var ( diffPreCommitCacheSize int diffSectorCacheSize int actorCacheSize int + addressCacheSize int tipsetMessageReceiptSizeEnv = "LILY_TIPSET_MSG_RECEIPT_CACHE_SIZE" executedTsCacheSizeEnv = "LILY_EXECUTED_TS_CACHE_SIZE" diffPreCommitCacheSizeEnv = "LILY_DIFF_PRECOMMIT_CACHE_SIZE" diffSectorCacheSizeEnv = "LILY_DIFF_SECTORS_CACHE_SIZE" actorCacheSizeEnv = "LILY_ACTOR_CACHE_SIZE" + addressCacheSizeEnv = "LILY_ADDRESS_CACHE_SIZE" ) func getCacheSizeFromEnv(env string, defaultValue int) int { @@ -67,6 +70,7 @@ func init() { diffPreCommitCacheSize = getCacheSizeFromEnv(diffPreCommitCacheSizeEnv, 500) diffSectorCacheSize = getCacheSizeFromEnv(diffSectorCacheSizeEnv, 500) actorCacheSize = getCacheSizeFromEnv(actorCacheSizeEnv, 5000) + addressCacheSize = getCacheSizeFromEnv(addressCacheSizeEnv, 4) } var _ tasks.DataSource = (*DataSource)(nil) @@ -104,6 +108,11 @@ func NewDataSource(node lens.API) (*DataSource, error) { return nil, err } + t.addressCache, err = lru.New(addressCacheSize) + if err != nil { + return nil, err + } + return t, nil } @@ -122,7 +131,8 @@ type DataSource struct { diffPreCommitCache *lru.Cache diffPreCommitGroup singleflight.Group - actorCache *lru.Cache + actorCache *lru.Cache + addressCache *lru.Cache } func (t *DataSource) MessageReceiptEvents(ctx context.Context, root cid.Cid) ([]types.Event, error) { @@ -253,7 +263,12 @@ func (t *DataSource) ActorInfo(ctx context.Context, addr address.Address, tsk ty actorInfo := tasks.ActorInfo{} if err == nil { if act.Address == nil { - act.Address = &addr + robustAddress, err := t.LookupRobustAddress(ctx, addr, tsk) + if err == nil { + act.Address = &robustAddress + } else { + act.Address = &addr + } } actorInfo.Actor = act actorName, actorFamily, err := util.ActorNameAndFamilyFromCode(act.Code) @@ -271,6 +286,67 @@ func (t *DataSource) ActorInfo(ctx context.Context, addr address.Address, tsk ty return &actorInfo, err } +func genIdAddressCacheKey(tsk types.TipSetKey) string { + key, keyErr := asKey(KeyPrefix{"IdRobustAddress"}, tsk) + if keyErr != nil { + return "IdRobustAddressDefaultkey" + } + return key +} + +func (t *DataSource) SetIdRobustAddressMap(ctx context.Context, tsk types.TipSetKey) error { + ctx, span := otel.Tracer("").Start(ctx, "DataSource.SetIdAddressMap") + if span.IsRecording() { + span.SetAttributes(attribute.String("tipset", tsk.String())) + } + defer span.End() + + key := genIdAddressCacheKey(tsk) + + initActor, err := t.Actor(ctx, initactor.Address, tsk) + if err != nil { + return err + } + + initState, err := initactor.Load(t.Store(), initActor) + if err != nil { + return err + } + + idRobustAddress := make(map[uint64]address.Address) + + _ = initState.ForEachActor(func(id abi.ActorID, addr address.Address) error { + idRobustAddress[uint64(id)] = addr + return nil + }) + + t.addressCache.Add(key, idRobustAddress) + + return nil +} + +func (t *DataSource) LookupRobustAddress(ctx context.Context, idAddr address.Address, tsk types.TipSetKey) (address.Address, error) { + robustAddress := address.Undef + + key := genIdAddressCacheKey(tsk) + value, found := t.addressCache.Get(key) + if found { + idRobustAddress := value.(map[uint64]address.Address) + idAddrDecoded, err := address.IDFromAddress(idAddr) + if err != nil { + return robustAddress, err + } + + address, exists := idRobustAddress[idAddrDecoded] + if exists { + return address, nil + } + } + + // Use the default way: StateLookup + return t.node.StateLookupRobustAddress(ctx, idAddr, tsk) +} + func (t *DataSource) MinerPower(ctx context.Context, addr address.Address, ts *types.TipSet) (*api.MinerPower, error) { ctx, span := otel.Tracer("").Start(ctx, "DataSource.MinerPower") if span.IsRecording() { diff --git a/chain/indexer/integrated/processor/state.go b/chain/indexer/integrated/processor/state.go index c4d3e144..9ca50604 100644 --- a/chain/indexer/integrated/processor/state.go +++ b/chain/indexer/integrated/processor/state.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" actorstypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/manifest" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" @@ -70,6 +71,7 @@ import ( // actor dump fevmactordumptask "github.com/filecoin-project/lily/tasks/periodic_actor_dump/fevm_actor" + mineractordumptask "github.com/filecoin-project/lily/tasks/periodic_actor_dump/miner_actor" "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/metrics" @@ -415,6 +417,15 @@ func (sp *StateProcessor) startActor(ctx context.Context, current, executed *typ return taskNames } +// isStoragePowerActor will check if the actor is storage power or not +func isStoragePowerActor(c cid.Cid) bool { + name, _, ok := actors.GetActorMetaByCode(c) + if ok { + return name == manifest.PowerKey + } + return false +} + // startPeriodicActorDump starts all TipSetsProcessor's in parallel, their results are emitted on the `results` channel. // A list containing all executed task names is returned. func (sp *StateProcessor) startPeriodicActorDump(ctx context.Context, current *types.TipSet, interval int, results chan *Result) []string { @@ -447,9 +458,18 @@ func (sp *StateProcessor) startPeriodicActorDump(ctx context.Context, current *t actors[manifest.EthAccountKey] = append(actors[manifest.EthAccountKey], actor) } else if builtin.IsPlaceholderActor(actor.Code) { actors[manifest.PlaceholderKey] = append(actors[manifest.PlaceholderKey], actor) + } else if isStoragePowerActor(actor.Code) { + // Power Actor + actors[manifest.PowerKey] = append(actors[manifest.PowerKey], actor) } } + // Set the Map to Cache + err := sp.api.SetIdRobustAddressMap(ctx, current.Key()) + if err != nil { + log.Errorf("Error at setting IdRobustAddressMap: %v", err) + } + for taskName, proc := range sp.periodicActorDumpProcessors { name := taskName p := proc @@ -789,6 +809,8 @@ func MakeProcessors(api tasks.DataSource, indexerTasks []string) (*IndexerProces // case tasktype.FEVMActorDump: out.PeriodicActorDumpProcessors[t] = fevmactordumptask.NewTask(api) + case tasktype.MinerActorDump: + out.PeriodicActorDumpProcessors[t] = mineractordumptask.NewTask(api) case BuiltinTaskName: out.ReportProcessors[t] = indexertask.NewTask(api) diff --git a/chain/indexer/tasktype/table_tasks.go b/chain/indexer/tasktype/table_tasks.go index 3882b2d6..227074d8 100644 --- a/chain/indexer/tasktype/table_tasks.go +++ b/chain/indexer/tasktype/table_tasks.go @@ -52,8 +52,9 @@ const ( FEVMContract = "fevm_contract" FEVMTrace = "fevm_trace" - // New task types + // New task types: full dump FEVMActorDump = "fevm_actor_dump" + MinerActorDump = "miner_actor_dump" ) var AllTableTasks = []string{ @@ -107,6 +108,7 @@ var AllTableTasks = []string{ FEVMContract, FEVMTrace, FEVMActorDump, + MinerActorDump, } var TableLookup = map[string]struct{}{ @@ -160,6 +162,7 @@ var TableLookup = map[string]struct{}{ FEVMContract: {}, FEVMTrace: {}, FEVMActorDump: {}, + MinerActorDump: {}, } var TableComment = map[string]string{ @@ -213,6 +216,7 @@ var TableComment = map[string]string{ FEVMContract: ``, FEVMTrace: ``, FEVMActorDump: ``, + MinerActorDump: ``, } var TableFieldComments = map[string]map[string]string{ @@ -323,4 +327,5 @@ var TableFieldComments = map[string]map[string]string{ FEVMContract: {}, FEVMTrace: {}, FEVMActorDump: {}, + MinerActorDump: {}, } diff --git a/chain/indexer/tasktype/tasks.go b/chain/indexer/tasktype/tasks.go index aa2cfe37..1a847a23 100644 --- a/chain/indexer/tasktype/tasks.go +++ b/chain/indexer/tasktype/tasks.go @@ -103,6 +103,7 @@ var TaskLookup = map[string][]string{ }, ActorDump: { FEVMActorDump, + MinerActorDump, }, } diff --git a/chain/indexer/tasktype/tasks_test.go b/chain/indexer/tasktype/tasks_test.go index d5dd5bdd..50d7f118 100644 --- a/chain/indexer/tasktype/tasks_test.go +++ b/chain/indexer/tasktype/tasks_test.go @@ -101,7 +101,7 @@ func TestMakeAllTaskAliasNames(t *testing.T) { } func TestMakeAllTaskNames(t *testing.T) { - const TotalTableTasks = 50 + const TotalTableTasks = 51 actual, err := tasktype.MakeTaskNames(tasktype.AllTableTasks) require.NoError(t, err) // if this test fails it means a new task name was added, update the above test diff --git a/lens/interface.go b/lens/interface.go index 06f6eb4d..fe361852 100644 --- a/lens/interface.go +++ b/lens/interface.go @@ -58,6 +58,7 @@ type ChainAPI interface { type StateAPI interface { StateGetActor(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) + StateLookupRobustAddress(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error) StateListActors(context.Context, types.TipSetKey) ([]address.Address, error) StateChangedActors(context.Context, cid.Cid, cid.Cid) (map[string]types.Actor, error) diff --git a/model/actordumps/miner_actor_dump.go b/model/actordumps/miner_actor_dump.go new file mode 100644 index 00000000..e816e09c --- /dev/null +++ b/model/actordumps/miner_actor_dump.go @@ -0,0 +1,138 @@ +package actordumps + +import ( + "context" + "encoding/json" + + "go.opencensus.io/tag" + "go.opentelemetry.io/otel" + + "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/metrics" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lotus/chain/types" +) + +type MinerActorDump struct { + tableName struct{} `pg:"miner_actor_dumps"` // nolint: structcheck + + Height int64 `pg:",pk,notnull,use_zero"` + MinerID string `pg:",pk,notnull"` + MinerAddress string `pg:",pk,notnull"` + StateRoot string `pg:",notnull"` + + // Miner Info + OwnerID string `pg:",notnull"` + OwnerAddress string `pg:",notnull"` + WorkerID string `pg:",notnull"` + WorkerAddress string `pg:",notnull"` + + ConsensusFaultedElapsed int64 `pg:",notnull,use_zero"` + + PeerID string `pg:",notnull"` + ControlAddresses string `pg:",type:jsonb"` + Beneficiary string `pg:",notnull"` + BeneficiaryAddress string `pg:",notnull"` + + SectorSize uint64 `pg:",use_zero"` + NumLiveSectors uint64 `pg:",use_zero"` + + // Claims + RawBytePower string `pg:"type:numeric,notnull"` + QualityAdjPower string `pg:"type:numeric,notnull"` + + // Fil Related Fields + // Locked Funds + TotalLockedFunds string `pg:"type:numeric,notnull"` + VestingFunds string `pg:"type:numeric,notnull"` + InitialPledge string `pg:"type:numeric,notnull"` + PreCommitDeposits string `pg:"type:numeric,notnull"` + + // Balance + AvailableBalance string `pg:"type:numeric,notnull"` + Balance string `pg:"type:numeric,notnull"` + + FeeDebt string `pg:"type:numeric,notnull"` +} + +func (m *MinerActorDump) Persist(ctx context.Context, s model.StorageBatch, _ model.Version) error { + ctx, span := otel.Tracer("").Start(ctx, "MinerActorDump.Persist") + defer span.End() + + ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "miner_actor_dumps")) + metrics.RecordCount(ctx, metrics.PersistModel, 1) + return s.PersistModel(ctx, m) +} + +func (m *MinerActorDump) UpdateMinerInfo(minerState miner.State) error { + minerInfo, err := minerState.Info() + if err != nil { + return err + } + + m.PeerID = string(minerInfo.PeerId) + m.WorkerID = minerInfo.Worker.String() + m.OwnerID = minerInfo.Owner.String() + m.ConsensusFaultedElapsed = int64(minerInfo.ConsensusFaultElapsed) + m.SectorSize = uint64(minerInfo.SectorSize) + m.Beneficiary = minerInfo.Beneficiary.String() + + var ctrlAddresses []string + for _, addr := range minerInfo.ControlAddresses { + ctrlAddresses = append(ctrlAddresses, addr.String()) + } + + b, err := json.Marshal(ctrlAddresses) + if err == nil { + m.ControlAddresses = string(b) + } + + num, err := minerState.NumLiveSectors() + if err == nil { + m.NumLiveSectors = num + } + + return err +} + +func (m *MinerActorDump) UpdateBalanceInfo(actor *types.ActorV5, minerState miner.State) error { + m.Balance = actor.Balance.String() + + availableBalance, err := minerState.AvailableBalance(actor.Balance) + if err != nil { + return err + } + m.AvailableBalance = availableBalance.String() + + feeDebt, err := minerState.FeeDebt() + if err != nil { + return err + } + m.FeeDebt = feeDebt.String() + + lockedFunds, err := minerState.LockedFunds() + if err != nil { + return err + } + m.InitialPledge = lockedFunds.InitialPledgeRequirement.String() + m.VestingFunds = lockedFunds.VestingFunds.String() + m.PreCommitDeposits = lockedFunds.PreCommitDeposits.String() + m.TotalLockedFunds = lockedFunds.TotalLockedFunds().String() + + return nil +} + +type MinerActorDumpList []*MinerActorDump + +func (ml MinerActorDumpList) Persist(ctx context.Context, s model.StorageBatch, _ model.Version) error { + ctx, span := otel.Tracer("").Start(ctx, "MinerActorDumpList.Persist") + defer span.End() + + ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "miner_actor_dumps")) + + if len(ml) == 0 { + return nil + } + metrics.RecordCount(ctx, metrics.PersistModel, len(ml)) + return s.PersistModel(ctx, ml) +} diff --git a/schemas/v1/33_miner_actor_dumps.go b/schemas/v1/33_miner_actor_dumps.go new file mode 100644 index 00000000..0673575f --- /dev/null +++ b/schemas/v1/33_miner_actor_dumps.go @@ -0,0 +1,44 @@ +package v1 + +func init() { + patches.Register( + 33, + ` + CREATE TABLE IF NOT EXISTS {{ .SchemaName | default "public"}}.miner_actor_dumps ( + height BIGINT NOT NULL, + miner_id TEXT, + miner_address TEXT, + state_root TEXT, + owner_id TEXT, + owner_address TEXT, + worker_id TEXT, + worker_address TEXT, + + consensus_faulted_elapsed BIGINT, + + peer_id TEXT, + control_addresses JSONB, + beneficiary TEXT, + beneficiary_address TEXT, + + sector_size BIGINT, + num_live_sectors BIGINT, + + raw_byte_power NUMERIC, + quality_adj_power NUMERIC, + total_locked_funds NUMERIC, + vesting_funds NUMERIC, + initial_pledge NUMERIC, + pre_commit_deposits NUMERIC, + available_balance NUMERIC, + balance NUMERIC, + fee_debt NUMERIC, + + PRIMARY KEY(height, miner_id, miner_address) + ); + CREATE INDEX IF NOT EXISTS miner_actor_dumps_height_idx ON {{ .SchemaName | default "public"}}.miner_actor_dumps USING btree (height); + CREATE INDEX IF NOT EXISTS miner_actor_dumps_miner_id_idx ON {{ .SchemaName | default "public"}}.miner_actor_dumps USING hash (miner_id); + CREATE INDEX IF NOT EXISTS miner_actor_dumps_miner_address_idx ON {{ .SchemaName | default "public"}}.miner_actor_dumps USING hash (miner_address); +`, + ) +} diff --git a/storage/sql.go b/storage/sql.go index 34a9d2af..1b2f797b 100644 --- a/storage/sql.go +++ b/storage/sql.go @@ -102,6 +102,7 @@ var Models = []interface{}{ (*fevm.FEVMContract)(nil), (*fevm.FEVMTrace)(nil), (*actordumps.FEVMActorDump)(nil), + (*actordumps.MinerActorDump)(nil), } var log = logging.Logger("lily/storage") diff --git a/tasks/api.go b/tasks/api.go index 61d2bdf7..8be5a549 100644 --- a/tasks/api.go +++ b/tasks/api.go @@ -84,4 +84,7 @@ type DataSource interface { ChainGetMessagesInTipset(ctx context.Context, tsk types.TipSetKey) ([]api.Message, error) EthGetTransactionByHash(ctx context.Context, txHash *ethtypes.EthHash) (*ethtypes.EthTx, error) StateListActors(ctx context.Context, tsk types.TipSetKey) ([]address.Address, error) + + SetIdRobustAddressMap(ctx context.Context, tsk types.TipSetKey) error + LookupRobustAddress(ctx context.Context, idAddr address.Address, tsk types.TipSetKey) (address.Address, error) } diff --git a/tasks/periodic_actor_dump/miner_actor/tasks.go b/tasks/periodic_actor_dump/miner_actor/tasks.go new file mode 100644 index 00000000..94d363f9 --- /dev/null +++ b/tasks/periodic_actor_dump/miner_actor/tasks.go @@ -0,0 +1,150 @@ +package mineractordump + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/manifest" + "github.com/filecoin-project/lotus/chain/types" + + logging "github.com/ipfs/go-log/v2" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + + builtinminer "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/chain/actors/builtin/power" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/model/actordumps" + visormodel "github.com/filecoin-project/lily/model/visor" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("lily/tasks/mineractordump") + +type Task struct { + node tasks.DataSource +} + +func NewTask(node tasks.DataSource) *Task { + return &Task{ + node: node, + } +} + +func (p *Task) updateAddressFromID(ctx context.Context, current *types.TipSet, minerDumpObj *actordumps.MinerActorDump) error { + // Owner Address + ownerAddr, err := address.NewFromString(minerDumpObj.OwnerID) + if err != nil { + return err + } + ownerActor, err := p.node.ActorInfo(ctx, ownerAddr, current.Key()) + if err != nil { + return err + } + minerDumpObj.OwnerAddress = ownerActor.Actor.Address.String() + + // Worker Address + workerAddr, err := address.NewFromString(minerDumpObj.WorkerID) + if err != nil { + return err + } + workerActor, err := p.node.ActorInfo(ctx, workerAddr, current.Key()) + if err != nil { + return err + } + minerDumpObj.WorkerAddress = workerActor.Actor.Address.String() + + // Beneficiary Address + beneficiaryAddr, err := address.NewFromString(minerDumpObj.Beneficiary) + if err != nil { + return err + } + beneficiaryWorkerActor, err := p.node.ActorInfo(ctx, beneficiaryAddr, current.Key()) + if err != nil { + return err + } + minerDumpObj.BeneficiaryAddress = beneficiaryWorkerActor.Actor.Address.String() + + return nil +} + +func (p *Task) ProcessPeriodicActorDump(ctx context.Context, current *types.TipSet, actors tasks.ActorStatesByType) (model.Persistable, *visormodel.ProcessingReport, error) { + _, span := otel.Tracer("").Start(ctx, "ProcessPeriodicActorDump") + if span.IsRecording() { + span.SetAttributes( + attribute.String("current", current.String()), + attribute.Int64("current_height", int64(current.Height())), + attribute.String("processor", "miner_actor_state_dump"), + ) + } + defer span.End() + + report := &visormodel.ProcessingReport{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + } + + log.Infof("Size of Power Actors: %v", len(actors[manifest.PowerKey])) + + out := make(actordumps.MinerActorDumpList, 0) + errs := []error{} + for _, actor := range actors[manifest.PowerKey] { + powerState, err := power.Load(p.node.Store(), actor) + if err != nil { + log.Errorf("Error at loading power state: [actor cid: %v] err: %v", actor.Code.String(), err) + errs = append(errs, err) + continue + } + + err = powerState.ForEachClaim(func(miner address.Address, claim power.Claim) error { + minerDumpObj := &actordumps.MinerActorDump{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + MinerID: miner.String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + } + + // Update the minerInfo Field into dump model + minerActor, err := p.node.ActorInfo(ctx, miner, current.Key()) + if err != nil { + return err + } + minerDumpObj.MinerAddress = minerActor.Actor.Address.String() + + minerState, err := builtinminer.Load(p.node.Store(), minerActor.Actor) + if err != nil { + return err + } + + err = minerDumpObj.UpdateMinerInfo(minerState) + if err != nil { + return err + } + + err = minerDumpObj.UpdateBalanceInfo(minerActor.Actor, minerState) + if err != nil { + return err + } + + err = p.updateAddressFromID(ctx, current, minerDumpObj) + if err != nil { + log.Error("Error at getting getting the actor address by actor id: %v", err) + } + out = append(out, minerDumpObj) + + return nil + }) + + if err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + report.ErrorsDetected = fmt.Errorf("%v", errs) + } + + return model.PersistableList{out}, report, nil +}