Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(ledger/store): fill bloom filter at store init #457

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions pkg/storage/sqlstorage/accounts.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,83 @@
return sb, t
}

func (s *Store) GetAccountAddresses(ctx context.Context, q ledger.AccountsQuery) (api.Cursor[core.AccountAddress], error) {
sb, t := s.buildAccountsQuery(q)
sb.Select("address")
sb.OrderBy("address desc")

if q.AfterAddress != "" {
sb.Where(sb.L("address", q.AfterAddress))
t.AfterAddress = q.AfterAddress
}

Check warning on line 126 in pkg/storage/sqlstorage/accounts.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/accounts.go#L124-L126

Added lines #L124 - L126 were not covered by tests

// We fetch an additional account to know if there is more
sb.Limit(int(q.PageSize + 1))
t.PageSize = q.PageSize
sb.Offset(int(q.Offset))

executor, err := s.executorProvider(ctx)
if err != nil {
return api.Cursor[core.AccountAddress]{}, err
}

Check warning on line 136 in pkg/storage/sqlstorage/accounts.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/accounts.go#L135-L136

Added lines #L135 - L136 were not covered by tests

sqlq, args := sb.BuildWithFlavor(s.schema.Flavor())
rows, err := executor.QueryContext(ctx, sqlq, args...)
if err != nil {
return api.Cursor[core.AccountAddress]{}, s.error(err)
}

Check warning on line 142 in pkg/storage/sqlstorage/accounts.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/accounts.go#L141-L142

Added lines #L141 - L142 were not covered by tests
defer rows.Close()

addresses := make([]core.AccountAddress, 0)
for rows.Next() {
var accountAddress core.AccountAddress
if err := rows.Scan(&accountAddress); err != nil {
return api.Cursor[core.AccountAddress]{}, err
}

Check warning on line 150 in pkg/storage/sqlstorage/accounts.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/accounts.go#L149-L150

Added lines #L149 - L150 were not covered by tests

addresses = append(addresses, accountAddress)
}
if rows.Err() != nil {
return api.Cursor[core.AccountAddress]{}, rows.Err()
}

Check warning on line 156 in pkg/storage/sqlstorage/accounts.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/accounts.go#L155-L156

Added lines #L155 - L156 were not covered by tests

var previous, next string
if q.Offset > 0 {
offset := int(q.Offset) - int(q.PageSize)
if offset < 0 {
t.Offset = 0

Check warning on line 162 in pkg/storage/sqlstorage/accounts.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/accounts.go#L162

Added line #L162 was not covered by tests
} else {
t.Offset = uint(offset)
}
raw, err := json.Marshal(t)
if err != nil {
return api.Cursor[core.AccountAddress]{}, s.error(err)
}

Check warning on line 169 in pkg/storage/sqlstorage/accounts.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/accounts.go#L168-L169

Added lines #L168 - L169 were not covered by tests
previous = base64.RawURLEncoding.EncodeToString(raw)
}

if len(addresses) == int(q.PageSize+1) {
addresses = addresses[:len(addresses)-1]
t.Offset = q.Offset + q.PageSize
raw, err := json.Marshal(t)
if err != nil {
return api.Cursor[core.AccountAddress]{}, s.error(err)
}

Check warning on line 179 in pkg/storage/sqlstorage/accounts.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/accounts.go#L178-L179

Added lines #L178 - L179 were not covered by tests
next = base64.RawURLEncoding.EncodeToString(raw)
}

hasMore := next != ""
return api.Cursor[core.AccountAddress]{
PageSize: int(q.PageSize),
HasMore: hasMore,
Previous: previous,
Next: next,
Data: addresses,
PageSizeDeprecated: int(q.PageSize),
HasMoreDeprecated: &hasMore,
}, nil
}

func (s *Store) GetAccounts(ctx context.Context, q ledger.AccountsQuery) (api.Cursor[core.Account], error) {
accounts := make([]core.Account, 0)

Expand Down
46 changes: 45 additions & 1 deletion pkg/storage/sqlstorage/store_ledger.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"os"
"strconv"
Expand Down Expand Up @@ -61,7 +63,49 @@
return false, err
}

return Migrate(ctx, s.schema, migrations...)
modified, err := Migrate(ctx, s.schema, migrations...)
if err != nil {
return modified, err
}

Check warning on line 69 in pkg/storage/sqlstorage/store_ledger.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/store_ledger.go#L68-L69

Added lines #L68 - L69 were not covered by tests

hasMore := true
q := ledger.NewAccountsQuery()
for hasMore {
addresses, err := s.GetAccountAddresses(ctx, *q)
if err != nil {
return modified, err
}

Check warning on line 77 in pkg/storage/sqlstorage/store_ledger.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/store_ledger.go#L76-L77

Added lines #L76 - L77 were not covered by tests

for _, address := range addresses.Data {
fmt.Println("ADDING ADDRESS", string(address))
s.bloom.AddString(string(address))
}

hasMore = addresses.HasMore

if hasMore {
res, err := base64.RawURLEncoding.DecodeString(addresses.Next)
if err != nil {
return modified, err
}

Check warning on line 90 in pkg/storage/sqlstorage/store_ledger.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/store_ledger.go#L89-L90

Added lines #L89 - L90 were not covered by tests

token := AccPaginationToken{}
if err := json.Unmarshal(res, &token); err != nil {
return modified, err
}

Check warning on line 95 in pkg/storage/sqlstorage/store_ledger.go

View check run for this annotation

Codecov / codecov/patch

pkg/storage/sqlstorage/store_ledger.go#L94-L95

Added lines #L94 - L95 were not covered by tests

q = q.
WithOffset(token.Offset).
WithAfterAddress(token.AfterAddress).
WithAddressFilter(token.AddressRegexpFilter).
WithBalanceFilter(token.BalanceFilter).
WithBalanceOperatorFilter(token.BalanceOperatorFilter).
WithMetadataFilter(token.MetadataFilter).
WithPageSize(token.PageSize)
}
}

return modified, err
}

func (s *Store) Close(ctx context.Context) error {
Expand Down
Loading