Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Upgrade to v1.4.0-beta.1 & implement key-only Iteration #450

Merged
merged 11 commits into from
Aug 30, 2023
Merged

Upgrade to v1.4.0-beta.1 & implement key-only Iteration #450

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
94b6319
Update cosmwasm version
chipshort Aug 25, 2023
41d7ced
Implement GoStorage::{next_key, next_value}
chipshort Aug 25, 2023
61bfe19
Remove _db suffix
chipshort Aug 28, 2023
4f4ce1d
Add function name to iterator vtable error message
chipshort Aug 28, 2023
57da21e
Use if-let instead of match
chipshort Aug 28, 2023
7410295
Use let-else
webmaster128 Aug 28, 2023
b7d3f0f
Adjust cache sizes
chipshort Aug 28, 2023
6409d9d
Fix comment
chipshort Aug 28, 2023
c928670
Rename db function type names
chipshort Aug 28, 2023
f84a632
Simplify next_key_or_val signature
chipshort Aug 28, 2023
ac29b8d
Use cosmwasm beta release
chipshort Aug 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions internal/api/bindings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,12 +254,22 @@ typedef struct iterator_t {
} iterator_t;

typedef struct Iterator_vtable {
int32_t (*next_db)(struct iterator_t,
struct gas_meter_t*,
uint64_t*,
struct UnmanagedVector*,
struct UnmanagedVector*,
struct UnmanagedVector*);
int32_t (*next)(struct iterator_t,
struct gas_meter_t*,
uint64_t*,
struct UnmanagedVector*,
struct UnmanagedVector*,
struct UnmanagedVector*);
int32_t (*next_key)(struct iterator_t,
struct gas_meter_t*,
uint64_t*,
struct UnmanagedVector*,
struct UnmanagedVector*);
int32_t (*next_value)(struct iterator_t,
struct gas_meter_t*,
uint64_t*,
struct UnmanagedVector*,
struct UnmanagedVector*);
} Iterator_vtable;

typedef struct GoIter {
Expand Down
57 changes: 56 additions & 1 deletion internal/api/callbacks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ typedef GoError (*remove_db_fn)(db_t *ptr, gas_meter_t *gas_meter, uint64_t *use
typedef GoError (*scan_db_fn)(db_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, U8SliceView start, U8SliceView end, int32_t order, GoIter *out, UnmanagedVector *errOut);
// iterator
typedef GoError (*next_db_fn)(iterator_t idx, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *key, UnmanagedVector *val, UnmanagedVector *errOut);
typedef GoError (*next_key_db_fn)(iterator_t idx, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *key, UnmanagedVector *errOut);
typedef GoError (*next_value_db_fn)(iterator_t idx, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *val, UnmanagedVector *errOut);
// and api
typedef GoError (*humanize_address_fn)(api_t *ptr, U8SliceView src, UnmanagedVector *dest, UnmanagedVector *errOut, uint64_t *used_gas);
typedef GoError (*canonicalize_address_fn)(api_t *ptr, U8SliceView src, UnmanagedVector *dest, UnmanagedVector *errOut, uint64_t *used_gas);
Expand All @@ -25,6 +27,8 @@ GoError cDelete_cgo(db_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, U8Sli
GoError cScan_cgo(db_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, U8SliceView start, U8SliceView end, int32_t order, GoIter *out, UnmanagedVector *errOut);
// iterator
GoError cNext_cgo(iterator_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *key, UnmanagedVector *val, UnmanagedVector *errOut);
GoError cNextKey_cgo(iterator_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *key, UnmanagedVector *errOut);
GoError cNextValue_cgo(iterator_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *val, UnmanagedVector *errOut);
// api
GoError cHumanAddress_cgo(api_t *ptr, U8SliceView src, UnmanagedVector *dest, UnmanagedVector *errOut, uint64_t *used_gas);
GoError cCanonicalAddress_cgo(api_t *ptr, U8SliceView src, UnmanagedVector *dest, UnmanagedVector *errOut, uint64_t *used_gas);
Expand Down Expand Up @@ -128,7 +132,9 @@ func buildDB(state *DBState, gm *types.GasMeter) C.Db {
}

var iterator_vtable = C.Iterator_vtable{
next_db: (C.next_db_fn)(C.cNext_cgo),
next: (C.next_db_fn)(C.cNext_cgo),
next_key: (C.next_key_db_fn)(C.cNextKey_cgo),
next_value: (C.next_value_db_fn)(C.cNextValue_cgo),
chipshort marked this conversation as resolved.
Show resolved Hide resolved
}

// An iterator including referenced objects is 117 bytes large (calculated using https://github.com/DmitriyVTitov/size).
Expand Down Expand Up @@ -310,6 +316,55 @@ func cNext(ref C.iterator_t, gasMeter *C.gas_meter_t, usedGas *cu64, key *C.Unma
return C.GoError_None
}

//export cNextKey
func cNextKey(ref C.iterator_t, gasMeter *C.gas_meter_t, usedGas *cu64, key *C.UnmanagedVector, errOut *C.UnmanagedVector) (ret C.GoError) {
return nextPart(ref, gasMeter, usedGas, key, errOut, func(iter types.Iterator) []byte { return iter.Key() })
}

//export cNextValue
func cNextValue(ref C.iterator_t, gasMeter *C.gas_meter_t, usedGas *cu64, value *C.UnmanagedVector, errOut *C.UnmanagedVector) (ret C.GoError) {
return nextPart(ref, gasMeter, usedGas, value, errOut, func(iter types.Iterator) []byte { return iter.Value() })
}

// nextPart is a helper function that contains the shared code for key- and value-only iteration.
func nextPart(ref C.iterator_t, gasMeter *C.gas_meter_t, usedGas *cu64, output *C.UnmanagedVector, errOut *C.UnmanagedVector, valFn func(types.Iterator) []byte) (ret C.GoError) {
// typical usage of iterator
// for ; itr.Valid(); itr.Next() {
// k, v := itr.Key(); itr.Value()
// ...
// }

defer recoverPanic(&ret)
if ref.call_id == 0 || gasMeter == nil || usedGas == nil || output == nil || errOut == nil {
// we received an invalid pointer
return C.GoError_BadArgument
}
if !(*output).is_none || !(*errOut).is_none {
panic("Got a non-none UnmanagedVector we're about to override. This is a bug because someone has to drop the old one.")
}

gm := *(*types.GasMeter)(unsafe.Pointer(gasMeter))
iter := retrieveIterator(uint64(ref.call_id), uint64(ref.iterator_index))
if iter == nil {
panic("Unable to retrieve iterator.")
}
if !iter.Valid() {
// end of iterator, return as no-op, nil key is considered end
chipshort marked this conversation as resolved.
Show resolved Hide resolved
return C.GoError_None
}

gasBefore := gm.GasConsumed()
// call Next at the end, upon creation we have first data loaded
out := valFn(iter)
// check iter.Error() ????
iter.Next()
gasAfter := gm.GasConsumed()
*usedGas = (cu64)(gasAfter - gasBefore)
chipshort marked this conversation as resolved.
Show resolved Hide resolved

*output = newUnmanagedVector(out)
return C.GoError_None
}

var api_vtable = C.GoApi_vtable{
humanize_address: (C.humanize_address_fn)(C.cHumanAddress_cgo),
canonicalize_address: (C.canonicalize_address_fn)(C.cCanonicalAddress_cgo),
Expand Down
8 changes: 8 additions & 0 deletions internal/api/callbacks_cgo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ GoError cDelete(db_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, U8SliceVi
GoError cScan(db_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, U8SliceView start, U8SliceView end, int32_t order, GoIter *out, UnmanagedVector *errOut);
// imports (iterator)
GoError cNext(iterator_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *key, UnmanagedVector *val, UnmanagedVector *errOut);
GoError cNextKey(iterator_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *key, UnmanagedVector *errOut);
GoError cNextValue(iterator_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *value, UnmanagedVector *errOut);
// imports (api)
GoError cHumanAddress(api_t *ptr, U8SliceView src, UnmanagedVector *dest, UnmanagedVector *errOut, uint64_t *used_gas);
GoError cCanonicalAddress(api_t *ptr, U8SliceView src, UnmanagedVector *dest, UnmanagedVector *errOut, uint64_t *used_gas);
Expand All @@ -35,6 +37,12 @@ GoError cScan_cgo(db_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, U8Slice
GoError cNext_cgo(iterator_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *key, UnmanagedVector *val, UnmanagedVector *errOut) {
return cNext(ptr, gas_meter, used_gas, key, val, errOut);
}
GoError cNextKey_cgo(iterator_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *key, UnmanagedVector *errOut) {
return cNextKey(ptr, gas_meter, used_gas, key, errOut);
}
GoError cNextValue_cgo(iterator_t *ptr, gas_meter_t *gas_meter, uint64_t *used_gas, UnmanagedVector *val, UnmanagedVector *errOut) {
return cNextValue(ptr, gas_meter, used_gas, val, errOut);
}

// Gateway functions (api)
GoError cCanonicalAddress_cgo(api_t *ptr, U8SliceView src, UnmanagedVector *dest, UnmanagedVector *errOut, uint64_t *used_gas) {
Expand Down
33 changes: 17 additions & 16 deletions internal/api/lib_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,8 @@ func TestGetMetrics(t *testing.T) {
require.Equal(t, uint32(0), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
t.Log(t, metrics.SizeMemoryCache)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)

// Instantiate 2
msg2 := []byte(`{"verifier": "fred", "beneficiary": "susi"}`)
Expand All @@ -273,7 +274,7 @@ func TestGetMetrics(t *testing.T) {
require.Equal(t, uint32(1), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)

// Pin
err = Pin(cache, checksum)
Expand All @@ -282,12 +283,12 @@ func TestGetMetrics(t *testing.T) {
// GetMetrics 5
metrics, err = GetMetrics(cache)
assert.NoError(t, err)
require.Equal(t, uint32(2), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint32(1), metrics.HitsMemoryCache)
require.Equal(t, uint32(2), metrics.HitsFsCache)
require.Equal(t, uint64(1), metrics.ElementsPinnedMemoryCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizePinnedMemoryCache, 0.2)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizePinnedMemoryCache, 0.25)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)

// Instantiate 3
msg3 := []byte(`{"verifier": "fred", "beneficiary": "bert"}`)
Expand All @@ -298,12 +299,12 @@ func TestGetMetrics(t *testing.T) {
metrics, err = GetMetrics(cache)
assert.NoError(t, err)
require.Equal(t, uint32(1), metrics.HitsPinnedMemoryCache)
require.Equal(t, uint32(2), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint32(1), metrics.HitsMemoryCache)
require.Equal(t, uint32(2), metrics.HitsFsCache)
require.Equal(t, uint64(1), metrics.ElementsPinnedMemoryCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizePinnedMemoryCache, 0.2)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizePinnedMemoryCache, 0.25)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)

// Unpin
err = Unpin(cache, checksum)
Expand All @@ -313,12 +314,12 @@ func TestGetMetrics(t *testing.T) {
metrics, err = GetMetrics(cache)
assert.NoError(t, err)
require.Equal(t, uint32(1), metrics.HitsPinnedMemoryCache)
require.Equal(t, uint32(2), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint32(1), metrics.HitsMemoryCache)
require.Equal(t, uint32(2), metrics.HitsFsCache)
require.Equal(t, uint64(0), metrics.ElementsPinnedMemoryCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.Equal(t, uint64(0), metrics.SizePinnedMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)

// Instantiate 4
msg4 := []byte(`{"verifier": "fred", "beneficiary": "jeff"}`)
Expand All @@ -329,12 +330,12 @@ func TestGetMetrics(t *testing.T) {
metrics, err = GetMetrics(cache)
assert.NoError(t, err)
require.Equal(t, uint32(1), metrics.HitsPinnedMemoryCache)
require.Equal(t, uint32(3), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint32(2), metrics.HitsMemoryCache)
require.Equal(t, uint32(2), metrics.HitsFsCache)
require.Equal(t, uint64(0), metrics.ElementsPinnedMemoryCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.Equal(t, uint64(0), metrics.SizePinnedMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)
}

func TestInstantiate(t *testing.T) {
Expand Down
34 changes: 17 additions & 17 deletions lib_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func TestStoreCode(t *testing.T) {

wasm := []byte{0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00}
_, err := vm.StoreCode(wasm)
require.ErrorContains(t, err, "Error during static Wasm validation: Wasm contract doesn't have a memory section")
require.ErrorContains(t, err, "Error during static Wasm validation: Wasm contract must contain exactly one memory")
}

// No Wasm
Expand Down Expand Up @@ -270,7 +270,7 @@ func TestGetMetrics(t *testing.T) {
require.Equal(t, uint32(0), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)

// Instantiate 2
msg2 := []byte(`{"verifier": "fred", "beneficiary": "susi"}`)
Expand All @@ -284,7 +284,7 @@ func TestGetMetrics(t *testing.T) {
require.Equal(t, uint32(1), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)

// Pin
err = vm.Pin(checksum)
Expand All @@ -293,12 +293,12 @@ func TestGetMetrics(t *testing.T) {
// GetMetrics 5
metrics, err = vm.GetMetrics()
assert.NoError(t, err)
require.Equal(t, uint32(2), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint32(1), metrics.HitsMemoryCache)
require.Equal(t, uint32(2), metrics.HitsFsCache)
require.Equal(t, uint64(1), metrics.ElementsPinnedMemoryCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizePinnedMemoryCache, 0.2)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizePinnedMemoryCache, 0.25)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)

// Instantiate 3
msg3 := []byte(`{"verifier": "fred", "beneficiary": "bert"}`)
Expand All @@ -310,12 +310,12 @@ func TestGetMetrics(t *testing.T) {
metrics, err = vm.GetMetrics()
assert.NoError(t, err)
require.Equal(t, uint32(1), metrics.HitsPinnedMemoryCache)
require.Equal(t, uint32(2), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint32(1), metrics.HitsMemoryCache)
require.Equal(t, uint32(2), metrics.HitsFsCache)
require.Equal(t, uint64(1), metrics.ElementsPinnedMemoryCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizePinnedMemoryCache, 0.2)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizePinnedMemoryCache, 0.25)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)

// Unpin
err = vm.Unpin(checksum)
Expand All @@ -325,12 +325,12 @@ func TestGetMetrics(t *testing.T) {
metrics, err = vm.GetMetrics()
assert.NoError(t, err)
require.Equal(t, uint32(1), metrics.HitsPinnedMemoryCache)
require.Equal(t, uint32(2), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint32(1), metrics.HitsMemoryCache)
require.Equal(t, uint32(2), metrics.HitsFsCache)
require.Equal(t, uint64(0), metrics.ElementsPinnedMemoryCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.Equal(t, uint64(0), metrics.SizePinnedMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)

// Instantiate 4
msg4 := []byte(`{"verifier": "fred", "beneficiary": "jeff"}`)
Expand All @@ -342,10 +342,10 @@ func TestGetMetrics(t *testing.T) {
metrics, err = vm.GetMetrics()
assert.NoError(t, err)
require.Equal(t, uint32(1), metrics.HitsPinnedMemoryCache)
require.Equal(t, uint32(3), metrics.HitsMemoryCache)
require.Equal(t, uint32(1), metrics.HitsFsCache)
require.Equal(t, uint32(2), metrics.HitsMemoryCache)
require.Equal(t, uint32(2), metrics.HitsFsCache)
require.Equal(t, uint64(0), metrics.ElementsPinnedMemoryCache)
require.Equal(t, uint64(1), metrics.ElementsMemoryCache)
require.Equal(t, uint64(0), metrics.SizePinnedMemoryCache)
require.InEpsilon(t, 4075417, metrics.SizeMemoryCache, 0.2)
require.InEpsilon(t, 2832576, metrics.SizeMemoryCache, 0.25)
}
Loading