Skip to content

Commit d9cab1c

Browse files
committed
upgrade to pebble v2.1.0
1 parent 6324481 commit d9cab1c

File tree

4 files changed

+45
-45
lines changed

4 files changed

+45
-45
lines changed

bench/store_bench_test.go

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func newPebbleSubject(b *testing.B) func() (indexer.Interface, error) {
8585
pebbleOpts := &pb2.Options{
8686
BytesPerSync: 10 << 20, // 10 MiB
8787
WALBytesPerSync: 10 << 20, // 10 MiB
88-
MaxConcurrentCompactions: func() int { return 10 },
88+
CompactionConcurrencyRange: func() (int, int) { return 1, 10 },
8989
MemTableSize: 64 << 20, // 64 MiB
9090
MemTableStopWritesThreshold: 4,
9191
LBaseMaxBytes: 64 << 20, // 64 MiB
@@ -96,21 +96,15 @@ func newPebbleSubject(b *testing.B) func() (indexer.Interface, error) {
9696

9797
pebbleOpts.Experimental.ReadCompactionRate = 10 << 20 // 20 MiB
9898

99-
const numLevels = 7
100-
pebbleOpts.Levels = make([]pb2.LevelOptions, numLevels)
101-
for i := 0; i < numLevels; i++ {
102-
l := &pebbleOpts.Levels[i]
103-
l.BlockSize = 32 << 10 // 32 KiB
104-
l.IndexBlockSize = 256 << 10 // 256 KiB
105-
l.FilterPolicy = bloom.FilterPolicy(10)
106-
l.FilterType = pb2.TableFilter
107-
if i > 0 {
108-
l.TargetFileSize = pebbleOpts.Levels[i-1].TargetFileSize * 2
109-
}
110-
l.EnsureDefaults()
99+
for i := range pebbleOpts.Levels {
100+
pebbleOpts.Levels[i].BlockSize = 32 << 10 // 32 KiB
101+
pebbleOpts.Levels[i].IndexBlockSize = 256 << 10 // 256 KiB
102+
pebbleOpts.Levels[i].FilterPolicy = bloom.FilterPolicy(10)
103+
pebbleOpts.Levels[i].FilterType = pb2.TableFilter
111104
}
112-
pebbleOpts.Levels[numLevels-1].FilterPolicy = nil
105+
pebbleOpts.Levels[len(pebbleOpts.Levels)-1].FilterPolicy = nil
113106
pebbleOpts.Cache = pb2.NewCache(1 << 30) // 1 GiB
107+
pebbleOpts.EnsureDefaults()
114108

115109
return pebble.New(b.TempDir(), pebbleOpts)
116110
}

go.mod

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/ipni/go-indexer-core
33
go 1.24.0
44

55
require (
6-
github.com/cockroachdb/pebble/v2 v2.0.6
6+
github.com/cockroachdb/pebble/v2 v2.1.0
77
github.com/gammazero/radixtree v0.4.0
88
github.com/ipfs/go-cid v0.5.0
99
github.com/ipfs/go-log/v2 v2.8.1
@@ -18,12 +18,13 @@ require (
1818
)
1919

2020
require (
21-
github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect
21+
github.com/DataDog/zstd v1.5.7 // indirect
22+
github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 // indirect
23+
github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect
2224
github.com/beorn7/perks v1.0.1 // indirect
2325
github.com/cespare/xxhash/v2 v2.3.0 // indirect
24-
github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect
26+
github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect
2527
github.com/cockroachdb/errors v1.11.3 // indirect
26-
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
2728
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
2829
github.com/cockroachdb/redact v1.1.5 // indirect
2930
github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 // indirect
@@ -40,6 +41,7 @@ require (
4041
github.com/kr/text v0.2.0 // indirect
4142
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
4243
github.com/mattn/go-isatty v0.0.20 // indirect
44+
github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect
4345
github.com/minio/sha256-simd v1.0.1 // indirect
4446
github.com/multiformats/go-base32 v0.1.0 // indirect
4547
github.com/multiformats/go-base36 v0.2.0 // indirect

go.sum

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
22
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
3-
github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE=
4-
github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
3+
github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
4+
github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
5+
github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657 h1:8XBWWQD+vFF+JqOsm16t0Kab1a7YWV8+GISVEP8AuZ8=
6+
github.com/RaduBerinde/axisds v0.0.0-20250419182453-5135a0650657/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y=
7+
github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk=
8+
github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc=
59
github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo=
610
github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo=
711
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -11,20 +15,18 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
1115
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
1216
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
1317
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
14-
github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 h1:bvJv505UUfjzbaIPdNS4AEkHreDqQk6yuNpsdRHpwFA=
15-
github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac=
16-
github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056 h1:slXychO2uDM6hYRu4c0pD0udNI8uObfeKN6UInWViS8=
17-
github.com/cockroachdb/datadriven v1.0.3-0.20240530155848-7682d40af056/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
18+
github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU=
19+
github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac=
20+
github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI=
21+
github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g=
1822
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
1923
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
20-
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
21-
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
2224
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
2325
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
2426
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
2527
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
26-
github.com/cockroachdb/pebble/v2 v2.0.6 h1:eL54kX2AKp1ePJ/8vq4IO3xIEPpvVjlSP12dlLYilyE=
27-
github.com/cockroachdb/pebble/v2 v2.0.6/go.mod h1:un1DXG73PKw3F7Ndd30YactyvsFviI9Fuhe0tENdnyA=
28+
github.com/cockroachdb/pebble/v2 v2.1.0 h1:6KZvjSpWcEXZUvlLzTRC7T1A2G7r+bFskIzggklxixo=
29+
github.com/cockroachdb/pebble/v2 v2.1.0/go.mod h1:Aza05DCCc05ghIJZkB4Q/axv/JK9wx5cFwWcnhG0eGw=
2830
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
2931
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
3032
github.com/cockroachdb/swiss v0.0.0-20250624142022-d6e517c1d961 h1:Nua446ru3juLHLZd4AwKNzClZgL1co3pUPGv3o8FlcA=
@@ -105,6 +107,8 @@ github.com/libp2p/go-libp2p v0.43.0 h1:b2bg2cRNmY4HpLK8VHYQXLX2d3iND95OjodLFymvq
105107
github.com/libp2p/go-libp2p v0.43.0/go.mod h1:IiSqAXDyP2sWH+J2gs43pNmB/y4FOi2XQPbsb+8qvzc=
106108
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
107109
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
110+
github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw=
111+
github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec=
108112
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
109113
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
110114
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=

metrics/pebble_metrics.go

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,11 @@ var (
6262
" compaction. Such files are compacted in a rewrite compaction"+
6363
" when no other compactions are picked.", stats.UnitDimensionless)
6464

65-
// l0NumFiles is the total number of files in L0. The number of L0 files should not be in the high thousands.
66-
// High values indicate heavy write load that is causing accumulation of files in level 0. These files are not
65+
// l0TablesCount is the total count of sstables in L0. The number of L0 sstables should not be in the high thousands.
66+
// High values indicate heavy write load that is causing accumulation of sstables in level 0. These sstables are not
6767
// being compacted quickly enough to lower levels, resulting in a misshapen LSM.
68-
l0NumFiles = stats.Int64("ipni/pebble/compact_l0_num_files", "The total number of files in L0. The number of L0 files should not be in the high thousands."+
69-
" High values indicate heavy write load that is causing accumulation of files in level 0. These files are not"+
68+
l0TablesCount = stats.Int64("ipni/pebble/compact_l0_tables_count", "The total count of sstables in L0. The number of L0 sstables should not be in the high thousands."+
69+
" High values indicate heavy write load that is causing accumulation of sstables in level 0. These sstables are not"+
7070
" being compacted quickly enough to lower levels, resulting in a misshapen LSM.", stats.UnitDimensionless)
7171
)
7272

@@ -120,8 +120,8 @@ var (
120120
Measure: compactMarkedFiles,
121121
Aggregation: view.LastValue(),
122122
}
123-
l0NumFilesView = &view.View{
124-
Measure: l0NumFiles,
123+
l0TablesCountView = &view.View{
124+
Measure: l0TablesCount,
125125
Aggregation: view.LastValue(),
126126
}
127127
)
@@ -139,7 +139,7 @@ var PebbleViews = []*view.View{
139139
compactInProgressBytesView,
140140
compactNumInProgressView,
141141
compactMarkedFilesView,
142-
l0NumFilesView,
142+
l0TablesCountView,
143143
}
144144

145145
// ObservePebbleMetrics is used to periodically report metrics from the pebble database
@@ -176,7 +176,7 @@ func reportMetrics(m *pebble.Metrics) {
176176
compactInProgressBytes.M(int64(m.Compact.InProgressBytes)),
177177
compactNumInProgress.M(int64(m.Compact.NumInProgress)),
178178
compactMarkedFiles.M(int64(m.Compact.MarkedFiles)),
179-
l0NumFiles.M(int64(m.Levels[0].NumFiles)))
179+
l0TablesCount.M(int64(m.Levels[0].TablesCount)))
180180

181181
// Block cache metrics
182182
stats.RecordWithOptions(ctx,
@@ -195,20 +195,20 @@ func reportMetrics(m *pebble.Metrics) {
195195
stats.WithMeasurements(cacheMisses.M(m.BlockCache.Misses)),
196196
stats.WithTags(tag.Insert(cacheTag, "block")))
197197

198-
// Table cache metrics
198+
// File cache metrics
199199
stats.RecordWithOptions(ctx,
200-
stats.WithMeasurements(cacheCount.M(m.TableCache.Count)),
201-
stats.WithTags(tag.Insert(cacheTag, "table")))
200+
stats.WithMeasurements(cacheCount.M(m.FileCache.TableCount+m.FileCache.BlobFileCount)),
201+
stats.WithTags(tag.Insert(cacheTag, "file")))
202202

203203
stats.RecordWithOptions(ctx,
204-
stats.WithMeasurements(cacheSize.M(m.TableCache.Size)),
205-
stats.WithTags(tag.Insert(cacheTag, "table")))
204+
stats.WithMeasurements(cacheSize.M(m.FileCache.Size)),
205+
stats.WithTags(tag.Insert(cacheTag, "file")))
206206

207207
stats.RecordWithOptions(ctx,
208-
stats.WithMeasurements(cacheHits.M(m.TableCache.Hits)),
209-
stats.WithTags(tag.Insert(cacheTag, "table")))
208+
stats.WithMeasurements(cacheHits.M(m.FileCache.Hits)),
209+
stats.WithTags(tag.Insert(cacheTag, "file")))
210210

211211
stats.RecordWithOptions(ctx,
212-
stats.WithMeasurements(cacheMisses.M(m.TableCache.Misses)),
213-
stats.WithTags(tag.Insert(cacheTag, "table")))
212+
stats.WithMeasurements(cacheMisses.M(m.FileCache.Misses)),
213+
stats.WithTags(tag.Insert(cacheTag, "file")))
214214
}

0 commit comments

Comments
 (0)