Skip to content
Closed
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
55 commits
Select commit Hold shift + click to select a range
3090cbf
add blob submission metrics
gupadhyaya Sep 12, 2025
b25c77f
fix tests
gupadhyaya Sep 12, 2025
c451259
fix build errors
gupadhyaya Sep 12, 2025
931387a
fix api
gupadhyaya Sep 12, 2025
43e6746
shorten
gupadhyaya Sep 12, 2025
b4f3dde
Merge branch 'main' into blob_submission_metrics
gupadhyaya Sep 12, 2025
0487333
remove duplicates
gupadhyaya Sep 16, 2025
6be3ead
fix format issue
gupadhyaya Sep 16, 2025
17a30e2
Fix bitswap compilation issues and enable metrics in tastora framework
gupadhyaya Sep 17, 2025
6bc2316
Remove HashOnRead method from BlockstoreWithMetrics
gupadhyaya Sep 17, 2025
962f115
Update tastora framework to use local Docker image with metrics
gupadhyaya Sep 17, 2025
7198aa5
Remove interface compliance check for BlockstoreWithMetrics
gupadhyaya Sep 17, 2025
80e2a6a
fix coreaccessor with new metrics instead of nil, fx.invoke issue
gupadhyaya Sep 17, 2025
af3953a
fix blob metrics
gupadhyaya Sep 17, 2025
2af16ef
go.mod changes
gupadhyaya Sep 17, 2025
06e3513
Merge branch 'main' into blob_submission_metrics
gupadhyaya Sep 17, 2025
4659fbe
go mod tidy
gupadhyaya Sep 17, 2025
871bd90
blob metrics fixes
gupadhyaya Sep 18, 2025
ccbb3a3
fix
gupadhyaya Sep 18, 2025
e00d3c7
remove unnecessary method
gupadhyaya Sep 18, 2025
e98716f
remove debug stuff
gupadhyaya Sep 18, 2025
ab8d1b2
WithMetrics
gupadhyaya Sep 18, 2025
0f2a3cf
remove passing nil metrics
gupadhyaya Sep 18, 2025
5c1a2f6
private metrics, stop method
gupadhyaya Sep 18, 2025
88ba0e0
cleanup
gupadhyaya Sep 18, 2025
88e78ba
Merge branch 'main' into blob_submission_metrics
gupadhyaya Sep 18, 2025
510b48c
blob metrics changes
gupadhyaya Sep 18, 2025
0044277
remove nil checks
gupadhyaya Sep 19, 2025
ab99226
get rid of duplicate legacy state metrics
gupadhyaya Sep 19, 2025
29c9a4e
optimizing metrics recording
gupadhyaya Sep 19, 2025
1ba9c6c
error out if metrics fail
gupadhyaya Sep 19, 2025
c604d7b
named error
gupadhyaya Sep 19, 2025
74a42d1
skip resp name
gupadhyaya Sep 19, 2025
878d1bf
error as bool/enum to avoid cardinality explosion
gupadhyaya Sep 19, 2025
9356b77
golangci-lint fix
gupadhyaya Sep 19, 2025
06a3961
unify error counting
gupadhyaya Sep 19, 2025
b6f7a3d
remove magic strings
gupadhyaya Sep 22, 2025
96b085a
golangci-lint fixes
gupadhyaya Sep 23, 2025
f110c2d
further fix golangci-lint
gupadhyaya Sep 23, 2025
533f70e
fixing docker security issue
gupadhyaya Sep 23, 2025
344da58
Revert "fixing docker security issue"
gupadhyaya Sep 23, 2025
f7d2d64
Merge origin/main into blob_submission_metrics
gupadhyaya Oct 14, 2025
0de2e67
Fix compilation error after merge
gupadhyaya Oct 14, 2025
380ea9e
Apply goimports-reviser formatting to fix import organization
gupadhyaya Oct 14, 2025
5ac90dc
Fix import ordering in state/core_access.go
gupadhyaya Oct 14, 2025
b81bfc0
Fix remaining issues after merge
gupadhyaya Oct 14, 2025
4495e04
minor
gupadhyaya Oct 14, 2025
5006f04
go mod tidy
gupadhyaya Oct 14, 2025
0d3d4e4
remove unnecessary metrics
gupadhyaya Oct 21, 2025
70ad967
Merge branch 'main' into blob_submission_metrics
gupadhyaya Oct 21, 2025
c373bd9
Remove accidentally committed grafana and metrics files
gupadhyaya Oct 21, 2025
5ac205d
Fix TestListenerWithWrongChainRPC cleanup issue
gupadhyaya Oct 21, 2025
03d26e7
Merge remote-tracking branch 'origin/main' into blob_submission_metrics
gupadhyaya Oct 28, 2025
ace5b72
fix
gupadhyaya Oct 28, 2025
24a6adc
minor fix
gupadhyaya Oct 28, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion api/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ func (c *Client) initTxClient(
trustedHeadGetter{remote: c.Header},
conn,
submitCfg.Network.String(),
nil, // metrics not available in client context
)
if err != nil {
return err
Expand All @@ -108,7 +109,7 @@ func (c *Client) initTxClient(
c.State = core

// setup blob submission service using core
blobSvc := blob.NewService(core, nil, nil, nil)
blobSvc := blob.NewService(core, nil, nil, nil, nil) // metrics not available in client context
err = blobSvc.Start(ctx)
if err != nil {
return err
Expand Down
297 changes: 297 additions & 0 deletions blob/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,297 @@
package blob

import (
"context"
"errors"
"sync/atomic"
"time"

"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
"go.uber.org/fx"
)

var meter = otel.Meter("blob")

// Metrics tracks blob-related metrics
type Metrics struct {
// Submission metrics
submissionCounter metric.Int64Counter
submissionDuration metric.Float64Histogram
submissionErrors metric.Int64Counter
submissionBlobCount metric.Int64Counter
submissionBlobSize metric.Int64Counter

// Retrieval metrics
retrievalCounter metric.Int64Counter
retrievalDuration metric.Float64Histogram
retrievalErrors metric.Int64Counter
retrievalNotFound metric.Int64Counter

// Proof metrics
proofCounter metric.Int64Counter
proofDuration metric.Float64Histogram
proofErrors metric.Int64Counter

// Internal counters (thread-safe)
totalSubmissions atomic.Int64
totalSubmissionErrors atomic.Int64
totalRetrievals atomic.Int64
totalRetrievalErrors atomic.Int64
totalProofs atomic.Int64
totalProofErrors atomic.Int64
}

// WithMetrics registers blob metrics
func WithMetrics(lc fx.Lifecycle) (*Metrics, error) {
// Submission metrics
submissionCounter, err := meter.Int64Counter(
"blob_submission_total",
metric.WithDescription("Total number of blob submissions"),
)
if err != nil {
return nil, err
}

submissionDuration, err := meter.Float64Histogram(
"blob_submission_duration_seconds",
metric.WithDescription("Duration of blob submission operations"),
metric.WithUnit("s"),
)
if err != nil {
return nil, err
}

submissionErrors, err := meter.Int64Counter(
"blob_submission_errors_total",
metric.WithDescription("Total number of blob submission errors"),
)
if err != nil {
return nil, err
}

submissionBlobCount, err := meter.Int64Counter(
"blob_submission_blob_count_total",
metric.WithDescription("Total number of blobs submitted"),
)
if err != nil {
return nil, err
}

submissionBlobSize, err := meter.Int64Counter(
"blob_submission_blob_size_bytes_total",
metric.WithDescription("Total size of blobs submitted in bytes"),
)
if err != nil {
return nil, err
}

// Retrieval metrics
retrievalCounter, err := meter.Int64Counter(
"blob_retrieval_total",
metric.WithDescription("Total number of blob retrieval operations"),
)
if err != nil {
return nil, err
}

retrievalDuration, err := meter.Float64Histogram(
"blob_retrieval_duration_seconds",
metric.WithDescription("Duration of blob retrieval operations"),
metric.WithUnit("s"),
)
if err != nil {
return nil, err
}

retrievalErrors, err := meter.Int64Counter(
"blob_retrieval_errors_total",
metric.WithDescription("Total number of blob retrieval errors"),
)
if err != nil {
return nil, err
}

retrievalNotFound, err := meter.Int64Counter(
"blob_retrieval_not_found_total",
metric.WithDescription("Total number of blob not found errors"),
)
if err != nil {
return nil, err
}

// Proof metrics
proofCounter, err := meter.Int64Counter(
"blob_proof_total",
metric.WithDescription("Total number of blob proof operations"),
)
if err != nil {
return nil, err
}

proofDuration, err := meter.Float64Histogram(
"blob_proof_duration_seconds",
metric.WithDescription("Duration of blob proof operations"),
metric.WithUnit("s"),
)
if err != nil {
return nil, err
}

proofErrors, err := meter.Int64Counter(
"blob_proof_errors_total",
metric.WithDescription("Total number of blob proof errors"),
)
if err != nil {
return nil, err
}

metrics := &Metrics{
submissionCounter: submissionCounter,
submissionDuration: submissionDuration,
submissionErrors: submissionErrors,
submissionBlobCount: submissionBlobCount,
submissionBlobSize: submissionBlobSize,
retrievalCounter: retrievalCounter,
retrievalDuration: retrievalDuration,
retrievalErrors: retrievalErrors,
retrievalNotFound: retrievalNotFound,
proofCounter: proofCounter,
proofDuration: proofDuration,
proofErrors: proofErrors,
}

// Register observable metrics
submissionTotal, err := meter.Int64ObservableCounter(
"blob_submission_total_observable",
metric.WithDescription("Observable total number of blob submissions"),
)
if err != nil {
return nil, err
}

retrievalTotal, err := meter.Int64ObservableCounter(
"blob_retrieval_total_observable",
metric.WithDescription("Observable total number of blob retrievals"),
)
if err != nil {
return nil, err
}

proofTotal, err := meter.Int64ObservableCounter(
"blob_proof_total_observable",
metric.WithDescription("Observable total number of blob proofs"),
)
if err != nil {
return nil, err
}

callback := func(_ context.Context, observer metric.Observer) error {
observer.ObserveInt64(submissionTotal, metrics.totalSubmissions.Load())
observer.ObserveInt64(retrievalTotal, metrics.totalRetrievals.Load())
observer.ObserveInt64(proofTotal, metrics.totalProofs.Load())
return nil
}

clientReg, err := meter.RegisterCallback(callback, submissionTotal, retrievalTotal, proofTotal)
if err != nil {
return nil, err
}

lc.Append(fx.Hook{
OnStop: func(context.Context) error {
return clientReg.Unregister()
},
})

return metrics, nil
}

// ObserveSubmission records blob submission metrics
func (m *Metrics) ObserveSubmission(
ctx context.Context,
duration time.Duration,
blobCount int,
totalSize int64,
err error,
) {
if m == nil {
return
}

// Update counters
m.totalSubmissions.Add(1)
if err != nil {
m.totalSubmissionErrors.Add(1)
}

// Record metrics
attrs := []attribute.KeyValue{
attribute.Int("blob_count", blobCount),
attribute.Int64("total_size_bytes", totalSize),
}

if err != nil {
attrs = append(attrs, attribute.String("error", err.Error()))
m.submissionErrors.Add(ctx, 1, metric.WithAttributes(attrs...))
} else {
m.submissionCounter.Add(ctx, 1, metric.WithAttributes(attrs...))
}

m.submissionDuration.Record(ctx, duration.Seconds(), metric.WithAttributes(attrs...))
m.submissionBlobCount.Add(ctx, int64(blobCount), metric.WithAttributes(attrs...))
m.submissionBlobSize.Add(ctx, totalSize, metric.WithAttributes(attrs...))
}

// ObserveRetrieval records blob retrieval metrics
func (m *Metrics) ObserveRetrieval(ctx context.Context, duration time.Duration, err error) {
if m == nil {
return
}

// Update counters
m.totalRetrievals.Add(1)
if err != nil {
m.totalRetrievalErrors.Add(1)
}

// Record metrics
attrs := []attribute.KeyValue{}
if err != nil {
attrs = append(attrs, attribute.String("error", err.Error()))
if errors.Is(err, ErrBlobNotFound) {
m.retrievalNotFound.Add(ctx, 1, metric.WithAttributes(attrs...))
} else {
m.retrievalErrors.Add(ctx, 1, metric.WithAttributes(attrs...))
}
} else {
m.retrievalCounter.Add(ctx, 1, metric.WithAttributes(attrs...))
}

m.retrievalDuration.Record(ctx, duration.Seconds(), metric.WithAttributes(attrs...))
}

// ObserveProof records blob proof metrics
func (m *Metrics) ObserveProof(ctx context.Context, duration time.Duration, err error) {
if m == nil {
return
}

// Update counters
m.totalProofs.Add(1)
if err != nil {
m.totalProofErrors.Add(1)
}

// Record metrics
attrs := []attribute.KeyValue{}
if err != nil {
attrs = append(attrs, attribute.String("error", err.Error()))
m.proofErrors.Add(ctx, 1, metric.WithAttributes(attrs...))
} else {
m.proofCounter.Add(ctx, 1, metric.WithAttributes(attrs...))
}

m.proofDuration.Record(ctx, duration.Seconds(), metric.WithAttributes(attrs...))
}
Loading
Loading