Skip to content

Commit

Permalink
Remove deprecated storage backends
Browse files Browse the repository at this point in the history 
Ever since we published ADR #2 which introduced LocalBlobAccess,
ShardingBlobAccess and MirroredBlobAccess, the writing has been on the
wall that there would no longer be any place for backends such as Redis
and SizeDistinguishing.

The main reason I kept these two backends around, was because people in
the community were still interested in seeing S3 support reappear.
Because I have not received any serious proposal from members within the
community to do this in a way that it meets the standards for inclusion,
I think we've now reached the point where we can assume no work is going
to happen in this area in the short term.

Furthermore, the HTTP backend was mainly of use to talk to bazel-remote,
but as far as I know, that also supports gRPC perfectly fine nowadays.
There is absolutely no reason to prefer HTTP over gRPC. The latter
supports doing bulk existence checks, querying capabilities, while the
former does not.
  • Loading branch information
@EdSchouten
EdSchouten committed Sep 3, 2023
1 parent a9ac0c7 commit e483745
Show file tree
Hide file tree
Showing 12 changed files with 916 additions and 2,200 deletions.
9 changes: 5 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@ does, however, facilitate remote execution by allowing execution
requests to be forwarded to a separate remote execution service.

This storage daemon can be configured to use a whole series of backends.
Examples include networked systems like Redis, but also a local on-disk
storage backend that writes data to a large file, using a hash table as
an index. This storage backend is self-cleaning; no garbage collection
is needed. The [schema of the storage configuration file](https://github.com/buildbarn/bb-storage/blob/master/pkg/proto/configuration/blobstore/blobstore.proto)
Examples include a backend that forwards traffic over gRPC, but also a
local on-disk storage backend that writes data to a large file, using a
hash table as an index. This storage backend is self-cleaning; no
garbage collection is needed. The
[schema of the storage configuration file](https://github.com/buildbarn/bb-storage/blob/master/pkg/proto/configuration/blobstore/blobstore.proto)
gives a good overview of which storage backends are available and how
they can be configured.

Expand Down
10 changes: 0 additions & 10 deletions internal/mock/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,14 +238,6 @@ gomock(
package = "mock",
)

gomock(
name = "redis",
out = "redis.go",
interfaces = ["RedisClient"],
library = "//pkg/blobstore",
package = "mock",
)

gomock(
name = "remoteexecution",
out = "remoteexecution.go",
Expand Down Expand Up @@ -305,7 +297,6 @@ go_library(
"grpc_go.go",
"jwt.go",
"random.go",
"redis.go",
"remoteexecution.go",
"trace.go",
"util.go",
Expand All @@ -331,7 +322,6 @@ go_library(
"//pkg/util",
"@com_github_aws_aws_sdk_go_v2_service_s3//:s3",
"@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",
"@com_github_go_redis_redis_v8//:redis",
"@com_github_golang_mock//gomock",
"@com_github_google_uuid//:uuid",
"@com_google_cloud_go_longrunning//autogen/longrunningpb",
Expand Down
6 changes: 0 additions & 6 deletions pkg/blobstore/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,12 @@ go_library(
"existence_caching_blob_access.go",
"fsac_read_buffer_factory.go",
"hierarchical_instance_names_blob_access.go",
"http_blob_access.go",
"icas_read_buffer_factory.go",
"iscc_read_buffer_factory.go",
"metrics_blob_access.go",
"read_buffer_factory.go",
"read_canarying_blob_access.go",
"redis_blob_access.go",
"reference_expanding_blob_access.go",
"size_distinguishing_blob_access.go",
"validation_caching_read_buffer_factory.go",
"zip_reading_blob_access.go",
"zip_writing_blob_access.go",
Expand All @@ -47,15 +44,13 @@ go_library(
"@com_github_aws_aws_sdk_go_v2//aws",
"@com_github_aws_aws_sdk_go_v2_service_s3//:s3",
"@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",
"@com_github_go_redis_redis_v8//:redis",
"@com_github_klauspost_compress//zstd",
"@com_github_prometheus_client_golang//prometheus",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//status",
"@org_golang_google_protobuf//encoding/protowire",
"@org_golang_google_protobuf//proto",
"@org_golang_google_protobuf//types/known/timestamppb",
"@org_golang_x_sync//errgroup",
],
)

Expand All @@ -70,7 +65,6 @@ go_test(
"existence_caching_blob_access_test.go",
"hierarchical_instance_names_blob_access_test.go",
"read_canarying_blob_access_test.go",
"redis_blob_access_test.go",
"reference_expanding_blob_access_test.go",
"validation_caching_read_buffer_factory_test.go",
"zip_reading_blob_access_test.go",
Expand Down
2 changes: 0 additions & 2 deletions pkg/blobstore/configuration/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,6 @@ go_library(
"@com_github_aws_aws_sdk_go_v2_service_s3//:s3",
"@com_github_bazelbuild_remote_apis//build/bazel/remote/execution/v2:execution",
"@com_github_fxtlabs_primes//:primes",
"@com_github_go_redis_redis_extra_redisotel//:redisotel",
"@com_github_go_redis_redis_v8//:redis",
"@com_github_google_uuid//:uuid",
"@com_google_cloud_go_storage//:storage",
"@org_golang_google_grpc//codes",
Expand Down
142 changes: 0 additions & 142 deletions pkg/blobstore/configuration/new_blob_access.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package configuration
import (
"archive/zip"
"context"
"net/http"
"os"
"sync"
"time"
Expand All @@ -20,15 +19,12 @@ import (
"github.com/buildbarn/bb-storage/pkg/eviction"
"github.com/buildbarn/bb-storage/pkg/filesystem"
"github.com/buildbarn/bb-storage/pkg/grpc"
bb_http "github.com/buildbarn/bb-storage/pkg/http"
"github.com/buildbarn/bb-storage/pkg/program"
pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/blobstore"
digest_pb "github.com/buildbarn/bb-storage/pkg/proto/configuration/digest"
"github.com/buildbarn/bb-storage/pkg/random"
"github.com/buildbarn/bb-storage/pkg/util"
"github.com/fxtlabs/primes"
"github.com/go-redis/redis/extra/redisotel"
"github.com/go-redis/redis/v8"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
Expand All @@ -42,12 +38,6 @@ type BlobAccessInfo struct {
DigestKeyFormat digest.KeyFormat
}

func newRedisClient(opt *redis.Options) *redis.Client {
client := redis.NewClient(opt)
client.AddHook(redisotel.TracingHook{})
return client
}

func newCachedReadBufferFactory(cacheConfiguration *digest_pb.ExistenceCacheConfiguration, baseReadBufferFactory blobstore.ReadBufferFactory, digestKeyFormat digest.KeyFormat) (blobstore.ReadBufferFactory, error) {
if cacheConfiguration == nil {
// No caching enabled.
Expand Down Expand Up @@ -93,125 +83,6 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration *
BlobAccess: readcaching.NewReadCachingBlobAccess(slow.BlobAccess, fast.BlobAccess, replicator),
DigestKeyFormat: slow.DigestKeyFormat,
}, "read_caching", nil
case *pb.BlobAccessConfiguration_Redis:
tlsConfig, err := util.NewTLSConfigFromClientConfiguration(backend.Redis.Tls)
if err != nil {
return BlobAccessInfo{}, "", util.StatusWrap(err, "Failed to obtain TLS configuration")
}

var replicationTimeout time.Duration
if backend.Redis.ReplicationTimeout != nil {
if err := backend.Redis.ReplicationTimeout.CheckValid(); err != nil {
return BlobAccessInfo{}, "", util.StatusWrap(err, "Failed to obtain replication timeout")
}
replicationTimeout = backend.Redis.ReplicationTimeout.AsDuration()
}

var dialTimeout time.Duration
if backend.Redis.DialTimeout != nil {
if err := backend.Redis.DialTimeout.CheckValid(); err != nil {
return BlobAccessInfo{}, "", util.StatusWrap(err, "Failed to obtain dial timeout configuration")
}
dialTimeout = backend.Redis.DialTimeout.AsDuration()
}

var readTimeout time.Duration
if backend.Redis.ReadTimeout != nil {
if err := backend.Redis.ReadTimeout.CheckValid(); err != nil {
return BlobAccessInfo{}, "", util.StatusWrap(err, "Failed to obtain read timeout configuration")
}
readTimeout = backend.Redis.ReadTimeout.AsDuration()
}

var writeTimeout time.Duration
if backend.Redis.WriteTimeout != nil {
if err := backend.Redis.WriteTimeout.CheckValid(); err != nil {
return BlobAccessInfo{}, "", util.StatusWrap(err, "Failed to obtain write timeout configuration")
}
writeTimeout = backend.Redis.WriteTimeout.AsDuration()
}

var redisClient blobstore.RedisClient
switch mode := backend.Redis.Mode.(type) {
case *pb.RedisBlobAccessConfiguration_Clustered:
// Gather retry configuration (min/max delay and overall retry attempts)
minRetryDur := time.Millisecond * 32
if mode.Clustered.MinimumRetryBackoff != nil {
if err := mode.Clustered.MinimumRetryBackoff.CheckValid(); err != nil {
return BlobAccessInfo{}, "", util.StatusWrap(err, "Failed to obtain minimum retry back off configuration")
}
minRetryDur = mode.Clustered.MinimumRetryBackoff.AsDuration()
}

maxRetryDur := time.Millisecond * 2048
if mode.Clustered.MaximumRetryBackoff != nil {
if err := mode.Clustered.MaximumRetryBackoff.CheckValid(); err != nil {
return BlobAccessInfo{}, "", util.StatusWrap(err, "Failed to obtain maximum retry back off")
}
maxRetryDur = mode.Clustered.MaximumRetryBackoff.AsDuration()
}

maxRetries := 16 // Default will be 16
if mode.Clustered.MaximumRetries != 0 {
maxRetries = int(mode.Clustered.MaximumRetries)
}

redisClient = redis.NewClusterClient(
&redis.ClusterOptions{
Addrs: mode.Clustered.Endpoints,
TLSConfig: tlsConfig,
ReadOnly: true,
MaxRetries: maxRetries,
MinRetryBackoff: minRetryDur,
MaxRetryBackoff: maxRetryDur,
DialTimeout: dialTimeout,
ReadTimeout: readTimeout,
WriteTimeout: writeTimeout,
NewClient: newRedisClient,
})

case *pb.RedisBlobAccessConfiguration_Single:
redisClient = newRedisClient(
&redis.Options{
Addr: mode.Single.Endpoint,
Password: mode.Single.Password,
DB: int(mode.Single.Db),
TLSConfig: tlsConfig,
DialTimeout: dialTimeout,
ReadTimeout: readTimeout,
WriteTimeout: writeTimeout,
})
default:
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Redis configuration must either be clustered or single server")
}

digestKeyFormat := creator.GetBaseDigestKeyFormat()
return BlobAccessInfo{
BlobAccess: blobstore.NewRedisBlobAccess(
redisClient,
readBufferFactory,
digestKeyFormat,
backend.Redis.ReplicationCount,
replicationTimeout,
creator.GetDefaultCapabilitiesProvider()),
DigestKeyFormat: digestKeyFormat,
}, "redis", nil
case *pb.BlobAccessConfiguration_Http:
roundTripper, err := bb_http.NewRoundTripperFromConfiguration(backend.Http.Client)
if err != nil {
return BlobAccessInfo{}, "", util.StatusWrap(err, "Failed to create HTTP client")
}
return BlobAccessInfo{
BlobAccess: blobstore.NewHTTPBlobAccess(
backend.Http.Address,
storageTypeName,
readBufferFactory,
&http.Client{
Transport: bb_http.NewMetricsRoundTripper(roundTripper, "HTTPBlobAccess"),
},
creator.GetDefaultCapabilitiesProvider()),
DigestKeyFormat: digest.KeyWithInstance,
}, "remote", nil
case *pb.BlobAccessConfiguration_Sharding:
backends := make([]blobstore.BlobAccess, 0, len(backend.Sharding.Shards))
weights := make([]uint32, 0, len(backend.Sharding.Shards))
Expand Down Expand Up @@ -250,19 +121,6 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration *
backend.Sharding.HashInitialization),
DigestKeyFormat: *combinedDigestKeyFormat,
}, "sharding", nil
case *pb.BlobAccessConfiguration_SizeDistinguishing:
small, err := nc.NewNestedBlobAccess(backend.SizeDistinguishing.Small, creator)
if err != nil {
return BlobAccessInfo{}, "", err
}
large, err := nc.NewNestedBlobAccess(backend.SizeDistinguishing.Large, creator)
if err != nil {
return BlobAccessInfo{}, "", err
}
return BlobAccessInfo{
BlobAccess: blobstore.NewSizeDistinguishingBlobAccess(small.BlobAccess, large.BlobAccess, backend.SizeDistinguishing.CutoffSizeBytes),
DigestKeyFormat: small.DigestKeyFormat.Combine(large.DigestKeyFormat),
}, "size_distinguishing", nil
case *pb.BlobAccessConfiguration_Mirrored:
backendA, err := nc.NewNestedBlobAccess(backend.Mirrored.BackendA, creator)
if err != nil {
Expand Down
Loading

0 comments on commit e483745

Please sign in to comment.